1/* $NetBSD: locore.S,v 1.200 2025/05/06 04:34:58 imil Exp $ */ 2 3/* 4 * Copyright-o-rama! 5 */ 6 7/* 8 * Copyright (c) 1998, 2000, 2004, 2006, 2007, 2009, 2016 9 * The NetBSD Foundation, Inc., All rights reserved. 10 * 11 * This code is derived from software contributed to The NetBSD Foundation 12 * by Charles M. Hannum, by Andrew Doran and by Maxime Villard. 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 24 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 25 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 26 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 27 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 28 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 29 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 30 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 31 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 32 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 33 * POSSIBILITY OF SUCH DAMAGE. 34 */ 35 36/* 37 * Copyright (c) 2006 Manuel Bouyer. 38 * 39 * Redistribution and use in source and binary forms, with or without 40 * modification, are permitted provided that the following conditions 41 * are met: 42 * 1. Redistributions of source code must retain the above copyright 43 * notice, this list of conditions and the following disclaimer. 44 * 2. Redistributions in binary form must reproduce the above copyright 45 * notice, this list of conditions and the following disclaimer in the 46 * documentation and/or other materials provided with the distribution. 47 * 48 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 49 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 50 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 51 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 52 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 53 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 54 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 55 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 56 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 57 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 58 * 59 */ 60 61/* 62 * Copyright (c) 2001 Wasabi Systems, Inc. 63 * All rights reserved. 64 * 65 * Written by Frank van der Linden for Wasabi Systems, Inc. 66 * 67 * Redistribution and use in source and binary forms, with or without 68 * modification, are permitted provided that the following conditions 69 * are met: 70 * 1. Redistributions of source code must retain the above copyright 71 * notice, this list of conditions and the following disclaimer. 72 * 2. Redistributions in binary form must reproduce the above copyright 73 * notice, this list of conditions and the following disclaimer in the 74 * documentation and/or other materials provided with the distribution. 75 * 3. All advertising materials mentioning features or use of this software 76 * must display the following acknowledgement: 77 * This product includes software developed for the NetBSD Project by 78 * Wasabi Systems, Inc. 79 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 80 * or promote products derived from this software without specific prior 81 * written permission. 82 * 83 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 84 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 85 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 86 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 87 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 88 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 89 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 90 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 91 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 92 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 93 * POSSIBILITY OF SUCH DAMAGE. 94 */ 95 96/*- 97 * Copyright (c) 1990 The Regents of the University of California. 98 * All rights reserved. 99 * 100 * This code is derived from software contributed to Berkeley by 101 * William Jolitz. 102 * 103 * Redistribution and use in source and binary forms, with or without 104 * modification, are permitted provided that the following conditions 105 * are met: 106 * 1. Redistributions of source code must retain the above copyright 107 * notice, this list of conditions and the following disclaimer. 108 * 2. Redistributions in binary form must reproduce the above copyright 109 * notice, this list of conditions and the following disclaimer in the 110 * documentation and/or other materials provided with the distribution. 111 * 3. Neither the name of the University nor the names of its contributors 112 * may be used to endorse or promote products derived from this software 113 * without specific prior written permission. 114 * 115 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 116 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 117 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 118 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 119 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 120 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 121 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 122 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 123 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 124 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 125 * SUCH DAMAGE. 126 * 127 * @(#)locore.s 7.3 (Berkeley) 5/13/91 128 */ 129 130#include <machine/asm.h> 131__KERNEL_RCSID(0, "$NetBSD: locore.S,v 1.200 2025/05/06 04:34:58 imil Exp $"); 132 133#include "opt_copy_symtab.h" 134#include "opt_ddb.h" 135#include "opt_modular.h" 136#include "opt_multiboot.h" 137#include "opt_realmem.h" 138#include "opt_xen.h" 139 140#include "assym.h" 141#include "lapic.h" 142#include "ioapic.h" 143#include "ksyms.h" 144 145#include <sys/errno.h> 146#include <sys/syscall.h> 147 148#include <machine/segments.h> 149#include <machine/specialreg.h> 150#include <machine/trap.h> 151#include <machine/i82489reg.h> 152#include <machine/frameasm.h> 153#include <machine/i82489reg.h> 154#include <machine/cputypes.h> 155 156#ifndef XENPV 157#include <machine/multiboot.h> 158#endif 159 160/* Get definitions for IOM_BEGIN, IOM_END, and IOM_SIZE */ 161#include <dev/isa/isareg.h> 162 163#ifndef XENPV 164#define _RELOC(x) ((x) - KERNBASE) 165#else 166#define _RELOC(x) ((x)) 167#endif /* XENPV */ 168#define RELOC(x) _RELOC(_C_LABEL(x)) 169 170/* 32bit version of PTE_NX */ 171#define PTE_NX32 0x80000000 172 173#ifndef PAE 174#define PROC0_PDIR_OFF 0 175#else 176#define PROC0_L3_OFF 0 177#define PROC0_PDIR_OFF 1 * PAGE_SIZE 178#endif 179 180#define PROC0_STK_OFF (PROC0_PDIR_OFF + PDP_SIZE * PAGE_SIZE) 181#define PROC0_PTP1_OFF (PROC0_STK_OFF + UPAGES * PAGE_SIZE) 182 183/* 184 * fillkpt - Fill in a kernel page table 185 * eax = pte (page frame | control | status) 186 * ebx = page table address 187 * ecx = number of pages to map 188 * 189 * For PAE, each entry is 8 bytes long: we must set the 4 upper bytes to 0. 190 * This is done by the first instruction of fillkpt. In the non-PAE case, this 191 * instruction just clears the page table entry. 192 */ 193#define fillkpt \ 194 cmpl $0,%ecx ; /* zero-sized? */ \ 195 je 2f ; \ 1961: movl $0,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: 0 */ \ 197 movl %eax,(%ebx) ; /* store phys addr */ \ 198 addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \ 199 addl $PAGE_SIZE,%eax ; /* next phys page */ \ 200 loop 1b ; \ 2012: ; 202 203/* 204 * fillkpt_nox - Same as fillkpt, but sets the NX/XD bit. 205 */ 206#define fillkpt_nox \ 207 cmpl $0,%ecx ; /* zero-sized? */ \ 208 je 2f ; \ 209 pushl %ebp ; \ 210 movl RELOC(nox_flag),%ebp ; \ 2111: movl %ebp,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: NX */ \ 212 movl %eax,(%ebx) ; /* store phys addr */ \ 213 addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \ 214 addl $PAGE_SIZE,%eax ; /* next phys page */ \ 215 loop 1b ; \ 216 popl %ebp ; \ 2172: ; 218 219/* 220 * fillkpt_blank - Fill in a kernel page table with blank entries 221 * ebx = page table address 222 * ecx = number of pages to map 223 */ 224#define fillkpt_blank \ 225 cmpl $0,%ecx ; /* zero-sized? */ \ 226 je 2f ; \ 2271: movl $0,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: 0 */ \ 228 movl $0,(%ebx) ; /* lower 32 bits: 0 */ \ 229 addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \ 230 loop 1b ; \ 2312: ; 232 233/* 234 * killkpt - Destroy a kernel page table 235 * ebx = page table address 236 * ecx = number of pages to destroy 237 */ 238#define killkpt \ 2391: movl $0,(PDE_SIZE-4)(%ebx) ; /* upper bits (for PAE) */ \ 240 movl $0,(%ebx) ; \ 241 addl $PDE_SIZE,%ebx ; \ 242 loop 1b ; 243 244/* record boot start cycle count */ 245#define getstarttsc \ 246 rdtsc ; \ 247 movl %eax, RELOC(starttsc_lo) ; \ 248 movl %edx, RELOC(starttsc_hi) ; 249 250#ifdef XEN 251#define __ASSEMBLY__ 252#include <xen/include/public/arch-x86/cpuid.h> 253#include <xen/include/public/elfnote.h> 254#include <xen/include/public/xen.h> 255 256#define ELFNOTE(name, type, desctype, descdata...) \ 257.pushsection .note.name, "a", @note ; \ 258 .align 4 ; \ 259 .long 2f - 1f /* namesz */ ; \ 260 .long 4f - 3f /* descsz */ ; \ 261 .long type ; \ 2621:.asciz #name ; \ 2632:.align 4 ; \ 2643:desctype descdata ; \ 2654:.align 4 ; \ 266.popsection 267 268/* 269 * Xen guest identifier and loader selection 270 */ 271.section __xen_guest 272 ELFNOTE(Xen, XEN_ELFNOTE_GUEST_OS, .asciz, "NetBSD") 273 ELFNOTE(Xen, XEN_ELFNOTE_GUEST_VERSION, .asciz, "4.99") 274 ELFNOTE(Xen, XEN_ELFNOTE_XEN_VERSION, .asciz, "xen-3.0") 275 ELFNOTE(Xen, XEN_ELFNOTE_VIRT_BASE, .long, KERNBASE) 276#ifdef XENPV 277 ELFNOTE(Xen, XEN_ELFNOTE_PADDR_OFFSET, .long, KERNBASE) 278 ELFNOTE(Xen, XEN_ELFNOTE_ENTRY, .long, start) 279#else 280 ELFNOTE(Xen, XEN_ELFNOTE_PADDR_OFFSET, .long, 0) 281 ELFNOTE(Xen, XEN_ELFNOTE_PHYS32_ENTRY, .long, RELOC(start_pvh)) 282#endif /* XENPV */ 283 ELFNOTE(Xen, XEN_ELFNOTE_HYPERCALL_PAGE, .long, hypercall_page) 284 ELFNOTE(Xen, XEN_ELFNOTE_HV_START_LOW, .long, HYPERVISOR_VIRT_START) 285 ELFNOTE(Xen, XEN_ELFNOTE_FEATURES, .asciz, "writable_descriptor_tables|auto_translated_physmap|supervisor_mode_kernel|hvm_callback_vector") 286 ELFNOTE(Xen, XEN_ELFNOTE_PAE_MODE, .asciz, "yes") 287 ELFNOTE(Xen, XEN_ELFNOTE_L1_MFN_VALID, .quad, PTE_P, PTE_P)\ 288 ELFNOTE(Xen, XEN_ELFNOTE_LOADER, .asciz, "generic") 289 ELFNOTE(Xen, XEN_ELFNOTE_SUSPEND_CANCEL, .long, 0) 290#if NKSYMS > 0 || defined(DDB) || defined(MODULAR) 291 ELFNOTE(Xen, XEN_ELFNOTE_BSD_SYMTAB, .asciz, "yes") 292#endif 293#endif /* XEN */ 294 295/* 296 * Initialization 297 */ 298 .data 299 300 .globl _C_LABEL(tablesize) 301 .globl _C_LABEL(nox_flag) 302 .globl _C_LABEL(cputype) 303 .globl _C_LABEL(cpuid_level) 304 .globl _C_LABEL(esym) 305 .globl _C_LABEL(eblob) 306 .globl _C_LABEL(atdevbase) 307 .globl _C_LABEL(PDPpaddr) 308 .globl _C_LABEL(lwp0uarea) 309 .globl _C_LABEL(gdt) 310 .globl _C_LABEL(idt) 311 312 .type _C_LABEL(tablesize), @object 313_C_LABEL(tablesize): .long 0 314END(tablesize) 315 .type _C_LABEL(nox_flag), @object 316LABEL(nox_flag) .long 0 /* 32bit NOX flag, set if supported */ 317END(nox_flag) 318 .type _C_LABEL(cputype), @object 319LABEL(cputype) .long 0 /* are we 80486, Pentium, or.. */ 320END(cputype) 321 .type _C_LABEL(cpuid_level), @object 322LABEL(cpuid_level) .long -1 /* max. level accepted by cpuid instr */ 323END(cpuid_level) 324 .type _C_LABEL(atdevbase), @object 325LABEL(atdevbase) .long 0 /* location of start of iomem in virt */ 326END(atdevbase) 327 .type _C_LABEL(lwp0uarea), @object 328LABEL(lwp0uarea) .long 0 329END(lwp0uarea) 330 .type _C_LABEL(PDPpaddr), @object 331LABEL(PDPpaddr) .long 0 /* paddr of PDP, for libkvm */ 332END(PDPpaddr) 333 .type _C_LABEL(starttsc_lo), @object 334LABEL(starttsc_lo) .long 0 /* low part of rdtsc */ 335END(starttsc_lo) 336 .type _C_LABEL(starttsc_hi), @object 337LABEL(starttsc_hi) .long 0 /* high part of rdtsc */ 338END(starttsc_hi) 339 340 /* Space for the temporary stack */ 341 .globl _C_LABEL(tmpstk) 342 .size tmpstk, tmpstk - . 343 .space 512 344tmpstk: 345#ifdef XENPV 346 .align PAGE_SIZE, 0x0 /* Align on page boundary */ 347LABEL(tmpgdt) 348 .space PAGE_SIZE /* Xen expects a page */ 349END(tmpgdt) 350#endif /* XENPV */ 351 352 .text 353 .globl _C_LABEL(kernel_text) 354 .set _C_LABEL(kernel_text),KERNTEXTOFF 355 356ENTRY(start) 357#ifndef XENPV 358 359 getstarttsc 360 361 /* Warm boot */ 362 movw $0x1234,0x472 363 364#if defined(MULTIBOOT) 365 jmp 1f 366 367 .align 4 368 .globl Multiboot_Header 369_C_LABEL(Multiboot_Header): 370#define MULTIBOOT_HEADER_FLAGS (MULTIBOOT_HEADER_WANT_MEMORY) 371 .long MULTIBOOT_HEADER_MAGIC 372 .long MULTIBOOT_HEADER_FLAGS 373 .long -(MULTIBOOT_HEADER_MAGIC + MULTIBOOT_HEADER_FLAGS) 374 375 .align 8 376 .globl Multiboot2_Header 377_C_LABEL(Multiboot2_Header): 378 .long MULTIBOOT2_HEADER_MAGIC 379 .long MULTIBOOT2_ARCHITECTURE_I386 380 .long Multiboot2_Header_end - Multiboot2_Header 381 .long -(MULTIBOOT2_HEADER_MAGIC + MULTIBOOT2_ARCHITECTURE_I386 \ 382 + (Multiboot2_Header_end - Multiboot2_Header)) 383 384 .long 1 /* MULTIBOOT_HEADER_TAG_INFORMATION_REQUEST */ 385 .long 12 /* sizeof(multiboot_header_tag_information_request) */ 386 /* + sizeof(uint32_t) * requests */ 387 .long 4 /* MULTIBOOT_TAG_TYPE_BASIC_MEMINFO */ 388 .long 0 /* pad for 8 bytes alignment */ 389 390 .long 8 /* MULTIBOOT_HEADER_TAG_ENTRY_ADDRESS_EFI32 */ 391 .long 12 /* sizeof(struct multiboot_tag_efi32) */ 392 .long efi_multiboot2_loader - KERNBASE 393 .long 0 /* pad for 8 bytes alignment */ 394 395#if notyet 396 /* 397 * Could be used to get an early console for debug, 398 * but this is broken. 399 */ 400 .long 7 /* MULTIBOOT_HEADER_TAG_EFI_BS */ 401 .long 8 /* sizeof(struct multiboot_tag) */ 402#endif 403 404 .long 0 /* MULTIBOOT_HEADER_TAG_END */ 405 .long 8 /* sizeof(struct multiboot_tag) */ 406 .globl Multiboot2_Header_end 407_C_LABEL(Multiboot2_Header_end): 408 4091: 410 /* Check if we are being executed by a Multiboot-compliant boot 411 * loader. */ 412 cmpl $MULTIBOOT_INFO_MAGIC,%eax 413 je multiboot1_loader 414 415 cmpl $MULTIBOOT2_BOOTLOADER_MAGIC,%eax 416 je multiboot2_loader 417 418 jmp 1f 419 420multiboot1_loader: 421 /* 422 * Indeed, a multiboot-compliant boot loader executed us. We switch 423 * to the temporary stack, and copy the received Multiboot information 424 * structure into kernel's data space to process it later -- after we 425 * are relocated. It will be safer to run complex C code than doing it 426 * at this point. 427 */ 428 movl $_RELOC(tmpstk),%esp 429 pushl %ebx /* Address of Multiboot information */ 430 call _C_LABEL(multiboot1_pre_reloc) 431 addl $4,%esp 432 jmp .Lstart_common 433 434efi_multiboot2_loader: 435 /* 436 * EFI32 multiboot2 entry point. We are left here without 437 * stack and with no idea of where we were loaded in memory. 438 * The only inputs are 439 * %eax MULTIBOOT2_BOOTLOADER_MAGIC 440 * %ebx pointer to multiboot_info 441 * 442 * Here we will copy the kernel to 0x100000 (KERNTEXTOFF - KERNBASE) 443 * as almost all the code in locore.S assume it is there. Once done, 444 * we join the main start code .This is derived from 445 * src/sys/arch/i386/stand/efiboot/bootia32/startprog32.S 446 */ 447 448 cli 449 450 /* 451 * Discover our load address and store it in %edx 452 */ 453 movl $_RELOC(tmpstk),%esp 454 call next 455next: popl %edx 456 subl $(next - efi_multiboot2_loader), %edx 457 458 /* 459 * Save multiboot_info for later. We cannot use 460 * temporary stack for that since we are going to 461 * overwrite it. 462 */ 463 movl %ebx, (multiboot2_info_ptr - efi_multiboot2_loader)(%edx) 464 465 /* 466 * Get relocated multiboot2_loader entry point in %ebx 467 */ 468 movl $(KERNTEXTOFF - KERNBASE), %ebx 469 addl $(multiboot2_loader - start), %ebx 470 471 /* Copy kernel */ 472 movl $(KERNTEXTOFF - KERNBASE), %edi /* dest */ 473 movl %edx, %esi 474 subl $(efi_multiboot2_loader - start), %esi /* src */ 475 movl $(__kernel_end - kernel_text), %ecx /* size */ 476#if defined(NO_OVERLAP) 477 movl %ecx, %eax 478#else 479 movl %edi, %eax 480 subl %esi, %eax 481 cmpl %ecx, %eax /* overlapping? */ 482 movl %ecx, %eax 483 jb .Lbackwards 484#endif 485 /* nope, copy forwards. */ 486 shrl $2, %ecx /* copy by words */ 487 rep 488 movsl 489 and $3, %eax /* any bytes left? */ 490 jnz .Ltrailing 491 jmp .Lcopy_done 492 493.Ltrailing: 494 cmp $2, %eax 495 jb 11f 496 movw (%esi), %ax 497 movw %ax, (%edi) 498 je .Lcopy_done 499 movb 2(%esi), %al 500 movb %al, 2(%edi) 501 jmp .Lcopy_done 50211: movb (%esi), %al 503 movb %al, (%edi) 504 jmp .Lcopy_done 505 506#if !defined(NO_OVERLAP) 507.Lbackwards: 508 addl %ecx, %edi /* copy backwards. */ 509 addl %ecx, %esi 510 and $3, %eax /* any fractional bytes? */ 511 jnz .Lback_align 512.Lback_aligned: 513 shrl $2, %ecx 514 subl $4, %esi 515 subl $4, %edi 516 std 517 rep 518 movsl 519 cld 520 jmp .Lcopy_done 521 522.Lback_align: 523 sub %eax, %esi 524 sub %eax, %edi 525 cmp $2, %eax 526 jb 11f 527 je 12f 528 movb 2(%esi), %al 529 movb %al, 2(%edi) 53012: movw (%esi), %ax 531 movw %ax, (%edi) 532 jmp .Lback_aligned 53311: movb (%esi), %al 534 movb %al, (%edi) 535 jmp .Lback_aligned 536#endif 537 /* End of copy kernel */ 538.Lcopy_done: 539 cld /* LynxOS depends on it */ 540 541 /* Prepare jump address */ 542 lea (efi_multiboot2_loader32a - efi_multiboot2_loader)(%edx), %eax 543 movl %eax, (efi_multiboot2_loader32r - efi_multiboot2_loader)(%edx) 544 545 /* Setup GDT */ 546 lea (gdt - efi_multiboot2_loader)(%edx), %eax 547 movl %eax, (gdtrr - efi_multiboot2_loader)(%edx) 548 lgdt (gdtr - efi_multiboot2_loader)(%edx) 549 550 /* Jump to set %cs */ 551 ljmp *(efi_multiboot2_loader32r - efi_multiboot2_loader)(%edx) 552 553 .align 4 554efi_multiboot2_loader32a: 555 movl $0x10, %eax /* #define DATA_SEGMENT 0x10 */ 556 movw %ax, %ds 557 movw %ax, %es 558 movw %ax, %fs 559 movw %ax, %gs 560 movw %ax, %ss 561 562 /* Already set new stack pointer */ 563 movl %esp, %ebp 564 565 /* Disable Paging in CR0 */ 566 movl %cr0, %eax 567 andl $(~CR0_PG), %eax 568 movl %eax, %cr0 569 570 /* Disable PAE in CR4 */ 571 movl %cr4, %eax 572 andl $(~CR4_PAE), %eax 573 movl %eax, %cr4 574 575 jmp efi_multiboot2_loader32b 576 577 .align 4 578efi_multiboot2_loader32b: 579 xor %eax, %eax 580 movl %ebx, (efi_multiboot2_loader32r - efi_multiboot2_loader)(%edx) 581 /* 582 * Reload multiboot info from target location 583 */ 584 movl _RELOC(multiboot2_info_ptr), %ebx 585 ljmp *(efi_multiboot2_loader32r - efi_multiboot2_loader)(%edx) 586 587 .align 16 588efi_multiboot2_loader32r: 589 .long 0 590 .long 0x08 /* #define CODE_SEGMENT 0x08 */ 591 .align 16 592gdt: 593 .long 0, 0 594 .byte 0xff, 0xff, 0x00, 0x00, 0x00, 0x9f, 0xcf, 0x00 595 .byte 0xff, 0xff, 0x00, 0x00, 0x00, 0x93, 0xcf, 0x00 596gdtr: 597 .word gdtr - gdt 598gdtrr: 599 .quad 0 600multiboot2_info_ptr: 601 .long 0 602 603 .align 16 604multiboot2_loader: 605 movl $_RELOC(tmpstk),%esp 606 pushl %ebx /* Address of Multiboot information */ 607 call _C_LABEL(multiboot2_pre_reloc) 608 addl $4,%esp 609 jmp .Lstart_common 610#endif /* MULTIBOOT */ 611 6121: 613 /* 614 * At this point, we know that a NetBSD-specific boot loader 615 * booted this kernel. 616 * 617 * Load parameters from the stack (32 bits): 618 * boothowto, [bootdev], bootinfo, esym, biosextmem, biosbasemem 619 * We are not interested in 'bootdev'. 620 */ 621 622 addl $4,%esp /* Discard return address to boot loader */ 623 call _C_LABEL(native_loader) 624 addl $24,%esp 625 626.Lstart_common: 627 /* First, reset the PSL. */ 628 pushl $PSL_MBO 629 popfl 630 631 /* Clear segment registers; always null in proc0. */ 632 xorl %eax,%eax 633 movw %ax,%fs 634 movw %ax,%gs 635 636 /* Find out our CPU type. */ 637 638try386: /* Try to toggle alignment check flag; does not exist on 386. */ 639 pushfl 640 popl %eax 641 movl %eax,%ecx 642 orl $PSL_AC,%eax 643 pushl %eax 644 popfl 645 pushfl 646 popl %eax 647 xorl %ecx,%eax 648 andl $PSL_AC,%eax 649 pushl %ecx 650 popfl 651 652 testl %eax,%eax 653 jnz try486 654 655 /* 656 * Try the test of a NexGen CPU -- ZF will not change on a DIV 657 * instruction on a NexGen, it will on an i386. Documented in 658 * Nx586 Processor Recognition Application Note, NexGen, Inc. 659 */ 660 movl $0x5555,%eax 661 xorl %edx,%edx 662 movl $2,%ecx 663 divl %ecx 664 jnz is386 665 666isnx586: 667 /* 668 * Don't try cpuid, as Nx586s reportedly don't support the 669 * PSL_ID bit. 670 */ 671 movl $CPU_NX586,RELOC(cputype) 672 jmp 2f 673 674is386: 675 movl $CPU_386,RELOC(cputype) 676 jmp 2f 677 678try486: /* Try to toggle identification flag; does not exist on early 486s. */ 679 pushfl 680 popl %eax 681 movl %eax,%ecx 682 xorl $PSL_ID,%eax 683 pushl %eax 684 popfl 685 pushfl 686 popl %eax 687 xorl %ecx,%eax 688 andl $PSL_ID,%eax 689 pushl %ecx 690 popfl 691 692 testl %eax,%eax 693 jnz try586 694is486: movl $CPU_486,RELOC(cputype) 695 /* 696 * Check Cyrix CPU 697 * Cyrix CPUs do not change the undefined flags following 698 * execution of the divide instruction which divides 5 by 2. 699 * 700 * Note: CPUID is enabled on M2, so it passes another way. 701 */ 702 pushfl 703 movl $0x5555, %eax 704 xorl %edx, %edx 705 movl $2, %ecx 706 clc 707 divl %ecx 708 jnc trycyrix486 709 popfl 710 jmp 2f 711trycyrix486: 712 movl $CPU_6x86,RELOC(cputype) /* set CPU type */ 713 /* 714 * Check for Cyrix 486 CPU by seeing if the flags change during a 715 * divide. This is documented in the Cx486SLC/e SMM Programmer's 716 * Guide. 717 */ 718 xorl %edx,%edx 719 cmpl %edx,%edx /* set flags to known state */ 720 pushfl 721 popl %ecx /* store flags in ecx */ 722 movl $-1,%eax 723 movl $4,%ebx 724 divl %ebx /* do a long division */ 725 pushfl 726 popl %eax 727 xorl %ecx,%eax /* are the flags different? */ 728 testl $0x8d5,%eax /* only check C|PF|AF|Z|N|V */ 729 jne 2f /* yes; must be Cyrix 6x86 CPU */ 730 movl $CPU_486DLC,RELOC(cputype) /* set CPU type */ 731 732#ifndef CYRIX_CACHE_WORKS 733 /* Disable caching of the ISA hole only. */ 734 invd 735 movb $CCR0,%al /* Configuration Register index (CCR0) */ 736 outb %al,$0x22 737 inb $0x23,%al 738 orb $(CCR0_NC1|CCR0_BARB),%al 739 movb %al,%ah 740 movb $CCR0,%al 741 outb %al,$0x22 742 movb %ah,%al 743 outb %al,$0x23 744 invd 745#else /* CYRIX_CACHE_WORKS */ 746 /* Set cache parameters */ 747 invd /* Start with guaranteed clean cache */ 748 movb $CCR0,%al /* Configuration Register index (CCR0) */ 749 outb %al,$0x22 750 inb $0x23,%al 751 andb $~CCR0_NC0,%al 752#ifndef CYRIX_CACHE_REALLY_WORKS 753 orb $(CCR0_NC1|CCR0_BARB),%al 754#else 755 orb $CCR0_NC1,%al 756#endif 757 movb %al,%ah 758 movb $CCR0,%al 759 outb %al,$0x22 760 movb %ah,%al 761 outb %al,$0x23 762 /* clear non-cacheable region 1 */ 763 movb $(NCR1+2),%al 764 outb %al,$0x22 765 movb $NCR_SIZE_0K,%al 766 outb %al,$0x23 767 /* clear non-cacheable region 2 */ 768 movb $(NCR2+2),%al 769 outb %al,$0x22 770 movb $NCR_SIZE_0K,%al 771 outb %al,$0x23 772 /* clear non-cacheable region 3 */ 773 movb $(NCR3+2),%al 774 outb %al,$0x22 775 movb $NCR_SIZE_0K,%al 776 outb %al,$0x23 777 /* clear non-cacheable region 4 */ 778 movb $(NCR4+2),%al 779 outb %al,$0x22 780 movb $NCR_SIZE_0K,%al 781 outb %al,$0x23 782 /* enable caching in CR0 */ 783 movl %cr0,%eax 784 andl $~(CR0_CD|CR0_NW),%eax 785 movl %eax,%cr0 786 invd 787#endif /* CYRIX_CACHE_WORKS */ 788 789 jmp 2f 790 791try586: /* Use the `cpuid' instruction. */ 792 xorl %eax,%eax 793 cpuid 794 movl %eax,RELOC(cpuid_level) 795 796 /* 797 * Retrieve the NX/XD flag. We use the 32bit version of PTE_NX. 798 */ 799 movl $0x80000001,%eax 800 cpuid 801 andl $CPUID_NOX,%edx 802 jz no_NOX 803 movl $PTE_NX32,RELOC(nox_flag) 804no_NOX: 805 8062: 807 /* 808 * Finished with old stack; load new %esp now instead of later so we 809 * can trace this code without having to worry about the trace trap 810 * clobbering the memory test or the zeroing of the bss+bootstrap page 811 * tables. 812 * 813 * The boot program should check: 814 * text+data <= &stack_variable - more_space_for_stack 815 * text+data+bss+pad+space_for_page_tables <= end_of_memory 816 * 817 * XXX: the gdt is in the carcass of the boot program so clearing 818 * the rest of memory is still not possible. 819 */ 820 movl $_RELOC(tmpstk),%esp 821 822/* 823 * There are two different layouts possible, depending on whether PAE is 824 * enabled or not. 825 * 826 * If PAE is not enabled, there are two levels of pages: PD -> PT. They will 827 * be referred to as: L2 -> L1. L2 is 1 page long. The BOOTSTRAP TABLES have 828 * the following layout: 829 * +-----+------------+----+ 830 * | L2 -> PROC0 STK -> L1 | 831 * +-----+------------+----+ 832 * 833 * If PAE is enabled, there are three levels of pages: PDP -> PD -> PT. They 834 * will be referred to as: L3 -> L2 -> L1. L3 is 1 page long, L2 is 4 page 835 * long. The BOOTSTRAP TABLES have the following layout: 836 * +-----+-----+------------+----+ 837 * | L3 -> L2 -> PROC0 STK -> L1 | 838 * +-----+-----+------------+----+ 839 * 840 * Virtual address space of the kernel in both cases: 841 * +------+--------+------+-----+--------+---------------------+----------- 842 * | TEXT | RODATA | DATA | BSS | [SYMS] | [PRELOADED MODULES] | BOOTSTRAP 843 * +------+--------+------+-----+--------+---------------------+----------- 844 * (1) (2) (3) 845 * 846 * -------+-------------+ 847 * TABLES | ISA I/O MEM | 848 * -------+-------------+ 849 * (4) 850 * 851 * PROC0 STK is obviously not linked as a page level. It just happens to be 852 * caught between L2 and L1. 853 * 854 * Important note: the kernel segments are properly 4k-aligned 855 * (see kern.ldscript), so there's no need to enforce alignment. 856 */ 857 858 /* Find end of kernel image; brings us on (1). */ 859 movl $RELOC(__kernel_end),%edi 860 861#if (NKSYMS || defined(DDB) || defined(MODULAR)) && !defined(makeoptions_COPY_SYMTAB) 862 /* Save the symbols (if loaded); brings us on (2). */ 863 movl RELOC(esym),%eax 864 testl %eax,%eax 865 jz 1f 866 subl $KERNBASE,%eax 867 movl %eax,%edi 8681: 869#endif 870 871 /* Skip over any modules/blobs; brings us on (3). */ 872 movl RELOC(eblob),%eax 873 testl %eax,%eax 874 jz 1f 875 subl $KERNBASE,%eax 876 movl %eax,%edi 8771: 878 879 /* We are on (3). Align up for BOOTSTRAP TABLES. */ 880 movl %edi,%esi 881 addl $PGOFSET,%esi 882 andl $~PGOFSET,%esi 883 884 /* nkptp[1] = (esi + ~L2_FRAME) >> L2_SHIFT + 1; */ 885 movl %esi,%eax 886 addl $~L2_FRAME,%eax 887 shrl $L2_SHIFT,%eax 888 incl %eax /* one more PTP for VAs stolen by bootstrap */ 8891: movl %eax,RELOC(nkptp)+1*4 890 891 /* tablesize = (PDP_SIZE + UPAGES + nkptp[1]) << PGSHIFT; */ 892 addl $(PDP_SIZE+UPAGES),%eax 893#ifdef PAE 894 incl %eax /* one more page for L3 */ 895 shll $PGSHIFT+1,%eax /* PTP tables are twice larger with PAE */ 896#else 897 shll $PGSHIFT,%eax 898#endif 899 movl %eax,RELOC(tablesize) 900 901 /* Ensure that nkptp[1] covers BOOTSTRAP TABLES, ie: 902 * (esi + tablesize) >> L2_SHIFT + 1 < nkptp[1] */ 903 addl %esi,%eax 904 addl $~L2_FRAME,%eax 905 shrl $L2_SHIFT,%eax 906 incl %eax 907 cmpl %eax,RELOC(nkptp)+1*4 908 jnz 1b 909 910 /* Now, zero out the BOOTSTRAP TABLES (before filling them in). */ 911 movl %esi,%edi 912 xorl %eax,%eax 913 cld 914 movl RELOC(tablesize),%ecx 915 shrl $2,%ecx 916 rep 917 stosl /* copy eax -> edi */ 918 919/* 920 * Build the page tables and levels. We go from L1 to L2/L3, and link the levels 921 * together. Note: RELOC computes &addr - KERNBASE in 32 bits; the value can't 922 * be > 4G, or we can't deal with it anyway, since we are in 32bit mode. 923 */ 924 /* 925 * Build L1. 926 */ 927 leal (PROC0_PTP1_OFF)(%esi),%ebx 928 929 /* Skip the area below the kernel text. */ 930 movl $(KERNTEXTOFF - KERNBASE),%ecx 931 shrl $PGSHIFT,%ecx 932 fillkpt_blank 933 934 /* Map the kernel text RX. */ 935 movl $(KERNTEXTOFF - KERNBASE),%eax /* start of TEXT */ 936 movl $RELOC(__rodata_start),%ecx 937 subl %eax,%ecx 938 shrl $PGSHIFT,%ecx 939 orl $(PTE_P),%eax 940 fillkpt 941 942 /* Map the kernel rodata R. */ 943 movl $RELOC(__rodata_start),%eax 944 movl $RELOC(__data_start),%ecx 945 subl %eax,%ecx 946 shrl $PGSHIFT,%ecx 947 orl $(PTE_P),%eax 948 fillkpt_nox 949 950 /* Map the kernel data+bss RW. */ 951 movl $RELOC(__data_start),%eax 952 movl $RELOC(__kernel_end),%ecx 953 subl %eax,%ecx 954 shrl $PGSHIFT,%ecx 955 orl $(PTE_P|PTE_W),%eax 956 fillkpt_nox 957 958 /* Map [SYMS]+[PRELOADED MODULES] RW. */ 959 movl $RELOC(__kernel_end),%eax 960 movl %esi,%ecx /* start of BOOTSTRAP TABLES */ 961 subl %eax,%ecx 962 shrl $PGSHIFT,%ecx 963 orl $(PTE_P|PTE_W),%eax 964 fillkpt_nox 965 966 /* Map the BOOTSTRAP TABLES RW. */ 967 movl %esi,%eax /* start of BOOTSTRAP TABLES */ 968 movl RELOC(tablesize),%ecx /* length of BOOTSTRAP TABLES */ 969 shrl $PGSHIFT,%ecx 970 orl $(PTE_P|PTE_W),%eax 971 fillkpt_nox 972 973 /* We are on (4). Map ISA I/O MEM RW. */ 974 movl $IOM_BEGIN,%eax 975 movl $IOM_SIZE,%ecx /* size of ISA I/O MEM */ 976 shrl $PGSHIFT,%ecx 977 orl $(PTE_P|PTE_W/*|PTE_PCD*/),%eax 978 fillkpt_nox 979 980 /* 981 * Build L2 for identity mapping. Linked to L1. 982 */ 983 leal (PROC0_PDIR_OFF)(%esi),%ebx 984 leal (PROC0_PTP1_OFF)(%esi),%eax 985 orl $(PTE_P|PTE_W),%eax 986 movl RELOC(nkptp)+1*4,%ecx 987 fillkpt 988 989 /* Set up L2 entries for actual kernel mapping */ 990 leal (PROC0_PDIR_OFF + L2_SLOT_KERNBASE * PDE_SIZE)(%esi),%ebx 991 leal (PROC0_PTP1_OFF)(%esi),%eax 992 orl $(PTE_P|PTE_W),%eax 993 movl RELOC(nkptp)+1*4,%ecx 994 fillkpt 995 996 /* Install recursive top level PDE */ 997 leal (PROC0_PDIR_OFF + PDIR_SLOT_PTE * PDE_SIZE)(%esi),%ebx 998 leal (PROC0_PDIR_OFF)(%esi),%eax 999 orl $(PTE_P|PTE_W),%eax 1000 movl $PDP_SIZE,%ecx 1001 fillkpt_nox 1002 1003#ifdef PAE 1004 /* 1005 * Build L3. Linked to L2. 1006 */ 1007 leal (PROC0_L3_OFF)(%esi),%ebx 1008 leal (PROC0_PDIR_OFF)(%esi),%eax 1009 orl $(PTE_P),%eax 1010 movl $PDP_SIZE,%ecx 1011 fillkpt 1012 1013 /* Enable PAE mode */ 1014 movl %cr4,%eax 1015 orl $CR4_PAE,%eax 1016 movl %eax,%cr4 1017#endif 1018 1019 /* Save physical address of L2. */ 1020 leal (PROC0_PDIR_OFF)(%esi),%eax 1021 movl %eax,RELOC(PDPpaddr) 1022 1023 /* 1024 * Startup checklist: 1025 * 1. Load %cr3 with pointer to L2 (or L3 for PAE). 1026 */ 1027 movl %esi,%eax 1028 movl %eax,%cr3 1029 1030 /* 1031 * 2. Set NOX in EFER, if available. 1032 */ 1033 movl RELOC(nox_flag),%ebx 1034 cmpl $0,%ebx 1035 je skip_NOX 1036 movl $MSR_EFER,%ecx 1037 rdmsr 1038 xorl %eax,%eax 1039 orl $(EFER_NXE),%eax 1040 wrmsr 1041skip_NOX: 1042 1043 /* 1044 * 3. Enable paging and the rest of it. 1045 */ 1046 movl %cr0,%eax 1047 orl $(CR0_PE|CR0_PG|CR0_NE|CR0_TS|CR0_MP|CR0_WP|CR0_AM),%eax 1048 movl %eax,%cr0 1049 1050 pushl $begin /* jump to high mem */ 1051 ret 1052 1053begin: 1054 /* 1055 * We have arrived. There's no need anymore for the identity mapping in 1056 * low memory, remove it. 1057 */ 1058 movl _C_LABEL(nkptp)+1*4,%ecx 1059 leal (PROC0_PDIR_OFF)(%esi),%ebx /* old, phys address of PDIR */ 1060 addl $(KERNBASE), %ebx /* new, virt address of PDIR */ 1061 killkpt 1062 1063 /* Relocate atdevbase. */ 1064 movl $KERNBASE,%edx 1065 addl _C_LABEL(tablesize),%edx 1066 addl %esi,%edx 1067 movl %edx,_C_LABEL(atdevbase) 1068 1069 /* Set up bootstrap stack. */ 1070 leal (PROC0_STK_OFF+KERNBASE)(%esi),%eax 1071 movl %eax,_C_LABEL(lwp0uarea) 1072 leal (USPACE-FRAMESIZE)(%eax),%esp 1073 movl %esi,PCB_CR3(%eax) /* pcb->pcb_cr3 */ 1074 xorl %ebp,%ebp /* mark end of frames */ 1075 1076#if defined(MULTIBOOT) 1077 /* It is now safe to parse the Multiboot information structure 1078 * we saved before from C code. Note that we cannot delay its 1079 * parsing any more because initgdt (called below) needs to make 1080 * use of this information. 1081 * We call both multiboot 1 and 2 flavors, they now if they 1082 * have something to do on their own. 1083 */ 1084 call _C_LABEL(multiboot1_post_reloc) 1085 call _C_LABEL(multiboot2_post_reloc) 1086#endif 1087 1088 /* 1089 * Initialize a temporary GDT (Global Descriptor Table) on the 1090 * stack and make the segment registers to use it. 1091 * 1092 * This creates a segment descriptor for the CPU-local segment 1093 * and loads %fs with its segment selector to set up addressing 1094 * for %fs. Thus, after this point, CPUVAR(...), curcpu(), and 1095 * curlwp will work. 1096 * 1097 * Later, we will replace this temporary GDT on the stack by a 1098 * permanent GDT allocated with uvm_km in gdt_init. 1099 * 1100 * XXX Intel recommends ensuring the GDT address is aligned on 1101 * an 8-byte boundary for performance. Perhaps not an issue 1102 * early at boot, but maybe worth doing? 1103 * 1104 * Intel 64 and IA-32 Architectures, Software Developer's 1105 * Manual, Volume 3: System Programming Guide, Order 1106 * Number 325383, April 2022, Sec. 3.5.1 `Segment 1107 * Descriptor Tables', p. 3-15: 1108 * 1109 * The base address of the GDT should be aligned 1110 * on an eight-byte boundary to yield the best 1111 * processor performance. 1112 */ 1113 subl $NGDT*8, %esp /* space for temporary gdt */ 1114 pushl %esp 1115 call _C_LABEL(initgdt) 1116 addl $4,%esp 1117 1118 movl _C_LABEL(tablesize),%eax 1119 addl %esi,%eax /* skip past stack and page tables */ 1120 1121#ifdef PAE 1122 pushl $0 /* init386() expects a 64 bits paddr_t with PAE */ 1123#endif 1124 pushl %eax 1125#if defined(XEN) && !defined(XENPV) 1126 call _C_LABEL(init_xen_early) 1127#endif 1128 call _C_LABEL(init_bootspace) 1129 call _C_LABEL(init386) 1130 addl $PDE_SIZE,%esp /* pop paddr_t */ 1131 addl $NGDT*8,%esp /* pop temporary gdt */ 1132 1133 call _C_LABEL(main) 1134#else /* XENPV */ 1135 /* First, reset the PSL. */ 1136 pushl $PSL_MBO 1137 popfl 1138 1139 cld 1140 1141 /* 1142 * Xen info: 1143 * - %esp -> stack, *theoretically* the last used page by Xen bootstrap 1144 */ 1145 movl %esp,%ebx 1146 movl $_RELOC(tmpstk),%esp 1147 1148 /* Clear BSS. */ 1149 xorl %eax,%eax 1150 movl $RELOC(__bss_start),%edi 1151 movl $RELOC(_end),%ecx 1152 subl %edi,%ecx 1153 rep 1154 stosb 1155 1156 /* Copy the necessary stuff from start_info structure. */ 1157 /* We need to copy shared_info early, so that sti/cli work */ 1158 movl $RELOC(start_info_union),%edi 1159 movl $(PAGE_SIZE / 4),%ecx 1160 rep 1161 movsl 1162 1163 /* Clear segment registers. */ 1164 xorl %eax,%eax 1165 movw %ax,%fs 1166 movw %ax,%gs 1167 1168 xorl %eax,%eax 1169 cpuid 1170 movl %eax,RELOC(cpuid_level) 1171 1172 movl $VM_GUEST_XENPV, RELOC(vm_guest) 1173 1174 /* 1175 * Use a temporary GDT page. We'll re-add it to uvm(9) once we're done 1176 * using it. 1177 */ 1178 movl $RELOC(tmpgdt),%eax 1179 pushl %eax /* start of temporary gdt */ 1180 call _C_LABEL(initgdt) 1181 addl $4,%esp 1182 1183 call xen_locore 1184 1185 /* 1186 * The first VA available is returned by xen_locore in %eax. We 1187 * use it as the UAREA, and set up the stack here. 1188 */ 1189 movl %eax,%esi 1190 movl %esi,_C_LABEL(lwp0uarea) 1191 leal (USPACE-FRAMESIZE)(%eax),%esp 1192 xorl %ebp,%ebp /* mark end of frames */ 1193 1194 /* Set first_avail after the DUMMY PAGE (see xen_locore). */ 1195 addl $(USPACE+PAGE_SIZE),%esi 1196 subl $KERNBASE,%esi /* init386 wants a physical address */ 1197 1198 pushl $0 /* init386() expects a 64 bits paddr_t with PAE */ 1199 pushl %esi 1200 call _C_LABEL(init_bootspace) 1201 call _C_LABEL(init386) 1202 addl $PDE_SIZE,%esp /* pop paddr_t */ 1203 call _C_LABEL(main) 1204#endif /* XENPV */ 1205END(start) 1206 1207#if defined(XEN) 1208#ifndef XENPV 1209/* entry point for Xen PVH */ 1210ENTRY(start_pvh) 1211 getstarttsc 1212 /* Xen doesn't start us with a valid gdt */ 1213 movl $RELOC(gdtdesc_xenpvh), %eax 1214 lgdt (%eax) 1215 jmp $GSEL(GCODE_SEL, SEL_KPL), $RELOC(.Lreload_cs) 1216 1217.Lreload_cs: 1218 movw $GSEL(GDATA_SEL, SEL_KPL), %ax 1219 movw %ax, %ds 1220 movw %ax, %es 1221 movw %ax, %ss 1222 1223 /* we need a valid stack */ 1224 movl $RELOC(tmpstk),%esp 1225 1226 /* clear BSS */ 1227 xorl %eax,%eax 1228 movl $RELOC(__bss_start),%edi 1229 movl $RELOC(_end),%ecx 1230 subl %edi,%ecx 1231 rep 1232 stosb 1233 1234 /* 1235 * Here, we have 2 cases : 1236 * 1237 * 1) We have been started by Xen 1238 * 2) We have been started by another VMM (Qemu, Firecracker, ...) 1239 * 1240 * The main difference is that, when we are started by Xen, 1241 * %ebx (addr of the hvm_start_info structure) is pointing to a 1242 * location that will be mapped correctly later. 1243 * 1244 * In the second case, we have to copy this structure (and all 1245 * the information contained in it) to a location that will be 1246 * mapped later : __kernel_end 1247 * 1248 * To distinguish between the 2 cases, we'll use the 'cpuid' instruction 1249 */ 1250 1251 push %ebx 1252 xorl %eax, %eax 1253 cpuid 1254 cmpl $0x1, %eax /* Check if we can call CPUID with eax=1 */ 1255 jb .start_genpvh 1256 xorl %eax, %eax 1257 inc %eax 1258 cpuid 1259 shr $31, %ecx 1260 testb $1, %cl /* Check if bit 31 of ECX (hypervisor) is set */ 1261 jz .start_genpvh 1262 xorl %eax, %eax 1263 inc %eax 1264 shl $30, %eax 1265 cpuid /* Calling cpuid with eax=0x40000000 */ 1266 cmp $XEN_CPUID_SIGNATURE_EBX, %ebx /* "VneX" */ 1267 je .start_xen 1268 1269 /* We have been started by a VMM that is *not* Xen */ 1270 1271.start_genpvh: 1272 1273 /* First, copy the hvm_start_info structure to __kernel_end */ 1274 pop %ebx 1275 movl %ebx, %esi 1276 movl $RELOC(__kernel_end), %edi 1277 movl $HVM_START_INFO_SIZE, %ecx 1278 shrl $2, %ecx 1279 rep movsl 1280 1281 /* Copy cmdline_paddr after hvm_start_info */ 1282 movl CMDLINE_PADDR(%ebx), %esi 1283 movl $RELOC(__kernel_end), %ecx 1284 movl %edi, CMDLINE_PADDR(%ecx) /* Set new cmdline_paddr in hvm_start_info */ 1285 .cmdline_copy: 1286 movb (%esi), %al 1287 movsb 1288 cmp $0, %al 1289 jne .cmdline_copy 1290 1291 /* Copy memmap_paddr after cmdline (only if hvm_start_info->version != 0) */ 1292 xorl %eax, %eax 1293 cmpl START_INFO_VERSION(%ebx), %eax 1294 je .reload_ebx 1295 movl MMAP_PADDR(%ebx), %esi 1296 movl $RELOC(__kernel_end), %ecx 1297 movl %edi, MMAP_PADDR(%ecx) /* Set new memmap_paddr in hvm_start_info */ 1298 movl MMAP_ENTRIES(%ebx), %eax /* Get memmap_entries */ 1299 movl $MMAP_ENTRY_SIZE, %ebx 1300 mull %ebx /* eax * ebx => edx:eax */ 1301 movl %eax, %ecx 1302 shrl $2, %ecx 1303 rep movsl 1304 1305.reload_ebx: 1306 movl $RELOC(__kernel_end), %ebx 1307 1308 /* announce ourself */ 1309 movl $VM_GUEST_GENPVH, RELOC(vm_guest) 1310 1311 jmp .save_hvm_start_paddr 1312 1313.start_xen: 1314 pop %ebx 1315 movl $VM_GUEST_XENPVH, RELOC(vm_guest) 1316 1317.save_hvm_start_paddr: 1318 /* 1319 * save addr of the hvm_start_info structure. This is also the end 1320 * of the symbol table 1321 /* 1322 * save addr of the hvm_start_info structure. This is also the end 1323 * of the symbol table 1324 */ 1325 movl %ebx, RELOC(hvm_start_paddr) 1326 movl %ebx, %eax 1327 addl $KERNBASE,%eax 1328 movl $RELOC(esym),%ebp 1329 movl %eax,(%ebp) 1330 /* get a page for HYPERVISOR_shared_info */ 1331 /* this is only needed if we are running on Xen */ 1332 cmpl $VM_GUEST_XENPVH, RELOC(vm_guest) 1333 jne .add_hvm_start_info_page 1334 addl $PAGE_SIZE, %ebx 1335 addl $PGOFSET,%ebx 1336 andl $~PGOFSET,%ebx 1337 movl $RELOC(HYPERVISOR_shared_info_pa),%ebp 1338 movl %ebx,(%ebp) 1339 /* XXX assume hvm_start_info+dependant structure fits in a single page */ 1340.add_hvm_start_info_page: 1341 addl $PAGE_SIZE, %ebx 1342 addl $PGOFSET,%ebx 1343 andl $~PGOFSET,%ebx 1344 addl $KERNBASE,%ebx 1345 movl $RELOC(eblob),%ebp 1346 movl %ebx,(%ebp) 1347 1348 jmp .Lstart_common 1349END(start_pvh) 1350 .align 8 1351gdtdesc_xenpvh: 1352 .word gdt_xenpvhend - gdt_xenpvh 1353 .long RELOC(gdt_xenpvh) 1354 .word 0 1355gdt_xenpvh: 1356 .long 0 # null descriptor 1357 .long 0 1358 .long 0x0000ffff # %cs 1359 .long 0x00cf9a00 1360 .long 0x0000ffff # %ds, %es, %ss 1361 .long 0x00cf9200 1362gdt_xenpvhend: 1363 .align 4 1364#endif /* !XENPV */ 1365 1366 1367/* space for the hypercall call page */ 1368#define HYPERCALL_PAGE_OFFSET 0x1000 1369.align HYPERCALL_PAGE_OFFSET 1370ENTRY(hypercall_page) /* Returns -1, on HYPERVISOR_xen_version() */ 1371.skip (__HYPERVISOR_xen_version*32), 0x90 1372 movl $-1, %eax 1373 retl 1374.align HYPERCALL_PAGE_OFFSET, 0x90 1375END(hypercall_page) 1376 1377#ifdef XENPV 1378/* 1379 * void lgdt_finish(void); 1380 * Finish load a new GDT pointer (do any necessary cleanup). 1381 * XXX It's somewhat questionable whether reloading all the segment registers 1382 * is necessary, since the actual descriptor data is not changed except by 1383 * process creation and exit, both of which clean up via task switches. OTOH, 1384 * this only happens at run time when the GDT is resized. 1385 */ 1386/* LINTSTUB: Func: void lgdt_finish(void) */ 1387ENTRY(lgdt_finish) 1388 movl $GSEL(GDATA_SEL, SEL_KPL),%eax 1389 movw %ax,%ds 1390 movw %ax,%es 1391 movw %ax,%gs 1392 movw %ax,%ss 1393 movl $GSEL(GCPU_SEL, SEL_KPL),%eax 1394 movw %ax,%fs 1395 /* Reload code selector by doing intersegment return. */ 1396 popl %eax 1397 pushl $GSEL(GCODE_SEL, SEL_KPL) 1398 pushl %eax 1399 lret 1400END(lgdt_finish) 1401 1402#endif /* XENPV */ 1403#endif /* XEN */ 1404 1405/* 1406 * void lwp_trampoline(void); 1407 * 1408 * This is a trampoline function pushed onto the stack of a newly created 1409 * process in order to do some additional setup. The trampoline is entered by 1410 * cpu_switchto()ing to the process, so we abuse the callee-saved 1411 * registers used by cpu_switchto() to store the information about the 1412 * stub to call. 1413 * NOTE: This function does not have a normal calling sequence! 1414 */ 1415ENTRY(lwp_trampoline) 1416 movl %ebp,%edi /* for .Lsyscall_checkast */ 1417 xorl %ebp,%ebp 1418 pushl %edi 1419 pushl %eax 1420 call _C_LABEL(lwp_startup) 1421 addl $8,%esp 1422 pushl %ebx 1423 call *%esi 1424 addl $4,%esp 1425 jmp .Lsyscall_checkast 1426 /* NOTREACHED */ 1427END(lwp_trampoline) 1428 1429/* 1430 * sigcode() 1431 * 1432 * Signal trampoline; copied to top of user stack. Used only for 1433 * compatibility with old releases of NetBSD. 1434 */ 1435ENTRY(sigcode) 1436 /* 1437 * Handler has returned here as if we called it. The sigcontext 1438 * is on the stack after the 3 args "we" pushed. 1439 */ 1440 leal 12(%esp),%eax /* get pointer to sigcontext */ 1441 movl %eax,4(%esp) /* put it in the argument slot */ 1442 /* fake return address already there */ 1443 movl $SYS_compat_16___sigreturn14,%eax 1444 int $0x80 /* enter kernel with args on stack */ 1445 movl $SYS_exit,%eax 1446 int $0x80 /* exit if sigreturn fails */ 1447 .globl _C_LABEL(esigcode) 1448_C_LABEL(esigcode): 1449END(sigcode) 1450 1451/* 1452 * int setjmp(label_t *) 1453 * 1454 * Used primarily by DDB. 1455 */ 1456ENTRY(setjmp) 1457 movl 4(%esp),%eax 1458 movl %ebx,(%eax) /* save ebx */ 1459 movl %esp,4(%eax) /* save esp */ 1460 movl %ebp,8(%eax) /* save ebp */ 1461 movl %esi,12(%eax) /* save esi */ 1462 movl %edi,16(%eax) /* save edi */ 1463 movl (%esp),%edx /* get rta */ 1464 movl %edx,20(%eax) /* save eip */ 1465 xorl %eax,%eax /* return 0 */ 1466 ret 1467END(setjmp) 1468 1469/* 1470 * int longjmp(label_t *) 1471 * 1472 * Used primarily by DDB. 1473 */ 1474ENTRY(longjmp) 1475 movl 4(%esp),%eax 1476 movl (%eax),%ebx /* restore ebx */ 1477 movl 4(%eax),%esp /* restore esp */ 1478 movl 8(%eax),%ebp /* restore ebp */ 1479 movl 12(%eax),%esi /* restore esi */ 1480 movl 16(%eax),%edi /* restore edi */ 1481 movl 20(%eax),%edx /* get rta */ 1482 movl %edx,(%esp) /* put in return frame */ 1483 movl $1,%eax /* return 1 */ 1484 ret 1485END(longjmp) 1486 1487/* 1488 * void dumpsys(void) 1489 * 1490 * Mimic cpu_switchto() for postmortem debugging. 1491 */ 1492ENTRY(dumpsys) 1493 pushl %ebx /* set up fake switchframe */ 1494 pushl %esi /* and save context */ 1495 pushl %edi 1496 movl %esp,_C_LABEL(dumppcb)+PCB_ESP 1497 movl %ebp,_C_LABEL(dumppcb)+PCB_EBP 1498 call _C_LABEL(dodumpsys) /* dump! */ 1499 addl $(3*4), %esp /* unwind switchframe */ 1500 ret 1501END(dumpsys) 1502 1503/* 1504 * struct lwp *cpu_switchto(struct lwp *oldlwp, struct lwp *newlwp, 1505 * bool returning) 1506 * 1507 * 1. save context of oldlwp. 1508 * 2. restore context of newlwp. 1509 * 1510 * Note that the stack frame layout is known to "struct switchframe" in 1511 * <machine/frame.h> and to the code in cpu_lwp_fork() which initializes 1512 * it for a new lwp. 1513 */ 1514ENTRY(cpu_switchto) 1515 pushl %ebx 1516 pushl %esi 1517 pushl %edi 1518 1519 movl 16(%esp),%esi /* oldlwp */ 1520 movl 20(%esp),%edi /* newlwp */ 1521 movl 24(%esp),%edx /* returning */ 1522 1523 /* Save old context. */ 1524 movl L_PCB(%esi),%eax 1525 movl %esp,PCB_ESP(%eax) 1526 movl %ebp,PCB_EBP(%eax) 1527 1528 /* Switch to newlwp's stack. */ 1529 movl L_PCB(%edi),%ebx 1530 movl PCB_EBP(%ebx),%ebp 1531 movl PCB_ESP(%ebx),%esp 1532 1533 /* 1534 * Issue XCHG, rather than MOV, to set ci_curlwp := newlwp in 1535 * order to coordinate mutex_exit on this CPU with 1536 * mutex_vector_enter on another CPU. 1537 * 1538 * 1. Any prior mutex_exit by oldlwp must be visible to other 1539 * CPUs before we set ci_curlwp := newlwp on this one, 1540 * requiring a store-before-store barrier. 1541 * 1542 * (This is always guaranteed by the x86 memory model, TSO, 1543 * but other architectures require a explicit barrier before 1544 * the store to ci->ci_curlwp.) 1545 * 1546 * 2. ci_curlwp := newlwp must be visible on all other CPUs 1547 * before any subsequent mutex_exit by newlwp can even test 1548 * whether there might be waiters, requiring a 1549 * store-before-load barrier. 1550 * 1551 * (This is the only ordering x86 TSO ever requires any kind 1552 * of barrier for -- in this case, we take advantage of the 1553 * sequential consistency implied by XCHG to obviate the 1554 * need for MFENCE or something.) 1555 * 1556 * See kern_mutex.c for details -- this is necessary for 1557 * adaptive mutexes to detect whether the lwp is on the CPU in 1558 * order to safely block without requiring atomic r/m/w in 1559 * mutex_exit. 1560 */ 1561 movl %edi,%ecx 1562 xchgl %ecx,CPUVAR(CURLWP) 1563 1564#ifdef XENPV 1565 /* if we are there, we're obviously not in user context. 1566 * reset ci_xen_clockf_* in case the splx() at the end of mi_switch() 1567 * triggers a deffered call do xen_timer_handler() 1568 */ 1569 movb $0, CPUVAR(XEN_CLOCKF_USERMODE) 1570 movl $_C_LABEL(cpu_switchto), CPUVAR(XEN_CLOCKF_PC) 1571#endif 1572 1573 /* Skip the rest if returning to a pinned LWP. */ 1574 testl %edx,%edx 1575 jnz switch_return 1576 1577 /* Switch ring0 stack */ 1578#ifdef XENPV 1579 pushl %edi 1580 call _C_LABEL(i386_switch_context) 1581 addl $4,%esp 1582#else 1583 movl PCB_ESP0(%ebx),%eax 1584 movl CPUVAR(TSS),%ecx 1585 movl %eax,TSS_ESP0(%ecx) 1586#endif 1587 1588 /* Switch the dbregs. */ 1589 pushl %edi 1590 pushl %esi 1591 call _C_LABEL(x86_dbregs_switch) 1592 addl $8,%esp 1593 1594 /* Switch the FPU. */ 1595 pushl %edx 1596 pushl %edi 1597 pushl %esi 1598 call _C_LABEL(fpu_switch) 1599 addl $8,%esp 1600 popl %edx 1601 1602 /* Don't bother with the rest if switching to a system process. */ 1603 testl $LW_SYSTEM,L_FLAG(%edi) 1604 jnz switch_return 1605 1606#ifndef XENPV 1607 /* Restore thread-private %fs/%gs descriptors. */ 1608 movl CPUVAR(GDT),%ecx 1609 movl PCB_FSD(%ebx),%eax 1610 movl PCB_FSD+4(%ebx),%edx 1611 movl %eax,(GUFS_SEL*8)(%ecx) 1612 movl %edx,(GUFS_SEL*8+4)(%ecx) 1613 movl PCB_GSD(%ebx),%eax 1614 movl PCB_GSD+4(%ebx),%edx 1615 movl %eax,(GUGS_SEL*8)(%ecx) 1616 movl %edx,(GUGS_SEL*8+4)(%ecx) 1617#endif /* !XENPV */ 1618 1619 /* Switch I/O bitmap */ 1620 movl PCB_IOMAP(%ebx),%eax 1621 orl %eax,%eax 1622 jnz .Lcopy_iobitmap 1623 movl CPUVAR(TSS),%eax 1624 movl $(IOMAP_INVALOFF << 16),TSS_IOBASE(%eax) 1625.Liobitmap_done: 1626 1627 /* Is this process using RAS (restartable atomic sequences)? */ 1628 movl L_PROC(%edi),%eax 1629 cmpl $0,P_RASLIST(%eax) 1630 je no_RAS 1631 1632 /* Handle restartable atomic sequences (RAS). */ 1633 movl L_MD_REGS(%edi),%ecx 1634 pushl TF_EIP(%ecx) 1635 pushl %eax 1636 call _C_LABEL(ras_lookup) 1637 addl $8,%esp 1638 cmpl $-1,%eax 1639 je no_RAS 1640 movl L_MD_REGS(%edi),%ecx 1641 movl %eax,TF_EIP(%ecx) 1642no_RAS: 1643 1644#ifdef XENPV 1645 pushl %edi 1646 call _C_LABEL(i386_tls_switch) 1647 addl $4,%esp 1648#endif 1649 1650switch_return: 1651 /* Return to the new LWP, returning 'oldlwp' in %eax. */ 1652 movl %esi,%eax 1653 popl %edi 1654 popl %esi 1655 popl %ebx 1656 ret 1657 1658.Lcopy_iobitmap: 1659 /* Copy I/O bitmap. */ 1660 incl _C_LABEL(pmap_iobmp_evcnt)+EV_COUNT 1661 movl $(IOMAPSIZE/4),%ecx 1662 pushl %esi 1663 pushl %edi 1664 movl %eax,%esi /* pcb_iomap */ 1665 movl CPUVAR(TSS),%edi 1666 leal TSS_IOMAP(%edi),%edi 1667 rep 1668 movsl 1669 popl %edi 1670 popl %esi 1671 movl CPUVAR(TSS),%eax 1672 movl $(IOMAP_VALIDOFF << 16),TSS_IOBASE(%eax) 1673 jmp .Liobitmap_done 1674END(cpu_switchto) 1675 1676/* 1677 * void savectx(struct pcb *pcb); 1678 * 1679 * Update pcb, saving current processor state. 1680 */ 1681ENTRY(savectx) 1682 movl 4(%esp),%edx /* edx = pcb */ 1683 movl %esp,PCB_ESP(%edx) 1684 movl %ebp,PCB_EBP(%edx) 1685 ret 1686END(savectx) 1687 1688/* 1689 * syscall() 1690 * 1691 * Trap gate entry for syscall 1692 */ 1693IDTVEC(syscall) 1694 pushl $2 /* size of instruction for restart */ 1695 pushl $T_ASTFLT /* trap # for doing ASTs */ 1696 INTRENTRY 1697 STI(%eax) 1698 1699#ifdef DIAGNOSTIC 1700 movzbl CPUVAR(ILEVEL),%ebx 1701 testl %ebx,%ebx 1702 jz 1f 1703 pushl $5f 1704 call _C_LABEL(panic) 1705 addl $4,%esp 1706#ifdef DDB 1707 int $3 1708#endif 17091: 1710#endif /* DIAGNOSTIC */ 1711 1712 addl $1,CPUVAR(NSYSCALL) /* count it atomically */ 1713 adcl $0,CPUVAR(NSYSCALL)+4 /* count it atomically */ 1714 movl CPUVAR(CURLWP),%edi 1715 movl L_PROC(%edi),%edx 1716 movl %esp,L_MD_REGS(%edi) /* save pointer to frame */ 1717 pushl %esp 1718 call *P_MD_SYSCALL(%edx) /* get pointer to syscall() function */ 1719 addl $4,%esp 1720.Lsyscall_checkast: 1721 /* Check for ASTs on exit to user mode. */ 1722 CLI(%eax) 1723 movl L_MD_ASTPENDING(%edi), %eax 1724 orl CPUVAR(WANT_PMAPLOAD), %eax 1725 jnz 9f 1726 1727 HANDLE_DEFERRED_FPU 1728 1729#ifdef XENPV 1730 STIC(%eax) 1731 jz 14f 1732 call _C_LABEL(stipending) 1733 testl %eax,%eax 1734 jz 14f 1735 /* process pending interrupts */ 1736 CLI(%eax) 1737 movzbl CPUVAR(ILEVEL), %ebx 1738 movl $.Lsyscall_resume, %esi /* address to resume loop at */ 1739.Lsyscall_resume: 1740 movl %ebx,%eax /* get cpl */ 1741 movl CPUVAR(IUNMASK)(,%eax,4),%eax 1742 andl CPUVAR(IPENDING),%eax /* any non-masked bits left? */ 1743 jz 17f 1744 bsrl %eax,%eax 1745 btrl %eax,CPUVAR(IPENDING) 1746 movl CPUVAR(ISOURCES)(,%eax,4),%eax 1747 jmp *IS_RESUME(%eax) 174817: movb %bl, CPUVAR(ILEVEL) /* restore cpl */ 1749 jmp .Lsyscall_checkast 175014: 1751#endif /* XENPV */ 1752 1753#ifdef DIAGNOSTIC 1754 cmpb $IPL_NONE,CPUVAR(ILEVEL) 1755 jne 3f 1756#endif 1757 1758 INTRFASTEXIT 1759 1760#ifdef DIAGNOSTIC 17613: STI(%eax) 1762 pushl $4f 1763 call _C_LABEL(panic) 1764 addl $4,%esp 1765 pushl $IPL_NONE 1766 call _C_LABEL(spllower) 1767 addl $4,%esp 1768 jmp .Lsyscall_checkast 17694: .asciz "SPL NOT LOWERED ON SYSCALL EXIT\n" 17705: .asciz "SPL NOT ZERO ON SYSCALL ENTRY\n" 1771#endif 1772 17739: 1774 cmpl $0, CPUVAR(WANT_PMAPLOAD) 1775 jz 10f 1776 STI(%eax) 1777 call _C_LABEL(pmap_load) 1778 jmp .Lsyscall_checkast /* re-check ASTs */ 177910: 1780 /* Always returning to user mode here. */ 1781 movl $0, L_MD_ASTPENDING(%edi) 1782 STI(%eax) 1783 /* Pushed T_ASTFLT into tf_trapno on entry. */ 1784 pushl %esp 1785 call _C_LABEL(trap) 1786 addl $4,%esp 1787 jmp .Lsyscall_checkast /* re-check ASTs */ 1788IDTVEC_END(syscall) 1789 1790/* 1791 * int npx586bug1(int a, int b) 1792 * Used when checking for the FDIV bug on first generations pentiums. 1793 * Anything 120MHz or above is fine. 1794 */ 1795ENTRY(npx586bug1) 1796 fildl 4(%esp) /* x */ 1797 fildl 8(%esp) /* y */ 1798 fld %st(1) 1799 fdiv %st(1),%st /* x/y */ 1800 fmulp %st,%st(1) /* (x/y)*y */ 1801 fsubrp %st,%st(1) /* x-(x/y)*y */ 1802 pushl $0 1803 fistpl (%esp) 1804 popl %eax 1805 ret 1806END(npx586bug1) 1807 1808ENTRY(intrfastexit) 1809 movw TF_GS(%esp),%gs 1810 movw TF_FS(%esp),%fs 1811 movw TF_ES(%esp),%es 1812 movw TF_DS(%esp),%ds 1813 movl TF_EDI(%esp),%edi 1814 movl TF_ESI(%esp),%esi 1815 movl TF_EBP(%esp),%ebp 1816 movl TF_EBX(%esp),%ebx 1817 movl TF_EDX(%esp),%edx 1818 movl TF_ECX(%esp),%ecx 1819 movl TF_EAX(%esp),%eax 1820 addl $(TF_PUSHSIZE+8),%esp 1821 iret 1822END(intrfastexit) 1823 1824 .section .rodata 1825 1826 /* 1827 * Hotpatch templates. 1828 */ 1829 1830LABEL(hp_nolock) 1831 nop 1832LABEL(hp_nolock_end) 1833 1834LABEL(hp_retfence) 1835 lfence 1836LABEL(hp_retfence_end) 1837 1838LABEL(hp_clac) 1839 clac 1840LABEL(hp_clac_end) 1841 1842LABEL(hp_stac) 1843 stac 1844LABEL(hp_stac_end) 1845