[Midnightbsd-cvs] src [9991] trunk/sys/i386/xen: sync with freebsd
laffer1 at midnightbsd.org
laffer1 at midnightbsd.org
Sat May 26 18:08:47 EDT 2018
Revision: 9991
http://svnweb.midnightbsd.org/src/?rev=9991
Author: laffer1
Date: 2018-05-26 18:08:46 -0400 (Sat, 26 May 2018)
Log Message:
-----------
sync with freebsd
Modified Paths:
--------------
trunk/sys/i386/xbox/pic16l.s
trunk/sys/i386/xbox/xbox.c
trunk/sys/i386/xbox/xboxfb.c
trunk/sys/i386/xen/clock.c
trunk/sys/i386/xen/exception.s
trunk/sys/i386/xen/locore.s
trunk/sys/i386/xen/mp_machdep.c
trunk/sys/i386/xen/mptable.c
trunk/sys/i386/xen/pmap.c
trunk/sys/i386/xen/xen_clock_util.c
trunk/sys/i386/xen/xen_machdep.c
trunk/sys/i386/xen/xen_rtc.c
Property Changed:
----------------
trunk/sys/i386/xbox/pic16l.s
trunk/sys/i386/xen/exception.s
trunk/sys/i386/xen/locore.s
Modified: trunk/sys/i386/xbox/pic16l.s
===================================================================
--- trunk/sys/i386/xbox/pic16l.s 2018-05-26 22:00:12 UTC (rev 9990)
+++ trunk/sys/i386/xbox/pic16l.s 2018-05-26 22:08:46 UTC (rev 9991)
@@ -1,3 +1,4 @@
+/* $MidnightBSD$ */
/*-
* Copyright (c) 2005 Rink Springer
* All rights reserved.
@@ -24,7 +25,7 @@
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
- * $FreeBSD$
+ * $FreeBSD: stable/10/sys/i386/xbox/pic16l.s 152219 2005-11-09 03:55:40Z imp $
*/
#include <machine/asmacros.h>
Property changes on: trunk/sys/i386/xbox/pic16l.s
___________________________________________________________________
Added: svn:keywords
## -0,0 +1 ##
+MidnightBSD=%H
\ No newline at end of property
Modified: trunk/sys/i386/xbox/xbox.c
===================================================================
--- trunk/sys/i386/xbox/xbox.c 2018-05-26 22:00:12 UTC (rev 9990)
+++ trunk/sys/i386/xbox/xbox.c 2018-05-26 22:08:46 UTC (rev 9991)
@@ -1,3 +1,4 @@
+/* $MidnightBSD$ */
/*-
* Copyright (c) 2005 Rink Springer
* All rights reserved.
@@ -24,7 +25,7 @@
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
- * $FreeBSD$
+ * $FreeBSD: stable/10/sys/i386/xbox/xbox.c 177253 2008-03-16 10:58:09Z rwatson $
*/
#include <sys/param.h>
#include <sys/systm.h>
Modified: trunk/sys/i386/xbox/xboxfb.c
===================================================================
--- trunk/sys/i386/xbox/xboxfb.c 2018-05-26 22:00:12 UTC (rev 9990)
+++ trunk/sys/i386/xbox/xboxfb.c 2018-05-26 22:08:46 UTC (rev 9991)
@@ -1,3 +1,4 @@
+/* $MidnightBSD$ */
/*-
* Copyright (c) 2005, 2006 Rink Springer <rink at il.fontys.nl>
* All rights reserved.
@@ -26,7 +27,7 @@
*/
#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
+__FBSDID("$FreeBSD: stable/10/sys/i386/xbox/xboxfb.c 233707 2012-03-30 19:10:14Z jhb $");
/*
* This is the syscon(4)-ized version of the Xbox Frame Buffer driver. It
@@ -54,7 +55,7 @@
#include <vm/pmap.h>
#include <machine/bus.h>
#include <machine/xbox.h>
-#include <machine/legacyvar.h>
+#include <x86/legacyvar.h>
#include <dev/fb/fbreg.h>
#include <dev/fb/gfb.h>
#include <dev/syscons/syscons.h>
Modified: trunk/sys/i386/xen/clock.c
===================================================================
--- trunk/sys/i386/xen/clock.c 2018-05-26 22:00:12 UTC (rev 9990)
+++ trunk/sys/i386/xen/clock.c 2018-05-26 22:08:46 UTC (rev 9991)
@@ -1,3 +1,4 @@
+/* $MidnightBSD$ */
/*-
* Copyright (c) 1990 The Regents of the University of California.
* All rights reserved.
@@ -37,7 +38,7 @@
*/
#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
+__FBSDID("$FreeBSD: stable/10/sys/i386/xen/clock.c 255046 2013-08-29 23:11:58Z gibbs $");
/* #define DELAYDEBUG */
/*
@@ -79,16 +80,15 @@
#include <x86/isa/isa.h>
#include <isa/rtc.h>
-#include <xen/xen_intr.h>
#include <vm/vm.h>
#include <vm/pmap.h>
#include <machine/pmap.h>
#include <xen/hypervisor.h>
-#include <machine/xen/xen-os.h>
+#include <xen/xen-os.h>
#include <machine/xen/xenfunc.h>
#include <xen/interface/vcpu.h>
#include <machine/cpu.h>
-#include <machine/xen/xen_clock_util.h>
+#include <xen/xen_intr.h>
/*
* 32-bit time_t's can't reach leap years before 1904 or after 2036, so we
@@ -117,6 +117,7 @@
mtx_init(&clock_lock, "clk", NULL, MTX_SPIN | MTX_NOPROFILE)
#define RTC_LOCK mtx_lock_spin(&clock_lock)
#define RTC_UNLOCK mtx_unlock_spin(&clock_lock)
+#define NS_PER_TICK (1000000000ULL/hz)
int adjkerntz; /* local offset from GMT in seconds */
int clkintr_pending;
@@ -124,21 +125,11 @@
int psdiv = 1;
int wall_cmos_clock;
u_int timer_freq = TIMER_FREQ;
-static int independent_wallclock;
-static int xen_disable_rtc_set;
static u_long cyc2ns_scale;
-static struct timespec shadow_tv;
-static uint32_t shadow_tv_version; /* XXX: lazy locking */
static uint64_t processed_system_time; /* stime (ns) at last processing. */
-static const u_char daysinmonth[] = {31,28,31,30,31,30,31,31,30,31,30,31};
+extern volatile uint64_t xen_timer_last_time;
-SYSCTL_INT(_machdep, OID_AUTO, independent_wallclock,
- CTLFLAG_RW, &independent_wallclock, 0, "");
-SYSCTL_INT(_machdep, OID_AUTO, xen_disable_rtc_set,
- CTLFLAG_RW, &xen_disable_rtc_set, 1, "");
-
-
#define do_div(n,base) ({ \
unsigned long __upper, __low, __high, __mod, __base; \
__base = (base); \
@@ -154,12 +145,6 @@
})
-#define NS_PER_TICK (1000000000ULL/hz)
-
-#define rdtscll(val) \
- __asm__ __volatile__("rdtsc" : "=A" (val))
-
-
/* convert from cycles(64bits) => nanoseconds (64bits)
* basic equation:
* ns = cycles / (freq / ns_per_sec)
@@ -182,201 +167,13 @@
static inline unsigned long long cycles_2_ns(unsigned long long cyc)
{
- return (cyc * cyc2ns_scale) >> CYC2NS_SCALE_FACTOR;
+ return ((cyc * cyc2ns_scale) >> CYC2NS_SCALE_FACTOR);
}
-/*
- * Scale a 64-bit delta by scaling and multiplying by a 32-bit fraction,
- * yielding a 64-bit result.
- */
-static inline uint64_t
-scale_delta(uint64_t delta, uint32_t mul_frac, int shift)
-{
- uint64_t product;
- uint32_t tmp1, tmp2;
-
- if ( shift < 0 )
- delta >>= -shift;
- else
- delta <<= shift;
-
- __asm__ (
- "mul %5 ; "
- "mov %4,%%eax ; "
- "mov %%edx,%4 ; "
- "mul %5 ; "
- "xor %5,%5 ; "
- "add %4,%%eax ; "
- "adc %5,%%edx ; "
- : "=A" (product), "=r" (tmp1), "=r" (tmp2)
- : "a" ((uint32_t)delta), "1" ((uint32_t)(delta >> 32)), "2" (mul_frac) );
-
- return product;
-}
-
-static uint64_t
-get_nsec_offset(struct shadow_time_info *shadow)
-{
- uint64_t now, delta;
- rdtscll(now);
- delta = now - shadow->tsc_timestamp;
- return scale_delta(delta, shadow->tsc_to_nsec_mul, shadow->tsc_shift);
-}
-
-static void update_wallclock(void)
-{
- shared_info_t *s = HYPERVISOR_shared_info;
-
- do {
- shadow_tv_version = s->wc_version;
- rmb();
- shadow_tv.tv_sec = s->wc_sec;
- shadow_tv.tv_nsec = s->wc_nsec;
- rmb();
- }
- while ((s->wc_version & 1) | (shadow_tv_version ^ s->wc_version));
-
-}
-
-static void
-add_uptime_to_wallclock(void)
-{
- struct timespec ut;
-
- xen_fetch_uptime(&ut);
- timespecadd(&shadow_tv, &ut);
-}
-
-/*
- * Reads a consistent set of time-base values from Xen, into a shadow data
- * area. Must be called with the xtime_lock held for writing.
- */
-static void __get_time_values_from_xen(void)
-{
- shared_info_t *s = HYPERVISOR_shared_info;
- struct vcpu_time_info *src;
- struct shadow_time_info *dst;
- uint32_t pre_version, post_version;
-
- src = &s->vcpu_info[smp_processor_id()].time;
- dst = &per_cpu(shadow_time, smp_processor_id());
-
- spinlock_enter();
- do {
- pre_version = dst->version = src->version;
- rmb();
- dst->tsc_timestamp = src->tsc_timestamp;
- dst->system_timestamp = src->system_time;
- dst->tsc_to_nsec_mul = src->tsc_to_system_mul;
- dst->tsc_shift = src->tsc_shift;
- rmb();
- post_version = src->version;
- }
- while ((pre_version & 1) | (pre_version ^ post_version));
-
- dst->tsc_to_usec_mul = dst->tsc_to_nsec_mul / 1000;
- spinlock_exit();
-}
-
-
-static inline int time_values_up_to_date(int cpu)
-{
- struct vcpu_time_info *src;
- struct shadow_time_info *dst;
-
- src = &HYPERVISOR_shared_info->vcpu_info[cpu].time;
- dst = &per_cpu(shadow_time, cpu);
-
- rmb();
- return (dst->version == src->version);
-}
-
-static unsigned xen_get_timecount(struct timecounter *tc);
-
-static struct timecounter xen_timecounter = {
- xen_get_timecount, /* get_timecount */
- 0, /* no poll_pps */
- ~0u, /* counter_mask */
- 0, /* frequency */
- "ixen", /* name */
- 0 /* quality */
-};
-
-static struct eventtimer xen_et;
-
-struct xen_et_state {
- int mode;
-#define MODE_STOP 0
-#define MODE_PERIODIC 1
-#define MODE_ONESHOT 2
- int64_t period;
- int64_t next;
-};
-
-static DPCPU_DEFINE(struct xen_et_state, et_state);
-
-static int
-clkintr(void *arg)
-{
- int64_t now;
- int cpu = smp_processor_id();
- struct shadow_time_info *shadow = &per_cpu(shadow_time, cpu);
- struct xen_et_state *state = DPCPU_PTR(et_state);
-
- do {
- __get_time_values_from_xen();
- now = shadow->system_timestamp + get_nsec_offset(shadow);
- } while (!time_values_up_to_date(cpu));
-
- /* Process elapsed ticks since last call. */
- processed_system_time = now;
- if (state->mode == MODE_PERIODIC) {
- while (now >= state->next) {
- state->next += state->period;
- if (xen_et.et_active)
- xen_et.et_event_cb(&xen_et, xen_et.et_arg);
- }
- HYPERVISOR_set_timer_op(state->next + 50000);
- } else if (state->mode == MODE_ONESHOT) {
- if (xen_et.et_active)
- xen_et.et_event_cb(&xen_et, xen_et.et_arg);
- }
- /*
- * Take synchronised time from Xen once a minute if we're not
- * synchronised ourselves, and we haven't chosen to keep an independent
- * time base.
- */
-
- if (shadow_tv_version != HYPERVISOR_shared_info->wc_version &&
- !independent_wallclock) {
- printf("[XEN] hypervisor wallclock nudged; nudging TOD.\n");
- update_wallclock();
- add_uptime_to_wallclock();
- tc_setclock(&shadow_tv);
- }
-
- /* XXX TODO */
- return (FILTER_HANDLED);
-}
static uint32_t
getit(void)
{
- struct shadow_time_info *shadow;
- uint64_t time;
- uint32_t local_time_version;
-
- shadow = &per_cpu(shadow_time, smp_processor_id());
-
- do {
- local_time_version = shadow->version;
- barrier();
- time = shadow->system_timestamp + get_nsec_offset(shadow);
- if (!time_values_up_to_date(smp_processor_id()))
- __get_time_values_from_xen(/*cpu */);
- barrier();
- } while (local_time_version != shadow->version);
-
- return (time);
+ return (xen_timer_last_time);
}
@@ -480,39 +277,13 @@
#endif
}
-
-/*
- * Restore all the timers non-atomically (XXX: should be atomically).
- *
- * This function is called from pmtimer_resume() to restore all the timers.
- * This should not be necessary, but there are broken laptops that do not
- * restore all the timers on resume.
- */
void
-timer_restore(void)
-{
- struct xen_et_state *state = DPCPU_PTR(et_state);
-
- /* Get timebases for new environment. */
- __get_time_values_from_xen();
-
- /* Reset our own concept of passage of system time. */
- processed_system_time = per_cpu(shadow_time, 0).system_timestamp;
- state->next = processed_system_time;
-}
-
-void
startrtclock()
{
- unsigned long long alarm;
uint64_t __cpu_khz;
uint32_t cpu_khz;
struct vcpu_time_info *info;
- /* initialize xen values */
- __get_time_values_from_xen();
- processed_system_time = per_cpu(shadow_time, 0).system_timestamp;
-
__cpu_khz = 1000000ULL << 32;
info = &HYPERVISOR_shared_info->vcpu_info[0].time;
@@ -530,12 +301,6 @@
set_cyc2ns_scale(cpu_khz/1000);
tsc_freq = cpu_khz * 1000;
-
- timer_freq = 1000000000LL;
- xen_timecounter.tc_frequency = timer_freq >> 9;
- tc_init(&xen_timecounter);
-
- rdtscll(alarm);
}
/*
@@ -594,8 +359,10 @@
int s;
dom0_op_t op;
struct shadow_time_info *shadow;
+ struct pcpu *pc;
- shadow = &per_cpu(shadow_time, smp_processor_id());
+ pc = pcpu_find(smp_processor_id());
+ shadow = &pc->pc_shadow_time;
if (xen_disable_rtc_set)
return;
@@ -767,50 +534,6 @@
}
#endif
-static int
-xen_et_start(struct eventtimer *et,
- struct bintime *first, struct bintime *period)
-{
- struct xen_et_state *state = DPCPU_PTR(et_state);
- struct shadow_time_info *shadow;
- int64_t fperiod;
-
- __get_time_values_from_xen();
-
- if (period != NULL) {
- state->mode = MODE_PERIODIC;
- state->period = (1000000000LL *
- (uint32_t)(period->frac >> 32)) >> 32;
- if (period->sec != 0)
- state->period += 1000000000LL * period->sec;
- } else {
- state->mode = MODE_ONESHOT;
- state->period = 0;
- }
- if (first != NULL) {
- fperiod = (1000000000LL * (uint32_t)(first->frac >> 32)) >> 32;
- if (first->sec != 0)
- fperiod += 1000000000LL * first->sec;
- } else
- fperiod = state->period;
-
- shadow = &per_cpu(shadow_time, smp_processor_id());
- state->next = shadow->system_timestamp + get_nsec_offset(shadow);
- state->next += fperiod;
- HYPERVISOR_set_timer_op(state->next + 50000);
- return (0);
-}
-
-static int
-xen_et_stop(struct eventtimer *et)
-{
- struct xen_et_state *state = DPCPU_PTR(et_state);
-
- state->mode = MODE_STOP;
- HYPERVISOR_set_timer_op(0);
- return (0);
-}
-
/*
* Start clocks running.
*/
@@ -817,80 +540,16 @@
void
cpu_initclocks(void)
{
- unsigned int time_irq;
- int error;
-
- HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, 0, NULL);
- error = bind_virq_to_irqhandler(VIRQ_TIMER, 0, "cpu0:timer",
- clkintr, NULL, NULL, INTR_TYPE_CLK, &time_irq);
- if (error)
- panic("failed to register clock interrupt\n");
- /* should fast clock be enabled ? */
-
- bzero(&xen_et, sizeof(xen_et));
- xen_et.et_name = "ixen";
- xen_et.et_flags = ET_FLAGS_PERIODIC | ET_FLAGS_ONESHOT |
- ET_FLAGS_PERCPU;
- xen_et.et_quality = 600;
- xen_et.et_frequency = 0;
- xen_et.et_min_period.sec = 0;
- xen_et.et_min_period.frac = 0x00400000LL << 32;
- xen_et.et_max_period.sec = 2;
- xen_et.et_max_period.frac = 0;
- xen_et.et_start = xen_et_start;
- xen_et.et_stop = xen_et_stop;
- xen_et.et_priv = NULL;
- et_register(&xen_et);
-
cpu_initclocks_bsp();
}
-int
-ap_cpu_initclocks(int cpu)
-{
- char buf[MAXCOMLEN + 1];
- unsigned int time_irq;
- int error;
-
- HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL);
- snprintf(buf, sizeof(buf), "cpu%d:timer", cpu);
- error = bind_virq_to_irqhandler(VIRQ_TIMER, cpu, buf,
- clkintr, NULL, NULL, INTR_TYPE_CLK, &time_irq);
- if (error)
- panic("failed to register clock interrupt\n");
-
- return (0);
-}
-
-static uint32_t
-xen_get_timecount(struct timecounter *tc)
-{
- uint64_t clk;
- struct shadow_time_info *shadow;
- shadow = &per_cpu(shadow_time, smp_processor_id());
-
- __get_time_values_from_xen();
-
- clk = shadow->system_timestamp + get_nsec_offset(shadow);
-
- return (uint32_t)(clk >> 9);
-
-}
-
/* Return system time offset by ticks */
uint64_t
get_system_time(int ticks)
{
- return processed_system_time + (ticks * NS_PER_TICK);
+ return (processed_system_time + (ticks * NS_PER_TICK));
}
-void
-idle_block(void)
-{
-
- HYPERVISOR_sched_op(SCHEDOP_block, 0);
-}
-
int
timer_spkr_acquire(void)
{
Modified: trunk/sys/i386/xen/exception.s
===================================================================
--- trunk/sys/i386/xen/exception.s 2018-05-26 22:00:12 UTC (rev 9990)
+++ trunk/sys/i386/xen/exception.s 2018-05-26 22:08:46 UTC (rev 9991)
@@ -27,7 +27,8 @@
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
- * $FreeBSD$
+ * $FreeBSD: stable/10/sys/i386/xen/exception.s 255040 2013-08-29 19:52:18Z gibbs $
+ * $MidnightBSD$
*/
#include "opt_apic.h"
@@ -168,7 +169,7 @@
jb critical_region_fixup
10: pushl %esp
- call evtchn_do_upcall
+ call xen_intr_handle_upcall
addl $4,%esp
/*
Property changes on: trunk/sys/i386/xen/exception.s
___________________________________________________________________
Added: svn:keywords
## -0,0 +1 ##
+MidnightBSD=%H
\ No newline at end of property
Modified: trunk/sys/i386/xen/locore.s
===================================================================
--- trunk/sys/i386/xen/locore.s 2018-05-26 22:00:12 UTC (rev 9990)
+++ trunk/sys/i386/xen/locore.s 2018-05-26 22:08:46 UTC (rev 9991)
@@ -30,7 +30,8 @@
* SUCH DAMAGE.
*
* from: @(#)locore.s 7.3 (Berkeley) 5/13/91
- * $FreeBSD$
+ * $FreeBSD: stable/10/sys/i386/xen/locore.s 254671 2013-08-22 20:07:06Z gibbs $
+ * $MidnightBSD$
*
* originally from: locore.s, by William F. Jolitz
*
@@ -76,7 +77,7 @@
ELFNOTE(Xen, XEN_ELFNOTE_PADDR_OFFSET, .long, KERNBASE)
ELFNOTE(Xen, XEN_ELFNOTE_ENTRY, .long, btext)
ELFNOTE(Xen, XEN_ELFNOTE_HYPERCALL_PAGE, .long, hypercall_page)
- ELFNOTE(Xen, XEN_ELFNOTE_HV_START_LOW, .long, HYPERVISOR_VIRT_START)
+ ELFNOTE(Xen, XEN_ELFNOTE_HV_START_LOW, .long, XEN_HYPERVISOR_VIRT_START)
#if 0
ELFNOTE(Xen, XEN_ELFNOTE_FEATURES, .asciz, "writable_page_tables|writable_descriptor_tables|auto_translated_physmap|pae_pgdir_above_4gb|supervisor_mode_kernel")
#endif
Property changes on: trunk/sys/i386/xen/locore.s
___________________________________________________________________
Added: svn:keywords
## -0,0 +1 ##
+MidnightBSD=%H
\ No newline at end of property
Modified: trunk/sys/i386/xen/mp_machdep.c
===================================================================
--- trunk/sys/i386/xen/mp_machdep.c 2018-05-26 22:00:12 UTC (rev 9990)
+++ trunk/sys/i386/xen/mp_machdep.c 2018-05-26 22:08:46 UTC (rev 9991)
@@ -1,3 +1,4 @@
+/* $MidnightBSD$ */
/*-
* Copyright (c) 1996, by Steve Passe
* Copyright (c) 2008, by Kip Macy
@@ -25,7 +26,7 @@
*/
#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
+__FBSDID("$FreeBSD: stable/10/sys/i386/xen/mp_machdep.c 278423 2015-02-08 23:04:32Z marius $");
#include "opt_apic.h"
#include "opt_cpu.h"
@@ -64,6 +65,7 @@
#include <sys/mutex.h>
#include <sys/pcpu.h>
#include <sys/proc.h>
+#include <sys/rwlock.h>
#include <sys/sched.h>
#include <sys/smp.h>
#include <sys/sysctl.h>
@@ -84,26 +86,61 @@
#include <machine/specialreg.h>
#include <machine/pcpu.h>
-
-
-#include <machine/xen/xen-os.h>
+#include <xen/xen-os.h>
#include <xen/evtchn.h>
#include <xen/xen_intr.h>
#include <xen/hypervisor.h>
#include <xen/interface/vcpu.h>
+/*---------------------------- Extern Declarations ---------------------------*/
+extern struct pcpu __pcpu[];
+extern void Xhypervisor_callback(void);
+extern void failsafe_callback(void);
+extern void pmap_lazyfix_action(void);
+
+/*--------------------------- Forward Declarations ---------------------------*/
+static driver_filter_t smp_reschedule_interrupt;
+static driver_filter_t smp_call_function_interrupt;
+static void assign_cpu_ids(void);
+static void set_interrupt_apic_ids(void);
+static int start_all_aps(void);
+static int start_ap(int apic_id);
+static void release_aps(void *dummy);
+
+/*---------------------------------- Macros ----------------------------------*/
+#define IPI_TO_IDX(ipi) ((ipi) - APIC_IPI_INTS)
+
+/*-------------------------------- Local Types -------------------------------*/
+typedef void call_data_func_t(uintptr_t , uintptr_t);
+
+struct cpu_info {
+ int cpu_present:1;
+ int cpu_bsp:1;
+ int cpu_disabled:1;
+};
+
+struct xen_ipi_handler
+{
+ driver_filter_t *filter;
+ const char *description;
+};
+
+enum {
+ RESCHEDULE_VECTOR,
+ CALL_FUNCTION_VECTOR,
+};
+
+/*-------------------------------- Global Data -------------------------------*/
+static u_int hyperthreading_cpus;
+static cpuset_t hyperthreading_cpus_mask;
+
int mp_naps; /* # of Applications processors */
int boot_cpu_id = -1; /* designated BSP */
-extern struct pcpu __pcpu[];
-
static int bootAP;
static union descriptor *bootAPgdt;
-static char resched_name[NR_CPUS][15];
-static char callfunc_name[NR_CPUS][15];
-
/* Free these after use */
void *bootstacks[MAXCPU];
@@ -114,8 +151,6 @@
vm_offset_t smp_tlb_addr2;
volatile int smp_tlb_wait;
-typedef void call_data_func_t(uintptr_t , uintptr_t);
-
static u_int logical_cpus;
static volatile cpuset_t ipi_nmi_pending;
@@ -129,11 +164,7 @@
* Store data from cpu_add() until later in the boot when we actually setup
* the APs.
*/
-struct cpu_info {
- int cpu_present:1;
- int cpu_bsp:1;
- int cpu_disabled:1;
-} static cpu_info[MAX_APIC_ID + 1];
+static struct cpu_info cpu_info[MAX_APIC_ID + 1];
int cpu_apic_ids[MAXCPU];
int apic_cpuids[MAX_APIC_ID + 1];
@@ -143,19 +174,17 @@
static int cpu_logical;
static int cpu_cores;
-static void assign_cpu_ids(void);
-static void set_interrupt_apic_ids(void);
-int start_all_aps(void);
-static int start_ap(int apic_id);
-static void release_aps(void *dummy);
+static const struct xen_ipi_handler xen_ipis[] =
+{
+ [RESCHEDULE_VECTOR] = { smp_reschedule_interrupt, "resched" },
+ [CALL_FUNCTION_VECTOR] = { smp_call_function_interrupt,"callfunc" }
+};
-static u_int hyperthreading_cpus;
-static cpuset_t hyperthreading_cpus_mask;
+/*------------------------------- Per-CPU Data -------------------------------*/
+DPCPU_DEFINE(xen_intr_handle_t, ipi_handle[nitems(xen_ipis)]);
+DPCPU_DEFINE(struct vcpu_info *, vcpu_info);
-extern void Xhypervisor_callback(void);
-extern void failsafe_callback(void);
-extern void pmap_lazyfix_action(void);
-
+/*------------------------------ Implementation ------------------------------*/
struct cpu_group *
cpu_topo(void)
{
@@ -354,12 +383,12 @@
*/
static call_data_func_t *ipi_vectors[6] =
{
- iv_rendezvous,
- iv_invltlb,
- iv_invlpg,
- iv_invlrng,
- iv_invlcache,
- iv_lazypmap,
+ iv_rendezvous,
+ iv_invltlb,
+ iv_invlpg,
+ iv_invlrng,
+ iv_invlcache,
+ iv_lazypmap,
};
/*
@@ -413,10 +442,11 @@
atomic_t *finished = &call_data->finished;
/* We only handle function IPIs, not bitmap IPIs */
- if (call_data->func_id < APIC_IPI_INTS || call_data->func_id > IPI_BITMAP_VECTOR)
+ if (call_data->func_id < APIC_IPI_INTS ||
+ call_data->func_id > IPI_BITMAP_VECTOR)
panic("invalid function id %u", call_data->func_id);
- func = ipi_vectors[call_data->func_id - APIC_IPI_INTS];
+ func = ipi_vectors[IPI_TO_IDX(call_data->func_id)];
/*
* Notify initiating CPU that I've grabbed the data and am
* about to execute the function
@@ -460,50 +490,46 @@
}
static int
-xen_smp_intr_init(unsigned int cpu)
+xen_smp_cpu_init(unsigned int cpu)
{
- int rc;
- unsigned int irq;
-
- per_cpu(resched_irq, cpu) = per_cpu(callfunc_irq, cpu) = -1;
+ xen_intr_handle_t *ipi_handle;
+ const struct xen_ipi_handler *ipi;
+ int idx, rc;
- sprintf(resched_name[cpu], "resched%u", cpu);
- rc = bind_ipi_to_irqhandler(RESCHEDULE_VECTOR,
- cpu,
- resched_name[cpu],
- smp_reschedule_interrupt,
- INTR_TYPE_TTY, &irq);
+ ipi_handle = DPCPU_ID_GET(cpu, ipi_handle);
+ for (ipi = xen_ipis, idx = 0; idx < nitems(xen_ipis); ipi++, idx++) {
- printf("[XEN] IPI cpu=%d irq=%d vector=RESCHEDULE_VECTOR (%d)\n",
- cpu, irq, RESCHEDULE_VECTOR);
-
- per_cpu(resched_irq, cpu) = irq;
+ /*
+ * The PCPU variable pc_device is not initialized on i386 PV,
+ * so we have to use the root_bus device in order to setup
+ * the IPIs.
+ */
+ rc = xen_intr_alloc_and_bind_ipi(root_bus, cpu,
+ ipi->filter, INTR_TYPE_TTY, &ipi_handle[idx]);
+ if (rc != 0) {
+ printf("Unable to allocate a XEN IPI port. "
+ "Error %d\n", rc);
+ break;
+ }
+ xen_intr_describe(ipi_handle[idx], "%s", ipi->description);
+ }
- sprintf(callfunc_name[cpu], "callfunc%u", cpu);
- rc = bind_ipi_to_irqhandler(CALL_FUNCTION_VECTOR,
- cpu,
- callfunc_name[cpu],
- smp_call_function_interrupt,
- INTR_TYPE_TTY, &irq);
- if (rc < 0)
- goto fail;
- per_cpu(callfunc_irq, cpu) = irq;
+ for (;idx < nitems(xen_ipis); idx++)
+ ipi_handle[idx] = NULL;
- printf("[XEN] IPI cpu=%d irq=%d vector=CALL_FUNCTION_VECTOR (%d)\n",
- cpu, irq, CALL_FUNCTION_VECTOR);
+ if (rc == 0)
+ return (0);
-
- if ((cpu != 0) && ((rc = ap_cpu_initclocks(cpu)) != 0))
- goto fail;
+ /* Either all are successfully mapped, or none at all. */
+ for (idx = 0; idx < nitems(xen_ipis); idx++) {
+ if (ipi_handle[idx] == NULL)
+ continue;
- return 0;
+ xen_intr_unbind(ipi_handle[idx]);
+ ipi_handle[idx] = NULL;
+ }
- fail:
- if (per_cpu(resched_irq, cpu) >= 0)
- unbind_from_irqhandler(per_cpu(resched_irq, cpu));
- if (per_cpu(callfunc_irq, cpu) >= 0)
- unbind_from_irqhandler(per_cpu(callfunc_irq, cpu));
- return rc;
+ return (rc);
}
static void
@@ -512,9 +538,19 @@
int i;
for (i = 0; i < mp_ncpus; i++)
- xen_smp_intr_init(i);
+ xen_smp_cpu_init(i);
}
+static void
+xen_smp_intr_setup_cpus(void *unused)
+{
+ int i;
+
+ for (i = 0; i < mp_ncpus; i++)
+ DPCPU_ID_SET(i, vcpu_info,
+ &HYPERVISOR_shared_info->vcpu_info[i]);
+}
+
#define MTOPSIZE (1<<(14 + PAGE_SHIFT))
/*
@@ -563,22 +599,13 @@
for (addr = 0; addr < NKPT * NBPDR - 1; addr += PAGE_SIZE)
invlpg(addr);
- /* set up FPU state on the AP */
- npxinit();
#if 0
-
- /* set up SSE registers */
- enable_sse();
+ /* set up SSE/NX */
+ initializecpu();
#endif
-#if 0 && defined(PAE)
- /* Enable the PTE no-execute bit. */
- if ((amd_feature & AMDID_NX) != 0) {
- uint64_t msr;
- msr = rdmsr(MSR_EFER) | EFER_NXE;
- wrmsr(MSR_EFER, msr);
- }
-#endif
+ /* set up FPU state on the AP */
+ npxinit(false);
#if 0
/* A quick check from sanity claus */
if (PCPU_GET(apic_id) != lapic_id()) {
@@ -620,7 +647,6 @@
if (smp_cpus == mp_ncpus) {
/* enable IPI's, tlb shootdown, freezes etc */
atomic_store_rel_int(&smp_started, 1);
- smp_active = 1; /* historic */
}
mtx_unlock_spin(&ap_boot_mtx);
@@ -745,8 +771,10 @@
/* Get per-cpu data */
pc = &__pcpu[bootAP];
pcpu_init(pc, bootAP, sizeof(struct pcpu));
- dpcpu_init((void *)kmem_alloc(kernel_map, DPCPU_SIZE), bootAP);
+ dpcpu_init((void *)kmem_malloc(kernel_arena, DPCPU_SIZE,
+ M_WAITOK | M_ZERO), bootAP);
pc->pc_apic_id = cpu_apic_ids[bootAP];
+ pc->pc_vcpu_id = cpu_apic_ids[bootAP];
pc->pc_prvspace = pc;
pc->pc_curthread = 0;
@@ -786,7 +814,7 @@
pmap_invalidate_range(kernel_pmap, 0, NKPT * NBPDR - 1);
/* number of APs actually started */
- return mp_naps;
+ return (mp_naps);
}
extern uint8_t *pcpu_boot_stack;
@@ -804,6 +832,7 @@
}
}
+extern struct rwlock pvh_global_lock;
extern int nkpt;
static void
cpu_initialize_context(unsigned int cpu)
@@ -810,12 +839,11 @@
{
/* vcpu_guest_context_t is too large to allocate on the stack.
* Hence we allocate statically and protect it with a lock */
- vm_page_t m[4];
+ vm_page_t m[NPGPTD + 2];
static vcpu_guest_context_t ctxt;
vm_offset_t boot_stack;
vm_offset_t newPTD;
vm_paddr_t ma[NPGPTD];
- static int color;
int i;
/*
@@ -825,7 +853,7 @@
*
*/
for (i = 0; i < NPGPTD + 2; i++) {
- m[i] = vm_page_alloc(NULL, color++,
+ m[i] = vm_page_alloc(NULL, 0,
VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED |
VM_ALLOC_ZERO);
@@ -832,8 +860,8 @@
pmap_zero_page(m[i]);
}
- boot_stack = kmem_alloc_nofault(kernel_map, 1);
- newPTD = kmem_alloc_nofault(kernel_map, NPGPTD);
+ boot_stack = kva_alloc(PAGE_SIZE);
+ newPTD = kva_alloc(NPGPTD * PAGE_SIZE);
ma[0] = VM_PAGE_TO_MACH(m[0])|PG_V;
#ifdef PAE
@@ -855,7 +883,7 @@
nkpt*sizeof(vm_paddr_t));
pmap_qremove(newPTD, 4);
- kmem_free(kernel_map, newPTD, 4);
+ kva_free(newPTD, 4 * PAGE_SIZE);
/*
* map actual idle stack to boot_stack
*/
@@ -863,7 +891,7 @@
xen_pgdpt_pin(VM_PAGE_TO_MACH(m[NPGPTD + 1]));
- vm_page_lock_queues();
+ rw_wlock(&pvh_global_lock);
for (i = 0; i < 4; i++) {
int pdir = (PTDPTDI + i) / NPDEPG;
int curoffset = (PTDPTDI + i) % NPDEPG;
@@ -873,7 +901,7 @@
ma[i]);
}
PT_UPDATES_FLUSH();
- vm_page_unlock_queues();
+ rw_wunlock(&pvh_global_lock);
memset(&ctxt, 0, sizeof(ctxt));
ctxt.flags = VGCF_IN_KERNEL;
@@ -891,7 +919,8 @@
smp_trap_init(ctxt.trap_ctxt);
ctxt.ldt_ents = 0;
- ctxt.gdt_frames[0] = (uint32_t)((uint64_t)vtomach(bootAPgdt) >> PAGE_SHIFT);
+ ctxt.gdt_frames[0] =
+ (uint32_t)((uint64_t)vtomach(bootAPgdt) >> PAGE_SHIFT);
ctxt.gdt_ents = 512;
#ifdef __i386__
@@ -951,12 +980,19 @@
/* Wait up to 5 seconds for it to start. */
for (ms = 0; ms < 5000; ms++) {
if (mp_naps > cpus)
- return 1; /* return SUCCESS */
+ return (1); /* return SUCCESS */
DELAY(1000);
}
- return 0; /* return FAILURE */
+ return (0); /* return FAILURE */
}
+static void
+ipi_pcpu(int cpu, u_int ipi)
+{
+ KASSERT((ipi <= nitems(xen_ipis)), ("invalid IPI"));
+ xen_intr_signal(DPCPU_ID_GET(cpu, ipi_handle[ipi]));
+}
+
/*
* send an IPI to a specific CPU.
*/
@@ -1010,7 +1046,8 @@
}
static void
-smp_targeted_tlb_shootdown(cpuset_t mask, u_int vector, vm_offset_t addr1, vm_offset_t addr2)
+smp_targeted_tlb_shootdown(cpuset_t mask, u_int vector, vm_offset_t addr1,
+ vm_offset_t addr2)
{
int cpu, ncpu, othercpus;
struct _call_data data;
@@ -1038,7 +1075,7 @@
ipi_all_but_self(vector);
} else {
ncpu = 0;
- while ((cpu = cpusetobj_ffs(&mask)) != 0) {
+ while ((cpu = CPU_FFS(&mask)) != 0) {
cpu--;
CPU_CLR(cpu, &mask);
CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu,
@@ -1131,7 +1168,7 @@
if (ipi == IPI_STOP_HARD)
CPU_OR_ATOMIC(&ipi_nmi_pending, &cpus);
- while ((cpu = cpusetobj_ffs(&cpus)) != 0) {
+ while ((cpu = CPU_FFS(&cpus)) != 0) {
cpu--;
CPU_CLR(cpu, &cpus);
CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu, ipi);
@@ -1230,6 +1267,31 @@
}
/*
+ * Handlers for TLB related IPIs
+ *
+ * On i386 Xen PV this are no-ops since this port doesn't support SMP.
+ */
+void
+invltlb_handler(void)
+{
+}
+
+void
+invlpg_handler(void)
+{
+}
+
+void
+invlrng_handler(void)
+{
+}
+
+void
+invlcache_handler(void)
+{
+}
+
+/*
* This is called once the rest of the system is up and running and we're
* ready to let the AP's out of the pen.
*/
@@ -1244,5 +1306,5 @@
ia32_pause();
}
SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL);
-SYSINIT(start_ipis, SI_SUB_INTR, SI_ORDER_ANY, xen_smp_intr_init_cpus, NULL);
-
+SYSINIT(start_ipis, SI_SUB_SMP, SI_ORDER_ANY, xen_smp_intr_init_cpus, NULL);
+SYSINIT(start_cpu, SI_SUB_INTR, SI_ORDER_ANY, xen_smp_intr_setup_cpus, NULL);
Modified: trunk/sys/i386/xen/mptable.c
===================================================================
--- trunk/sys/i386/xen/mptable.c 2018-05-26 22:00:12 UTC (rev 9990)
+++ trunk/sys/i386/xen/mptable.c 2018-05-26 22:08:46 UTC (rev 9991)
@@ -1,3 +1,4 @@
+/* $MidnightBSD$ */
/*-
* Copyright (c) 2003 John Baldwin <jhb at FreeBSD.org>
* Copyright (c) 1996, by Steve Passe
@@ -25,7 +26,7 @@
*/
#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
+__FBSDID("$FreeBSD: stable/10/sys/i386/xen/mptable.c 256073 2013-10-05 23:11:01Z gibbs $");
#include <sys/param.h>
#include <sys/bus.h>
@@ -40,7 +41,7 @@
#include <machine/apicvar.h>
#include <xen/hypervisor.h>
-#include <machine/xen/xen-os.h>
+#include <xen/xen-os.h>
#include <machine/smp.h>
#include <xen/interface/vcpu.h>
@@ -87,6 +88,8 @@
mptable_setup_local(void)
{
+ PCPU_SET(apic_id, 0);
+ PCPU_SET(vcpu_id, 0);
return (0);
}
Modified: trunk/sys/i386/xen/pmap.c
===================================================================
--- trunk/sys/i386/xen/pmap.c 2018-05-26 22:00:12 UTC (rev 9990)
+++ trunk/sys/i386/xen/pmap.c 2018-05-26 22:08:46 UTC (rev 9991)
@@ -1,3 +1,4 @@
+/* $MidnightBSD$ */
/*-
* Copyright (c) 1991 Regents of the University of California.
* All rights reserved.
@@ -75,18 +76,11 @@
*/
#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
+__FBSDID("$FreeBSD: stable/10/sys/i386/xen/pmap.c 273136 2014-10-15 14:07:24Z kib $");
/*
* Manages physical address maps.
*
- * In addition to hardware address maps, this
- * module is called upon to provide software-use-only
- * maps which may or may not be stored in the same
- * form as hardware maps. These pseudo-maps are
- * used to store intermediate results from copy
- * operations to and from address spaces.
- *
* Since the information managed by this module is
* also stored by the logical address mapping module,
* this module may throw away valid virtual-to-physical
@@ -118,6 +112,7 @@
#include <sys/msgbuf.h>
#include <sys/mutex.h>
#include <sys/proc.h>
+#include <sys/rwlock.h>
#include <sys/sf_buf.h>
#include <sys/sx.h>
#include <sys/vmmeter.h>
@@ -179,7 +174,6 @@
#define PMAP_INLINE
#endif
-#define PV_STATS
#ifdef PV_STATS
#define PV_STAT(x) do { x ; } while (0)
#else
@@ -228,6 +222,13 @@
static int pat_works; /* Is page attribute table sane? */
/*
+ * This lock is defined as static in other pmap implementations. It cannot,
+ * however, be defined as static here, because it is (ab)used to serialize
+ * queued page table changes in other sources files.
+ */
+struct rwlock pvh_global_lock;
+
+/*
* Data for the pv entry allocation mechanism
*/
static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks);
@@ -249,9 +250,9 @@
caddr_t CADDR2;
};
static struct sysmaps sysmaps_pcpu[MAXCPU];
-static pt_entry_t *CMAP3;
+pt_entry_t *CMAP3;
caddr_t ptvmmap = 0;
-static caddr_t CADDR3;
+caddr_t CADDR3;
struct msgbuf *msgbufp = 0;
/*
@@ -298,9 +299,9 @@
static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va,
vm_page_t m);
-static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags);
+static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, u_int flags);
-static vm_page_t _pmap_allocpte(pmap_t pmap, u_int ptepindex, int flags);
+static vm_page_t _pmap_allocpte(pmap_t pmap, u_int ptepindex, u_int flags);
static void _pmap_unwire_ptp(pmap_t pmap, vm_page_t m, vm_page_t *free);
static pt_entry_t *pmap_pte_quick(pmap_t pmap, vm_offset_t va);
static void pmap_pte_release(pt_entry_t *pte);
@@ -388,6 +389,12 @@
#endif
CPU_FILL(&kernel_pmap->pm_active); /* don't allow deactivation */
TAILQ_INIT(&kernel_pmap->pm_pvchunk);
+
+ /*
+ * Initialize the global pv list lock.
+ */
+ rw_init_flags(&pvh_global_lock, "pmap pv global", RW_RECURSE);
+
LIST_INIT(&allpmaps);
mtx_init(&allpmaps_lock, "allpmaps", NULL, MTX_SPIN);
mtx_lock_spin(&allpmaps_lock);
@@ -614,8 +621,7 @@
pv_entry_high_water = 9 * (pv_entry_max / 10);
pv_maxchunks = MAX(pv_entry_max / _NPCPV, maxproc);
- pv_chunkbase = (struct pv_chunk *)kmem_alloc_nofault(kernel_map,
- PAGE_SIZE * pv_maxchunks);
+ pv_chunkbase = (struct pv_chunk *)kva_alloc(PAGE_SIZE * pv_maxchunks);
if (pv_chunkbase == NULL)
panic("pmap_init: not enough kvm for pv chunks");
pmap_ptelist_init(&pv_vafree, pv_chunkbase, pv_maxchunks);
@@ -883,15 +889,19 @@
#define PMAP_CLFLUSH_THRESHOLD (2 * 1024 * 1024)
void
-pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva)
+pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva, boolean_t force)
{
- KASSERT((sva & PAGE_MASK) == 0,
- ("pmap_invalidate_cache_range: sva not page-aligned"));
- KASSERT((eva & PAGE_MASK) == 0,
- ("pmap_invalidate_cache_range: eva not page-aligned"));
+ if (force) {
+ sva &= ~(vm_offset_t)cpu_clflush_line_size;
+ } else {
+ KASSERT((sva & PAGE_MASK) == 0,
+ ("pmap_invalidate_cache_range: sva not page-aligned"));
+ KASSERT((eva & PAGE_MASK) == 0,
+ ("pmap_invalidate_cache_range: eva not page-aligned"));
+ }
- if (cpu_feature & CPUID_SS)
+ if ((cpu_feature & CPUID_SS) != 0 && !force)
; /* If "Self Snoop" is supported, do nothing. */
else if ((cpu_feature & CPUID_CLFSH) != 0 &&
eva - sva < PMAP_CLFLUSH_THRESHOLD) {
@@ -967,9 +977,7 @@
mtx_lock(&PMAP2mutex);
newpf = *pde & PG_FRAME;
if ((*PMAP2 & PG_FRAME) != newpf) {
- vm_page_lock_queues();
PT_SET_MA(PADDR2, newpf | PG_V | PG_A | PG_M);
- vm_page_unlock_queues();
CTR3(KTR_PMAP, "pmap_pte: pmap=%p va=0x%x newpte=0x%08x",
pmap, va, (*PMAP2 & 0xffffffff));
}
@@ -989,9 +997,9 @@
if ((pt_entry_t *)((vm_offset_t)pte & ~PAGE_MASK) == PADDR2) {
CTR1(KTR_PMAP, "pmap_pte_release: pte=0x%jx",
*PMAP2);
- vm_page_lock_queues();
+ rw_wlock(&pvh_global_lock);
PT_SET_VA(PMAP2, 0, TRUE);
- vm_page_unlock_queues();
+ rw_wunlock(&pvh_global_lock);
mtx_unlock(&PMAP2mutex);
}
}
@@ -1011,7 +1019,7 @@
* scans are across different pmaps. It is very wasteful
* to do an entire invltlb for checking a single mapping.
*
- * If the given pmap is not the current pmap, vm_page_queue_mtx
+ * If the given pmap is not the current pmap, pvh_global_lock
* must be held and curthread pinned to a CPU.
*/
static pt_entry_t *
@@ -1027,7 +1035,7 @@
/* are we current address space or kernel? */
if (pmap_is_current(pmap))
return (vtopte(va));
- mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+ rw_assert(&pvh_global_lock, RA_WLOCKED);
KASSERT(curthread->td_pinned > 0, ("curthread not pinned"));
newpf = *pde & PG_FRAME;
if ((*PMAP1 & PG_FRAME) != newpf) {
@@ -1309,7 +1317,7 @@
CTR2(KTR_PMAP, "pmap_qremove: sva=0x%x count=%d", sva, count);
va = sva;
- vm_page_lock_queues();
+ rw_wlock(&pvh_global_lock);
critical_enter();
while (count-- > 0) {
pmap_kremove(va);
@@ -1318,7 +1326,7 @@
PT_UPDATES_FLUSH();
pmap_invalidate_range(kernel_pmap, sva, va);
critical_exit();
- vm_page_unlock_queues();
+ rw_wunlock(&pvh_global_lock);
}
/***************************************************
@@ -1331,7 +1339,8 @@
while (free != NULL) {
m = free;
- free = m->right;
+ free = (void *)m->object;
+ m->object = NULL;
vm_page_free_zero(m);
}
}
@@ -1389,7 +1398,7 @@
* Put page on a list so that it is released after
* *ALL* TLB shootdown is done
*/
- m->right = *free;
+ m->object = (void *)*free;
*free = m;
}
@@ -1448,17 +1457,13 @@
mtx_lock(&createdelete_lock);
#endif
- PMAP_LOCK_INIT(pmap);
-
/*
* No need to allocate page table space yet but we do need a valid
* page directory table.
*/
if (pmap->pm_pdir == NULL) {
- pmap->pm_pdir = (pd_entry_t *)kmem_alloc_nofault(kernel_map,
- NBPTD);
+ pmap->pm_pdir = (pd_entry_t *)kva_alloc(NBPTD);
if (pmap->pm_pdir == NULL) {
- PMAP_LOCK_DESTROY(pmap);
#ifdef HAMFISTED_LOCKING
mtx_unlock(&createdelete_lock);
#endif
@@ -1465,7 +1470,7 @@
return (0);
}
#ifdef PAE
- pmap->pm_pdpt = (pd_entry_t *)kmem_alloc_nofault(kernel_map, 1);
+ pmap->pm_pdpt = (pd_entry_t *)kva_alloc(1);
#endif
}
@@ -1521,7 +1526,7 @@
#ifdef PAE
PT_SET_MA(pmap->pm_pdpt, *vtopte((vm_offset_t)pmap->pm_pdpt) & ~PG_RW);
#endif
- vm_page_lock_queues();
+ rw_wlock(&pvh_global_lock);
xen_flush_queue();
xen_pgdpt_pin(VM_PAGE_TO_MACH(ptdpg[NPGPTD]));
for (i = 0; i < NPGPTD; i++) {
@@ -1529,7 +1534,7 @@
PT_SET_VA_MA(&pmap->pm_pdir[PTDPTDI + i], ma | PG_V | PG_A, FALSE);
}
xen_flush_queue();
- vm_page_unlock_queues();
+ rw_wunlock(&pvh_global_lock);
CPU_ZERO(&pmap->pm_active);
TAILQ_INIT(&pmap->pm_pvchunk);
bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
@@ -1545,25 +1550,21 @@
* mapped correctly.
*/
static vm_page_t
-_pmap_allocpte(pmap_t pmap, u_int ptepindex, int flags)
+_pmap_allocpte(pmap_t pmap, u_int ptepindex, u_int flags)
{
vm_paddr_t ptema;
vm_page_t m;
- KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT ||
- (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK,
- ("_pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK"));
-
/*
* Allocate a page table page.
*/
if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ |
VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) {
- if (flags & M_WAITOK) {
+ if ((flags & PMAP_ENTER_NOSLEEP) == 0) {
PMAP_UNLOCK(pmap);
- vm_page_unlock_queues();
+ rw_wunlock(&pvh_global_lock);
VM_WAIT;
- vm_page_lock_queues();
+ rw_wlock(&pvh_global_lock);
PMAP_LOCK(pmap);
}
@@ -1594,16 +1595,12 @@
}
static vm_page_t
-pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags)
+pmap_allocpte(pmap_t pmap, vm_offset_t va, u_int flags)
{
u_int ptepindex;
pd_entry_t ptema;
vm_page_t m;
- KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT ||
- (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK,
- ("pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK"));
-
/*
* Calculate pagetable page index
*/
@@ -1643,7 +1640,7 @@
CTR3(KTR_PMAP, "pmap_allocpte: pmap=%p va=0x%08x flags=0x%x",
pmap, va, flags);
m = _pmap_allocpte(pmap, ptepindex, flags);
- if (m == NULL && (flags & M_WAITOK))
+ if (m == NULL && (flags & PMAP_ENTER_NOSLEEP) == 0)
goto retry;
KASSERT(pmap->pm_pdir[ptepindex], ("ptepindex=%d did not get mapped", ptepindex));
@@ -1702,7 +1699,7 @@
spins = 50000000;
/* Find least significant set bit. */
- lsb = cpusetobj_ffs(&mask);
+ lsb = CPU_FFS(&mask);
MPASS(lsb != 0);
lsb--;
CPU_SETOF(lsb, &mask);
@@ -1818,7 +1815,6 @@
#ifdef PAE
pmap_qremove((vm_offset_t)pmap->pm_pdpt, 1);
#endif
- PMAP_LOCK_DESTROY(pmap);
#ifdef HAMFISTED_LOCKING
mtx_unlock(&createdelete_lock);
@@ -1894,7 +1890,7 @@
pmap_zero_page(nkpg);
ptppaddr = VM_PAGE_TO_PHYS(nkpg);
newpdir = (pd_entry_t) (ptppaddr | PG_V | PG_RW | PG_A | PG_M);
- vm_page_lock_queues();
+ rw_wlock(&pvh_global_lock);
PD_SET_VA(kernel_pmap, (kernel_vm_end >> PDRSHIFT), newpdir, TRUE);
mtx_lock_spin(&allpmaps_lock);
LIST_FOREACH(pmap, &allpmaps, pm_list)
@@ -1901,7 +1897,7 @@
PD_SET_VA(pmap, (kernel_vm_end >> PDRSHIFT), newpdir, TRUE);
mtx_unlock_spin(&allpmaps_lock);
- vm_page_unlock_queues();
+ rw_wunlock(&pvh_global_lock);
kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK;
if (kernel_vm_end - 1 >= kernel_map->max_offset) {
@@ -2034,7 +2030,7 @@
vm_page_dirty(m);
if ((tpte & PG_A) != 0)
vm_page_aflag_set(m, PGA_REFERENCED);
- TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
+ TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
if (TAILQ_EMPTY(&m->md.pv_list))
vm_page_aflag_clear(m, PGA_WRITEABLE);
pc->pc_map[field] |= 1UL << bit;
@@ -2086,7 +2082,7 @@
}
if (m_pc == NULL && pv_vafree != 0 && free != NULL) {
m_pc = free;
- free = m_pc->right;
+ free = (void *)m_pc->object;
/* Recycle a freed page table page. */
m_pc->wire_count = 1;
atomic_add_int(&cnt.v_wire_count, 1);
@@ -2104,7 +2100,7 @@
struct pv_chunk *pc;
int idx, field, bit;
- mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+ rw_assert(&pvh_global_lock, RA_WLOCKED);
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
PV_STAT(pv_entry_frees++);
PV_STAT(pv_entry_spare++);
@@ -2164,7 +2160,7 @@
vm_page_t m;
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
- mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+ rw_assert(&pvh_global_lock, RA_WLOCKED);
PV_STAT(pv_entry_allocs++);
pv_entry_count++;
if (pv_entry_count > pv_entry_high_water)
@@ -2234,10 +2230,10 @@
{
pv_entry_t pv;
- mtx_assert(&vm_page_queue_mtx, MA_OWNED);
- TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) {
+ rw_assert(&pvh_global_lock, RA_WLOCKED);
+ TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
if (pmap == PV_PMAP(pv) && va == pv->pv_va) {
- TAILQ_REMOVE(&pvh->pv_list, pv, pv_list);
+ TAILQ_REMOVE(&pvh->pv_list, pv, pv_next);
break;
}
}
@@ -2258,7 +2254,7 @@
pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va)
{
- mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+ rw_assert(&pvh_global_lock, RA_WLOCKED);
pmap_pvh_free(&m->md, pmap, va);
if (TAILQ_EMPTY(&m->md.pv_list))
vm_page_aflag_clear(m, PGA_WRITEABLE);
@@ -2273,11 +2269,11 @@
pv_entry_t pv;
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
- mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+ rw_assert(&pvh_global_lock, RA_WLOCKED);
if (pv_entry_count < pv_entry_high_water &&
(pv = get_pv_entry(pmap, TRUE)) != NULL) {
pv->pv_va = va;
- TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
+ TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
return (TRUE);
} else
return (FALSE);
@@ -2295,7 +2291,7 @@
CTR3(KTR_PMAP, "pmap_remove_pte: pmap=%p *ptq=0x%x va=0x%x",
pmap, (u_long)*ptq, va);
- mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+ rw_assert(&pvh_global_lock, RA_WLOCKED);
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
oldpte = *ptq;
PT_SET_VA_MA(ptq, 0, TRUE);
@@ -2332,7 +2328,7 @@
CTR2(KTR_PMAP, "pmap_remove_page: pmap=%p va=0x%x",
pmap, va);
- mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+ rw_assert(&pvh_global_lock, RA_WLOCKED);
KASSERT(curthread->td_pinned > 0, ("curthread not pinned"));
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
if ((pte = pmap_pte_quick(pmap, va)) == NULL || (*pte & PG_V) == 0)
@@ -2370,7 +2366,7 @@
anyvalid = 0;
- vm_page_lock_queues();
+ rw_wlock(&pvh_global_lock);
sched_pin();
PMAP_LOCK(pmap);
@@ -2447,7 +2443,7 @@
if (anyvalid)
pmap_invalidate_all(pmap);
sched_unpin();
- vm_page_unlock_queues();
+ rw_wunlock(&pvh_global_lock);
PMAP_UNLOCK(pmap);
pmap_free_zero_pages(free);
}
@@ -2476,7 +2472,7 @@
KASSERT((m->oflags & VPO_UNMANAGED) == 0,
("pmap_remove_all: page %p is not managed", m));
free = NULL;
- vm_page_lock_queues();
+ rw_wlock(&pvh_global_lock);
sched_pin();
while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
pmap = PV_PMAP(pv);
@@ -2499,7 +2495,7 @@
vm_page_dirty(m);
pmap_unuse_pt(pmap, pv->pv_va, &free);
pmap_invalidate_page(pmap, pv->pv_va);
- TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
+ TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
free_pv_entry(pmap, pv);
PMAP_UNLOCK(pmap);
}
@@ -2508,7 +2504,7 @@
if (*PMAP1)
PT_SET_MA(PADDR1, 0);
sched_unpin();
- vm_page_unlock_queues();
+ rw_wunlock(&pvh_global_lock);
pmap_free_zero_pages(free);
}
@@ -2543,7 +2539,7 @@
anychanged = 0;
- vm_page_lock_queues();
+ rw_wlock(&pvh_global_lock);
sched_pin();
PMAP_LOCK(pmap);
for (; sva < eva; sva = pdnxt) {
@@ -2627,7 +2623,7 @@
if (anychanged)
pmap_invalidate_all(pmap);
sched_unpin();
- vm_page_unlock_queues();
+ rw_wunlock(&pvh_global_lock);
PMAP_UNLOCK(pmap);
}
@@ -2643,9 +2639,9 @@
* or lose information. That is, this routine must actually
* insert this page into the given map NOW.
*/
-void
-pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t access, vm_page_t m,
- vm_prot_t prot, boolean_t wired)
+int
+pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
+ u_int flags, int8_t psind __unused)
{
pd_entry_t *pde;
pt_entry_t *pte;
@@ -2653,22 +2649,23 @@
pv_entry_t pv;
vm_paddr_t opa, pa;
vm_page_t mpte, om;
- boolean_t invlva;
+ boolean_t invlva, wired;
- CTR6(KTR_PMAP, "pmap_enter: pmap=%08p va=0x%08x access=0x%x ma=0x%08x prot=0x%x wired=%d",
- pmap, va, access, VM_PAGE_TO_MACH(m), prot, wired);
+ CTR5(KTR_PMAP,
+ "pmap_enter: pmap=%08p va=0x%08x ma=0x%08x prot=0x%x flags=0x%x",
+ pmap, va, VM_PAGE_TO_MACH(m), prot, flags);
va = trunc_page(va);
KASSERT(va <= VM_MAX_KERNEL_ADDRESS, ("pmap_enter: toobig"));
KASSERT(va < UPT_MIN_ADDRESS || va >= UPT_MAX_ADDRESS,
("pmap_enter: invalid to pmap_enter page table pages (va: 0x%x)",
va));
- KASSERT((m->oflags & (VPO_UNMANAGED | VPO_BUSY)) != 0 ||
- VM_OBJECT_LOCKED(m->object),
- ("pmap_enter: page %p is not busy", m));
+ if ((m->oflags & VPO_UNMANAGED) == 0 && !vm_page_xbusied(m))
+ VM_OBJECT_ASSERT_LOCKED(m->object);
mpte = NULL;
+ wired = (flags & PMAP_ENTER_WIRED) != 0;
- vm_page_lock_queues();
+ rw_wlock(&pvh_global_lock);
PMAP_LOCK(pmap);
sched_pin();
@@ -2677,7 +2674,15 @@
* resident, we are creating it here.
*/
if (va < VM_MAXUSER_ADDRESS) {
- mpte = pmap_allocpte(pmap, va, M_WAITOK);
+ mpte = pmap_allocpte(pmap, va, flags);
+ if (mpte == NULL) {
+ KASSERT((flags & PMAP_ENTER_NOSLEEP) != 0,
+ ("pmap_allocpte failed with sleep allowed"));
+ sched_unpin();
+ rw_wunlock(&pvh_global_lock);
+ PMAP_UNLOCK(pmap);
+ return (KERN_RESOURCE_SHORTAGE);
+ }
}
pde = pmap_pde(pmap, va);
@@ -2767,7 +2772,7 @@
if (pv == NULL)
pv = get_pv_entry(pmap, FALSE);
pv->pv_va = va;
- TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
+ TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
pa |= PG_MANAGED;
} else if (pv != NULL)
free_pv_entry(pmap, pv);
@@ -2841,8 +2846,9 @@
if (*PMAP1)
PT_SET_VA_MA(PMAP1, 0, TRUE);
sched_unpin();
- vm_page_unlock_queues();
+ rw_wunlock(&pvh_global_lock);
PMAP_UNLOCK(pmap);
+ return (KERN_SUCCESS);
}
/*
@@ -2867,11 +2873,12 @@
multicall_entry_t *mclp = mcl;
int error, count = 0;
- VM_OBJECT_LOCK_ASSERT(m_start->object, MA_OWNED);
+ VM_OBJECT_ASSERT_LOCKED(m_start->object);
+
psize = atop(end - start);
mpte = NULL;
m = m_start;
- vm_page_lock_queues();
+ rw_wlock(&pvh_global_lock);
PMAP_LOCK(pmap);
while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
mpte = pmap_enter_quick_locked(&mclp, &count, pmap, start + ptoa(diff), m,
@@ -2888,7 +2895,7 @@
error = HYPERVISOR_multicall(mcl, count);
KASSERT(error == 0, ("bad multicall %d", error));
}
- vm_page_unlock_queues();
+ rw_wunlock(&pvh_global_lock);
PMAP_UNLOCK(pmap);
}
@@ -2911,12 +2918,12 @@
CTR4(KTR_PMAP, "pmap_enter_quick: pmap=%p va=0x%x m=%p prot=0x%x",
pmap, va, m, prot);
- vm_page_lock_queues();
+ rw_wlock(&pvh_global_lock);
PMAP_LOCK(pmap);
(void)pmap_enter_quick_locked(&mclp, &count, pmap, va, m, prot, NULL);
if (count)
HYPERVISOR_multicall(&mcl, count);
- vm_page_unlock_queues();
+ rw_wunlock(&pvh_global_lock);
PMAP_UNLOCK(pmap);
}
@@ -2962,7 +2969,7 @@
KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva ||
(m->oflags & VPO_UNMANAGED) != 0,
("pmap_enter_quick_locked: managed mapping within the clean submap"));
- mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+ rw_assert(&pvh_global_lock, RA_WLOCKED);
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
/*
@@ -2996,7 +3003,7 @@
mpte->wire_count++;
} else {
mpte = _pmap_allocpte(pmap, ptepindex,
- M_NOWAIT);
+ PMAP_ENTER_NOSLEEP);
if (mpte == NULL)
return (mpte);
}
@@ -3107,7 +3114,7 @@
vm_page_t p;
int pat_mode;
- VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
+ VM_OBJECT_ASSERT_WLOCKED(object);
KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG,
("pmap_object_init_pt: non-device object"));
if (pseflag &&
@@ -3167,40 +3174,58 @@
}
/*
- * Routine: pmap_change_wiring
- * Function: Change the wiring attribute for a map/virtual-address
- * pair.
- * In/out conditions:
- * The mapping must already exist in the pmap.
+ * Clear the wired attribute from the mappings for the specified range of
+ * addresses in the given pmap. Every valid mapping within that range
+ * must have the wired attribute set. In contrast, invalid mappings
+ * cannot have the wired attribute set, so they are ignored.
+ *
+ * The wired attribute of the page table entry is not a hardware feature,
+ * so there is no need to invalidate any TLB entries.
*/
void
-pmap_change_wiring(pmap_t pmap, vm_offset_t va, boolean_t wired)
+pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
{
+ vm_offset_t pdnxt;
+ pd_entry_t *pde;
pt_entry_t *pte;
- vm_page_lock_queues();
+ CTR3(KTR_PMAP, "pmap_unwire: pmap=%p sva=0x%x eva=0x%x", pmap, sva,
+ eva);
+ rw_wlock(&pvh_global_lock);
+ sched_pin();
PMAP_LOCK(pmap);
- pte = pmap_pte(pmap, va);
-
- if (wired && !pmap_pte_w(pte)) {
- PT_SET_VA_MA((pte), *(pte) | PG_W, TRUE);
- pmap->pm_stats.wired_count++;
- } else if (!wired && pmap_pte_w(pte)) {
- PT_SET_VA_MA((pte), *(pte) & ~PG_W, TRUE);
- pmap->pm_stats.wired_count--;
+ for (; sva < eva; sva = pdnxt) {
+ pdnxt = (sva + NBPDR) & ~PDRMASK;
+ if (pdnxt < sva)
+ pdnxt = eva;
+ pde = pmap_pde(pmap, sva);
+ if ((*pde & PG_V) == 0)
+ continue;
+ if ((*pde & PG_PS) != 0)
+ panic("pmap_unwire: unexpected PG_PS in pde %#jx",
+ (uintmax_t)*pde);
+ if (pdnxt > eva)
+ pdnxt = eva;
+ for (pte = pmap_pte_quick(pmap, sva); sva != pdnxt; pte++,
+ sva += PAGE_SIZE) {
+ if ((*pte & PG_V) == 0)
+ continue;
+ if ((*pte & PG_W) == 0)
+ panic("pmap_unwire: pte %#jx is missing PG_W",
+ (uintmax_t)*pte);
+ PT_SET_VA_MA(pte, *pte & ~PG_W, FALSE);
+ pmap->pm_stats.wired_count--;
+ }
}
-
- /*
- * Wiring is not a hardware characteristic so there is no need to
- * invalidate TLB.
- */
- pmap_pte_release(pte);
+ if (*PMAP1)
+ PT_CLEAR_VA(PMAP1, FALSE);
+ PT_UPDATES_FLUSH();
+ sched_unpin();
+ rw_wunlock(&pvh_global_lock);
PMAP_UNLOCK(pmap);
- vm_page_unlock_queues();
}
-
/*
* Copy the range specified by src_addr/len
* from the source map to the range dst_addr/len
@@ -3235,7 +3260,7 @@
mtx_lock(&createdelete_lock);
#endif
- vm_page_lock_queues();
+ rw_wlock(&pvh_global_lock);
if (dst_pmap < src_pmap) {
PMAP_LOCK(dst_pmap);
PMAP_LOCK(src_pmap);
@@ -3287,7 +3312,7 @@
*/
if ((ptetemp & PG_MANAGED) != 0) {
dstmpte = pmap_allocpte(dst_pmap, addr,
- M_NOWAIT);
+ PMAP_ENTER_NOSLEEP);
if (dstmpte == NULL)
goto out;
dst_pte = pmap_pte_quick(dst_pmap, addr);
@@ -3325,7 +3350,7 @@
out:
PT_UPDATES_FLUSH();
sched_unpin();
- vm_page_unlock_queues();
+ rw_wunlock(&pvh_global_lock);
PMAP_UNLOCK(src_pmap);
PMAP_UNLOCK(dst_pmap);
@@ -3444,6 +3469,8 @@
mtx_unlock(&sysmaps->lock);
}
+int unmapped_buf_allowed = 1;
+
void
pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[],
vm_offset_t b_offset, int xfersize)
@@ -3501,8 +3528,8 @@
KASSERT((m->oflags & VPO_UNMANAGED) == 0,
("pmap_page_exists_quick: page %p is not managed", m));
rv = FALSE;
- vm_page_lock_queues();
- TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
+ rw_wlock(&pvh_global_lock);
+ TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) {
if (PV_PMAP(pv) == pmap) {
rv = TRUE;
break;
@@ -3511,7 +3538,7 @@
if (loops >= 16)
break;
}
- vm_page_unlock_queues();
+ rw_wunlock(&pvh_global_lock);
return (rv);
}
@@ -3532,9 +3559,9 @@
count = 0;
if ((m->oflags & VPO_UNMANAGED) != 0)
return (count);
- vm_page_lock_queues();
+ rw_wlock(&pvh_global_lock);
sched_pin();
- TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
+ TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) {
pmap = PV_PMAP(pv);
PMAP_LOCK(pmap);
pte = pmap_pte_quick(pmap, pv->pv_va);
@@ -3543,7 +3570,7 @@
PMAP_UNLOCK(pmap);
}
sched_unpin();
- vm_page_unlock_queues();
+ rw_wunlock(&pvh_global_lock);
return (count);
}
@@ -3585,7 +3612,7 @@
printf("warning: pmap_remove_pages called with non-current pmap\n");
return;
}
- vm_page_lock_queues();
+ rw_wlock(&pvh_global_lock);
KASSERT(pmap_is_current(pmap), ("removing pages from non-current pmap"));
PMAP_LOCK(pmap);
sched_pin();
@@ -3639,7 +3666,7 @@
if (tpte & PG_M)
vm_page_dirty(m);
- TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
+ TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
if (TAILQ_EMPTY(&m->md.pv_list))
vm_page_aflag_clear(m, PGA_WRITEABLE);
@@ -3665,7 +3692,7 @@
sched_unpin();
pmap_invalidate_all(pmap);
- vm_page_unlock_queues();
+ rw_wunlock(&pvh_global_lock);
PMAP_UNLOCK(pmap);
pmap_free_zero_pages(free);
}
@@ -3689,17 +3716,16 @@
rv = FALSE;
/*
- * If the page is not VPO_BUSY, then PGA_WRITEABLE cannot be
+ * If the page is not exclusive busied, then PGA_WRITEABLE cannot be
* concurrently set while the object is locked. Thus, if PGA_WRITEABLE
* is clear, no PTEs can have PG_M set.
*/
- VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
- if ((m->oflags & VPO_BUSY) == 0 &&
- (m->aflags & PGA_WRITEABLE) == 0)
+ VM_OBJECT_ASSERT_WLOCKED(m->object);
+ if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0)
return (rv);
- vm_page_lock_queues();
+ rw_wlock(&pvh_global_lock);
sched_pin();
- TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
+ TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) {
pmap = PV_PMAP(pv);
PMAP_LOCK(pmap);
pte = pmap_pte_quick(pmap, pv->pv_va);
@@ -3711,7 +3737,7 @@
if (*PMAP1)
PT_SET_MA(PADDR1, 0);
sched_unpin();
- vm_page_unlock_queues();
+ rw_wunlock(&pvh_global_lock);
return (rv);
}
@@ -3758,9 +3784,9 @@
KASSERT((m->oflags & VPO_UNMANAGED) == 0,
("pmap_is_referenced: page %p is not managed", m));
rv = FALSE;
- vm_page_lock_queues();
+ rw_wlock(&pvh_global_lock);
sched_pin();
- TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
+ TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) {
pmap = PV_PMAP(pv);
PMAP_LOCK(pmap);
pte = pmap_pte_quick(pmap, pv->pv_va);
@@ -3772,7 +3798,7 @@
if (*PMAP1)
PT_SET_MA(PADDR1, 0);
sched_unpin();
- vm_page_unlock_queues();
+ rw_wunlock(&pvh_global_lock);
return (rv);
}
@@ -3783,9 +3809,9 @@
for (i = 0; i < npages; i++) {
pt_entry_t *pte;
pte = pmap_pte(pmap, (vm_offset_t)(va + i*PAGE_SIZE));
- vm_page_lock_queues();
+ rw_wlock(&pvh_global_lock);
pte_store(pte, xpmap_mtop(*pte & ~(PG_RW|PG_M)));
- vm_page_unlock_queues();
+ rw_wunlock(&pvh_global_lock);
PMAP_MARK_PRIV(xpmap_mtop(*pte));
pmap_pte_release(pte);
}
@@ -3799,9 +3825,9 @@
pt_entry_t *pte;
pte = pmap_pte(pmap, (vm_offset_t)(va + i*PAGE_SIZE));
PMAP_MARK_UNPRIV(xpmap_mtop(*pte));
- vm_page_lock_queues();
+ rw_wlock(&pvh_global_lock);
pte_store(pte, xpmap_mtop(*pte) | (PG_RW|PG_M));
- vm_page_unlock_queues();
+ rw_wunlock(&pvh_global_lock);
pmap_pte_release(pte);
}
}
@@ -3820,17 +3846,16 @@
("pmap_remove_write: page %p is not managed", m));
/*
- * If the page is not VPO_BUSY, then PGA_WRITEABLE cannot be set by
- * another thread while the object is locked. Thus, if PGA_WRITEABLE
- * is clear, no page table entries need updating.
+ * If the page is not exclusive busied, then PGA_WRITEABLE cannot be
+ * set by another thread while the object is locked. Thus,
+ * if PGA_WRITEABLE is clear, no page table entries need updating.
*/
- VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
- if ((m->oflags & VPO_BUSY) == 0 &&
- (m->aflags & PGA_WRITEABLE) == 0)
+ VM_OBJECT_ASSERT_WLOCKED(m->object);
+ if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0)
return;
- vm_page_lock_queues();
+ rw_wlock(&pvh_global_lock);
sched_pin();
- TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
+ TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) {
pmap = PV_PMAP(pv);
PMAP_LOCK(pmap);
pte = pmap_pte_quick(pmap, pv->pv_va);
@@ -3859,7 +3884,7 @@
if (*PMAP1)
PT_SET_MA(PADDR1, 0);
sched_unpin();
- vm_page_unlock_queues();
+ rw_wunlock(&pvh_global_lock);
}
/*
@@ -3884,14 +3909,14 @@
KASSERT((m->oflags & VPO_UNMANAGED) == 0,
("pmap_ts_referenced: page %p is not managed", m));
- vm_page_lock_queues();
+ rw_wlock(&pvh_global_lock);
sched_pin();
if ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
pvf = pv;
do {
- pvn = TAILQ_NEXT(pv, pv_list);
- TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
- TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
+ pvn = TAILQ_NEXT(pv, pv_next);
+ TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
+ TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
pmap = PV_PMAP(pv);
PMAP_LOCK(pmap);
pte = pmap_pte_quick(pmap, pv->pv_va);
@@ -3909,11 +3934,77 @@
if (*PMAP1)
PT_SET_MA(PADDR1, 0);
sched_unpin();
- vm_page_unlock_queues();
+ rw_wunlock(&pvh_global_lock);
return (rtval);
}
/*
+ * Apply the given advice to the specified range of addresses within the
+ * given pmap. Depending on the advice, clear the referenced and/or
+ * modified flags in each mapping and set the mapped page's dirty field.
+ */
+void
+pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)
+{
+ pd_entry_t oldpde;
+ pt_entry_t *pte;
+ vm_offset_t pdnxt;
+ vm_page_t m;
+ boolean_t anychanged;
+
+ if (advice != MADV_DONTNEED && advice != MADV_FREE)
+ return;
+ anychanged = FALSE;
+ rw_wlock(&pvh_global_lock);
+ sched_pin();
+ PMAP_LOCK(pmap);
+ for (; sva < eva; sva = pdnxt) {
+ pdnxt = (sva + NBPDR) & ~PDRMASK;
+ if (pdnxt < sva)
+ pdnxt = eva;
+ oldpde = pmap->pm_pdir[sva >> PDRSHIFT];
+ if ((oldpde & (PG_PS | PG_V)) != PG_V)
+ continue;
+ if (pdnxt > eva)
+ pdnxt = eva;
+ for (pte = pmap_pte_quick(pmap, sva); sva != pdnxt; pte++,
+ sva += PAGE_SIZE) {
+ if ((*pte & (PG_MANAGED | PG_V)) != (PG_MANAGED |
+ PG_V))
+ continue;
+ else if ((*pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
+ if (advice == MADV_DONTNEED) {
+ /*
+ * Future calls to pmap_is_modified()
+ * can be avoided by making the page
+ * dirty now.
+ */
+ m = PHYS_TO_VM_PAGE(xpmap_mtop(*pte) &
+ PG_FRAME);
+ vm_page_dirty(m);
+ }
+ PT_SET_VA_MA(pte, *pte & ~(PG_M | PG_A), TRUE);
+ } else if ((*pte & PG_A) != 0)
+ PT_SET_VA_MA(pte, *pte & ~PG_A, TRUE);
+ else
+ continue;
+ if ((*pte & PG_G) != 0)
+ pmap_invalidate_page(pmap, sva);
+ else
+ anychanged = TRUE;
+ }
+ }
+ PT_UPDATES_FLUSH();
+ if (*PMAP1)
+ PT_SET_VA_MA(PMAP1, 0, TRUE);
+ if (anychanged)
+ pmap_invalidate_all(pmap);
+ sched_unpin();
+ rw_wunlock(&pvh_global_lock);
+ PMAP_UNLOCK(pmap);
+}
+
+/*
* Clear the modify bits on the specified physical page.
*/
void
@@ -3925,20 +4016,20 @@
KASSERT((m->oflags & VPO_UNMANAGED) == 0,
("pmap_clear_modify: page %p is not managed", m));
- VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
- KASSERT((m->oflags & VPO_BUSY) == 0,
- ("pmap_clear_modify: page %p is busy", m));
+ VM_OBJECT_ASSERT_WLOCKED(m->object);
+ KASSERT(!vm_page_xbusied(m),
+ ("pmap_clear_modify: page %p is exclusive busied", m));
/*
* If the page is not PGA_WRITEABLE, then no PTEs can have PG_M set.
* If the object containing the page is locked and the page is not
- * VPO_BUSY, then PGA_WRITEABLE cannot be concurrently set.
+ * exclusive busied, then PGA_WRITEABLE cannot be concurrently set.
*/
if ((m->aflags & PGA_WRITEABLE) == 0)
return;
- vm_page_lock_queues();
+ rw_wlock(&pvh_global_lock);
sched_pin();
- TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
+ TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) {
pmap = PV_PMAP(pv);
PMAP_LOCK(pmap);
pte = pmap_pte_quick(pmap, pv->pv_va);
@@ -3954,45 +4045,10 @@
PMAP_UNLOCK(pmap);
}
sched_unpin();
- vm_page_unlock_queues();
+ rw_wunlock(&pvh_global_lock);
}
/*
- * pmap_clear_reference:
- *
- * Clear the reference bit on the specified physical page.
- */
-void
-pmap_clear_reference(vm_page_t m)
-{
- pv_entry_t pv;
- pmap_t pmap;
- pt_entry_t *pte;
-
- KASSERT((m->oflags & VPO_UNMANAGED) == 0,
- ("pmap_clear_reference: page %p is not managed", m));
- vm_page_lock_queues();
- sched_pin();
- TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
- pmap = PV_PMAP(pv);
- PMAP_LOCK(pmap);
- pte = pmap_pte_quick(pmap, pv->pv_va);
- if ((*pte & PG_A) != 0) {
- /*
- * Regardless of whether a pte is 32 or 64 bits
- * in size, PG_A is among the least significant
- * 32 bits.
- */
- PT_SET_VA_MA(pte, *pte & ~PG_A, FALSE);
- pmap_invalidate_page(pmap, pv->pv_va);
- }
- PMAP_UNLOCK(pmap);
- }
- sched_unpin();
- vm_page_unlock_queues();
-}
-
-/*
* Miscellaneous support routines follow
*/
@@ -4009,13 +4065,13 @@
vm_size_t tmpsize;
offset = pa & PAGE_MASK;
- size = roundup(offset + size, PAGE_SIZE);
+ size = round_page(offset + size);
pa = pa & PG_FRAME;
if (pa < KERNLOAD && pa + size <= KERNLOAD)
va = KERNBASE + pa;
else
- va = kmem_alloc_nofault(kernel_map, size);
+ va = kva_alloc(size);
if (!va)
panic("pmap_mapdev: Couldn't alloc kernel virtual memory");
@@ -4022,7 +4078,7 @@
for (tmpsize = 0; tmpsize < size; tmpsize += PAGE_SIZE)
pmap_kenter_attr(va + tmpsize, pa + tmpsize, mode);
pmap_invalidate_range(kernel_pmap, va, va + tmpsize);
- pmap_invalidate_cache_range(va, va + size);
+ pmap_invalidate_cache_range(va, va + size, FALSE);
return ((void *)(va + offset));
}
@@ -4049,8 +4105,8 @@
return;
base = trunc_page(va);
offset = va & PAGE_MASK;
- size = roundup(offset + size, PAGE_SIZE);
- kmem_free(kernel_map, base, size);
+ size = round_page(offset + size);
+ kva_free(base, size);
}
/*
@@ -4141,7 +4197,7 @@
base = trunc_page(va);
offset = va & PAGE_MASK;
- size = roundup(offset + size, PAGE_SIZE);
+ size = round_page(offset + size);
/* Only supported on kernel virtual addresses. */
if (base <= VM_MAXUSER_ADDRESS)
@@ -4190,7 +4246,7 @@
*/
if (changed) {
pmap_invalidate_range(kernel_pmap, base, tmpva);
- pmap_invalidate_cache_range(base, tmpva);
+ pmap_invalidate_cache_range(base, tmpva, FALSE);
}
return (0);
}
@@ -4455,7 +4511,7 @@
printf("pa %x", pa);
m = PHYS_TO_VM_PAGE(pa);
- TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
+ TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) {
pmap = PV_PMAP(pv);
printf(" -> pmap %p, va %x", (void *)pmap, pv->pv_va);
pads(pmap);
Modified: trunk/sys/i386/xen/xen_clock_util.c
===================================================================
--- trunk/sys/i386/xen/xen_clock_util.c 2018-05-26 22:00:12 UTC (rev 9990)
+++ trunk/sys/i386/xen/xen_clock_util.c 2018-05-26 22:08:46 UTC (rev 9991)
@@ -1,3 +1,4 @@
+/* $MidnightBSD$ */
/*-
* Copyright (c) 2009 Adrian Chadd
* All rights reserved.
Modified: trunk/sys/i386/xen/xen_machdep.c
===================================================================
--- trunk/sys/i386/xen/xen_machdep.c 2018-05-26 22:00:12 UTC (rev 9990)
+++ trunk/sys/i386/xen/xen_machdep.c 2018-05-26 22:08:46 UTC (rev 9991)
@@ -1,3 +1,4 @@
+/* $MidnightBSD$ */
/*
*
* Copyright (c) 2004 Christian Limpach.
@@ -31,7 +32,7 @@
*/
#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
+__FBSDID("$FreeBSD: stable/10/sys/i386/xen/xen_machdep.c 271132 2014-09-04 20:47:14Z emaste $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -44,9 +45,11 @@
#include <sys/kernel.h>
#include <sys/proc.h>
#include <sys/reboot.h>
+#include <sys/rwlock.h>
#include <sys/sysproto.h>
+#include <sys/boot.h>
-#include <machine/xen/xen-os.h>
+#include <xen/xen-os.h>
#include <vm/vm.h>
#include <vm/pmap.h>
@@ -95,6 +98,8 @@
xen_pfn_t *xen_pfn_to_mfn_frame_list_list;
int preemptable, init_first;
extern unsigned int avail_space;
+int xen_vector_callback_enabled = 0;
+enum xen_domain_type xen_domain_type = XEN_PV_DOMAIN;
void ni_cli(void);
void ni_sti(void);
@@ -128,6 +133,12 @@
);
}
+void
+force_evtchn_callback(void)
+{
+ (void)HYPERVISOR_xen_version(0, NULL);
+}
+
/*
* Modify the cmd_line by converting ',' to NULLs so that it is in a format
* suitable for the static env vars.
@@ -140,30 +151,12 @@
/* Skip leading spaces */
for (; *cmd_line == ' '; cmd_line++);
- printk("xen_setbootenv(): cmd_line='%s'\n", cmd_line);
+ xc_printf("xen_setbootenv(): cmd_line='%s'\n", cmd_line);
for (cmd_line_next = cmd_line; strsep(&cmd_line_next, ",") != NULL;);
return cmd_line;
}
-static struct
-{
- const char *ev;
- int mask;
-} howto_names[] = {
- {"boot_askname", RB_ASKNAME},
- {"boot_single", RB_SINGLE},
- {"boot_nosync", RB_NOSYNC},
- {"boot_halt", RB_ASKNAME},
- {"boot_serial", RB_SERIAL},
- {"boot_cdrom", RB_CDROM},
- {"boot_gdb", RB_GDB},
- {"boot_gdb_pause", RB_RESERVED1},
- {"boot_verbose", RB_VERBOSE},
- {"boot_multicons", RB_MULTIPLE},
- {NULL, 0}
-};
-
int
xen_boothowto(char *envp)
{
@@ -176,16 +169,16 @@
return howto;
}
-#define PRINTK_BUFSIZE 1024
+#define XC_PRINTF_BUFSIZE 1024
void
-printk(const char *fmt, ...)
+xc_printf(const char *fmt, ...)
{
__va_list ap;
int retval;
- static char buf[PRINTK_BUFSIZE];
+ static char buf[XC_PRINTF_BUFSIZE];
va_start(ap, fmt);
- retval = vsnprintf(buf, PRINTK_BUFSIZE - 1, fmt, ap);
+ retval = vsnprintf(buf, XC_PRINTF_BUFSIZE - 1, fmt, ap);
va_end(ap);
buf[retval] = 0;
(void)HYPERVISOR_console_write(buf, retval);
@@ -202,11 +195,11 @@
#ifdef SMP
/* per-cpu queues and indices */
#ifdef INVARIANTS
-static struct mmu_log xpq_queue_log[MAX_VIRT_CPUS][XPQUEUE_SIZE];
+static struct mmu_log xpq_queue_log[XEN_LEGACY_MAX_VCPUS][XPQUEUE_SIZE];
#endif
-static int xpq_idx[MAX_VIRT_CPUS];
-static mmu_update_t xpq_queue[MAX_VIRT_CPUS][XPQUEUE_SIZE];
+static int xpq_idx[XEN_LEGACY_MAX_VCPUS];
+static mmu_update_t xpq_queue[XEN_LEGACY_MAX_VCPUS][XPQUEUE_SIZE];
#define XPQ_QUEUE_LOG xpq_queue_log[vcpu]
#define XPQ_QUEUE xpq_queue[vcpu]
@@ -238,9 +231,10 @@
if (_xpq_idx <= 1)
return;
- printk("xen_dump_queue(): %u entries\n", _xpq_idx);
+ xc_printf("xen_dump_queue(): %u entries\n", _xpq_idx);
for (i = 0; i < _xpq_idx; i++) {
- printk(" val: %llx ptr: %llx\n", XPQ_QUEUE[i].val, XPQ_QUEUE[i].ptr);
+ xc_printf(" val: %llx ptr: %llx\n", XPQ_QUEUE[i].val,
+ XPQ_QUEUE[i].ptr);
}
}
#endif
@@ -430,6 +424,8 @@
critical_exit();
}
+extern struct rwlock pvh_global_lock;
+
void
_xen_queue_pt_update(vm_paddr_t ptr, vm_paddr_t val, char *file, int line)
{
@@ -436,7 +432,7 @@
SET_VCPU();
if (__predict_true(gdtset))
- mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+ rw_assert(&pvh_global_lock, RA_WLOCKED);
KASSERT((ptr & 7) == 0, ("misaligned update"));
@@ -952,9 +948,10 @@
cur_space = xen_start_info->pt_base +
(l3_pages + l2_pages + l1_pages + 1)*PAGE_SIZE;
- printk("initvalues(): wooh - availmem=%x,%x\n", avail_space, cur_space);
+ xc_printf("initvalues(): wooh - availmem=%x,%x\n", avail_space,
+ cur_space);
- printk("KERNBASE=%x,pt_base=%x, VTOPFN(base)=%x, nr_pt_frames=%x\n",
+ xc_printf("KERNBASE=%x,pt_base=%x, VTOPFN(base)=%x, nr_pt_frames=%x\n",
KERNBASE,xen_start_info->pt_base, VTOPFN(xen_start_info->pt_base),
xen_start_info->nr_pt_frames);
xendebug_flags = 0; /* 0xffffffff; */
@@ -1004,7 +1001,7 @@
/* Map proc0's KSTACK */
proc0kstack = cur_space; cur_space += (KSTACK_PAGES * PAGE_SIZE);
- printk("proc0kstack=%u\n", proc0kstack);
+ xc_printf("proc0kstack=%u\n", proc0kstack);
/* vm86/bios stack */
cur_space += PAGE_SIZE;
@@ -1103,7 +1100,7 @@
shinfo = xen_start_info->shared_info;
PT_SET_MA(HYPERVISOR_shared_info, shinfo | PG_KERNEL);
- printk("#4\n");
+ xc_printf("#4\n");
xen_store_ma = (((vm_paddr_t)xen_start_info->store_mfn) << PAGE_SHIFT);
PT_SET_MA(xen_store, xen_store_ma | PG_KERNEL);
@@ -1110,11 +1107,11 @@
console_page_ma = (((vm_paddr_t)xen_start_info->console.domU.mfn) << PAGE_SHIFT);
PT_SET_MA(console_page, console_page_ma | PG_KERNEL);
- printk("#5\n");
+ xc_printf("#5\n");
set_iopl.iopl = 1;
PANIC_IF(HYPERVISOR_physdev_op(PHYSDEVOP_SET_IOPL, &set_iopl));
- printk("#6\n");
+ xc_printf("#6\n");
#if 0
/* add page table for KERNBASE */
xen_queue_pt_update(IdlePTDma + KPTDI*sizeof(vm_paddr_t),
@@ -1129,7 +1126,7 @@
#endif
xen_flush_queue();
cur_space += PAGE_SIZE;
- printk("#6\n");
+ xc_printf("#6\n");
#endif /* 0 */
#ifdef notyet
if (xen_start_info->flags & SIF_INITDOMAIN) {
@@ -1147,13 +1144,13 @@
i < (((vm_offset_t)&etext) & ~PAGE_MASK); i += PAGE_SIZE)
PT_SET_MA(i, VTOM(i) | PG_V | PG_A);
- printk("#7\n");
+ xc_printf("#7\n");
physfree = VTOP(cur_space);
init_first = physfree >> PAGE_SHIFT;
IdlePTD = (pd_entry_t *)VTOP(IdlePTD);
IdlePDPT = (pd_entry_t *)VTOP(IdlePDPT);
setup_xen_features();
- printk("#8, proc0kstack=%u\n", proc0kstack);
+ xc_printf("#8, proc0kstack=%u\n", proc0kstack);
}
@@ -1197,9 +1194,9 @@
/* Check the results of individual hypercalls. */
for (i = 0; i < nr_calls; i++)
- if (unlikely(call_list[i].result < 0))
+ if (__predict_false(call_list[i].result < 0))
ret++;
- if (unlikely(ret > 0))
+ if (__predict_false(ret > 0))
panic("%d multicall(s) failed: cpu %d\n",
ret, smp_processor_id());
Modified: trunk/sys/i386/xen/xen_rtc.c
===================================================================
--- trunk/sys/i386/xen/xen_rtc.c 2018-05-26 22:00:12 UTC (rev 9990)
+++ trunk/sys/i386/xen/xen_rtc.c 2018-05-26 22:08:46 UTC (rev 9991)
@@ -1,3 +1,4 @@
+/* $MidnightBSD$ */
/*-
* Copyright (c) 2009 Adrian Chadd
* All rights reserved.
More information about the Midnightbsd-cvs
mailing list