From 428b7ca290208e6b15d46440cd8d6309dc92dd2d Mon Sep 17 00:00:00 2001 From: "Justin T. Gibbs" Date: Fri, 20 Sep 2013 05:06:03 +0000 Subject: Add support for suspend/resume/migration operations when running as a Xen PVHVM guest. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Submitted by: Roger Pau Monné Sponsored by: Citrix Systems R&D Reviewed by: gibbs Approved by: re (blanket Xen) MFC after: 2 weeks sys/amd64/amd64/mp_machdep.c: sys/i386/i386/mp_machdep.c: - Make sure that are no MMU related IPIs pending on migration. - Reset pending IPI_BITMAP on resume. - Init vcpu_info on resume. sys/amd64/include/intr_machdep.h: sys/i386/include/intr_machdep.h: sys/x86/acpica/acpi_wakeup.c: sys/x86/x86/intr_machdep.c: sys/x86/isa/atpic.c: sys/x86/x86/io_apic.c: sys/x86/x86/local_apic.c: - Add a "suspend_cancelled" parameter to pic_resume(). For the Xen PIC, restoration of interrupt services differs between the aborted suspend and normal resume cases, so we must provide this information. sys/dev/acpica/acpi_timer.c: sys/dev/xen/timer/timer.c: sys/timetc.h: - Don't swap out "suspend safe" timers across a suspend/resume cycle. This includes the Xen PV and ACPI timers. sys/dev/xen/control/control.c: - Perform proper suspend/resume process for PVHVM: - Suspend all APs before going into suspension, this allows us to reset the vcpu_info on resume for each AP. - Reset shared info page and callback on resume. sys/dev/xen/timer/timer.c: - Implement suspend/resume support for the PV timer. Since FreeBSD doesn't perform a per-cpu resume of the timer, we need to call smp_rendezvous in order to correctly resume the timer on each CPU. sys/dev/xen/xenpci/xenpci.c: - Don't reset the PCI interrupt on each suspend/resume. sys/kern/subr_smp.c: - When suspending a PVHVM domain make sure there are no MMU IPIs in-flight, or we will get a lockup on resume due to the fact that pending event channels are not carried over on migration. - Implement a generic version of restart_cpus that can be used by suspended and stopped cpus. sys/x86/xen/hvm.c: - Implement resume support for the hypercall page and shared info. - Clear vcpu_info so it can be reset by APs when resuming from suspension. sys/dev/xen/xenpci/xenpci.c: sys/x86/xen/hvm.c: sys/x86/xen/xen_intr.c: - Support UP kernel configurations. sys/x86/xen/xen_intr.c: - Properly rebind per-cpus VIRQs and IPIs on resume. --- sys/x86/xen/hvm.c | 95 +++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 78 insertions(+), 17 deletions(-) (limited to 'sys/x86/xen/hvm.c') diff --git a/sys/x86/xen/hvm.c b/sys/x86/xen/hvm.c index 0404fe9af080..b0c2df627bda 100644 --- a/sys/x86/xen/hvm.c +++ b/sys/x86/xen/hvm.c @@ -58,6 +58,7 @@ __FBSDID("$FreeBSD$"); #include /*--------------------------- Forward Declarations ---------------------------*/ +#ifdef SMP static driver_filter_t xen_smp_rendezvous_action; static driver_filter_t xen_invltlb; static driver_filter_t xen_invlpg; @@ -70,6 +71,7 @@ static driver_filter_t xen_ipi_bitmap_handler; static driver_filter_t xen_cpustop_handler; static driver_filter_t xen_cpususpend_handler; static driver_filter_t xen_cpustophard_handler; +#endif /*---------------------------- Extern Declarations ---------------------------*/ /* Variables used by mp_machdep to perform the MMU related IPIs */ @@ -93,6 +95,12 @@ extern void pmap_lazyfix_action(void); #define IPI_TO_IDX(ipi) ((ipi) - APIC_IPI_INTS) /*-------------------------------- Local Types -------------------------------*/ +enum xen_hvm_init_type { + XEN_HVM_INIT_COLD, + XEN_HVM_INIT_CANCELLED_SUSPEND, + XEN_HVM_INIT_RESUME +}; + struct xen_ipi_handler { driver_filter_t *filter; @@ -104,6 +112,7 @@ enum xen_domain_type xen_domain_type = XEN_NATIVE; static MALLOC_DEFINE(M_XENHVM, "xen_hvm", "Xen HVM PV Support"); +#ifdef SMP static struct xen_ipi_handler xen_ipis[] = { [IPI_TO_IDX(IPI_RENDEZVOUS)] = { xen_smp_rendezvous_action, "r" }, @@ -119,6 +128,7 @@ static struct xen_ipi_handler xen_ipis[] = [IPI_TO_IDX(IPI_SUSPEND)] = { xen_cpususpend_handler, "sp" }, [IPI_TO_IDX(IPI_STOP_HARD)] = { xen_cpustophard_handler, "sth" }, }; +#endif /** * If non-zero, the hypervisor has been configured to use a direct @@ -129,13 +139,16 @@ int xen_vector_callback_enabled; /*------------------------------- Per-CPU Data -------------------------------*/ DPCPU_DEFINE(struct vcpu_info, vcpu_local_info); DPCPU_DEFINE(struct vcpu_info *, vcpu_info); +#ifdef SMP DPCPU_DEFINE(xen_intr_handle_t, ipi_handle[nitems(xen_ipis)]); +#endif /*------------------ Hypervisor Access Shared Memory Regions -----------------*/ /** Hypercall table accessed via HYPERVISOR_*_op() methods. */ char *hypercall_stubs; shared_info_t *HYPERVISOR_shared_info; +#ifdef SMP /*---------------------------- XEN PV IPI Handlers ---------------------------*/ /* * This are C clones of the ASM functions found in apic_vector.s @@ -496,6 +509,7 @@ xen_init_ipis(void) /* Set the xen pv ipi ops to replace the native ones */ cpu_ops.ipi_vectored = xen_ipi_vectored; } +#endif /*---------------------- XEN Hypervisor Probe and Setup ----------------------*/ static uint32_t @@ -579,6 +593,9 @@ xen_hvm_set_callback(device_t dev) struct xen_hvm_param xhp; int irq; + if (xen_vector_callback_enabled) + return; + xhp.domid = DOMID_SELF; xhp.index = HVM_PARAM_CALLBACK_IRQ; if (xen_feature(XENFEAT_hvm_callback_vector) != 0) { @@ -637,41 +654,83 @@ xen_hvm_disable_emulated_devices(void) outw(XEN_MAGIC_IOPORT, XMI_UNPLUG_IDE_DISKS|XMI_UNPLUG_NICS); } +static void +xen_hvm_init(enum xen_hvm_init_type init_type) +{ + int error; + int i; + + if (init_type == XEN_HVM_INIT_CANCELLED_SUSPEND) + return; + + error = xen_hvm_init_hypercall_stubs(); + + switch (init_type) { + case XEN_HVM_INIT_COLD: + if (error != 0) + return; + + setup_xen_features(); + break; + case XEN_HVM_INIT_RESUME: + if (error != 0) + panic("Unable to init Xen hypercall stubs on resume"); + break; + default: + panic("Unsupported HVM initialization type"); + } + + /* Clear any stale vcpu_info. */ + CPU_FOREACH(i) + DPCPU_ID_SET(i, vcpu_info, NULL); + + xen_vector_callback_enabled = 0; + xen_domain_type = XEN_HVM_DOMAIN; + xen_hvm_init_shared_info_page(); + xen_hvm_set_callback(NULL); + xen_hvm_disable_emulated_devices(); +} + void xen_hvm_suspend(void) { } void -xen_hvm_resume(void) +xen_hvm_resume(bool suspend_cancelled) { - xen_hvm_init_hypercall_stubs(); - xen_hvm_init_shared_info_page(); + xen_hvm_init(suspend_cancelled ? + XEN_HVM_INIT_CANCELLED_SUSPEND : XEN_HVM_INIT_RESUME); + + /* Register vcpu_info area for CPU#0. */ + xen_hvm_init_cpu(); } static void -xen_hvm_init(void *dummy __unused) +xen_hvm_sysinit(void *arg __unused) { + xen_hvm_init(XEN_HVM_INIT_COLD); +} - if (xen_hvm_init_hypercall_stubs() != 0) - return; - - xen_domain_type = XEN_HVM_DOMAIN; - setup_xen_features(); - xen_hvm_init_shared_info_page(); - xen_hvm_set_callback(NULL); - xen_hvm_disable_emulated_devices(); -} - -void xen_hvm_init_cpu(void) +void +xen_hvm_init_cpu(void) { struct vcpu_register_vcpu_info info; struct vcpu_info *vcpu_info; int cpu, rc; - cpu = PCPU_GET(acpi_id); + if (DPCPU_GET(vcpu_info) != NULL) { + /* + * vcpu_info is already set. We're resuming + * from a failed migration and our pre-suspend + * configuration is still valid. + */ + return; + } + vcpu_info = DPCPU_PTR(vcpu_local_info); + cpu = PCPU_GET(acpi_id); info.mfn = vtophys(vcpu_info) >> PAGE_SHIFT; info.offset = vtophys(vcpu_info) - trunc_page(vtophys(vcpu_info)); @@ -682,6 +741,8 @@ void xen_hvm_init_cpu(void) DPCPU_SET(vcpu_info, vcpu_info); } -SYSINIT(xen_hvm_init, SI_SUB_HYPERVISOR, SI_ORDER_FIRST, xen_hvm_init, NULL); +SYSINIT(xen_hvm_init, SI_SUB_HYPERVISOR, SI_ORDER_FIRST, xen_hvm_sysinit, NULL); +#ifdef SMP SYSINIT(xen_init_ipis, SI_SUB_SMP, SI_ORDER_FIRST, xen_init_ipis, NULL); +#endif SYSINIT(xen_hvm_init_cpu, SI_SUB_INTR, SI_ORDER_FIRST, xen_hvm_init_cpu, NULL); -- cgit v1.2.3