Switch to contigmalloc in the Hyper-V code

In the Hyper-V drivers we need to allocate buffers shared between the
host and guest. This memory has been allocated with bus_dma, however
it doesn't use this correctly, e.g. it is missing calls to
bus_dmamap_sync. Along with this on arm64 we need this memory to be
mapped with the correct memory type that bus_dma may not use.

Switch to contigmalloc to allocate this memory as this will correctly
allocate cacheable memory.

Reviewed by:	Souradeep Chakrabarti <schakrabarti@microsoft.com>
Sponsored by:	Arm Ltd
Differential Revision:	https://reviews.freebsd.org/D40227

14 files changed, 86 insertions, 141 deletions

diff --git a/sys/dev/hyperv/include/hyperv_busdma.h b/sys/dev/hyperv/include/hyperv_busdma.h
index ff01b3e27a95..804e5c26a669 100644
--- a/sys/dev/hyperv/include/hyperv_busdma.h
+++ b/sys/dev/hyperv/include/hyperv_busdma.h
@@ -41,9 +41,5 @@ struct hyperv_dma {
 
 void		hyperv_dma_map_paddr(void *arg, bus_dma_segment_t *segs,
 		    int nseg, int error);
-void		*hyperv_dmamem_alloc(bus_dma_tag_t parent_dtag,
-		    bus_size_t alignment, bus_addr_t boundary, bus_size_t size,
-		    struct hyperv_dma *dma, int flags);
-void		hyperv_dmamem_free(struct hyperv_dma *dma, void *ptr);
 
 #endif	/* !_HYPERV_BUSDMA_H_ */
diff --git a/sys/dev/hyperv/netvsc/hn_nvs.c b/sys/dev/hyperv/netvsc/hn_nvs.c
index 4dbc28996617..e1769a806c89 100644
--- a/sys/dev/hyperv/netvsc/hn_nvs.c
+++ b/sys/dev/hyperv/netvsc/hn_nvs.c
@@ -43,6 +43,10 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 #include <sys/taskqueue.h>
 
+#include <vm/vm.h>
+#include <vm/vm_extern.h>
+#include <vm/pmap.h>
+
 #include <net/ethernet.h>
 #include <net/if.h>
 #include <net/if_var.h>
@@ -52,7 +56,6 @@ __FBSDID("$FreeBSD$");
 #include <netinet/tcp_lro.h>
 
 #include <dev/hyperv/include/hyperv.h>
-#include <dev/hyperv/include/hyperv_busdma.h>
 #include <dev/hyperv/include/vmbus.h>
 #include <dev/hyperv/include/vmbus_xact.h>
 
@@ -166,7 +169,8 @@ hn_nvs_conn_rxbuf(struct hn_softc *sc)
 	 * just share this RXBUF.
 	 */
 	error = vmbus_chan_gpadl_connect(sc->hn_prichan,
-	    sc->hn_rxbuf_dma.hv_paddr, rxbuf_size, &sc->hn_rxbuf_gpadl);
+	    pmap_kextract((vm_offset_t)sc->hn_rxbuf), rxbuf_size,
+	    &sc->hn_rxbuf_gpadl);
 	if (error) {
 		if_printf(sc->hn_ifp, "rxbuf gpadl conn failed: %d\n",
 		    error);
@@ -235,7 +239,8 @@ hn_nvs_conn_chim(struct hn_softc *sc)
 	 * Sub-channels just share this chimney sending buffer.
 	 */
 	error = vmbus_chan_gpadl_connect(sc->hn_prichan,
-  	    sc->hn_chim_dma.hv_paddr, HN_CHIM_SIZE, &sc->hn_chim_gpadl);
+	    pmap_kextract((vm_offset_t)sc->hn_chim), HN_CHIM_SIZE,
+	    &sc->hn_chim_gpadl);
 	if (error) {
 		if_printf(sc->hn_ifp, "chim gpadl conn failed: %d\n", error);
 		goto cleanup;
diff --git a/sys/dev/hyperv/netvsc/hn_rndis.c b/sys/dev/hyperv/netvsc/hn_rndis.c
index 58167312c9f3..cafe73ff519c 100644
--- a/sys/dev/hyperv/netvsc/hn_rndis.c
+++ b/sys/dev/hyperv/netvsc/hn_rndis.c
@@ -50,7 +50,6 @@ __FBSDID("$FreeBSD$");
 #include <netinet/tcp_lro.h>
 
 #include <dev/hyperv/include/hyperv.h>
-#include <dev/hyperv/include/hyperv_busdma.h>
 #include <dev/hyperv/include/vmbus.h>
 #include <dev/hyperv/include/vmbus_xact.h>
 
diff --git a/sys/dev/hyperv/netvsc/if_hn.c b/sys/dev/hyperv/netvsc/if_hn.c
index 5d7002b8f65f..aeb6ec22a703 100644
--- a/sys/dev/hyperv/netvsc/if_hn.c
+++ b/sys/dev/hyperv/netvsc/if_hn.c
@@ -85,6 +85,10 @@ __FBSDID("$FreeBSD$");
 #include <sys/eventhandler.h>
 #include <sys/epoch.h>
 
+#include <vm/vm.h>
+#include <vm/vm_extern.h>
+#include <vm/pmap.h>
+
 #include <machine/atomic.h>
 #include <machine/in_cksum.h>
 
@@ -4984,9 +4988,8 @@ hn_create_rx_data(struct hn_softc *sc, int ring_cnt)
 	 * - A large enough buffer is allocated, certain version of NVSes
 	 *   may further limit the usable space.
 	 */
-	sc->hn_rxbuf = hyperv_dmamem_alloc(bus_get_dma_tag(dev),
-	    PAGE_SIZE, 0, HN_RXBUF_SIZE, &sc->hn_rxbuf_dma,
-	    BUS_DMA_WAITOK | BUS_DMA_ZERO);
+	sc->hn_rxbuf = contigmalloc(HN_RXBUF_SIZE, M_DEVBUF, M_WAITOK | M_ZERO,
+	    0ul, ~0ul, PAGE_SIZE, 0);
 	if (sc->hn_rxbuf == NULL) {
 		device_printf(sc->hn_dev, "allocate rxbuf failed\n");
 		return (ENOMEM);
@@ -5016,9 +5019,8 @@ hn_create_rx_data(struct hn_softc *sc, int ring_cnt)
 	for (i = 0; i < sc->hn_rx_ring_cnt; ++i) {
 		struct hn_rx_ring *rxr = &sc->hn_rx_ring[i];
 
-		rxr->hn_br = hyperv_dmamem_alloc(bus_get_dma_tag(dev),
-		    PAGE_SIZE, 0, HN_TXBR_SIZE + HN_RXBR_SIZE,
-		    &rxr->hn_br_dma, BUS_DMA_WAITOK);
+		rxr->hn_br = contigmalloc(HN_TXBR_SIZE + HN_RXBR_SIZE, M_DEVBUF,
+		    M_WAITOK | M_ZERO, 0ul, ~0ul, PAGE_SIZE, 0);
 		if (rxr->hn_br == NULL) {
 			device_printf(dev, "allocate bufring failed\n");
 			return (ENOMEM);
@@ -5171,7 +5173,7 @@ hn_destroy_rx_data(struct hn_softc *sc)
 
 	if (sc->hn_rxbuf != NULL) {
 		if ((sc->hn_flags & HN_FLAG_RXBUF_REF) == 0)
-			hyperv_dmamem_free(&sc->hn_rxbuf_dma, sc->hn_rxbuf);
+			contigfree(sc->hn_rxbuf, HN_RXBUF_SIZE, M_DEVBUF);
 		else
 			device_printf(sc->hn_dev, "RXBUF is referenced\n");
 		sc->hn_rxbuf = NULL;
@@ -5186,7 +5188,8 @@ hn_destroy_rx_data(struct hn_softc *sc)
 		if (rxr->hn_br == NULL)
 			continue;
 		if ((rxr->hn_rx_flags & HN_RX_FLAG_BR_REF) == 0) {
-			hyperv_dmamem_free(&rxr->hn_br_dma, rxr->hn_br);
+			contigfree(rxr->hn_br, HN_TXBR_SIZE + HN_RXBR_SIZE,
+			    M_DEVBUF);
 		} else {
 			device_printf(sc->hn_dev,
 			    "%dth channel bufring is referenced", i);
@@ -5491,9 +5494,8 @@ hn_create_tx_data(struct hn_softc *sc, int ring_cnt)
 	 *
 	 * NOTE: It is shared by all channels.
 	 */
-	sc->hn_chim = hyperv_dmamem_alloc(bus_get_dma_tag(sc->hn_dev),
-	    PAGE_SIZE, 0, HN_CHIM_SIZE, &sc->hn_chim_dma,
-	    BUS_DMA_WAITOK | BUS_DMA_ZERO);
+	sc->hn_chim = contigmalloc(HN_CHIM_SIZE, M_DEVBUF, M_WAITOK | M_ZERO,
+	    0ul, ~0ul, PAGE_SIZE, 0);
 	if (sc->hn_chim == NULL) {
 		device_printf(sc->hn_dev, "allocate txbuf failed\n");
 		return (ENOMEM);
@@ -5689,7 +5691,7 @@ hn_destroy_tx_data(struct hn_softc *sc)
 
 	if (sc->hn_chim != NULL) {
 		if ((sc->hn_flags & HN_FLAG_CHIM_REF) == 0) {
-			hyperv_dmamem_free(&sc->hn_chim_dma, sc->hn_chim);
+			contigfree(sc->hn_chim, HN_CHIM_SIZE, M_DEVBUF);
 		} else {
 			device_printf(sc->hn_dev,
 			    "chimney sending buffer is referenced");
@@ -6263,7 +6265,7 @@ hn_chan_attach(struct hn_softc *sc, struct vmbus_channel *chan)
 	 * Open this channel
 	 */
 	cbr.cbr = rxr->hn_br;
-	cbr.cbr_paddr = rxr->hn_br_dma.hv_paddr;
+	cbr.cbr_paddr = pmap_kextract((vm_offset_t)rxr->hn_br);
 	cbr.cbr_txsz = HN_TXBR_SIZE;
 	cbr.cbr_rxsz = HN_RXBR_SIZE;
 	error = vmbus_chan_open_br(chan, &cbr, NULL, 0, hn_chan_callback, rxr);
diff --git a/sys/dev/hyperv/netvsc/if_hnvar.h b/sys/dev/hyperv/netvsc/if_hnvar.h
index b85513308267..cc1f53b91876 100644
--- a/sys/dev/hyperv/netvsc/if_hnvar.h
+++ b/sys/dev/hyperv/netvsc/if_hnvar.h
@@ -102,7 +102,6 @@ struct hn_rx_ring {
 	struct sysctl_oid *hn_rx_sysctl_tree;
 
 	void		*hn_br;		/* TX/RX bufring */
-	struct hyperv_dma hn_br_dma;
 
 	struct vmbus_channel *hn_chan;
 } __aligned(CACHE_LINE_SIZE);
@@ -242,10 +241,8 @@ struct hn_softc {
 
 	void			*hn_rxbuf;
 	uint32_t		hn_rxbuf_gpadl;
-	struct hyperv_dma	hn_rxbuf_dma;
 
 	uint32_t		hn_chim_gpadl;
-	struct hyperv_dma	hn_chim_dma;
 
 	uint32_t		hn_rndis_rid;
 	uint32_t		hn_ndis_ver;
diff --git a/sys/dev/hyperv/pcib/vmbus_pcib.c b/sys/dev/hyperv/pcib/vmbus_pcib.c
index d68ad6821ad8..a684e79fbd14 100644
--- a/sys/dev/hyperv/pcib/vmbus_pcib.c
+++ b/sys/dev/hyperv/pcib/vmbus_pcib.c
@@ -77,7 +77,6 @@ __FBSDID("$FreeBSD$");
 #include <dev/acpica/acpi_pcibvar.h>
 #endif
 #include <dev/hyperv/include/hyperv.h>
-#include <dev/hyperv/include/hyperv_busdma.h>
 #include <dev/hyperv/include/vmbus_xact.h>
 #include <dev/hyperv/vmbus/vmbus_reg.h>
 #include <dev/hyperv/vmbus/vmbus_chanvar.h>
diff --git a/sys/dev/hyperv/vmbus/amd64/hyperv_machdep.c b/sys/dev/hyperv/vmbus/amd64/hyperv_machdep.c
index c158f72eb9e7..9b9c240d3843 100644
--- a/sys/dev/hyperv/vmbus/amd64/hyperv_machdep.c
+++ b/sys/dev/hyperv/vmbus/amd64/hyperv_machdep.c
@@ -31,6 +31,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/conf.h>
 #include <sys/fcntl.h>
 #include <sys/kernel.h>
+#include <sys/malloc.h>
 #include <sys/systm.h>
 #include <sys/timetc.h>
 #include <sys/vdso.h>
@@ -41,6 +42,8 @@ __FBSDID("$FreeBSD$");
 #include <machine/specialreg.h>
 
 #include <vm/vm.h>
+#include <vm/vm_extern.h>
+#include <vm/pmap.h>
 
 #include <dev/hyperv/include/hyperv.h>
 #include <dev/hyperv/include/hyperv_busdma.h>
@@ -51,7 +54,6 @@ __FBSDID("$FreeBSD$");
 
 struct hyperv_reftsc_ctx {
 	struct hyperv_reftsc	*tsc_ref;
-	struct hyperv_dma	tsc_ref_dma;
 };
 
 static uint32_t			hyperv_tsc_vdso_timehands(
@@ -117,7 +119,7 @@ hyperv_tsc_mmap(struct cdev *dev __unused, vm_ooffset_t offset,
 	if (offset != 0)
 		return (EOPNOTSUPP);
 
-	*paddr = hyperv_ref_tsc.tsc_ref_dma.hv_paddr;
+	*paddr = pmap_kextract((vm_offset_t)hyperv_ref_tsc.tsc_ref);
 	return (0);
 }
 
@@ -208,18 +210,17 @@ hyperv_tsc_tcinit(void *dummy __unused)
 		return;
 	}
 
-	hyperv_ref_tsc.tsc_ref = hyperv_dmamem_alloc(NULL, PAGE_SIZE, 0,
-	    sizeof(struct hyperv_reftsc), &hyperv_ref_tsc.tsc_ref_dma,
-	    BUS_DMA_WAITOK | BUS_DMA_ZERO);
+	hyperv_ref_tsc.tsc_ref = contigmalloc(PAGE_SIZE, M_DEVBUF,
+	    M_WAITOK | M_ZERO, 0ul, ~0ul, PAGE_SIZE, 0);
 	if (hyperv_ref_tsc.tsc_ref == NULL) {
 		printf("hyperv: reftsc page allocation failed\n");
 		return;
 	}
 
 	orig = rdmsr(MSR_HV_REFERENCE_TSC);
-	val = MSR_HV_REFTSC_ENABLE | (orig & MSR_HV_REFTSC_RSVD_MASK) |
-	    ((hyperv_ref_tsc.tsc_ref_dma.hv_paddr >> PAGE_SHIFT) <<
-	     MSR_HV_REFTSC_PGSHIFT);
+	val = (pmap_kextract((vm_offset_t)hyperv_ref_tsc.tsc_ref) >>
+	    PAGE_SHIFT) << MSR_HV_REFTSC_PGSHIFT;
+	val |= MSR_HV_REFTSC_ENABLE | (orig & MSR_HV_REFTSC_RSVD_MASK);
 	wrmsr(MSR_HV_REFERENCE_TSC, val);
 
 	/* Register "enlightened" timecounter. */
diff --git a/sys/dev/hyperv/vmbus/hyperv_busdma.c b/sys/dev/hyperv/vmbus/hyperv_busdma.c
index 9550540014c4..81817d476d29 100644
--- a/sys/dev/hyperv/vmbus/hyperv_busdma.c
+++ b/sys/dev/hyperv/vmbus/hyperv_busdma.c
@@ -49,50 +49,3 @@ hyperv_dma_map_paddr(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 	*paddr = segs->ds_addr;
 }
 
-void *
-hyperv_dmamem_alloc(bus_dma_tag_t parent_dtag, bus_size_t alignment,
-    bus_addr_t boundary, bus_size_t size, struct hyperv_dma *dma, int flags)
-{
-	void *ret;
-	int error;
-
-	error = bus_dma_tag_create(parent_dtag, /* parent */
-	    alignment,		/* alignment */
-	    boundary,		/* boundary */
-	    BUS_SPACE_MAXADDR,	/* lowaddr */
-	    BUS_SPACE_MAXADDR,	/* highaddr */
-	    NULL, NULL,		/* filter, filterarg */
-	    size,		/* maxsize */
-	    1,			/* nsegments */
-	    size,		/* maxsegsize */
-	    0,			/* flags */
-	    NULL,		/* lockfunc */
-	    NULL,		/* lockfuncarg */
-	    &dma->hv_dtag);
-	if (error)
-		return NULL;
-
-	error = bus_dmamem_alloc(dma->hv_dtag, &ret,
-	    (flags & HYPERV_DMA_MASK) | BUS_DMA_COHERENT, &dma->hv_dmap);
-	if (error) {
-		bus_dma_tag_destroy(dma->hv_dtag);
-		return NULL;
-	}
-
-	error = bus_dmamap_load(dma->hv_dtag, dma->hv_dmap, ret, size,
-	    hyperv_dma_map_paddr, &dma->hv_paddr, BUS_DMA_NOWAIT);
-	if (error) {
-		bus_dmamem_free(dma->hv_dtag, ret, dma->hv_dmap);
-		bus_dma_tag_destroy(dma->hv_dtag);
-		return NULL;
-	}
-	return ret;
-}
-
-void
-hyperv_dmamem_free(struct hyperv_dma *dma, void *ptr)
-{
-	bus_dmamap_unload(dma->hv_dtag, dma->hv_dmap);
-	bus_dmamem_free(dma->hv_dtag, ptr, dma->hv_dmap);
-	bus_dma_tag_destroy(dma->hv_dtag);
-}
diff --git a/sys/dev/hyperv/vmbus/hyperv_common_reg.h b/sys/dev/hyperv/vmbus/hyperv_common_reg.h
index 2a6437390ca6..e303d263a7fe 100644
--- a/sys/dev/hyperv/vmbus/hyperv_common_reg.h
+++ b/sys/dev/hyperv/vmbus/hyperv_common_reg.h
@@ -61,10 +61,12 @@
 #define MSR_HV_SIEFP_ENABLE		0x0001ULL
 #define MSR_HV_SIEFP_RSVD_MASK		0x0ffeULL
 #define MSR_HV_SIEFP_PGSHIFT		12
+#define MSR_HV_SIEFP_PGMASK		(~0ULL << MSR_HV_SIEFP_PGSHIFT)
 
 #define MSR_HV_SIMP_ENABLE		0x0001ULL
 #define MSR_HV_SIMP_RSVD_MASK		0x0ffeULL
 #define MSR_HV_SIMP_PGSHIFT		12
+#define	MSR_HV_SIMP_PGMASK		(~0ULL << MSR_HV_SIMP_PGSHIFT)
 
 #define MSR_HV_SINT_VECTOR_MASK		0x00ffULL
 #define MSR_HV_SINT_RSVD1_MASK		0xff00ULL
diff --git a/sys/dev/hyperv/vmbus/vmbus.c b/sys/dev/hyperv/vmbus/vmbus.c
index c349f862d378..ae4c466d98d9 100644
--- a/sys/dev/hyperv/vmbus/vmbus.c
+++ b/sys/dev/hyperv/vmbus/vmbus.c
@@ -47,6 +47,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/taskqueue.h>
 
 #include <vm/vm.h>
+#include <vm/vm_extern.h>
 #include <vm/vm_param.h>
 #include <vm/pmap.h>
 
@@ -425,9 +426,9 @@ vmbus_connect(struct vmbus_softc *sc, uint32_t version)
 	req = vmbus_msghc_dataptr(mh);
 	req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_CONNECT;
 	req->chm_ver = version;
-	req->chm_evtflags = sc->vmbus_evtflags_dma.hv_paddr;
-	req->chm_mnf1 = sc->vmbus_mnf1_dma.hv_paddr;
-	req->chm_mnf2 = sc->vmbus_mnf2_dma.hv_paddr;
+	req->chm_evtflags = pmap_kextract((vm_offset_t)sc->vmbus_evtflags);
+	req->chm_mnf1 = pmap_kextract((vm_offset_t)sc->vmbus_mnf1);
+	req->chm_mnf2 = pmap_kextract((vm_offset_t)sc->vmbus_mnf2);
 
 	error = vmbus_msghc_exec(sc, mh);
 	if (error) {
@@ -744,17 +745,17 @@ vmbus_synic_setup(void *xsc)
 	 * Setup the SynIC message.
 	 */
 	orig = RDMSR(MSR_HV_SIMP);
-	val = MSR_HV_SIMP_ENABLE | (orig & MSR_HV_SIMP_RSVD_MASK) |
-	    ((VMBUS_PCPU_GET(sc, message_dma.hv_paddr, cpu) >> PAGE_SHIFT)
-		<< MSR_HV_SIMP_PGSHIFT);
+	val = pmap_kextract((vm_offset_t)VMBUS_PCPU_GET(sc, message, cpu)) &
+	    MSR_HV_SIMP_PGMASK;
+	val |= MSR_HV_SIMP_ENABLE | (orig & MSR_HV_SIMP_RSVD_MASK);
 	WRMSR(MSR_HV_SIMP, val);
 	/*
 	 * Setup the SynIC event flags.
 	 */
 	orig = RDMSR(MSR_HV_SIEFP);
-	val = MSR_HV_SIEFP_ENABLE | (orig & MSR_HV_SIEFP_RSVD_MASK) |
-	    ((VMBUS_PCPU_GET(sc, event_flags_dma.hv_paddr, cpu) >> PAGE_SHIFT)
-		<< MSR_HV_SIEFP_PGSHIFT);
+	val = pmap_kextract((vm_offset_t)VMBUS_PCPU_GET(sc, event_flags, cpu)) &
+	    MSR_HV_SIMP_PGMASK;
+	val |= MSR_HV_SIEFP_ENABLE | (orig & MSR_HV_SIEFP_RSVD_MASK);
 	WRMSR(MSR_HV_SIEFP, val);
 
 	/*
@@ -817,48 +818,43 @@ vmbus_synic_teardown(void *arg)
 static int
 vmbus_dma_alloc(struct vmbus_softc *sc)
 {
-	bus_dma_tag_t parent_dtag;
 	uint8_t *evtflags;
 	int cpu;
 
-	parent_dtag = bus_get_dma_tag(sc->vmbus_dev);
 	CPU_FOREACH(cpu) {
 		void *ptr;
 
 		/*
 		 * Per-cpu messages and event flags.
 		 */
-		ptr = hyperv_dmamem_alloc(parent_dtag, PAGE_SIZE, 0,
-		    PAGE_SIZE, VMBUS_PCPU_PTR(sc, message_dma, cpu),
-		    BUS_DMA_WAITOK | BUS_DMA_ZERO);
+		ptr = contigmalloc(PAGE_SIZE, M_DEVBUF, M_WAITOK | M_ZERO,
+		    0ul, ~0ul, PAGE_SIZE, 0);
 		if (ptr == NULL)
 			return ENOMEM;
 		VMBUS_PCPU_GET(sc, message, cpu) = ptr;
 
-		ptr = hyperv_dmamem_alloc(parent_dtag, PAGE_SIZE, 0,
-		    PAGE_SIZE, VMBUS_PCPU_PTR(sc, event_flags_dma, cpu),
-		    BUS_DMA_WAITOK | BUS_DMA_ZERO);
+		ptr = contigmalloc(PAGE_SIZE, M_DEVBUF, M_WAITOK | M_ZERO,
+		    0ul, ~0ul, PAGE_SIZE, 0);
 		if (ptr == NULL)
 			return ENOMEM;
 		VMBUS_PCPU_GET(sc, event_flags, cpu) = ptr;
 	}
 
-	evtflags = hyperv_dmamem_alloc(parent_dtag, PAGE_SIZE, 0,
-	    PAGE_SIZE, &sc->vmbus_evtflags_dma, BUS_DMA_WAITOK | BUS_DMA_ZERO);
+	evtflags = contigmalloc(PAGE_SIZE, M_DEVBUF, M_WAITOK | M_ZERO,
+	    0ul, ~0ul, PAGE_SIZE, 0);
 	if (evtflags == NULL)
 		return ENOMEM;
 	sc->vmbus_rx_evtflags = (u_long *)evtflags;
 	sc->vmbus_tx_evtflags = (u_long *)(evtflags + (PAGE_SIZE / 2));
 	sc->vmbus_evtflags = evtflags;
 
-	sc->vmbus_mnf1 = hyperv_dmamem_alloc(parent_dtag, PAGE_SIZE, 0,
-	    PAGE_SIZE, &sc->vmbus_mnf1_dma, BUS_DMA_WAITOK | BUS_DMA_ZERO);
+	sc->vmbus_mnf1 = contigmalloc(PAGE_SIZE, M_DEVBUF, M_WAITOK | M_ZERO,
+	    0ul, ~0ul, PAGE_SIZE, 0);
 	if (sc->vmbus_mnf1 == NULL)
 		return ENOMEM;
 
-	sc->vmbus_mnf2 = hyperv_dmamem_alloc(parent_dtag, PAGE_SIZE, 0,
-	    sizeof(struct vmbus_mnf), &sc->vmbus_mnf2_dma,
-	    BUS_DMA_WAITOK | BUS_DMA_ZERO);
+	sc->vmbus_mnf2 = contigmalloc(sizeof(struct vmbus_mnf), M_DEVBUF,
+	    M_WAITOK | M_ZERO, 0ul, ~0ul, PAGE_SIZE, 0);
 	if (sc->vmbus_mnf2 == NULL)
 		return ENOMEM;
 
@@ -871,31 +867,29 @@ vmbus_dma_free(struct vmbus_softc *sc)
 	int cpu;
 
 	if (sc->vmbus_evtflags != NULL) {
-		hyperv_dmamem_free(&sc->vmbus_evtflags_dma, sc->vmbus_evtflags);
+		contigfree(sc->vmbus_evtflags, PAGE_SIZE, M_DEVBUF);
 		sc->vmbus_evtflags = NULL;
 		sc->vmbus_rx_evtflags = NULL;
 		sc->vmbus_tx_evtflags = NULL;
 	}
 	if (sc->vmbus_mnf1 != NULL) {
-		hyperv_dmamem_free(&sc->vmbus_mnf1_dma, sc->vmbus_mnf1);
+		contigfree(sc->vmbus_mnf1, PAGE_SIZE, M_DEVBUF);
 		sc->vmbus_mnf1 = NULL;
 	}
 	if (sc->vmbus_mnf2 != NULL) {
-		hyperv_dmamem_free(&sc->vmbus_mnf2_dma, sc->vmbus_mnf2);
+		contigfree(sc->vmbus_mnf2, sizeof(struct vmbus_mnf), M_DEVBUF);
 		sc->vmbus_mnf2 = NULL;
 	}
 
 	CPU_FOREACH(cpu) {
 		if (VMBUS_PCPU_GET(sc, message, cpu) != NULL) {
-			hyperv_dmamem_free(
-			    VMBUS_PCPU_PTR(sc, message_dma, cpu),
-			    VMBUS_PCPU_GET(sc, message, cpu));
+			contigfree(VMBUS_PCPU_GET(sc, message, cpu), PAGE_SIZE,
+			    M_DEVBUF);
 			VMBUS_PCPU_GET(sc, message, cpu) = NULL;
 		}
 		if (VMBUS_PCPU_GET(sc, event_flags, cpu) != NULL) {
-			hyperv_dmamem_free(
-			    VMBUS_PCPU_PTR(sc, event_flags_dma, cpu),
-			    VMBUS_PCPU_GET(sc, event_flags, cpu));
+			contigfree(VMBUS_PCPU_GET(sc, event_flags, cpu),
+			    PAGE_SIZE, M_DEVBUF);
 			VMBUS_PCPU_GET(sc, event_flags, cpu) = NULL;
 		}
 	}
diff --git a/sys/dev/hyperv/vmbus/vmbus_chan.c b/sys/dev/hyperv/vmbus/vmbus_chan.c
index 032e06c47c95..ff7bbc7c0dc3 100644
--- a/sys/dev/hyperv/vmbus/vmbus_chan.c
+++ b/sys/dev/hyperv/vmbus/vmbus_chan.c
@@ -43,7 +43,10 @@ __FBSDID("$FreeBSD$");
 #include <machine/atomic.h>
 #include <machine/stdarg.h>
 
-#include <dev/hyperv/include/hyperv_busdma.h>
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <vm/vm_extern.h>
+
 #include <dev/hyperv/include/vmbus_xact.h>
 #include <dev/hyperv/vmbus/hyperv_var.h>
 #include <dev/hyperv/vmbus/vmbus_reg.h>
@@ -137,7 +140,8 @@ vmbus_chan_signal(const struct vmbus_channel *chan)
 	if (chan->ch_txflags & VMBUS_CHAN_TXF_HASMNF)
 		atomic_set_int(chan->ch_montrig, chan->ch_montrig_mask);
 	else
-		hypercall_signal_event(chan->ch_monprm_dma.hv_paddr);
+		hypercall_signal_event(pmap_kextract(
+		    (vm_offset_t)chan->ch_monprm));
 }
 
 static __inline void
@@ -340,16 +344,16 @@ vmbus_chan_open(struct vmbus_channel *chan, int txbr_size, int rxbr_size,
 	 * Allocate the TX+RX bufrings.
 	 */
 	KASSERT(chan->ch_bufring == NULL, ("bufrings are allocated"));
-	chan->ch_bufring = hyperv_dmamem_alloc(bus_get_dma_tag(chan->ch_dev),
-	    PAGE_SIZE, 0, txbr_size + rxbr_size, &chan->ch_bufring_dma,
-	    BUS_DMA_WAITOK);
+	chan->ch_bufring_size = txbr_size + rxbr_size;
+	chan->ch_bufring = contigmalloc(chan->ch_bufring_size, M_DEVBUF,
+	    M_WAITOK | M_ZERO, 0ul, ~0ul, PAGE_SIZE, 0);
 	if (chan->ch_bufring == NULL) {
 		vmbus_chan_printf(chan, "bufring allocation failed\n");
 		return (ENOMEM);
 	}
 
 	cbr.cbr = chan->ch_bufring;
-	cbr.cbr_paddr = chan->ch_bufring_dma.hv_paddr;
+	cbr.cbr_paddr = pmap_kextract((vm_offset_t)chan->ch_bufring);
 	cbr.cbr_txsz = txbr_size;
 	cbr.cbr_rxsz = rxbr_size;
 
@@ -367,8 +371,8 @@ vmbus_chan_open(struct vmbus_channel *chan, int txbr_size, int rxbr_size,
 			    "leak %d bytes memory\n", chan->ch_id,
 			    txbr_size + rxbr_size);
 		} else {
-			hyperv_dmamem_free(&chan->ch_bufring_dma,
-			    chan->ch_bufring);
+			contigfree(chan->ch_bufring, chan->ch_bufring_size,
+			    M_DEVBUF);
 		}
 		chan->ch_bufring = NULL;
 	}
@@ -938,7 +942,7 @@ disconnect:
 	 * Destroy the TX+RX bufrings.
 	 */
 	if (chan->ch_bufring != NULL) {
-		hyperv_dmamem_free(&chan->ch_bufring_dma, chan->ch_bufring);
+		contigfree(chan->ch_bufring, chan->ch_bufring_size, M_DEVBUF);
 		chan->ch_bufring = NULL;
 	}
 	return (error);
@@ -1630,9 +1634,8 @@ vmbus_chan_alloc(struct vmbus_softc *sc)
 
 	chan = malloc(sizeof(*chan), M_DEVBUF, M_WAITOK | M_ZERO);
 
-	chan->ch_monprm = hyperv_dmamem_alloc(bus_get_dma_tag(sc->vmbus_dev),
-	    HYPERCALL_PARAM_ALIGN, 0, sizeof(struct hyperv_mon_param),
-	    &chan->ch_monprm_dma, BUS_DMA_WAITOK | BUS_DMA_ZERO);
+	chan->ch_monprm = contigmalloc(sizeof(struct hyperv_mon_param),
+	    M_DEVBUF, M_WAITOK | M_ZERO, 0ul, ~0ul, HYPERCALL_PARAM_ALIGN, 0);
 	if (chan->ch_monprm == NULL) {
 		device_printf(sc->vmbus_dev, "monprm alloc failed\n");
 		free(chan, M_DEVBUF);
@@ -1671,7 +1674,7 @@ vmbus_chan_free(struct vmbus_channel *chan)
 	KASSERT(chan->ch_poll_intvl == 0, ("chan%u: polling is activated",
 	    chan->ch_id));
 
-	hyperv_dmamem_free(&chan->ch_monprm_dma, chan->ch_monprm);
+	contigfree(chan->ch_monprm, sizeof(struct hyperv_mon_param), M_DEVBUF);
 	mtx_destroy(&chan->ch_subchan_lock);
 	sx_destroy(&chan->ch_orphan_lock);
 	vmbus_rxbr_deinit(&chan->ch_rxbr);
diff --git a/sys/dev/hyperv/vmbus/vmbus_chanvar.h b/sys/dev/hyperv/vmbus/vmbus_chanvar.h
index b20b0119bc04..ba445ad2f82e 100644
--- a/sys/dev/hyperv/vmbus/vmbus_chanvar.h
+++ b/sys/dev/hyperv/vmbus/vmbus_chanvar.h
@@ -39,7 +39,6 @@
 #include <sys/taskqueue.h>
 
 #include <dev/hyperv/include/hyperv.h>
-#include <dev/hyperv/include/hyperv_busdma.h>
 #include <dev/hyperv/include/vmbus.h>
 #include <dev/hyperv/vmbus/vmbus_brvar.h>
 
@@ -99,7 +98,6 @@ struct vmbus_channel {
 	 */
 
 	struct hyperv_mon_param		*ch_monprm;
-	struct hyperv_dma		ch_monprm_dma;
 
 	uint32_t			ch_id;		/* channel id */
 	device_t			ch_dev;
@@ -127,7 +125,7 @@ struct vmbus_channel {
 	struct vmbus_channel		*ch_prichan;	/* owner primary chan */
 
 	void				*ch_bufring;	/* TX+RX bufrings */
-	struct hyperv_dma		ch_bufring_dma;
+	size_t				ch_bufring_size;
 	uint32_t			ch_bufring_gpadl;
 
 	struct task			ch_attach_task;	/* run in ch_mgmt_tq */
diff --git a/sys/dev/hyperv/vmbus/vmbus_var.h b/sys/dev/hyperv/vmbus/vmbus_var.h
index c89d26d7aaf5..c50eeaca956d 100644
--- a/sys/dev/hyperv/vmbus/vmbus_var.h
+++ b/sys/dev/hyperv/vmbus/vmbus_var.h
@@ -34,7 +34,6 @@
 #include <sys/taskqueue.h>
 #include <sys/rman.h>
 
-#include <dev/hyperv/include/hyperv_busdma.h>
 #include <dev/pci/pcivar.h>
 #include <dev/pci/pcib_private.h>
 
@@ -74,8 +73,6 @@ struct vmbus_pcpu_data {
 	struct vmbus_evtflags	*event_flags;	/* event flags from host */
 
 	/* Rarely used fields */
-	struct hyperv_dma	message_dma;	/* busdma glue */
-	struct hyperv_dma	event_flags_dma;/* busdma glue */
 	struct taskqueue	*event_tq;	/* event taskq */
 	struct taskqueue	*message_tq;	/* message taskq */
 	struct task		message_task;	/* message task */
@@ -105,11 +102,8 @@ struct vmbus_softc {
 
 	/* Shared memory for vmbus_{rx,tx}_evtflags */
 	void			*vmbus_evtflags;
-	struct hyperv_dma	vmbus_evtflags_dma;
 
 	void			*vmbus_mnf1;	/* monitored by VM, unused */
-	struct hyperv_dma	vmbus_mnf1_dma;
-	struct hyperv_dma	vmbus_mnf2_dma;
 
 	bool			vmbus_scandone;
 	struct task		vmbus_scandone_task;
diff --git a/sys/dev/hyperv/vmbus/vmbus_xact.c b/sys/dev/hyperv/vmbus/vmbus_xact.c
index 90bdba7e1058..df6d8b45f9b1 100644
--- a/sys/dev/hyperv/vmbus/vmbus_xact.c
+++ b/sys/dev/hyperv/vmbus/vmbus_xact.c
@@ -34,7 +34,10 @@ __FBSDID("$FreeBSD$");
 #include <sys/proc.h>
 #include <sys/systm.h>
 
-#include <dev/hyperv/include/hyperv_busdma.h>
+#include <vm/vm.h>
+#include <vm/vm_extern.h>
+#include <vm/pmap.h>
+
 #include <dev/hyperv/include/vmbus_xact.h>
 
 struct vmbus_xact {
@@ -42,7 +45,6 @@ struct vmbus_xact {
 	void				*x_priv;
 
 	void				*x_req;
-	struct hyperv_dma		x_req_dma;
 
 	const void			*x_resp;
 	size_t				x_resp_len;
@@ -88,8 +90,8 @@ vmbus_xact_alloc(struct vmbus_xact_ctx *ctx, bus_dma_tag_t parent_dtag)
 	xact->x_ctx = ctx;
 
 	/* XXX assume that page aligned is enough */
-	xact->x_req = hyperv_dmamem_alloc(parent_dtag, PAGE_SIZE, 0,
-	    ctx->xc_req_size, &xact->x_req_dma, BUS_DMA_WAITOK);
+	xact->x_req = contigmalloc(ctx->xc_req_size, M_DEVBUF,
+	    M_WAITOK | M_ZERO, 0ul, ~0ul, PAGE_SIZE, 0);
 	if (xact->x_req == NULL) {
 		free(xact, M_DEVBUF);
 		return (NULL);
@@ -105,7 +107,7 @@ static void
 vmbus_xact_free(struct vmbus_xact *xact)
 {
 
-	hyperv_dmamem_free(&xact->x_req_dma, xact->x_req);
+	contigfree(xact->x_req, xact->x_ctx->xc_req_size, M_DEVBUF);
 	free(xact->x_resp0, M_DEVBUF);
 	if (xact->x_priv != NULL)
 		free(xact->x_priv, M_DEVBUF);
@@ -243,7 +245,7 @@ bus_addr_t
 vmbus_xact_req_paddr(const struct vmbus_xact *xact)
 {
 
-	return (xact->x_req_dma.hv_paddr);
+	return (pmap_kextract((vm_offset_t)xact->x_req));
 }
 
 void *