Use the limited scatter-gather capabilities of ehci, ohci and uhci

host controllers to avoid the need to allocate any multi-page
physically contiguous memory blocks. This makes it possible to use
USB devices reliably on low-memory systems or when memory is too
fragmented for contiguous allocations to succeed.

The USB subsystem now uses bus_dmamap_load() directly on the buffers
supplied by USB peripheral drivers, so this also avoids having to
copy data back and forth before and after transfers. The ehci and
ohci controllers support scatter/gather as long as the buffer is
contiguous in the virtual address space. For uhci the hardware
cannot handle a physical address discontinuity within a USB packet,
so it is necessary to copy small memory fragments at times.

14 files changed, 812 insertions, 379 deletions

diff --git a/sys/dev/usb/ehci.c b/sys/dev/usb/ehci.c
index 056e553fde6b..addf5762b5c5 100644
--- a/sys/dev/usb/ehci.c
+++ b/sys/dev/usb/ehci.c
@@ -654,7 +654,7 @@ ehci_pcd(ehci_softc_t *sc, usbd_xfer_handle xfer)
 
 	pipe = xfer->pipe;
 
-	p = KERNADDR(&xfer->dmabuf, 0);
+	p = xfer->buffer;
 	m = min(sc->sc_noport, xfer->length * 8 - 1);
 	memset(p, 0, xfer->length);
 	for (i = 1; i <= m; i++) {
@@ -1742,7 +1742,7 @@ ehci_root_ctrl_start(usbd_xfer_handle xfer)
 	index = UGETW(req->wIndex);
 
 	if (len != 0)
-		buf = KERNADDR(&xfer->dmabuf, 0);
+		buf = xfer->buffer;
 
 #define C(x,y) ((x) | ((y) << 8))
 	switch(C(req->bRequest, req->bmRequestType)) {
@@ -2335,11 +2335,11 @@ ehci_alloc_sqtd_chain(struct ehci_pipe *epipe, ehci_softc_t *sc,
      ehci_soft_qtd_t *newinactive, ehci_soft_qtd_t **sp, ehci_soft_qtd_t **ep)
 {
 	ehci_soft_qtd_t *next, *cur;
-	ehci_physaddr_t dataphys, dataphyspage, dataphyslastpage, nextphys;
+	ehci_physaddr_t dataphys, nextphys;
 	u_int32_t qtdstatus;
-	int len, curlen, mps, offset;
-	int i, iscontrol;
-	usb_dma_t *dma = &xfer->dmabuf;
+	int adj, len, curlen, mps, offset, pagelen, seg, segoff;
+	int i, iscontrol, forceshort;
+	struct usb_dma_mapping *dma = &xfer->dmamap;
 
 	DPRINTFN(alen<4*4096,("ehci_alloc_sqtd_chain: start len=%d\n", alen));
 
@@ -2347,8 +2347,6 @@ ehci_alloc_sqtd_chain(struct ehci_pipe *epipe, ehci_softc_t *sc,
 	len = alen;
 	iscontrol = (epipe->pipe.endpoint->edesc->bmAttributes & UE_XFERTYPE) ==
 	    UE_CONTROL;
-	dataphys = DMAADDR(dma, 0);
-	dataphyslastpage = EHCI_PAGE(DMAADDR(dma, len - 1));
 	qtdstatus = EHCI_QTD_ACTIVE |
 	    EHCI_QTD_SET_PID(rd ? EHCI_QTD_PID_IN : EHCI_QTD_PID_OUT) |
 	    EHCI_QTD_SET_CERR(3)
@@ -2356,6 +2354,7 @@ ehci_alloc_sqtd_chain(struct ehci_pipe *epipe, ehci_softc_t *sc,
 	    /* BYTES set below */
 	    ;
 	mps = UGETW(epipe->pipe.endpoint->edesc->wMaxPacketSize);
+	forceshort = (xfer->flags & USBD_FORCE_SHORT_XFER) && len % mps == 0;
 	/*
 	 * The control transfer data stage always starts with a toggle of 1.
 	 * For other transfers we let the hardware track the toggle state.
@@ -2377,63 +2376,63 @@ ehci_alloc_sqtd_chain(struct ehci_pipe *epipe, ehci_softc_t *sc,
 		if (cur == NULL)
 			goto nomem;
 	}
+	seg = 0;
+	segoff = 0;
 	for (;;) {
-		dataphyspage = EHCI_PAGE(dataphys);
+		curlen = 0;
+
 		/* The EHCI hardware can handle at most 5 pages. */
-#if defined(__NetBSD__) || defined(__OpenBSD__)
-		if (dataphyslastpage - dataphyspage <
-		    EHCI_QTD_NBUFFERS * EHCI_PAGE_SIZE) {
-			/* we can handle it in this QTD */
-			curlen = len;
-		}
-#elif defined(__FreeBSD__)
-		/* XXX This is pretty broken: Because we do not allocate
-		 * a contiguous buffer (contiguous in physical pages) we
-		 * can only transfer one page in one go.
-		 * So check whether the start and end of the buffer are on
-		 * the same page.
-		 */
-		if (dataphyspage == dataphyslastpage) {
-			curlen = len;
+		for (i = 0; i < EHCI_QTD_NBUFFERS && curlen < len; i++) {
+			KASSERT(seg < dma->nsegs,
+			    ("ehci_alloc_sqtd_chain: overrun"));
+			dataphys = dma->segs[seg].ds_addr + segoff;
+			pagelen = dma->segs[seg].ds_len - segoff;
+			if (pagelen > len - curlen)
+				pagelen = len - curlen;
+			if (pagelen > EHCI_PAGE_SIZE -
+			    EHCI_PAGE_OFFSET(dataphys))
+				pagelen = EHCI_PAGE_SIZE -
+				    EHCI_PAGE_OFFSET(dataphys);
+			segoff += pagelen;
+			if (segoff >= dma->segs[seg].ds_len) {
+				KASSERT(segoff == dma->segs[seg].ds_len,
+				    ("ehci_alloc_sqtd_chain: overlap"));
+				seg++;
+				segoff = 0;
+			}
+
+			cur->qtd.qtd_buffer[i] = htole32(dataphys);
+			cur->qtd.qtd_buffer_hi[i] = 0;
+			curlen += pagelen;
+
+			/*
+			 * Must stop if there is any gap before or after
+			 * the page boundary.
+			 */
+			if (EHCI_PAGE_OFFSET(dataphys + pagelen) != 0)
+				break;
+			if (seg < dma->nsegs && EHCI_PAGE_OFFSET(segoff +
+			    dma->segs[seg].ds_addr) != 0)
+				break;
 		}
-#endif
-		else {
-#if defined(__NetBSD__) || defined(__OpenBSD__)
-			/* must use multiple TDs, fill as much as possible. */
-			curlen = EHCI_QTD_NBUFFERS * EHCI_PAGE_SIZE -
-				 EHCI_PAGE_OFFSET(dataphys);
-#ifdef DIAGNOSTIC
-			if (curlen > len) {
-				printf("ehci_alloc_sqtd_chain: curlen=0x%x "
-				       "len=0x%x offs=0x%x\n", curlen, len,
-				       EHCI_PAGE_OFFSET(dataphys));
-				printf("lastpage=0x%x page=0x%x phys=0x%x\n",
-				       dataphyslastpage, dataphyspage,
-				       dataphys);
-				curlen = len;
+		/* Adjust down to a multiple of mps if not at the end. */
+		if (curlen < len && curlen % mps != 0) {
+			adj = curlen % mps;
+			curlen -= adj;
+			KASSERT(curlen > 0,
+			    ("ehci_alloc_sqtd_chain: need to copy"));
+			segoff -= adj;
+			if (segoff < 0) {
+				seg--;
+				segoff += dma->segs[seg].ds_len;
 			}
-#endif
-#elif defined(__FreeBSD__)
-			/* See comment above (XXX) */
-			curlen = EHCI_PAGE_SIZE -
-				 EHCI_PAGE_MASK(dataphys);
-#endif
-			/* the length must be a multiple of the max size */
-			curlen -= curlen % mps;
-			DPRINTFN(1,("ehci_alloc_sqtd_chain: multiple QTDs, "
-				    "curlen=%d\n", curlen));
-#ifdef DIAGNOSTIC
-			if (curlen == 0)
-				panic("ehci_alloc_std: curlen == 0");
-#endif
+			KASSERT(seg >= 0 && segoff >= 0,
+			    ("ehci_alloc_sqtd_chain: adjust to mps"));
 		}
-		DPRINTFN(4,("ehci_alloc_sqtd_chain: dataphys=0x%08x "
-			    "dataphyslastpage=0x%08x len=%d curlen=%d\n",
-			    dataphys, dataphyslastpage,
-			    len, curlen));
+
 		len -= curlen;
 
-		if (len != 0) {
+		if (len != 0 || forceshort) {
 			next = ehci_alloc_sqtd(sc);
 			if (next == NULL)
 				goto nomem;
@@ -2443,19 +2442,6 @@ ehci_alloc_sqtd_chain(struct ehci_pipe *epipe, ehci_softc_t *sc,
 			nextphys = EHCI_NULL;
 		}
 
-		for (i = 0; i * EHCI_PAGE_SIZE < curlen; i++) {
-			ehci_physaddr_t a = dataphys + i * EHCI_PAGE_SIZE;
-			if (i != 0) /* use offset only in first buffer */
-				a = EHCI_PAGE(a);
-			cur->qtd.qtd_buffer[i] = htole32(a);
-			cur->qtd.qtd_buffer_hi[i] = 0;
-#ifdef DIAGNOSTIC
-			if (i >= EHCI_QTD_NBUFFERS) {
-				printf("ehci_alloc_sqtd_chain: i=%d\n", i);
-				goto nomem;
-			}
-#endif
-		}
 		cur->nextqtd = next;
 		cur->qtd.qtd_next = nextphys;
 		/* Make sure to stop after a short transfer. */
@@ -2464,22 +2450,23 @@ ehci_alloc_sqtd_chain(struct ehci_pipe *epipe, ehci_softc_t *sc,
 		    htole32(qtdstatus | EHCI_QTD_SET_BYTES(curlen));
 		cur->xfer = xfer;
 		cur->len = curlen;
-		DPRINTFN(10,("ehci_alloc_sqtd_chain: cbp=0x%08x end=0x%08x\n",
-			    dataphys, dataphys + curlen));
+		DPRINTFN(10,("ehci_alloc_sqtd_chain: curlen=%d\n", curlen));
 		if (iscontrol) {
 			/*
 			 * adjust the toggle based on the number of packets
 			 * in this qtd
 			 */
-			if (((curlen + mps - 1) / mps) & 1)
+			if ((((curlen + mps - 1) / mps) & 1) || curlen == 0)
 				qtdstatus ^= EHCI_QTD_TOGGLE_MASK;
 		}
-		if (len == 0)
-			break;
 		qtdstatus |= EHCI_QTD_ACTIVE;
+		if (len == 0) {
+			if (!forceshort)
+				break;
+			forceshort = 0;
+		}
 		DPRINTFN(10,("ehci_alloc_sqtd_chain: extend chain\n"));
 		offset += curlen;
-		dataphys = DMAADDR(dma, offset);
 		cur = next;
 	}
 	cur->qtd.qtd_status |= htole32(EHCI_QTD_IOC);
diff --git a/sys/dev/usb/ehci_pci.c b/sys/dev/usb/ehci_pci.c
index 204fad084514..6ed2614091e4 100644
--- a/sys/dev/usb/ehci_pci.c
+++ b/sys/dev/usb/ehci_pci.c
@@ -58,6 +58,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/module.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
 #include <sys/bus.h>
 #include <sys/queue.h>
 #include <sys/lockmgr.h>
@@ -420,6 +422,30 @@ ehci_pci_attach(device_t self)
 	}
 	sc->sc_ncomp = ncomp;
 
+	/* Allocate a parent dma tag for DMA maps */
+	err = bus_dma_tag_create(NULL, 1, 0, BUS_SPACE_MAXADDR_32BIT,
+	    BUS_SPACE_MAXADDR, NULL, NULL, BUS_SPACE_MAXSIZE_32BIT,
+	    USB_DMA_NSEG, BUS_SPACE_MAXSIZE_32BIT, 0, NULL, NULL,
+	    &sc->sc_bus.parent_dmatag);
+	if (err) {
+		device_printf(self, "Could not allocate parent DMA tag (%d)\n",
+		    err);
+		ehci_pci_detach(self);
+		return ENXIO;
+	}
+
+	/* Allocate a dma tag for transfer buffers */
+	err = bus_dma_tag_create(sc->sc_bus.parent_dmatag, 1, 0,
+	    BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL,
+	    BUS_SPACE_MAXSIZE_32BIT, USB_DMA_NSEG, BUS_SPACE_MAXSIZE_32BIT, 0,
+	    busdma_lock_mutex, &Giant, &sc->sc_bus.buffer_dmatag);
+	if (err) {
+		device_printf(self, "Could not allocate buffer DMA tag (%d)\n",
+		    err);
+		ehci_pci_detach(self);
+		return ENXIO;
+	}
+
 	ehci_pci_takecontroller(self);
 	err = ehci_init(sc);
 	if (!err) {
@@ -450,6 +476,10 @@ ehci_pci_detach(device_t self)
 	 */
 	if (sc->iot && sc->ioh)
 		bus_space_write_4(sc->iot, sc->ioh, EHCI_USBINTR, 0);
+	if (sc->sc_bus.parent_dmatag != NULL)
+		bus_dma_tag_destroy(sc->sc_bus.parent_dmatag);
+	if (sc->sc_bus.buffer_dmatag != NULL)
+		bus_dma_tag_destroy(sc->sc_bus.buffer_dmatag);
 
 	if (sc->irq_res && sc->ih) {
 		int err = bus_teardown_intr(self, sc->irq_res, sc->ih);
diff --git a/sys/dev/usb/ohci.c b/sys/dev/usb/ohci.c
index 0b84a32b872c..c8e6b17640db 100644
--- a/sys/dev/usb/ohci.c
+++ b/sys/dev/usb/ohci.c
@@ -217,6 +217,8 @@ Static void		ohci_device_isoc_done(usbd_xfer_handle);
 
 Static usbd_status	ohci_device_setintr(ohci_softc_t *sc,
 			    struct ohci_pipe *pipe, int ival);
+Static usbd_status	ohci_device_intr_insert(ohci_softc_t *sc,
+			    usbd_xfer_handle xfer);
 
 Static int		ohci_str(usb_string_descriptor_t *, int, const char *);
 
@@ -510,11 +512,11 @@ ohci_alloc_std_chain(struct ohci_pipe *opipe, ohci_softc_t *sc,
 		     ohci_soft_td_t *sp, ohci_soft_td_t **ep)
 {
 	ohci_soft_td_t *next, *cur;
-	ohci_physaddr_t dataphys;
+	ohci_physaddr_t dataphys, physend;
 	u_int32_t tdflags;
 	int offset = 0;
-	int len, curlen;
-	usb_dma_t *dma = &xfer->dmabuf;
+	int len, maxp, curlen, curlen2, seg, segoff;
+	struct usb_dma_mapping *dma = &xfer->dmamap;
 	u_int16_t flags = xfer->flags;
 
 	DPRINTFN(alen < 4096,("ohci_alloc_std_chain: start len=%d\n", alen));
@@ -522,22 +524,22 @@ ohci_alloc_std_chain(struct ohci_pipe *opipe, ohci_softc_t *sc,
 	len = alen;
 	cur = sp;
 
+	maxp = UGETW(opipe->pipe.endpoint->edesc->wMaxPacketSize);
 	tdflags = htole32(
 	    (rd ? OHCI_TD_IN : OHCI_TD_OUT) |
 	    (flags & USBD_SHORT_XFER_OK ? OHCI_TD_R : 0) |
 	    OHCI_TD_NOCC | OHCI_TD_TOGGLE_CARRY | OHCI_TD_SET_DI(6));
 
+	seg = 0;
+	segoff = 0;
 	for (;;) {
 		next = ohci_alloc_std(sc);
 		if (next == NULL)
 			goto nomem;
 
-		dataphys = DMAADDR(dma, offset);
-
 		/*
 		 * The OHCI hardware can handle at most one 4k crossing.
-		 * XXX - currently we only allocate contigous buffers, but
-		 * the OHCI spec says: If during the data transfer the buffer
+		 * The OHCI spec says: If during the data transfer the buffer
 		 * address contained in the HC's working copy of
 		 * CurrentBufferPointer crosses a 4K boundary, the upper 20
 		 * bits of Buffer End are copied to the working value of
@@ -545,34 +547,67 @@ ohci_alloc_std_chain(struct ohci_pipe *opipe, ohci_softc_t *sc,
 		 * be the 0th byte in the same 4K page that contains the
 		 * last byte of the buffer (the 4K boundary crossing may
 		 * occur within a data packet transfer.)
-		 *
-		 * If/when dma has multiple segments, this will need to
-		 * properly handle fragmenting TD's.
-		 * 
-		 * Note that if we are gathering data from multiple SMALL
-		 * segments, e.g. mbufs, we need to do special gymnastics,
-		 * e.g. bounce buffering or data aggregation,
-		 * BEFORE WE GET HERE because a bulk USB transfer must
-		 * consist of maximally sized packets right up to the end.
-		 * A shorter than maximal packet means that it is the end
-		 * of the transfer. If the data transfer length is a
-		 * multiple of the packet size, then a 0 byte
-		 * packet will be the signal of the end of transfer.
-		 * Since packets can't cross TDs this means that
-		 * each TD except the last one must cover an exact multiple
-		 * of the maximal packet length.
 		 */
-		if (OHCI_PAGE_OFFSET(dataphys) + len <= (2 * OHCI_PAGE_SIZE)) {
-			/* We can handle all that remains in this TD */
+		KASSERT(seg < dma->nsegs, ("ohci_alloc_std_chain: overrun"));
+		dataphys = dma->segs[seg].ds_addr + segoff;
+		curlen = dma->segs[seg].ds_len - segoff;
+		if (curlen > len)
 			curlen = len;
+		physend = dataphys + curlen - 1;
+		if (OHCI_PAGE(dataphys) != OHCI_PAGE(physend)) {
+			/* Truncate to two OHCI pages if there are more. */
+			if (curlen > 2 * OHCI_PAGE_SIZE -
+			    OHCI_PAGE_OFFSET(dataphys))
+				curlen = 2 * OHCI_PAGE_SIZE -
+				    OHCI_PAGE_OFFSET(dataphys);
+			if (curlen < len)
+				curlen -= curlen % maxp;
+			physend = dataphys + curlen - 1;
+		} else if (OHCI_PAGE_OFFSET(physend + 1) == 0 && curlen < len &&
+		    curlen + segoff == dma->segs[seg].ds_len) {
+			/* We can possibly include another segment. */
+			KASSERT(seg + 1 < dma->nsegs,
+			    ("ohci_alloc_std_chain: overrun2"));
+			seg++;
+
+			/* Determine how much of the second segment to use. */
+			curlen2 = dma->segs[seg].ds_len;
+			if (curlen + curlen2 > len)
+				curlen2 = len - curlen;
+			if (OHCI_PAGE(dma->segs[seg].ds_addr) !=
+			    OHCI_PAGE(dma->segs[seg].ds_addr + curlen2 - 1))
+				curlen2 = OHCI_PAGE_SIZE -
+				    OHCI_PAGE_OFFSET(dma->segs[seg].ds_addr);
+			if (curlen + curlen2 < len)
+				curlen2 -= (curlen + curlen2) % maxp;
+
+			if (curlen2 > 0) {
+				/* We can include a second segment */
+				segoff = curlen2;
+				physend = dma->segs[seg].ds_addr + curlen2 - 1;
+				curlen += curlen2;
+			} else {
+				/* Second segment not usable now. */
+				seg--;
+				segoff += curlen;
+			}
 		} else {
-			/* must use multiple TDs, fill as much as possible. */
-			curlen = 2 * OHCI_PAGE_SIZE -
-				 OHCI_PAGE_OFFSET(dataphys);
-			/* the length must be a multiple of the max size */
-			curlen -= curlen %
-			    UGETW(opipe->pipe.endpoint->edesc->wMaxPacketSize);
-			KASSERT((curlen != 0), ("ohci_alloc_std: curlen == 0"));
+			/* Simple case where there is just one OHCI page. */
+			segoff += curlen;
+		}
+		if (curlen == 0 && len != 0) {
+			/*
+			 * A maxp length packet would need to be split.
+			 * This shouldn't be possible if PAGE_SIZE >= 4k
+			 * and the buffer is contiguous in virtual memory.
+			 */
+			panic("ohci_alloc_std_chain: XXX need to copy");
+		}
+		if (segoff >= dma->segs[seg].ds_len) {
+			KASSERT(segoff == dma->segs[seg].ds_len,
+			    ("ohci_alloc_std_chain: overlap"));
+			seg++;
+			segoff = 0;
 		}
 		DPRINTFN(4,("ohci_alloc_std_chain: dataphys=0x%08x "
 			    "len=%d curlen=%d\n",
@@ -583,7 +618,7 @@ ohci_alloc_std_chain(struct ohci_pipe *opipe, ohci_softc_t *sc,
 		cur->td.td_cbp = htole32(dataphys);
 		cur->nexttd = next;
 		cur->td.td_nexttd = htole32(next->physaddr);
-		cur->td.td_be = htole32(DMAADDR(dma, offset + curlen - 1));
+		cur->td.td_be = htole32(physend);
 		cur->len = curlen;
 		cur->flags = OHCI_ADD_LEN;
 		cur->xfer = xfer;
@@ -1009,14 +1044,6 @@ void
 ohci_freex(struct usbd_bus *bus, usbd_xfer_handle xfer)
 {
 	struct ohci_softc *sc = (struct ohci_softc *)bus;
-	struct ohci_xfer *oxfer = (struct ohci_xfer *)xfer;
-	ohci_soft_itd_t *sitd;
-
-        if (oxfer->ohci_xfer_flags & OHCI_ISOC_DIRTY) {
-		for (sitd = xfer->hcpriv; sitd != NULL && sitd->xfer == xfer;
-		    sitd = sitd->nextitd)
-			ohci_free_sitd(sc, sitd);
-	}
 
 #ifdef DIAGNOSTIC
 	if (xfer->busy_free != XFER_BUSY) {
@@ -1535,6 +1562,11 @@ ohci_softintr(void *v)
 				if (sitd->flags & OHCI_CALL_DONE)
 					break;
 			}
+		 	for (sitd = xfer->hcpriv; sitd->xfer == xfer;
+			    sitd = next) {
+				next = sitd->nextitd;
+				ohci_free_sitd(sc, sitd);
+			}
 
 			s = splusb();
 			usb_transfer_complete(xfer);
@@ -1571,42 +1603,18 @@ ohci_device_intr_done(usbd_xfer_handle xfer)
 {
 	struct ohci_pipe *opipe = (struct ohci_pipe *)xfer->pipe;
 	ohci_softc_t *sc = (ohci_softc_t *)opipe->pipe.device->bus;
-	ohci_soft_ed_t *sed = opipe->sed;
-	ohci_soft_td_t *data, *tail;
-
+	usbd_status err;
 
 	DPRINTFN(10,("ohci_device_intr_done: xfer=%p, actlen=%d\n",
 		     xfer, xfer->actlen));
 
 	xfer->hcpriv = NULL;
-
 	if (xfer->pipe->repeat) {
-		data = opipe->tail.td;
-		tail = ohci_alloc_std(sc); /* XXX should reuse TD */
-		if (tail == NULL) {
-			xfer->status = USBD_NOMEM;
+		err = ohci_device_intr_insert(sc, xfer);
+		if (err) {
+			xfer->status = err;
 			return;
 		}
-		tail->xfer = NULL;
-
-		data->td.td_flags = htole32(
-			OHCI_TD_IN | OHCI_TD_NOCC |
-			OHCI_TD_SET_DI(1) | OHCI_TD_TOGGLE_CARRY);
-		if (xfer->flags & USBD_SHORT_XFER_OK)
-			data->td.td_flags |= htole32(OHCI_TD_R);
-		data->td.td_cbp = htole32(DMAADDR(&xfer->dmabuf, 0));
-		data->nexttd = tail;
-		data->td.td_nexttd = htole32(tail->physaddr);
-		data->td.td_be = htole32(le32toh(data->td.td_cbp) +
-			xfer->length - 1);
-		data->len = xfer->length;
-		data->xfer = xfer;
-		data->flags = OHCI_CALL_DONE | OHCI_ADD_LEN;
-		xfer->hcpriv = data;
-		xfer->actlen = 0;
-
-		sed->ed.ed_tailp = htole32(tail->physaddr);
-		opipe->tail.td = tail;
 	}
 }
 
@@ -1638,7 +1646,7 @@ ohci_rhsc(ohci_softc_t *sc, usbd_xfer_handle xfer)
 
 	pipe = xfer->pipe;
 
-	p = KERNADDR(&xfer->dmabuf, 0);
+	p = xfer->buffer;
 	m = min(sc->sc_noport, xfer->length * 8 - 1);
 	memset(p, 0, xfer->length);
 	for (i = 1; i <= m; i++) {
@@ -2284,6 +2292,7 @@ ohci_abort_xfer(usbd_xfer_handle xfer, usbd_status status)
 		usb_rem_task(xfer->pipe->device, &OXFER(xfer)->abort_task);
 		usb_transfer_complete(xfer);
 		splx(s);
+		return;
 	}
 
 	if (xfer->device->bus->intr_context || !curproc)
@@ -2519,7 +2528,7 @@ ohci_root_ctrl_start(usbd_xfer_handle xfer)
 	index = UGETW(req->wIndex);
 
 	if (len != 0)
-		buf = KERNADDR(&xfer->dmabuf, 0);
+		buf = xfer->buffer;
 
 #define C(x,y) ((x) | ((y) << 8))
 	switch(C(req->bRequest, req->bmRequestType)) {
@@ -3099,26 +3108,44 @@ Static usbd_status
 ohci_device_intr_start(usbd_xfer_handle xfer)
 {
 	struct ohci_pipe *opipe = (struct ohci_pipe *)xfer->pipe;
-	usbd_device_handle dev = opipe->pipe.device;
-	ohci_softc_t *sc = (ohci_softc_t *)dev->bus;
+	ohci_softc_t *sc = (ohci_softc_t *)opipe->pipe.device->bus;
 	ohci_soft_ed_t *sed = opipe->sed;
-	ohci_soft_td_t *data, *tail;
-	int len;
-	int s;
+	usbd_status err;
 
 	if (sc->sc_dying)
 		return (USBD_IOERROR);
 
-	DPRINTFN(3, ("ohci_device_intr_transfer: xfer=%p len=%d "
+	DPRINTFN(3, ("ohci_device_intr_start: xfer=%p len=%d "
 		     "flags=%d priv=%p\n",
 		     xfer, xfer->length, xfer->flags, xfer->priv));
 
 #ifdef DIAGNOSTIC
 	if (xfer->rqflags & URQ_REQUEST)
-		panic("ohci_device_intr_transfer: a request");
+		panic("ohci_device_intr_start: a request");
 #endif
 
-	len = xfer->length;
+	err = ohci_device_intr_insert(sc, xfer);
+	if (err)
+		return (err);
+
+	sed->ed.ed_flags &= htole32(~OHCI_ED_SKIP);
+
+	return (USBD_IN_PROGRESS);
+}
+
+/*
+ * Insert an interrupt transfer into an endpoint descriptor list
+ */
+Static usbd_status
+ohci_device_intr_insert(ohci_softc_t *sc, usbd_xfer_handle xfer)
+{
+	struct ohci_pipe *opipe = (struct ohci_pipe *)xfer->pipe;
+	ohci_soft_ed_t *sed = opipe->sed;
+	ohci_soft_td_t *data, *tail;
+	ohci_physaddr_t dataphys, physend;
+	int s;
+
+	DPRINTFN(4, ("ohci_device_intr_insert: xfer=%p", xfer));
 
 	data = opipe->tail.td;
 	tail = ohci_alloc_std(sc);
@@ -3131,18 +3158,43 @@ ohci_device_intr_start(usbd_xfer_handle xfer)
 		OHCI_TD_SET_DI(1) | OHCI_TD_TOGGLE_CARRY);
 	if (xfer->flags & USBD_SHORT_XFER_OK)
 		data->td.td_flags |= htole32(OHCI_TD_R);
-	data->td.td_cbp = htole32(DMAADDR(&xfer->dmabuf, 0));
+	/*
+	 * Assume a short mapping with no complications, which
+	 * should always be true for <= 4k buffers in contiguous
+	 * virtual memory. The data can take the following forms:
+	 *	1 segment in 1 OHCI page
+	 *	1 segment in 2 OHCI pages
+	 *	2 segments in 2 OHCI pages
+	 * (see comment in ohci_alloc_std_chain() for details)
+	 */
+	KASSERT(xfer->length > 0 && xfer->length <= OHCI_PAGE_SIZE,
+	    ("ohci_device_intr_insert: bad length %d", xfer->length));
+	dataphys = xfer->dmamap.segs[0].ds_addr;
+	physend = dataphys + xfer->length - 1;
+	if (xfer->dmamap.nsegs == 2) {
+		KASSERT(OHCI_PAGE_OFFSET(dataphys +
+		    xfer->dmamap.segs[0].ds_len) == 0,
+		    ("ohci_device_intr_insert: bad seg 0 termination"));
+		physend = xfer->dmamap.segs[1].ds_addr + xfer->length -
+		    xfer->dmamap.segs[0].ds_len - 1;
+	} else {
+		KASSERT(xfer->dmamap.nsegs == 1,
+		    ("ohci_device_intr_insert: bad seg count %d",
+		    (u_int)xfer->dmamap.nsegs));
+	}
+	data->td.td_cbp = htole32(dataphys);
 	data->nexttd = tail;
 	data->td.td_nexttd = htole32(tail->physaddr);
-	data->td.td_be = htole32(le32toh(data->td.td_cbp) + len - 1);
-	data->len = len;
+	data->td.td_be = htole32(physend);
+	data->len = xfer->length;
 	data->xfer = xfer;
 	data->flags = OHCI_CALL_DONE | OHCI_ADD_LEN;
 	xfer->hcpriv = data;
+	xfer->actlen = 0;
 
 #ifdef USB_DEBUG
 	if (ohcidebug > 5) {
-		DPRINTF(("ohci_device_intr_transfer:\n"));
+		DPRINTF(("ohci_device_intr_insert:\n"));
 		ohci_dump_ed(sed);
 		ohci_dump_tds(data);
 	}
@@ -3152,25 +3204,9 @@ ohci_device_intr_start(usbd_xfer_handle xfer)
 	s = splusb();
 	sed->ed.ed_tailp = htole32(tail->physaddr);
 	opipe->tail.td = tail;
-	sed->ed.ed_flags &= htole32(~OHCI_ED_SKIP);
-
-#if 0
-/*
- * This goes horribly wrong, printing thousands of descriptors,
- * because false references are followed due to the fact that the
- * TD is gone.
- */
-	if (ohcidebug > 5) {
-		usb_delay_ms(&sc->sc_bus, 5);
-		DPRINTF(("ohci_device_intr_transfer: status=%x\n",
-			 OREAD4(sc, OHCI_COMMAND_STATUS)));
-		ohci_dump_ed(sed);
-		ohci_dump_tds(data);
-	}
-#endif
 	splx(s);
 
-	return (USBD_IN_PROGRESS);
+	return (USBD_NORMAL_COMPLETION);
 }
 
 /* Abort a device control request. */
@@ -3325,10 +3361,10 @@ ohci_device_isoc_enter(usbd_xfer_handle xfer)
 	ohci_softc_t *sc = (ohci_softc_t *)dev->bus;
 	ohci_soft_ed_t *sed = opipe->sed;
 	struct iso *iso = &opipe->u.iso;
-	struct ohci_xfer *oxfer = (struct ohci_xfer *)xfer;
+	struct usb_dma_mapping *dma = &xfer->dmamap;
 	ohci_soft_itd_t *sitd, *nsitd;
-	ohci_physaddr_t buf, offs, noffs, bp0, tdphys;
-	int i, ncur, nframes;
+	ohci_physaddr_t dataphys, bp0, physend, prevpage;
+	int curlen, i, len, ncur, nframes, npages, seg, segoff;
 	int s;
 
 	DPRINTFN(1,("ohci_device_isoc_enter: used=%d next=%d xfer=%p "
@@ -3345,94 +3381,115 @@ ohci_device_isoc_enter(usbd_xfer_handle xfer)
 			    iso->next));
 	}
 
-	if (xfer->hcpriv) {
-		for (sitd = xfer->hcpriv; sitd != NULL && sitd->xfer == xfer;
-		    sitd = sitd->nextitd)
-			ohci_free_sitd(sc, sitd); /* Free ITDs in prev xfer*/
-
-		if (sitd == NULL) {
-			sitd = ohci_alloc_sitd(sc);
-			if (sitd == NULL)
-				panic("cant alloc isoc");
-			opipe->tail.itd = sitd;
-			tdphys = sitd->physaddr;
-			sed->ed.ed_flags |= htole32(OHCI_ED_SKIP); /* Stop*/
-			sed->ed.ed_headp =
-			sed->ed.ed_tailp = htole32(tdphys);
-			sed->ed.ed_flags &= htole32(~OHCI_ED_SKIP); /* Start.*/
-		}
-	}
-
 	sitd = opipe->tail.itd;
-	buf = DMAADDR(&xfer->dmabuf, 0);
-	bp0 = OHCI_PAGE(buf);
-	offs = OHCI_PAGE_OFFSET(buf);
 	nframes = xfer->nframes;
 	xfer->hcpriv = sitd;
-	for (i = ncur = 0; i < nframes; i++, ncur++) {
-		noffs = offs + xfer->frlengths[i];
-		if (ncur == OHCI_ITD_NOFFSET ||	/* all offsets used */
-		    OHCI_PAGE(buf + noffs) > bp0 + OHCI_PAGE_SIZE) { /* too many page crossings */
-
-			/* Allocate next ITD */
-			nsitd = ohci_alloc_sitd(sc);
-			if (nsitd == NULL) {
-				/* XXX what now? */
-				printf("%s: isoc TD alloc failed\n",
-				       USBDEVNAME(sc->sc_bus.bdev));
-				return;
+	seg = 0;
+	segoff = 0;
+	i = 0;
+	while (i < nframes) {
+		/*
+		 * Fill in as many ITD frames as possible.
+		 */
+		KASSERT(seg < dma->nsegs, ("ohci_device_isoc_enter: overrun"));
+		bp0 = dma->segs[seg].ds_addr + segoff;
+		sitd->itd.itd_bp0 = htole32(bp0);
+		prevpage = OHCI_PAGE(bp0);
+		npages = 1;
+
+		ncur = 0;
+		while (ncur < OHCI_ITD_NOFFSET && i < nframes) {
+			/* Find the frame start and end physical addresses. */
+			len = xfer->frlengths[i];
+			dataphys = dma->segs[seg].ds_addr + segoff;
+			curlen = dma->segs[seg].ds_len - segoff;
+			if (len > curlen) {
+				KASSERT(seg + 1 < dma->nsegs,
+				    ("ohci_device_isoc_enter: overrun2"));
+				seg++;
+				segoff = len - curlen;
+			} else {
+				segoff += len;
+			}
+			KASSERT(segoff <= dma->segs[seg].ds_len,
+			    ("ohci_device_isoc_enter: overrun3"));
+			physend = dma->segs[seg].ds_addr + segoff - 1;
+
+			/* Check if there would be more than 2 pages . */
+			if (OHCI_PAGE(dataphys) != prevpage) {
+				prevpage = OHCI_PAGE(dataphys);
+				npages++;
+			}
+			if (OHCI_PAGE(physend) != prevpage) {
+				prevpage = OHCI_PAGE(physend);
+				npages++;
+			}
+			if (npages > 2) {
+				/* We cannot fit this frame now. */
+				segoff -= len;
+				if (segoff < 0) {
+					seg--;
+					segoff += dma->segs[seg].ds_len;
+				}
+				break;
 			}
 
-			/* Fill current ITD */
+			sitd->itd.itd_be = htole32(physend);
+			sitd->itd.itd_offset[ncur] =
+			    htole16(OHCI_ITD_MK_OFFS(OHCI_PAGE(dataphys) ==
+			    OHCI_PAGE(bp0) ? 0 : 1, dataphys));
+			i++;
+			ncur++;
+		}
+		if (segoff >= dma->segs[seg].ds_len) {
+			KASSERT(segoff == dma->segs[seg].ds_len,
+			    ("ohci_device_isoc_enter: overlap"));
+			seg++;
+			segoff = 0;
+		}
+
+		/* Allocate next ITD */
+		nsitd = ohci_alloc_sitd(sc);
+		if (nsitd == NULL) {
+			/* XXX what now? */
+			printf("%s: isoc TD alloc failed\n",
+			       USBDEVNAME(sc->sc_bus.bdev));
+			return;
+		}
+
+		/* Fill out remaining fields of current ITD */
+		sitd->nextitd = nsitd;
+		sitd->itd.itd_nextitd = htole32(nsitd->physaddr);
+		sitd->xfer = xfer;
+		if (i < nframes) {
 			sitd->itd.itd_flags = htole32(
 				OHCI_ITD_NOCC |
 				OHCI_ITD_SET_SF(iso->next) |
 				OHCI_ITD_SET_DI(6) | /* delay intr a little */
 				OHCI_ITD_SET_FC(ncur));
-			sitd->itd.itd_bp0 = htole32(bp0);
-			sitd->nextitd = nsitd;
-			sitd->itd.itd_nextitd = htole32(nsitd->physaddr);
-			sitd->itd.itd_be = htole32(bp0 + offs - 1);
-			sitd->xfer = xfer;
 			sitd->flags = OHCI_ITD_ACTIVE;
-
-			sitd = nsitd;
-			iso->next = iso->next + ncur;
-			bp0 = OHCI_PAGE(buf + offs);
-			ncur = 0;
+		} else {
+			sitd->itd.itd_flags = htole32(
+				OHCI_ITD_NOCC |
+				OHCI_ITD_SET_SF(iso->next) |
+				OHCI_ITD_SET_DI(0) |
+				OHCI_ITD_SET_FC(ncur));
+			sitd->flags = OHCI_CALL_DONE | OHCI_ITD_ACTIVE;
 		}
-		sitd->itd.itd_offset[ncur] = htole16(OHCI_ITD_MK_OFFS(offs));
-		offs = noffs;
-	}
-	nsitd = ohci_alloc_sitd(sc);
-	if (nsitd == NULL) {
-		/* XXX what now? */
-		printf("%s: isoc TD alloc failed\n",
-		       USBDEVNAME(sc->sc_bus.bdev));
-		return;
+		iso->next += ncur;
+
+		sitd = nsitd;
 	}
-	/* Fixup last used ITD */
-	sitd->itd.itd_flags = htole32(
-		OHCI_ITD_NOCC |
-		OHCI_ITD_SET_SF(iso->next) |
-		OHCI_ITD_SET_DI(0) |
-		OHCI_ITD_SET_FC(ncur));
-	sitd->itd.itd_bp0 = htole32(bp0);
-	sitd->nextitd = nsitd;
-	sitd->itd.itd_nextitd = htole32(nsitd->physaddr);
-	sitd->itd.itd_be = htole32(bp0 + offs - 1);
-	sitd->xfer = xfer;
-	sitd->flags = OHCI_CALL_DONE | OHCI_ITD_ACTIVE;
-
-	iso->next = iso->next + ncur;
+
 	iso->inuse += nframes;
 
-	xfer->actlen = offs;	/* XXX pretend we did it all */
+	/* XXX pretend we did it all */
+	xfer->actlen = 0;
+	for (i = 0; i < nframes; i++)
+		xfer->actlen += xfer->frlengths[i];
 
 	xfer->status = USBD_IN_PROGRESS;
 
-	oxfer->ohci_xfer_flags |= OHCI_ISOC_DIRTY;
-
 #ifdef USB_DEBUG
 	if (ohcidebug > 5) {
 		DPRINTF(("ohci_device_isoc_enter: frame=%d\n",
@@ -3443,9 +3500,9 @@ ohci_device_isoc_enter(usbd_xfer_handle xfer)
 #endif
 
 	s = splusb();
-	opipe->tail.itd = nsitd;
+	opipe->tail.itd = sitd;
 	sed->ed.ed_flags &= htole32(~OHCI_ED_SKIP);
-	sed->ed.ed_tailp = htole32(nsitd->physaddr);
+	sed->ed.ed_tailp = htole32(sitd->physaddr);
 	splx(s);
 
 #ifdef USB_DEBUG
@@ -3493,7 +3550,7 @@ ohci_device_isoc_abort(usbd_xfer_handle xfer)
 	struct ohci_pipe *opipe = (struct ohci_pipe *)xfer->pipe;
 	ohci_softc_t *sc = (ohci_softc_t *)opipe->pipe.device->bus;
 	ohci_soft_ed_t *sed;
-	ohci_soft_itd_t *sitd, *tmp_sitd;
+	ohci_soft_itd_t *sitd, *sitdnext, *tmp_sitd;
 	int s,undone,num_sitds;
 
 	s = splusb();
@@ -3556,20 +3613,20 @@ ohci_device_isoc_abort(usbd_xfer_handle xfer)
 		}
 	} while( undone != 0 );
 
+	/* Free the sitds */
+ 	for (sitd = xfer->hcpriv; sitd->xfer == xfer;
+	    sitd = sitdnext) {
+		sitdnext = sitd->nextitd;
+		ohci_free_sitd(sc, sitd);
+	}
 
 	s = splusb();
 
 	/* Run callback. */
 	usb_transfer_complete(xfer);
 
-	if (sitd != NULL)
-		/*
-		 * Only if there is a `next' sitd in next xfer...
-		 * unlink this xfer's sitds.
-		 */
-		sed->ed.ed_headp = htole32(sitd->physaddr);
-	else
-		sed->ed.ed_headp = 0;
+	/* There is always a `next' sitd so link it up. */
+	sed->ed.ed_headp = htole32(sitd->physaddr);
 
 	sed->ed.ed_flags &= htole32(~OHCI_ED_SKIP); /* remove hardware skip */
 
diff --git a/sys/dev/usb/ohci_pci.c b/sys/dev/usb/ohci_pci.c
index 1afd9bcd3e65..1401a4b0316f 100644
--- a/sys/dev/usb/ohci_pci.c
+++ b/sys/dev/usb/ohci_pci.c
@@ -55,6 +55,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/module.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
 #include <sys/bus.h>
 #include <sys/queue.h>
 #include <machine/bus.h>
@@ -303,6 +305,30 @@ ohci_pci_attach(device_t self)
 		ohci_pci_detach(self);
 		return ENXIO;
 	}
+
+	/* Allocate a parent dma tag for DMA maps */
+	err = bus_dma_tag_create(NULL, 1, 0, BUS_SPACE_MAXADDR_32BIT,
+	    BUS_SPACE_MAXADDR, NULL, NULL, BUS_SPACE_MAXSIZE_32BIT,
+	    USB_DMA_NSEG, BUS_SPACE_MAXSIZE_32BIT, 0, NULL, NULL,
+	    &sc->sc_bus.parent_dmatag);
+	if (err) {
+		device_printf(self, "Could not allocate parent DMA tag (%d)\n",
+		    err);
+		ohci_pci_detach(self);
+		return ENXIO;
+	}
+	/* Allocate a dma tag for transfer buffers */
+	err = bus_dma_tag_create(sc->sc_bus.parent_dmatag, 1, 0,
+	    BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL,
+	    BUS_SPACE_MAXSIZE_32BIT, USB_DMA_NSEG, BUS_SPACE_MAXSIZE_32BIT, 0,
+	    busdma_lock_mutex, &Giant, &sc->sc_bus.buffer_dmatag);
+	if (err) {
+		device_printf(self, "Could not allocate transfer tag (%d)\n",
+		    err);
+		ohci_pci_detach(self);
+		return ENXIO;
+	}
+
 	err = ohci_init(sc);
 	if (!err) {
 		sc->sc_flags |= OHCI_SCFLG_DONEINIT;
@@ -327,6 +353,11 @@ ohci_pci_detach(device_t self)
 		sc->sc_flags &= ~OHCI_SCFLG_DONEINIT;
 	}
 
+	if (sc->sc_bus.parent_dmatag != NULL)
+		bus_dma_tag_destroy(sc->sc_bus.parent_dmatag);
+	if (sc->sc_bus.buffer_dmatag != NULL)
+		bus_dma_tag_destroy(sc->sc_bus.buffer_dmatag);
+
 	if (sc->irq_res && sc->ih) {
 		int err = bus_teardown_intr(self, sc->irq_res, sc->ih);
 
@@ -345,7 +376,8 @@ ohci_pci_detach(device_t self)
 		sc->irq_res = NULL;
 	}
 	if (sc->io_res) {
-		bus_release_resource(self, SYS_RES_MEMORY, PCI_CBMEM, sc->io_res);
+		bus_release_resource(self, SYS_RES_MEMORY, PCI_CBMEM,
+		    sc->io_res);
 		sc->io_res = NULL;
 		sc->iot = 0;
 		sc->ioh = 0;
diff --git a/sys/dev/usb/ohcireg.h b/sys/dev/usb/ohcireg.h
index c255a4b28f34..429beb8fbd83 100644
--- a/sys/dev/usb/ohcireg.h
+++ b/sys/dev/usb/ohcireg.h
@@ -220,7 +220,8 @@ typedef struct {
 	u_int16_t	itd_offset[OHCI_ITD_NOFFSET];	/* Buffer offsets */
 #define itd_pswn itd_offset				/* Packet Status Word*/
 #define OHCI_ITD_PAGE_SELECT	0x00001000
-#define OHCI_ITD_MK_OFFS(len)	(0xe000 | ((len) & 0x1fff))
+#define OHCI_ITD_MK_OFFS(page, off) \
+     (0xe000 | ((page) ? OHCI_ITD_PAGE_SELECT : 0) | ((off) & 0xfff))
 #define OHCI_ITD_PSW_LENGTH(x)	((x) & 0xfff)		/* Transfer length */
 #define OHCI_ITD_PSW_GET_CC(x)	((x) >> 12)		/* Condition Code */
 } ohci_itd_t;
diff --git a/sys/dev/usb/ohcivar.h b/sys/dev/usb/ohcivar.h
index 1daf083808b5..748030093e1f 100644
--- a/sys/dev/usb/ohcivar.h
+++ b/sys/dev/usb/ohcivar.h
@@ -157,9 +157,8 @@ struct ohci_xfer {
 	struct usb_task	abort_task;
 	u_int32_t ohci_xfer_flags;
 };
-#define OHCI_ISOC_DIRTY  0x01
-#define OHCI_XFER_ABORTING	0x02	/* xfer is aborting. */
-#define OHCI_XFER_ABORTWAIT	0x04	/* abort completion is being awaited. */
+#define OHCI_XFER_ABORTING	0x01	/* xfer is aborting. */
+#define OHCI_XFER_ABORTWAIT	0x02	/* abort completion is being awaited. */
 
 #define OXFER(xfer) ((struct ohci_xfer *)(xfer))
 
diff --git a/sys/dev/usb/sl811hs.c b/sys/dev/usb/sl811hs.c
index 2196e9f2586a..3ada66dd7426 100644
--- a/sys/dev/usb/sl811hs.c
+++ b/sys/dev/usb/sl811hs.c
@@ -395,7 +395,8 @@ slhci_attach(struct slhci_softc *sc)
 	sc->sc_bus.usbrev = USBREV_1_1;
 	sc->sc_bus.methods = &slhci_bus_methods;
 	sc->sc_bus.pipe_size = sizeof(struct slhci_pipe);
-	sc->sc_bus.dmatag = sc->sc_dmat;
+	sc->sc_bus.parent_dmatag = NULL; /* XXX */
+	sc->sc_bus.buffer_dmatag = NULL; /* XXX */
 	SIMPLEQ_INIT(&sc->sc_free_xfers);
 
 	usb_callout_init(sc->sc_poll_handle);
@@ -548,7 +549,7 @@ slhci_poll_hub(void *arg)
 	usb_callout(sc->sc_poll_handle, sc->sc_interval, slhci_poll_hub, xfer);
 
 	/* USB spec 11.13.3 (p.260) */
-	p = KERNADDR(&xfer->dmabuf, 0);
+	p = xfer->buffer;
 	p[0] = 0;
 	if ((sc->sc_flags & (SLF_INSERT | SLF_RESET))) {
 		p[0] = 2;
@@ -767,7 +768,7 @@ slhci_root_ctrl_start(usbd_xfer_handle xfer)
 	index = UGETW(req->wIndex);
 
 	if (len)
-		buf = KERNADDR(&xfer->dmabuf, 0);
+		buf = xfer->buffer;
 
 #ifdef SLHCI_DEBUG
 	if ((slhci_debug & D_TRACE))
@@ -1197,7 +1198,7 @@ slhci_device_ctrl_start(usbd_xfer_handle xfer)
 	actlen = 0;
 	len = UGETW(req->wLength);
 	if (len) {
-		buf = KERNADDR(&xfer->dmabuf, 0);
+		buf = xfer->buffer;
 		if (req->bmRequestType & UT_READ)
 			pid = SL11_PID_IN;
 		for (; actlen < len; ) {
@@ -1226,7 +1227,7 @@ slhci_device_ctrl_start(usbd_xfer_handle xfer)
 	if((slhci_debug & D_TRACE) && UGETW(req->wLength) > 0){
 		int i;
 		for(i=0; i < UGETW(req->wLength); i++)
-			printf("%02x", *(unsigned char*)(KERNADDR(&xfer->dmabuf, i)));
+			printf("%02x", ((unsigned char *)xfer->buffer)[i]);
 		printf(" ");
 	}
 #endif
@@ -1318,7 +1319,7 @@ slhci_poll_device(void *arg)
 	/* interrupt transfer */
 	pid = (UE_GET_DIR(pipe->endpoint->edesc->bEndpointAddress) == UE_DIR_IN)
 	    ? SL11_PID_IN : SL11_PID_OUT;
-	buf = KERNADDR(&xfer->dmabuf, 0);
+	buf = xfer->buffer;
 
 	r = slhci_transaction(sc, pipe, pid, xfer->length, buf, 0/*toggle*/);
 	if (r < 0) {
diff --git a/sys/dev/usb/sl811hsvar.h b/sys/dev/usb/sl811hsvar.h
index 2708aa2ba3b4..d7e143cfdc45 100644
--- a/sys/dev/usb/sl811hsvar.h
+++ b/sys/dev/usb/sl811hsvar.h
@@ -59,7 +59,6 @@ struct slhci_softc {
 	struct usbd_bus		 sc_bus;
 	bus_space_tag_t		 sc_iot;
 	bus_space_handle_t	 sc_ioh;
-	bus_dma_tag_t		 sc_dmat;
   
 #ifdef __FreeBSD__
 	void *ih; 
diff --git a/sys/dev/usb/uhci.c b/sys/dev/usb/uhci.c
index 6c27a1501d53..aee88b92d05b 100644
--- a/sys/dev/usb/uhci.c
+++ b/sys/dev/usb/uhci.c
@@ -189,6 +189,10 @@ Static uhci_soft_td_t  *uhci_alloc_std(uhci_softc_t *);
 Static void		uhci_free_std(uhci_softc_t *, uhci_soft_td_t *);
 Static uhci_soft_qh_t  *uhci_alloc_sqh(uhci_softc_t *);
 Static void		uhci_free_sqh(uhci_softc_t *, uhci_soft_qh_t *);
+Static usbd_status	uhci_aux_dma_alloc(uhci_softc_t *, uhci_soft_td_t *,
+			    void *data, int len);
+Static uhci_physaddr_t	uhci_aux_dma_prepare(uhci_soft_td_t *, int);
+Static void		uhci_aux_dma_complete(uhci_soft_td_t *, int);
 #if 0
 Static void		uhci_enter_ctl_q(uhci_softc_t *, uhci_soft_qh_t *,
 					 uhci_intr_info_t *);
@@ -198,7 +202,8 @@ Static void		uhci_exit_ctl_q(uhci_softc_t *, uhci_soft_qh_t *);
 Static void		uhci_free_std_chain(uhci_softc_t *,
 					    uhci_soft_td_t *, uhci_soft_td_t *);
 Static usbd_status	uhci_alloc_std_chain(struct uhci_pipe *,
-			    uhci_softc_t *, int, int, u_int16_t, usb_dma_t *,
+			    uhci_softc_t *, int, int, u_int16_t,
+			    usbd_xfer_handle xfer,
 			    uhci_soft_td_t **, uhci_soft_td_t **);
 Static void		uhci_poll_hub(void *);
 Static void		uhci_waitintr(uhci_softc_t *, usbd_xfer_handle);
@@ -206,6 +211,7 @@ Static void		uhci_check_intr(uhci_softc_t *, uhci_intr_info_t *);
 Static void		uhci_idone(uhci_intr_info_t *);
 
 Static void		uhci_abort_xfer(usbd_xfer_handle, usbd_status status);
+Static void		uhci_transfer_complete(usbd_xfer_handle xfer);
 
 Static void		uhci_timeout(void *);
 Static void		uhci_timeout_task(void *);
@@ -968,7 +974,8 @@ uhci_poll_hub(void *addr)
 {
 	usbd_xfer_handle xfer = addr;
 	usbd_pipe_handle pipe = xfer->pipe;
-	uhci_softc_t *sc = (uhci_softc_t *)pipe->device->bus;
+	usbd_device_handle dev = pipe->device;
+	uhci_softc_t *sc = (uhci_softc_t *)dev->bus;
 	int s;
 	u_char *p;
 
@@ -976,7 +983,7 @@ uhci_poll_hub(void *addr)
 
 	usb_callout(sc->sc_poll_handle, sc->sc_ival, uhci_poll_hub, xfer);
 
-	p = KERNADDR(&xfer->dmabuf, 0);
+	p = xfer->buffer;
 	p[0] = 0;
 	if (UREAD2(sc, UHCI_PORTSC1) & (UHCI_PORTSC_CSC|UHCI_PORTSC_OCIC))
 		p[0] |= 1<<1;
@@ -989,9 +996,9 @@ uhci_poll_hub(void *addr)
 	xfer->actlen = 1;
 	xfer->status = USBD_NORMAL_COMPLETION;
 	s = splusb();
-	xfer->device->bus->intr_context++;
-	usb_transfer_complete(xfer);
-	xfer->device->bus->intr_context--;
+	dev->bus->intr_context++;
+	uhci_transfer_complete(xfer);
+	dev->bus->intr_context--;
 	splx(s);
 }
 
@@ -1497,7 +1504,7 @@ uhci_idone(uhci_intr_info_t *ii)
 	}
 
  end:
-	usb_transfer_complete(xfer);
+	uhci_transfer_complete(xfer);
 	DPRINTFN(12, ("uhci_idone: ii=%p done\n", ii));
 }
 
@@ -1658,6 +1665,9 @@ uhci_alloc_std(uhci_softc_t *sc)
 			std = KERNADDR(&dma, offs);
 			std->physaddr = DMAADDR(&dma, offs);
 			std->link.std = sc->sc_freetds;
+			std->aux_dma.block = NULL;
+			std->aux_data = NULL;
+			std->aux_len = 0;
 			sc->sc_freetds = std;
 		}
 	}
@@ -1678,6 +1688,12 @@ uhci_free_std(uhci_softc_t *sc, uhci_soft_td_t *std)
 	}
 	std->td.td_token = htole32(TD_IS_FREE);
 #endif
+	if (std->aux_dma.block != NULL) {
+		usb_freemem(&sc->sc_bus, &std->aux_dma);
+		std->aux_dma.block = NULL;
+		std->aux_data = NULL;
+		std->aux_len = 0;
+	}
 	std->link.std = sc->sc_freetds;
 	sc->sc_freetds = std;
 }
@@ -1731,12 +1747,12 @@ uhci_free_std_chain(uhci_softc_t *sc, uhci_soft_td_t *std,
 
 usbd_status
 uhci_alloc_std_chain(struct uhci_pipe *upipe, uhci_softc_t *sc, int len,
-		     int rd, u_int16_t flags, usb_dma_t *dma,
+		     int rd, u_int16_t flags, usbd_xfer_handle xfer,
 		     uhci_soft_td_t **sp, uhci_soft_td_t **ep)
 {
-	uhci_soft_td_t *p, *lastp;
-	uhci_physaddr_t lastlink;
-	int i, ntd, l, tog, maxp;
+	struct usb_dma_mapping *dma = &xfer->dmamap;
+	uhci_soft_td_t *p, *prevp, *startp;
+	int err, i, ntd, l, tog, maxp, seg, segoff;
 	u_int32_t status;
 	int addr = upipe->pipe.device->address;
 	int endpt = upipe->pipe.endpoint->edesc->bEndpointAddress;
@@ -1759,29 +1775,31 @@ uhci_alloc_std_chain(struct uhci_pipe *upipe, uhci_softc_t *sc, int len,
 		return (USBD_NORMAL_COMPLETION);
 	}
 	tog = upipe->nexttoggle;
-	if (ntd % 2 == 0)
-		tog ^= 1;
-	upipe->nexttoggle = tog ^ 1;
-	lastp = NULL;
-	lastlink = UHCI_PTR_T;
-	ntd--;
+	prevp = NULL;
+	startp = NULL;
 	status = UHCI_TD_ZERO_ACTLEN(UHCI_TD_SET_ERRCNT(3) | UHCI_TD_ACTIVE);
 	if (upipe->pipe.device->speed == USB_SPEED_LOW)
 		status |= UHCI_TD_LS;
 	if (flags & USBD_SHORT_XFER_OK)
 		status |= UHCI_TD_SPD;
-	for (i = ntd; i >= 0; i--) {
+	seg = 0;
+	segoff = 0;
+	for (i = 0; i < ntd; i++) {
 		p = uhci_alloc_std(sc);
 		if (p == NULL) {
-			uhci_free_std_chain(sc, lastp, NULL);
+			uhci_free_std_chain(sc, startp, NULL);
 			return (USBD_NOMEM);
 		}
-		p->link.std = lastp;
-		p->td.td_link = htole32(lastlink | UHCI_PTR_VF | UHCI_PTR_TD);
-		lastp = p;
-		lastlink = p->physaddr;
+		p->link.std = NULL;
+		if (prevp != NULL) {
+			prevp->link.std = p;
+			prevp->td.td_link = htole32(p->physaddr | UHCI_PTR_VF |
+			    UHCI_PTR_TD);
+		} else {
+			startp = p;
+		}
 		p->td.td_status = htole32(status);
-		if (i == ntd) {
+		if (i == ntd - 1) {
 			/* last TD */
 			l = len % maxp;
 			if (l == 0 && !(flags & USBD_FORCE_SHORT_XFER))
@@ -1792,15 +1810,100 @@ uhci_alloc_std_chain(struct uhci_pipe *upipe, uhci_softc_t *sc, int len,
 		p->td.td_token =
 		    htole32(rd ? UHCI_TD_IN (l, endpt, addr, tog) :
 				 UHCI_TD_OUT(l, endpt, addr, tog));
-		p->td.td_buffer = htole32(DMAADDR(dma, i * maxp));
+
+		KASSERT(seg < dma->nsegs,
+		    ("uhci_alloc_std_chain: too few segments"));
+		if (l > dma->segs[seg].ds_len - segoff) {
+			/* UHCI can't handle non-contiguous data. */
+			err = uhci_aux_dma_alloc(sc, p, (char *)xfer->buffer +
+			    i * maxp, l);
+			if (err) {
+				uhci_free_std_chain(sc, startp, NULL);
+				return (err);
+			}
+			p->td.td_buffer = htole32(uhci_aux_dma_prepare(p, rd));
+			l -= dma->segs[seg].ds_len - segoff;
+			seg++;
+			KASSERT(seg < dma->nsegs,
+			    ("uhci_alloc_std_chain: too few segments 2"));
+			segoff = 0;
+		} else {
+			p->td.td_buffer = htole32(dma->segs[seg].ds_addr +
+			    segoff);
+		}
+		segoff += l;
+		if (segoff >= dma->segs[seg].ds_len) {
+			KASSERT(segoff == dma->segs[seg].ds_len,
+			    ("uhci_alloc_std_chain: overlap"));
+			if (i * maxp + l != len) {
+				seg++;
+				segoff = 0;
+			}
+		}
+		prevp = p;
 		tog ^= 1;
 	}
-	*sp = lastp;
+	prevp->td.td_link = htole32(UHCI_PTR_T | UHCI_PTR_VF | UHCI_PTR_TD);
+	upipe->nexttoggle = tog;
+	*sp = startp;
 	DPRINTFN(10, ("uhci_alloc_std_chain: nexttog=%d\n",
 		      upipe->nexttoggle));
 	return (USBD_NORMAL_COMPLETION);
 }
 
+/*
+ * Allocate a physically contiguous buffer to handle cases where UHCI
+ * cannot handle a packet because it is not physically contiguous.
+ * If the usb_dma_t was already allocated this just ensures it is
+ * large enough for the specified size.
+ */
+Static usbd_status
+uhci_aux_dma_alloc(uhci_softc_t *sc, uhci_soft_td_t *std, void *data, int len)
+{
+	int err, align;
+
+	if (std->aux_dma.block == NULL || std->aux_dma.block->size < len) {
+		/* Align to avoid crossing a page boundary. */
+		if (powerof2(len))
+			align = len;
+		else
+			align = 1 << fls(len);
+
+		if (std->aux_dma.block != NULL)
+			usb_freemem(&sc->sc_bus, &std->aux_dma);
+		std->aux_dma.block = NULL;
+		err = usb_allocmem(&sc->sc_bus, len, align, &std->aux_dma);
+		if (err)
+			return (err);
+	}
+	std->aux_data = data;
+	std->aux_len = len;
+
+	return (USBD_NORMAL_COMPLETION);
+}
+
+Static uhci_physaddr_t
+uhci_aux_dma_prepare(uhci_soft_td_t *std, int isread)
+{
+	if (!isread) {
+		bcopy(std->aux_data, KERNADDR(&std->aux_dma, 0), std->aux_len);
+		bus_dmamap_sync(std->aux_dma.block->tag,
+		    std->aux_dma.block->map, BUS_DMASYNC_PREWRITE);
+	}
+
+	return (DMAADDR(&std->aux_dma, 0));
+}
+
+Static void
+uhci_aux_dma_complete(uhci_soft_td_t *std, int isread)
+{
+	if (isread) {
+		bus_dmamap_sync(std->aux_dma.block->tag,
+		    std->aux_dma.block->map, BUS_DMASYNC_POSTREAD);
+		bcopy(KERNADDR(&std->aux_dma, 0), std->aux_data, std->aux_len);
+	}
+}
+
 void
 uhci_device_clear_toggle(usbd_pipe_handle pipe)
 {
@@ -1862,8 +1965,8 @@ uhci_device_bulk_start(usbd_xfer_handle xfer)
 	upipe->u.bulk.isread = isread;
 	upipe->u.bulk.length = len;
 
-	err = uhci_alloc_std_chain(upipe, sc, len, isread, xfer->flags,
-				   &xfer->dmabuf, &data, &dataend);
+	err = uhci_alloc_std_chain(upipe, sc, len, isread, xfer->flags, xfer,
+	    &data, &dataend);
 	if (err)
 		return (err);
 	dataend->td.td_status |= htole32(UHCI_TD_IOC);
@@ -1949,7 +2052,7 @@ uhci_abort_xfer(usbd_xfer_handle xfer, usbd_status status)
 		xfer->status = status;	/* make software ignore it */
 		usb_uncallout(xfer->timeout_handle, uhci_timeout, xfer);
 		usb_rem_task(xfer->pipe->device, &UXFER(xfer)->abort_task);
-		usb_transfer_complete(xfer);
+		uhci_transfer_complete(xfer);
 		splx(s);
 		return;
 	}
@@ -2019,10 +2122,50 @@ uhci_abort_xfer(usbd_xfer_handle xfer, usbd_status status)
 		uxfer->uhci_xfer_flags &= ~UHCI_XFER_ABORTWAIT;
 		wakeup(&uxfer->uhci_xfer_flags);
 	}
-	usb_transfer_complete(xfer);
+	uhci_transfer_complete(xfer);
 	splx(s);
 }
 
+/*
+ * Perform any UHCI-specific transfer completion operations, then
+ * call usb_transfer_complete().
+ */
+Static void
+uhci_transfer_complete(usbd_xfer_handle xfer)
+{
+	uhci_intr_info_t *ii = &UXFER(xfer)->iinfo;
+	struct uhci_pipe *upipe = (struct uhci_pipe *)xfer->pipe;
+	uhci_soft_td_t *p;
+	int i, isread, n;
+
+	/* XXX, must be an easier way to detect reads... */
+	isread = ((xfer->rqflags & URQ_REQUEST) &&
+	    (xfer->request.bmRequestType & UT_READ)) ||
+            (xfer->pipe->endpoint->edesc->bEndpointAddress & UE_DIR_IN);
+
+	/* Copy back from any auxillary buffers after a read operation. */
+	if (xfer->nframes == 0) {
+		for (p = ii->stdstart; p != NULL; p = p->link.std) {
+			if (p->aux_data != NULL)
+				uhci_aux_dma_complete(p, isread);
+		}
+	} else {
+		if (xfer->nframes != 0) {
+			/* Isoc transfer, do things differently. */
+			n = UXFER(xfer)->curframe;
+			for (i = 0; i < xfer->nframes; i++) {
+				p = upipe->u.iso.stds[n];
+				if (p->aux_data != NULL)
+					uhci_aux_dma_complete(p, isread);
+				if (++n >= UHCI_VFRAMELIST_COUNT)
+				n = 0;
+			}
+		}
+	}
+
+	usb_transfer_complete(xfer);
+}
+
 /* Close a device bulk pipe. */
 void
 uhci_device_bulk_close(usbd_pipe_handle pipe)
@@ -2122,9 +2265,8 @@ uhci_device_intr_start(usbd_xfer_handle xfer)
 
 	upipe->u.intr.isread = isread;
 
-	err = uhci_alloc_std_chain(upipe, sc, xfer->length, isread,
-				   xfer->flags, &xfer->dmabuf, &data,
-				   &dataend);
+	err = uhci_alloc_std_chain(upipe, sc, xfer->length, isread, xfer->flags,
+	     xfer, &data, &dataend);
 	if (err)
 		return (err);
 	dataend->td.td_status |= htole32(UHCI_TD_IOC);
@@ -2262,7 +2404,7 @@ uhci_device_request(usbd_xfer_handle xfer)
 	if (len != 0) {
 		upipe->nexttoggle = 1;
 		err = uhci_alloc_std_chain(upipe, sc, len, isread, xfer->flags,
-					   &xfer->dmabuf, &data, &dataend);
+		    xfer, &data, &dataend);
 		if (err)
 			return (err);
 		next = data;
@@ -2389,8 +2531,9 @@ uhci_device_isoc_enter(usbd_xfer_handle xfer)
 	uhci_softc_t *sc = (uhci_softc_t *)dev->bus;
 	struct iso *iso = &upipe->u.iso;
 	uhci_soft_td_t *std;
-	u_int32_t buf, len, status;
-	int s, i, next, nframes;
+	void *dataptr;
+	u_int32_t len, status;
+	int err, s, i, isread, next, nframes, seg, segoff;
 
 	DPRINTFN(5,("uhci_device_isoc_enter: used=%d next=%d xfer=%p "
 		    "nframes=%d\n",
@@ -2420,7 +2563,10 @@ uhci_device_isoc_enter(usbd_xfer_handle xfer)
 	xfer->status = USBD_IN_PROGRESS;
 	UXFER(xfer)->curframe = next;
 
-	buf = DMAADDR(&xfer->dmabuf, 0);
+	seg = 0;
+	segoff = 0;
+	dataptr = xfer->allocbuf; /* Normal buffers not possible for isoc? */
+	isread = xfer->pipe->endpoint->edesc->bEndpointAddress & UE_DIR_IN;
 	status = UHCI_TD_ZERO_ACTLEN(UHCI_TD_SET_ERRCNT(0) |
 				     UHCI_TD_ACTIVE |
 				     UHCI_TD_IOS);
@@ -2431,7 +2577,35 @@ uhci_device_isoc_enter(usbd_xfer_handle xfer)
 		if (++next >= UHCI_VFRAMELIST_COUNT)
 			next = 0;
 		len = xfer->frlengths[i];
-		std->td.td_buffer = htole32(buf);
+		KASSERT(seg < xfer->dmamap.nsegs,
+		    ("uhci_device_isoc_enter: too few segments"));
+		if (len + segoff > xfer->dmamap.segs[seg].ds_len) {
+			/* UHCI can't handle non-contiguous data. */
+			err = uhci_aux_dma_alloc(sc, std, dataptr, len);
+			/* XXX */
+			if (err)
+				printf("uhci_device_isoc_enter: aux alloc\n");
+			std->td.td_buffer = htole32(uhci_aux_dma_prepare(std,
+			    isread));
+			segoff += len;
+			while (segoff >= xfer->dmamap.segs[seg].ds_len) {
+				KASSERT(seg < xfer->dmamap.nsegs - 1 ||
+				    segoff == xfer->dmamap.segs[seg].ds_len,
+				    ("uhci_device_isoc_enter: overlap2"));
+				segoff -= xfer->dmamap.segs[seg].ds_len;
+				seg++;
+			}
+		} else {
+			std->td.td_buffer =
+			    htole32(xfer->dmamap.segs[seg].ds_addr + segoff);
+			segoff += len;
+			if (segoff >= xfer->dmamap.segs[seg].ds_len) {
+				KASSERT(segoff == xfer->dmamap.segs[seg].ds_len,
+				    ("uhci_device_isoc_enter: overlap"));
+				segoff = 0;
+				seg++;
+			}
+		}
 		if (i == nframes - 1)
 			status |= UHCI_TD_IOC;
 		std->td.td_status = htole32(status);
@@ -2443,7 +2617,7 @@ uhci_device_isoc_enter(usbd_xfer_handle xfer)
 			uhci_dump_td(std);
 		}
 #endif
-		buf += len;
+		dataptr = (char *)dataptr + len;
 	}
 	iso->next = next;
 	iso->inuse += xfer->nframes;
@@ -2542,7 +2716,7 @@ uhci_device_isoc_abort(usbd_xfer_handle xfer)
 	UXFER(xfer)->iinfo.isdone = 1;
 #endif
 	/* Run callback and remove from interrupt list. */
-	usb_transfer_complete(xfer);
+	uhci_transfer_complete(xfer);
 
 	splx(s);
 }
@@ -2721,8 +2895,8 @@ uhci_device_intr_done(usbd_xfer_handle xfer)
 
 		/* This alloc cannot fail since we freed the chain above. */
 		uhci_alloc_std_chain(upipe, sc, xfer->length,
-				     upipe->u.intr.isread, xfer->flags,
-				     &xfer->dmabuf, &data, &dataend);
+				     upipe->u.intr.isread, xfer->flags, xfer,
+				     &data, &dataend);
 		dataend->td.td_status |= htole32(UHCI_TD_IOC);
 
 #ifdef USB_DEBUG
@@ -3210,7 +3384,7 @@ uhci_root_ctrl_start(usbd_xfer_handle xfer)
 	index = UGETW(req->wIndex);
 
 	if (len != 0)
-		buf = KERNADDR(&xfer->dmabuf, 0);
+		buf = xfer->buffer;
 
 #define C(x,y) ((x) | ((y) << 8))
 	switch(C(req->bRequest, req->bmRequestType)) {
@@ -3500,7 +3674,7 @@ uhci_root_ctrl_start(usbd_xfer_handle xfer)
  ret:
 	xfer->status = err;
 	s = splusb();
-	usb_transfer_complete(xfer);
+	uhci_transfer_complete(xfer);
 	splx(s);
 	return (USBD_IN_PROGRESS);
 }
@@ -3536,7 +3710,7 @@ uhci_root_intr_abort(usbd_xfer_handle xfer)
 #ifdef DIAGNOSTIC
 	UXFER(xfer)->iinfo.isdone = 1;
 #endif
-	usb_transfer_complete(xfer);
+	uhci_transfer_complete(xfer);
 }
 
 usbd_status
diff --git a/sys/dev/usb/uhci_pci.c b/sys/dev/usb/uhci_pci.c
index c92273c590f5..bd4761fc0807 100644
--- a/sys/dev/usb/uhci_pci.c
+++ b/sys/dev/usb/uhci_pci.c
@@ -54,6 +54,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/module.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
 #include <sys/bus.h>
 #include <sys/queue.h>
 #if defined(__FreeBSD__)
@@ -348,6 +350,29 @@ uhci_pci_attach(device_t self)
 #endif
 	pci_write_config(self, PCI_LEGSUP, PCI_LEGSUP_USBPIRQDEN, 2);
 
+	/* Allocate a parent dma tag for DMA maps */
+	err = bus_dma_tag_create(NULL, 1, 0, BUS_SPACE_MAXADDR_32BIT,
+	    BUS_SPACE_MAXADDR, NULL, NULL, BUS_SPACE_MAXSIZE_32BIT,
+	    USB_DMA_NSEG, BUS_SPACE_MAXSIZE_32BIT, 0, NULL, NULL,
+	    &sc->sc_bus.parent_dmatag);
+	if (err) {
+		device_printf(self, "Could not allocate parent DMA tag (%d)\n",
+		    err);
+		uhci_pci_detach(self);
+		return ENXIO;
+	}
+	/* Allocate a dma tag for transfer buffers */
+	err = bus_dma_tag_create(sc->sc_bus.parent_dmatag, 1, 0,
+	    BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL,
+	    BUS_SPACE_MAXSIZE_32BIT, USB_DMA_NSEG, BUS_SPACE_MAXSIZE_32BIT, 0,
+	    busdma_lock_mutex, &Giant, &sc->sc_bus.buffer_dmatag);
+	if (err) {
+		device_printf(self, "Could not allocate transfer tag (%d)\n",
+		    err);
+		uhci_pci_detach(self);
+		return ENXIO;
+	}
+
 	err = uhci_init(sc);
 	if (!err) {
 		sc->sc_flags |= UHCI_SCFLG_DONEINIT;
@@ -372,6 +397,10 @@ uhci_pci_detach(device_t self)
 		sc->sc_flags &= ~UHCI_SCFLG_DONEINIT;
 	}
 
+	if (sc->sc_bus.parent_dmatag != NULL)
+		bus_dma_tag_destroy(sc->sc_bus.parent_dmatag);
+	if (sc->sc_bus.buffer_dmatag != NULL)
+		bus_dma_tag_destroy(sc->sc_bus.buffer_dmatag);
 
 	if (sc->irq_res && sc->ih) {
 		int err = bus_teardown_intr(self, sc->irq_res, sc->ih);
diff --git a/sys/dev/usb/uhcivar.h b/sys/dev/usb/uhcivar.h
index 3100db756c86..7e9fc804d56d 100644
--- a/sys/dev/usb/uhcivar.h
+++ b/sys/dev/usb/uhcivar.h
@@ -100,6 +100,9 @@ struct uhci_soft_td {
 	uhci_td_t td;			/* The real TD, must be first */
 	uhci_soft_td_qh_t link; 	/* soft version of the td_link field */
 	uhci_physaddr_t physaddr;	/* TD's physical address. */
+	usb_dma_t aux_dma;		/* Auxillary storage if needed. */
+	void *aux_data;			/* Original aux data virtual address. */
+	int aux_len;			/* Auxillary storage size. */
 };
 /*
  * Make the size such that it is a multiple of UHCI_TD_ALIGN.  This way
diff --git a/sys/dev/usb/usb_mem.c b/sys/dev/usb/usb_mem.c
index 70317bb6a55b..957fb92523f3 100644
--- a/sys/dev/usb/usb_mem.c
+++ b/sys/dev/usb/usb_mem.c
@@ -230,7 +230,7 @@ usb_block_freemem(usb_dma_block_t *p)
 usbd_status
 usb_allocmem(usbd_bus_handle bus, size_t size, size_t align, usb_dma_t *p)
 {
-	bus_dma_tag_t tag = bus->dmatag;
+	bus_dma_tag_t tag = bus->parent_dmatag;
 	usbd_status err;
 	struct usb_frag_dma *f;
 	usb_dma_block_t *b;
diff --git a/sys/dev/usb/usbdi.c b/sys/dev/usb/usbdi.c
index 968b393b7162..2e937ff04d3e 100644
--- a/sys/dev/usb/usbdi.c
+++ b/sys/dev/usb/usbdi.c
@@ -86,6 +86,10 @@ Static void usbd_start_next(usbd_pipe_handle pipe);
 Static usbd_status usbd_open_pipe_ival
 	(usbd_interface_handle, u_int8_t, u_int8_t, usbd_pipe_handle *, int);
 Static int usbd_xfer_isread(usbd_xfer_handle xfer);
+Static void usbd_start_transfer(void *arg, bus_dma_segment_t *segs, int nseg,
+    int error);
+Static void usbd_alloc_callback(void *arg, bus_dma_segment_t *segs, int nseg,
+    int error);
 
 Static int usbd_nbuses = 0;
 
@@ -282,7 +286,7 @@ usbd_status
 usbd_transfer(usbd_xfer_handle xfer)
 {
 	usbd_pipe_handle pipe = xfer->pipe;
-	usb_dma_t *dmap = &xfer->dmabuf;
+	struct usb_dma_mapping *dmap = &xfer->dmamap;
 	usbd_status err;
 	u_int size;
 	int s;
@@ -301,43 +305,36 @@ usbd_transfer(usbd_xfer_handle xfer)
 	size = xfer->length;
 	/* If there is no buffer, allocate one. */
 	if (!(xfer->rqflags & URQ_DEV_DMABUF) && size != 0) {
-		struct usbd_bus *bus = pipe->device->bus;
+		bus_dma_tag_t tag = pipe->device->bus->buffer_dmatag;
 
 #ifdef DIAGNOSTIC
 		if (xfer->rqflags & URQ_AUTO_DMABUF)
 			printf("usbd_transfer: has old buffer!\n");
 #endif
-		err = bus->methods->allocm(bus, dmap, size);
+		err = bus_dmamap_create(tag, 0, &dmap->map);
 		if (err)
-			return (err);
-		xfer->rqflags |= URQ_AUTO_DMABUF;
-	}
-
-	/* Copy data if going out. */
-	if (!(xfer->flags & USBD_NO_COPY) && size != 0 &&
-	    !usbd_xfer_isread(xfer))
-		memcpy(KERNADDR(dmap, 0), xfer->buffer, size);
-
-	err = pipe->methods->transfer(xfer);
-
-	if (err != USBD_IN_PROGRESS && err) {
-		/* The transfer has not been queued, so free buffer. */
-		if (xfer->rqflags & URQ_AUTO_DMABUF) {
-			struct usbd_bus *bus = pipe->device->bus;
+			return (USBD_NOMEM);
 
-			bus->methods->freem(bus, &xfer->dmabuf);
+		xfer->rqflags |= URQ_AUTO_DMABUF;
+		err = bus_dmamap_load(tag, dmap->map, xfer->buffer, size,
+		    usbd_start_transfer, xfer, 0);
+		if (err != 0 && err != EINPROGRESS) {
 			xfer->rqflags &= ~URQ_AUTO_DMABUF;
+			bus_dmamap_destroy(tag, dmap->map);
+			return (USBD_INVAL);
 		}
+	} else if (size != 0) {
+		usbd_start_transfer(xfer, dmap->segs, dmap->nsegs, 0);
+	} else {
+		usbd_start_transfer(xfer, NULL, 0, 0);
 	}
 
 	if (!(xfer->flags & USBD_SYNCHRONOUS))
-		return (err);
+		return (xfer->done ? 0 : USBD_IN_PROGRESS);
 
 	/* Sync transfer, wait for completion. */
-	if (err != USBD_IN_PROGRESS)
-		return (err);
 	s = splusb();
-	if (!xfer->done) {
+	while (!xfer->done) {
 		if (pipe->device->bus->use_polling)
 			panic("usbd_transfer: not done");
 		tsleep(xfer, PRIBIO, "usbsyn", 0);
@@ -346,6 +343,56 @@ usbd_transfer(usbd_xfer_handle xfer)
 	return (xfer->status);
 }
 
+Static void
+usbd_start_transfer(void *arg, bus_dma_segment_t *segs, int nseg, int error)
+{
+	usbd_xfer_handle xfer = arg;
+	usbd_pipe_handle pipe = xfer->pipe;
+	struct usb_dma_mapping *dmap = &xfer->dmamap;
+	bus_dma_tag_t tag = pipe->device->bus->buffer_dmatag;
+	int err, i;
+
+	if (error != 0) {
+		KASSERT(xfer->rqflags & URQ_AUTO_DMABUF,
+		    ("usbd_start_transfer: error with non-auto buf"));
+		if (nseg > 0)
+			bus_dmamap_unload(tag, dmap->map);
+		bus_dmamap_destroy(tag, dmap->map);
+		/* XXX */
+		usb_insert_transfer(xfer);
+		xfer->status = USBD_IOERROR;
+		usb_transfer_complete(xfer);
+		return;
+	}
+
+	if (segs != dmap->segs) {
+		for (i = 0; i < nseg; i++)
+			dmap->segs[i] = segs[i];
+	}
+	dmap->nsegs = nseg;
+
+	if (segs > 0 && !usbd_xfer_isread(xfer)) {
+		/* Copy data if it is not already in the correct buffer. */
+		if (!(xfer->flags & USBD_NO_COPY) && xfer->allocbuf != NULL &&
+		    xfer->buffer != xfer->allocbuf)
+			memcpy(xfer->allocbuf, xfer->buffer, xfer->length);
+		bus_dmamap_sync(tag, dmap->map, BUS_DMASYNC_PREWRITE);
+	}
+	err = pipe->methods->transfer(xfer);
+	if (err != USBD_IN_PROGRESS && err) {
+		if (xfer->rqflags & URQ_AUTO_DMABUF) {
+			bus_dmamap_unload(tag, dmap->map);
+			bus_dmamap_destroy(tag, dmap->map);
+			xfer->rqflags &= ~URQ_AUTO_DMABUF;
+		}
+		/* XXX */
+		usb_insert_transfer(xfer);
+		xfer->status = err;
+		usb_transfer_complete(xfer);
+		return;
+	}
+}
+
 /* Like usbd_transfer(), but waits for completion. */
 usbd_status
 usbd_sync_transfer(usbd_xfer_handle xfer)
@@ -354,42 +401,103 @@ usbd_sync_transfer(usbd_xfer_handle xfer)
 	return (usbd_transfer(xfer));
 }
 
+struct usbd_allocstate {
+	usbd_xfer_handle xfer;
+	int done;
+	int error;
+	int waiting;
+};
+
 void *
 usbd_alloc_buffer(usbd_xfer_handle xfer, u_int32_t size)
 {
-	struct usbd_bus *bus = xfer->device->bus;
+	struct usbd_allocstate allocstate;
+	struct usb_dma_mapping *dmap = &xfer->dmamap;
+	bus_dma_tag_t tag = xfer->device->bus->buffer_dmatag;
+	void *buf;
 	usbd_status err;
+	int error, s;
 
-#ifdef DIAGNOSTIC
-	if (xfer->rqflags & (URQ_DEV_DMABUF | URQ_AUTO_DMABUF))
-		printf("usbd_alloc_buffer: xfer already has a buffer\n");
-#endif
-	err = bus->methods->allocm(bus, &xfer->dmabuf, size);
+	KASSERT((xfer->rqflags & (URQ_DEV_DMABUF | URQ_AUTO_DMABUF)) == 0,
+	    ("usbd_alloc_buffer: xfer already has a buffer"));
+	err = bus_dmamap_create(tag, 0, &dmap->map);
 	if (err)
 		return (NULL);
+	buf = malloc(size, M_USB, M_WAITOK);
+
+	allocstate.xfer = xfer;
+	allocstate.done = 0;
+	allocstate.error = 0;
+	allocstate.waiting = 0;
+	error = bus_dmamap_load(tag, dmap->map, buf, size, usbd_alloc_callback,
+	    &allocstate, 0);
+	if (error && error != EINPROGRESS) {
+		bus_dmamap_destroy(tag, dmap->map);
+		free(buf, M_USB);
+		return (NULL);
+	}
+	if (error == EINPROGRESS) {
+		/* Wait for completion. */
+		s = splusb();
+		allocstate.waiting = 1;
+		while (!allocstate.done)
+			tsleep(&allocstate, PRIBIO, "usbdab", 0);
+		splx(s);
+		error = allocstate.error;
+	}
+	if (error) {
+		bus_dmamap_unload(tag, dmap->map);
+		bus_dmamap_destroy(tag, dmap->map);
+		free(buf, M_USB);
+		return (NULL);
+	}
+
+	xfer->allocbuf = buf;
 	xfer->rqflags |= URQ_DEV_DMABUF;
-	return (KERNADDR(&xfer->dmabuf, 0));
+	return (buf);
 }
 
 void
 usbd_free_buffer(usbd_xfer_handle xfer)
 {
-#ifdef DIAGNOSTIC
-	if (!(xfer->rqflags & (URQ_DEV_DMABUF | URQ_AUTO_DMABUF))) {
-		printf("usbd_free_buffer: no buffer\n");
-		return;
-	}
-#endif
-	xfer->rqflags &= ~(URQ_DEV_DMABUF | URQ_AUTO_DMABUF);
-	xfer->device->bus->methods->freem(xfer->device->bus, &xfer->dmabuf);
+	struct usb_dma_mapping *dmap = &xfer->dmamap;
+	bus_dma_tag_t tag = xfer->device->bus->buffer_dmatag;
+
+	KASSERT((xfer->rqflags & (URQ_DEV_DMABUF | URQ_AUTO_DMABUF)) ==
+	    URQ_DEV_DMABUF, ("usbd_free_buffer: no/auto buffer"));
+
+	xfer->rqflags &= ~URQ_DEV_DMABUF;
+	bus_dmamap_unload(tag, dmap->map);
+	bus_dmamap_destroy(tag, dmap->map);
+	free(xfer->allocbuf, M_USB);
+	xfer->allocbuf = NULL;
 }
 
 void *
 usbd_get_buffer(usbd_xfer_handle xfer)
 {
 	if (!(xfer->rqflags & URQ_DEV_DMABUF))
-		return (0);
-	return (KERNADDR(&xfer->dmabuf, 0));
+		return (NULL);
+	return (xfer->allocbuf);
+}
+
+Static void
+usbd_alloc_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error)
+{
+	struct usbd_allocstate *allocstate = arg;
+	usbd_xfer_handle xfer = allocstate->xfer;
+	struct usb_dma_mapping *dmap = &xfer->dmamap;
+	int i;
+
+	allocstate->error = error;
+	if (error == 0) {
+		for (i = 0; i < nseg; i++)
+			dmap->segs[i] = segs[i];
+		dmap->nsegs = nseg;
+	}
+	allocstate->done = 1;
+	if (allocstate->waiting)
+		wakeup(&allocstate);
 }
 
 usbd_xfer_handle
@@ -410,7 +518,7 @@ usbd_status
 usbd_free_xfer(usbd_xfer_handle xfer)
 {
 	DPRINTFN(5,("usbd_free_xfer: %p\n", xfer));
-	if (xfer->rqflags & (URQ_DEV_DMABUF | URQ_AUTO_DMABUF))
+	if (xfer->rqflags & URQ_DEV_DMABUF)
 		usbd_free_buffer(xfer);
 #if defined(__NetBSD__) && defined(DIAGNOSTIC)
 	if (callout_pending(&xfer->timeout_handle)) {
@@ -467,10 +575,14 @@ usbd_setup_isoc_xfer(usbd_xfer_handle xfer, usbd_pipe_handle pipe,
 		     usbd_private_handle priv, u_int16_t *frlengths,
 		     u_int32_t nframes, u_int16_t flags, usbd_callback callback)
 {
+	int i;
+
 	xfer->pipe = pipe;
 	xfer->priv = priv;
 	xfer->buffer = 0;
 	xfer->length = 0;
+	for (i = 0; i < nframes; i++)
+		xfer->length += frlengths[i];
 	xfer->actlen = 0;
 	xfer->flags = flags;
 	xfer->timeout = USBD_NO_TIMEOUT;
@@ -759,6 +871,7 @@ usbd_ar_pipe(usbd_pipe_handle pipe)
 			    pipe, xfer, pipe->methods));
 		/* Make the HC abort it (and invoke the callback). */
 		pipe->methods->abort(xfer);
+		KASSERT(SIMPLEQ_FIRST(&pipe->queue) != xfer, ("usbd_ar_pipe"));
 		/* XXX only for non-0 usbd_clear_endpoint_stall(pipe); */
 	}
 	pipe->aborting = 0;
@@ -770,7 +883,8 @@ void
 usb_transfer_complete(usbd_xfer_handle xfer)
 {
 	usbd_pipe_handle pipe = xfer->pipe;
-	usb_dma_t *dmap = &xfer->dmabuf;
+	struct usb_dma_mapping *dmap = &xfer->dmamap;
+	bus_dma_tag_t tag = pipe->device->bus->buffer_dmatag;
 	int sync = xfer->flags & USBD_SYNCHRONOUS;
 	int erred = xfer->status == USBD_CANCELLED ||
 	    xfer->status == USBD_TIMEOUT;
@@ -800,23 +914,19 @@ usb_transfer_complete(usbd_xfer_handle xfer)
 	if (polling)
 		pipe->running = 0;
 
-	if (!(xfer->flags & USBD_NO_COPY) && xfer->actlen != 0 &&
-	    usbd_xfer_isread(xfer)) {
-#ifdef DIAGNOSTIC
-		if (xfer->actlen > xfer->length) {
-			printf("usb_transfer_complete: actlen > len %d > %d\n",
-			       xfer->actlen, xfer->length);
-			xfer->actlen = xfer->length;
-		}
-#endif
-		memcpy(xfer->buffer, KERNADDR(dmap, 0), xfer->actlen);
+	if (xfer->actlen != 0 && usbd_xfer_isread(xfer)) {
+		bus_dmamap_sync(tag, dmap->map, BUS_DMASYNC_POSTREAD);
+		/* Copy data if it is not already in the correct buffer. */
+		if (!(xfer->flags & USBD_NO_COPY) && xfer->allocbuf != NULL &&
+		    xfer->buffer != xfer->allocbuf)
+			memcpy(xfer->buffer, xfer->allocbuf, xfer->actlen);
 	}
 
-	/* if we allocated the buffer in usbd_transfer() we free it here. */
+	/* if we mapped the buffer in usbd_transfer() we unmap it here. */
 	if (xfer->rqflags & URQ_AUTO_DMABUF) {
 		if (!repeat) {
-			struct usbd_bus *bus = pipe->device->bus;
-			bus->methods->freem(bus, dmap);
+			bus_dmamap_unload(tag, dmap->map);
+			bus_dmamap_destroy(tag, dmap->map);
 			xfer->rqflags &= ~URQ_AUTO_DMABUF;
 		}
 	}
@@ -824,11 +934,10 @@ usb_transfer_complete(usbd_xfer_handle xfer)
 	if (!repeat) {
 		/* Remove request from queue. */
 #ifdef DIAGNOSTIC
-		if (xfer != SIMPLEQ_FIRST(&pipe->queue))
-			printf("usb_transfer_complete: bad dequeue %p != %p\n",
-			       xfer, SIMPLEQ_FIRST(&pipe->queue));
 		xfer->busy_free = XFER_BUSY;
 #endif
+		KASSERT(SIMPLEQ_FIRST(&pipe->queue) == xfer,
+		    ("usb_transfer_complete: bad dequeue"));
 		SIMPLEQ_REMOVE_HEAD(&pipe->queue, next);
 	}
 	DPRINTFN(5,("usb_transfer_complete: repeat=%d new head=%p\n",
@@ -892,6 +1001,7 @@ usb_insert_transfer(usbd_xfer_handle xfer)
 	xfer->busy_free = XFER_ONQU;
 #endif
 	s = splusb();
+	KASSERT(SIMPLEQ_FIRST(&pipe->queue) != xfer, ("usb_insert_transfer"));
 	SIMPLEQ_INSERT_TAIL(&pipe->queue, xfer, next);
 	if (pipe->running)
 		err = USBD_IN_PROGRESS;
diff --git a/sys/dev/usb/usbdivar.h b/sys/dev/usb/usbdivar.h
index 42537daa4785..fe783fc8d683 100644
--- a/sys/dev/usb/usbdivar.h
+++ b/sys/dev/usb/usbdivar.h
@@ -130,7 +130,8 @@ struct usbd_bus {
 #endif
 #endif
 
-	bus_dma_tag_t		dmatag;	/* DMA tag */
+	bus_dma_tag_t		parent_dmatag;	/* Base DMA tag */
+	bus_dma_tag_t		buffer_dmatag;	/* Tag for transfer buffers */
 };
 
 struct usbd_device {
@@ -187,6 +188,15 @@ struct usbd_pipe {
 	struct usbd_pipe_methods *methods;
 };
 
+#define USB_DMA_NSEG (btoc(MAXPHYS) + 1)
+
+/* DMA-capable memory buffer. */
+struct usb_dma_mapping {
+	bus_dma_segment_t segs[USB_DMA_NSEG];	/* The physical segments. */
+	int nsegs;				/* Number of segments. */
+	bus_dmamap_t map;			/* DMA mapping. */
+};
+
 struct usbd_xfer {
 	struct usbd_pipe       *pipe;
 	void		       *priv;
@@ -214,7 +224,8 @@ struct usbd_xfer {
 
 	/* For memory allocation */
 	struct usbd_device     *device;
-	usb_dma_t		dmabuf;
+	struct usb_dma_mapping	dmamap;
+	void			*allocbuf;
 
 	int			rqflags;
 #define URQ_REQUEST	0x01