hyeprv/vmbus: Rework prplist sending.

MFC after:	1 week
Sponsored by:	Microsoft OSTC
Differential Revision:	https://reviews.freebsd.org/D7175

6 files changed, 92 insertions, 141 deletions

diff --git a/sys/dev/hyperv/include/hyperv.h b/sys/dev/hyperv/include/hyperv.h
index 5d9373e7310e..692aa36bd3c1 100644
--- a/sys/dev/hyperv/include/hyperv.h
+++ b/sys/dev/hyperv/include/hyperv.h
@@ -82,18 +82,6 @@ typedef uint8_t	hv_bool_uint8_t;
 #define VMBUS_VERSION_MAJOR(ver)	(((uint32_t)(ver)) >> 16)
 #define VMBUS_VERSION_MINOR(ver)	(((uint32_t)(ver)) & 0xffff)
 
-#define HV_MAX_MULTIPAGE_BUFFER_COUNT	32
-
-#define HV_ALIGN_UP(value, align)					\
-		(((value) & (align-1)) ?				\
-		    (((value) + (align-1)) & ~(align-1) ) : (value))
-
-#define HV_ALIGN_DOWN(value, align) ( (value) & ~(align-1) )
-
-#define HV_NUM_PAGES_SPANNED(addr, len)					\
-		((HV_ALIGN_UP(addr+len, PAGE_SIZE) -			\
-		    HV_ALIGN_DOWN(addr, PAGE_SIZE)) >> PAGE_SHIFT )
-
 struct hyperv_guid {
 	uint8_t		hv_guid[16];
 } __packed;
@@ -224,12 +212,6 @@ typedef struct {
 } __packed hv_vmbus_ring_buffer;
 
 typedef struct {
-	int		length;
-	int		offset;
-	uint64_t	pfn_array[HV_MAX_MULTIPAGE_BUFFER_COUNT];
-} __packed hv_vmbus_multipage_buffer;
-
-typedef struct {
 	hv_vmbus_ring_buffer*	ring_buffer;
 	struct mtx		ring_lock;
 	uint32_t		ring_data_size;	/* ring_size */
@@ -368,13 +350,6 @@ int		hv_vmbus_channel_send_packet(
 				hv_vmbus_packet_type	type,
 				uint32_t		flags);
 
-int		hv_vmbus_channel_send_packet_multipagebuffer(
-				hv_vmbus_channel*	    channel,
-				hv_vmbus_multipage_buffer*  multi_page_buffer,
-				void*			    buffer,
-				uint32_t		    buffer_len,
-				uint64_t		    request_id);
-
 int		hv_vmbus_channel_establish_gpadl(
 				hv_vmbus_channel*	channel,
 				/* must be phys and virt contiguous */
diff --git a/sys/dev/hyperv/include/vmbus.h b/sys/dev/hyperv/include/vmbus.h
index 19b2c86b9705..3931fc72d690 100644
--- a/sys/dev/hyperv/include/vmbus.h
+++ b/sys/dev/hyperv/include/vmbus.h
@@ -31,6 +31,15 @@
 
 #include <sys/param.h>
 
+/*
+ * GPA stuffs.
+ */
+struct vmbus_gpa_range {
+	uint32_t	gpa_len;
+	uint32_t	gpa_ofs;
+	uint64_t	gpa_page[0];
+} __packed;
+
 /* This is actually vmbus_gpa_range.gpa_page[1] */
 struct vmbus_gpa {
 	uint32_t	gpa_len;
@@ -39,11 +48,15 @@ struct vmbus_gpa {
 } __packed;
 
 #define VMBUS_CHAN_SGLIST_MAX	32
+#define VMBUS_CHAN_PRPLIST_MAX	32
 
 struct hv_vmbus_channel;
 
 int	vmbus_chan_send_sglist(struct hv_vmbus_channel *chan,
 	    struct vmbus_gpa sg[], int sglen, void *data, int dlen,
 	    uint64_t xactid);
+int	vmbus_chan_send_prplist(struct hv_vmbus_channel *chan,
+	    struct vmbus_gpa_range *prp, int prp_cnt, void *data, int dlen,
+	    uint64_t xactid);
 
 #endif	/* !_VMBUS_H_ */
diff --git a/sys/dev/hyperv/storvsc/hv_storvsc_drv_freebsd.c b/sys/dev/hyperv/storvsc/hv_storvsc_drv_freebsd.c
index 05abf2ac57ad..7b0a24d140a8 100644
--- a/sys/dev/hyperv/storvsc/hv_storvsc_drv_freebsd.c
+++ b/sys/dev/hyperv/storvsc/hv_storvsc_drv_freebsd.c
@@ -72,6 +72,7 @@ __FBSDID("$FreeBSD$");
 #include <cam/scsi/scsi_message.h>
 
 #include <dev/hyperv/include/hyperv.h>
+#include <dev/hyperv/include/vmbus.h>
 
 #include "hv_vstorage.h"
 #include "vmbus_if.h"
@@ -100,7 +101,7 @@ struct hv_sgl_page_pool{
 	boolean_t                is_init;
 } g_hv_sgl_page_pool;
 
-#define STORVSC_MAX_SG_PAGE_CNT STORVSC_MAX_IO_REQUESTS * HV_MAX_MULTIPAGE_BUFFER_COUNT
+#define STORVSC_MAX_SG_PAGE_CNT STORVSC_MAX_IO_REQUESTS * VMBUS_CHAN_PRPLIST_MAX
 
 enum storvsc_request_type {
 	WRITE_TYPE,
@@ -108,10 +109,16 @@ enum storvsc_request_type {
 	UNKNOWN_TYPE
 };
 
+struct hvs_gpa_range {
+	struct vmbus_gpa_range	gpa_range;
+	uint64_t		gpa_page[VMBUS_CHAN_PRPLIST_MAX];
+} __packed;
+
 struct hv_storvsc_request {
 	LIST_ENTRY(hv_storvsc_request) link;
 	struct vstor_packet	vstor_packet;
-	hv_vmbus_multipage_buffer data_buf;
+	int prp_cnt;
+	struct hvs_gpa_range prp_list;
 	void *sense_data;
 	uint8_t sense_info_len;
 	uint8_t retries;
@@ -674,21 +681,18 @@ hv_storvsc_io_request(struct storvsc_softc *sc,
 	
 	vstor_packet->u.vm_srb.sense_info_len = sense_buffer_size;
 
-	vstor_packet->u.vm_srb.transfer_len = request->data_buf.length;
+	vstor_packet->u.vm_srb.transfer_len =
+	    request->prp_list.gpa_range.gpa_len;
 
 	vstor_packet->operation = VSTOR_OPERATION_EXECUTESRB;
 
 	outgoing_channel = vmbus_select_outgoing_channel(sc->hs_chan);
 
 	mtx_unlock(&request->softc->hs_lock);
-	if (request->data_buf.length) {
-		ret = hv_vmbus_channel_send_packet_multipagebuffer(
-				outgoing_channel,
-				&request->data_buf,
-				vstor_packet,
-				VSTOR_PKT_SIZE,
-				(uint64_t)(uintptr_t)request);
-
+	if (request->prp_list.gpa_range.gpa_len) {
+		ret = vmbus_chan_send_prplist(outgoing_channel,
+		    &request->prp_list.gpa_range, request->prp_cnt,
+		    vstor_packet, VSTOR_PKT_SIZE, (uint64_t)(uintptr_t)request);
 	} else {
 		ret = hv_vmbus_channel_send_packet(
 			outgoing_channel,
@@ -954,7 +958,7 @@ storvsc_attach(device_t dev)
 
 		/*
 		 * Pre-create SG list, each SG list with
-		 * HV_MAX_MULTIPAGE_BUFFER_COUNT segments, each
+		 * VMBUS_CHAN_PRPLIST_MAX segments, each
 		 * segment has one page buffer
 		 */
 		for (i = 0; i < STORVSC_MAX_IO_REQUESTS; i++) {
@@ -962,10 +966,10 @@ storvsc_attach(device_t dev)
 			    M_DEVBUF, M_WAITOK|M_ZERO);
 
 			sgl_node->sgl_data =
-			    sglist_alloc(HV_MAX_MULTIPAGE_BUFFER_COUNT,
+			    sglist_alloc(VMBUS_CHAN_PRPLIST_MAX,
 			    M_WAITOK|M_ZERO);
 
-			for (j = 0; j < HV_MAX_MULTIPAGE_BUFFER_COUNT; j++) {
+			for (j = 0; j < VMBUS_CHAN_PRPLIST_MAX; j++) {
 				tmp_buff = malloc(PAGE_SIZE,
 				    M_DEVBUF, M_WAITOK|M_ZERO);
 
@@ -1052,7 +1056,7 @@ cleanup:
 	while (!LIST_EMPTY(&g_hv_sgl_page_pool.free_sgl_list)) {
 		sgl_node = LIST_FIRST(&g_hv_sgl_page_pool.free_sgl_list);
 		LIST_REMOVE(sgl_node, link);
-		for (j = 0; j < HV_MAX_MULTIPAGE_BUFFER_COUNT; j++) {
+		for (j = 0; j < VMBUS_CHAN_PRPLIST_MAX; j++) {
 			if (NULL !=
 			    (void*)sgl_node->sgl_data->sg_segs[j].ss_paddr) {
 				free((void*)sgl_node->sgl_data->sg_segs[j].ss_paddr, M_DEVBUF);
@@ -1115,7 +1119,7 @@ storvsc_detach(device_t dev)
 	while (!LIST_EMPTY(&g_hv_sgl_page_pool.free_sgl_list)) {
 		sgl_node = LIST_FIRST(&g_hv_sgl_page_pool.free_sgl_list);
 		LIST_REMOVE(sgl_node, link);
-		for (j = 0; j < HV_MAX_MULTIPAGE_BUFFER_COUNT; j++){
+		for (j = 0; j < VMBUS_CHAN_PRPLIST_MAX; j++){
 			if (NULL !=
 			    (void*)sgl_node->sgl_data->sg_segs[j].ss_paddr) {
 				free((void*)sgl_node->sgl_data->sg_segs[j].ss_paddr, M_DEVBUF);
@@ -1666,6 +1670,7 @@ create_storvsc_request(union ccb *ccb, struct hv_storvsc_request *reqp)
 	uint32_t pfn_num = 0;
 	uint32_t pfn;
 	uint64_t not_aligned_seg_bits = 0;
+	struct hvs_gpa_range *prplist;
 	
 	/* refer to struct vmscsi_req for meanings of these two fields */
 	reqp->vstor_packet.u.vm_srb.port =
@@ -1709,22 +1714,23 @@ create_storvsc_request(union ccb *ccb, struct hv_storvsc_request *reqp)
 		return (0);
 	}
 
-	reqp->data_buf.length = csio->dxfer_len;
+	prplist = &reqp->prp_list;
+	prplist->gpa_range.gpa_len = csio->dxfer_len;
 
 	switch (ccb->ccb_h.flags & CAM_DATA_MASK) {
 	case CAM_DATA_VADDR:
 	{
 		bytes_to_copy = csio->dxfer_len;
 		phys_addr = vtophys(csio->data_ptr);
-		reqp->data_buf.offset = phys_addr & PAGE_MASK;
+		prplist->gpa_range.gpa_ofs = phys_addr & PAGE_MASK;
 		
 		while (bytes_to_copy != 0) {
 			int bytes, page_offset;
 			phys_addr =
-			    vtophys(&csio->data_ptr[reqp->data_buf.length -
+			    vtophys(&csio->data_ptr[prplist->gpa_range.gpa_len -
 			    bytes_to_copy]);
 			pfn = phys_addr >> PAGE_SHIFT;
-			reqp->data_buf.pfn_array[pfn_num] = pfn;
+			prplist->gpa_page[pfn_num] = pfn;
 			page_offset = phys_addr & PAGE_MASK;
 
 			bytes = min(PAGE_SIZE - page_offset, bytes_to_copy);
@@ -1732,6 +1738,7 @@ create_storvsc_request(union ccb *ccb, struct hv_storvsc_request *reqp)
 			bytes_to_copy -= bytes;
 			pfn_num++;
 		}
+		reqp->prp_cnt = pfn_num;
 		break;
 	}
 
@@ -1748,10 +1755,10 @@ create_storvsc_request(union ccb *ccb, struct hv_storvsc_request *reqp)
 		printf("Storvsc: get SG I/O operation, %d\n",
 		    reqp->vstor_packet.u.vm_srb.data_in);
 
-		if (storvsc_sg_count > HV_MAX_MULTIPAGE_BUFFER_COUNT){
+		if (storvsc_sg_count > VMBUS_CHAN_PRPLIST_MAX){
 			printf("Storvsc: %d segments is too much, "
 			    "only support %d segments\n",
-			    storvsc_sg_count, HV_MAX_MULTIPAGE_BUFFER_COUNT);
+			    storvsc_sg_count, VMBUS_CHAN_PRPLIST_MAX);
 			return (EINVAL);
 		}
 
@@ -1804,10 +1811,10 @@ create_storvsc_request(union ccb *ccb, struct hv_storvsc_request *reqp)
  				phys_addr =
 					vtophys(storvsc_sglist[0].ds_addr);
 			}
-			reqp->data_buf.offset = phys_addr & PAGE_MASK;
+			prplist->gpa_range.gpa_ofs = phys_addr & PAGE_MASK;
 
 			pfn = phys_addr >> PAGE_SHIFT;
-			reqp->data_buf.pfn_array[0] = pfn;
+			prplist->gpa_page[0] = pfn;
 			
 			for (i = 1; i < storvsc_sg_count; i++) {
 				if (reqp->not_aligned_seg_bits & (1 << i)) {
@@ -1819,27 +1826,31 @@ create_storvsc_request(union ccb *ccb, struct hv_storvsc_request *reqp)
 				}
 
 				pfn = phys_addr >> PAGE_SHIFT;
-				reqp->data_buf.pfn_array[i] = pfn;
+				prplist->gpa_page[i] = pfn;
 			}
+			reqp->prp_cnt = i;
 		} else {
 			phys_addr = vtophys(storvsc_sglist[0].ds_addr);
 
-			reqp->data_buf.offset = phys_addr & PAGE_MASK;
+			prplist->gpa_range.gpa_ofs = phys_addr & PAGE_MASK;
 
 			for (i = 0; i < storvsc_sg_count; i++) {
 				phys_addr = vtophys(storvsc_sglist[i].ds_addr);
 				pfn = phys_addr >> PAGE_SHIFT;
-				reqp->data_buf.pfn_array[i] = pfn;
+				prplist->gpa_page[i] = pfn;
 			}
+			reqp->prp_cnt = i;
 
 			/* check the last segment cross boundary or not */
 			offset = phys_addr & PAGE_MASK;
 			if (offset) {
+				/* Add one more PRP entry */
 				phys_addr =
 				    vtophys(storvsc_sglist[i-1].ds_addr +
 				    PAGE_SIZE - offset);
 				pfn = phys_addr >> PAGE_SHIFT;
-				reqp->data_buf.pfn_array[i] = pfn;
+				prplist->gpa_page[i] = pfn;
+				reqp->prp_cnt++;
 			}
 			
 			reqp->bounce_sgl_count = 0;
diff --git a/sys/dev/hyperv/vmbus/hv_channel.c b/sys/dev/hyperv/vmbus/hv_channel.c
index 360370cd1d9e..3c9174cf7e43 100644
--- a/sys/dev/hyperv/vmbus/hv_channel.c
+++ b/sys/dev/hyperv/vmbus/hv_channel.c
@@ -712,78 +712,46 @@ vmbus_chan_send_sglist(struct hv_vmbus_channel *chan,
 	return error;
 }
 
-/**
- * @brief Send a multi-page buffer packet using a GPADL Direct packet type
- */
 int
-hv_vmbus_channel_send_packet_multipagebuffer(
-	hv_vmbus_channel*		channel,
-	hv_vmbus_multipage_buffer*	multi_page_buffer,
-	void*				buffer,
-	uint32_t			buffer_len,
-	uint64_t			request_id)
+vmbus_chan_send_prplist(struct hv_vmbus_channel *chan,
+    struct vmbus_gpa_range *prp, int prp_cnt, void *data, int dlen,
+    uint64_t xactid)
 {
+	struct vmbus_chanpkt_prplist pkt;
+	int pktlen, pad_pktlen, hlen, error;
+	struct iovec iov[4];
+	boolean_t send_evt;
+	uint64_t pad = 0;
 
-	int			ret = 0;
-	uint32_t		desc_size;
-	boolean_t		need_sig;
-	uint32_t		packet_len;
-	uint32_t		packet_len_aligned;
-	uint32_t		pfn_count;
-	uint64_t		aligned_data = 0;
-	struct iovec		iov[3];
-	hv_vmbus_channel_packet_multipage_buffer desc;
-
-	pfn_count =
-	    HV_NUM_PAGES_SPANNED(
-		    multi_page_buffer->offset,
-		    multi_page_buffer->length);
-
-	if ((pfn_count == 0) || (pfn_count > HV_MAX_MULTIPAGE_BUFFER_COUNT))
-	    return (EINVAL);
-	/*
-	 * Adjust the size down since hv_vmbus_channel_packet_multipage_buffer
-	 * is the largest size we support
-	 */
-	desc_size =
-	    sizeof(hv_vmbus_channel_packet_multipage_buffer) -
-		    ((HV_MAX_MULTIPAGE_BUFFER_COUNT - pfn_count) *
-			sizeof(uint64_t));
-	packet_len = desc_size + buffer_len;
-	packet_len_aligned = HV_ALIGN_UP(packet_len, sizeof(uint64_t));
-
-	/*
-	 * Setup the descriptor
-	 */
-	desc.type = HV_VMBUS_PACKET_TYPE_DATA_USING_GPA_DIRECT;
-	desc.flags = HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED;
-	desc.data_offset8 = desc_size >> 3; /* in 8-bytes granularity */
-	desc.length8 = (uint16_t) (packet_len_aligned >> 3);
-	desc.transaction_id = request_id;
-	desc.range_count = 1;
-
-	desc.range.length = multi_page_buffer->length;
-	desc.range.offset = multi_page_buffer->offset;
-
-	memcpy(desc.range.pfn_array, multi_page_buffer->pfn_array,
-		pfn_count * sizeof(uint64_t));
-
-	iov[0].iov_base = &desc;
-	iov[0].iov_len = desc_size;
-
-	iov[1].iov_base = buffer;
-	iov[1].iov_len = buffer_len;
+	KASSERT(prp_cnt < VMBUS_CHAN_PRPLIST_MAX,
+	    ("invalid prplist entry count %d", prp_cnt));
 
-	iov[2].iov_base = &aligned_data;
-	iov[2].iov_len = packet_len_aligned - packet_len;
+	hlen = __offsetof(struct vmbus_chanpkt_prplist,
+	    cp_range[0].gpa_page[prp_cnt]);
+	pktlen = hlen + dlen;
+	pad_pktlen = roundup2(pktlen, VMBUS_CHANPKT_SIZE_ALIGN);
 
-	ret = hv_ring_buffer_write(&channel->outbound, iov, 3, &need_sig);
+	pkt.cp_hdr.cph_type = HV_VMBUS_PACKET_TYPE_DATA_USING_GPA_DIRECT;
+	pkt.cp_hdr.cph_flags = HV_VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED;
+	pkt.cp_hdr.cph_data_ofs = hlen >> VMBUS_CHANPKT_SIZE_SHIFT;
+	pkt.cp_hdr.cph_len = pad_pktlen >> VMBUS_CHANPKT_SIZE_SHIFT;
+	pkt.cp_hdr.cph_xactid = xactid;
+	pkt.cp_rsvd = 0;
+	pkt.cp_range_cnt = 1;
 
-	/* TODO: We should determine if this is optional */
-	if (ret == 0 && need_sig)
-		vmbus_chan_send_event(channel);
+	iov[0].iov_base = &pkt;
+	iov[0].iov_len = sizeof(pkt);
+	iov[1].iov_base = prp;
+	iov[1].iov_len = __offsetof(struct vmbus_gpa_range, gpa_page[prp_cnt]);
+	iov[2].iov_base = data;
+	iov[2].iov_len = dlen;
+	iov[3].iov_base = &pad;
+	iov[3].iov_len = pad_pktlen - pktlen;
 
-	return (ret);
+	error = hv_ring_buffer_write(&chan->outbound, iov, 4, &send_evt);
+	if (!error && send_evt)
+		vmbus_chan_send_event(chan);
+	return error;
 }
 
 /**
diff --git a/sys/dev/hyperv/vmbus/hv_vmbus_priv.h b/sys/dev/hyperv/vmbus/hv_vmbus_priv.h
index 11bce5bbc713..a1bff12bfac8 100644
--- a/sys/dev/hyperv/vmbus/hv_vmbus_priv.h
+++ b/sys/dev/hyperv/vmbus/hv_vmbus_priv.h
@@ -42,20 +42,6 @@
 struct vmbus_softc;
 
 /*
- * The format must be the same as hv_vm_data_gpa_direct
- */
-typedef struct hv_vmbus_channel_packet_multipage_buffer {
-	uint16_t 			type;
-	uint16_t 			data_offset8;
-	uint16_t 			length8;
-	uint16_t 			flags;
-	uint64_t			transaction_id;
-	uint32_t 			reserved;
-	uint32_t			range_count; /* Always 1 in this case */
-	hv_vmbus_multipage_buffer	range;
-} __packed hv_vmbus_channel_packet_multipage_buffer;
-
-/*
  * Private, VM Bus functions
  */
 struct sysctl_ctx_list;
diff --git a/sys/dev/hyperv/vmbus/vmbus_reg.h b/sys/dev/hyperv/vmbus/vmbus_reg.h
index 70bc52e84336..2a94f15ed38a 100644
--- a/sys/dev/hyperv/vmbus/vmbus_reg.h
+++ b/sys/dev/hyperv/vmbus/vmbus_reg.h
@@ -109,15 +109,6 @@ CTASSERT(sizeof(struct vmbus_mnf) == PAGE_SIZE);
 #define VMBUS_CHAN_MAX		(VMBUS_EVTFLAG_LEN * VMBUS_EVTFLAGS_MAX)
 
 /*
- * GPA range.
- */
-struct vmbus_gpa_range {
-	uint32_t	gpa_len;
-	uint32_t	gpa_ofs;
-	uint64_t	gpa_page[];
-} __packed;
-
-/*
  * Channel packets
  */
 
@@ -143,6 +134,13 @@ struct vmbus_chanpkt_sglist {
 	struct vmbus_gpa cp_gpa[];
 } __packed;
 
+struct vmbus_chanpkt_prplist {
+	struct vmbus_chanpkt_hdr cp_hdr;
+	uint32_t	cp_rsvd;
+	uint32_t	cp_range_cnt;
+	struct vmbus_gpa_range cp_range[];
+} __packed;
+
 /*
  * Channel messages
  * - Embedded in vmbus_message.msg_data, e.g. response and notification.