diff options
author | Dimitry Andric <dim@FreeBSD.org> | 2016-08-16 20:19:05 +0000 |
---|---|---|
committer | Dimitry Andric <dim@FreeBSD.org> | 2016-08-16 20:19:05 +0000 |
commit | 27067774dce3388702a4cf744d7096c6fb71b688 (patch) | |
tree | 56300949abd050b3cd2e23d29210d2a567bc8aec /sys/dev | |
parent | 44be0a8ea517cbe7a9140bca20e1e93228ac0a04 (diff) | |
parent | 915a263ea24ab051bb57674a4d6f5ffe4ef0d5b5 (diff) | |
download | src-27067774dce3388702a4cf744d7096c6fb71b688.tar.gz src-27067774dce3388702a4cf744d7096c6fb71b688.zip |
Merge ^/head r303250 through r304235.
Notes
Notes:
svn path=/projects/clang390-import/; revision=304236
Diffstat (limited to 'sys/dev')
191 files changed, 20524 insertions, 14284 deletions
diff --git a/sys/dev/aic7xxx/aic7xxx_osm.c b/sys/dev/aic7xxx/aic7xxx_osm.c index a51177c97899..141cac1c8fb6 100644 --- a/sys/dev/aic7xxx/aic7xxx_osm.c +++ b/sys/dev/aic7xxx/aic7xxx_osm.c @@ -130,7 +130,7 @@ aic7770_map_registers(struct ahc_softc *ahc, u_int unused_ioport_arg) return ENOMEM; } ahc->platform_data->regs_res_type = SYS_RES_IOPORT; - ahc->platform_data->regs_res_id = rid, + ahc->platform_data->regs_res_id = rid; ahc->platform_data->regs = regs; ahc->tag = rman_get_bustag(regs); ahc->bsh = rman_get_bushandle(regs); diff --git a/sys/dev/ath/ath_hal/ar5212/ar5212_attach.c b/sys/dev/ath/ath_hal/ar5212/ar5212_attach.c index c7c77a9e2855..83b49139d65a 100644 --- a/sys/dev/ath/ath_hal/ar5212/ar5212_attach.c +++ b/sys/dev/ath/ath_hal/ar5212/ar5212_attach.c @@ -270,8 +270,8 @@ ar5212InitState(struct ath_hal_5212 *ahp, uint16_t devid, HAL_SOFTC sc, ahp->ah_acktimeout = (u_int) -1; ahp->ah_ctstimeout = (u_int) -1; ahp->ah_sifstime = (u_int) -1; - ahp->ah_txTrigLev = INIT_TX_FIFO_THRESHOLD, - ahp->ah_maxTxTrigLev = MAX_TX_FIFO_THRESHOLD, + ahp->ah_txTrigLev = INIT_TX_FIFO_THRESHOLD; + ahp->ah_maxTxTrigLev = MAX_TX_FIFO_THRESHOLD; OS_MEMCPY(&ahp->ah_bssidmask, defbssidmask, IEEE80211_ADDR_LEN); #undef N diff --git a/sys/dev/ath/ath_hal/ar5416/ar5416_attach.c b/sys/dev/ath/ath_hal/ar5416/ar5416_attach.c index 98482fde42aa..5b10c7c80fd9 100644 --- a/sys/dev/ath/ath_hal/ar5416/ar5416_attach.c +++ b/sys/dev/ath/ath_hal/ar5416/ar5416_attach.c @@ -103,8 +103,8 @@ ar5416InitState(struct ath_hal_5416 *ahp5416, uint16_t devid, HAL_SOFTC sc, ah->ah_configPCIE = ar5416ConfigPCIE; ah->ah_disablePCIE = ar5416DisablePCIE; ah->ah_perCalibration = ar5416PerCalibration; - ah->ah_perCalibrationN = ar5416PerCalibrationN, - ah->ah_resetCalValid = ar5416ResetCalValid, + ah->ah_perCalibrationN = ar5416PerCalibrationN; + ah->ah_resetCalValid = ar5416ResetCalValid; ah->ah_setTxPowerLimit = ar5416SetTxPowerLimit; ah->ah_setTxPower = ar5416SetTransmitPower; ah->ah_setBoardValues = ar5416SetBoardValues; diff --git a/sys/dev/ath/if_athioctl.h b/sys/dev/ath/if_athioctl.h index 7b69ab5fe6c2..73905d3fa0a4 100644 --- a/sys/dev/ath/if_athioctl.h +++ b/sys/dev/ath/if_athioctl.h @@ -303,8 +303,8 @@ struct ath_radiotap_vendor_hdr { /* 30 bytes */ /* At this point it should be 4 byte aligned */ uint32_t evm[ATH_RADIOTAP_MAX_EVM]; /* 5 * 4 = 20 */ - uint8_t rssi_ctl[ATH_RADIOTAP_MAX_CHAINS]; /* 4 */ - uint8_t rssi_ext[ATH_RADIOTAP_MAX_CHAINS]; /* 4 */ + uint8_t rssi_ctl[ATH_RADIOTAP_MAX_CHAINS]; /* 4 * 4 = 16 */ + uint8_t rssi_ext[ATH_RADIOTAP_MAX_CHAINS]; /* 4 * 4 = 16 */ uint8_t vh_phyerr_code; /* Phy error code, or 0xff */ uint8_t vh_rs_status; /* RX status */ diff --git a/sys/dev/auxio/auxio.c b/sys/dev/auxio/auxio.c index b104d5af5ee9..f559c173b867 100644 --- a/sys/dev/auxio/auxio.c +++ b/sys/dev/auxio/auxio.c @@ -98,7 +98,7 @@ __FBSDID("$FreeBSD$"); #define AUXIO_PCIO_NREG 5 struct auxio_softc { - struct device *sc_dev; + device_t sc_dev; int sc_nauxio; struct resource *sc_res[AUXIO_PCIO_NREG]; diff --git a/sys/dev/bktr/bktr_os.c b/sys/dev/bktr/bktr_os.c index aa41454f0cab..42069b39db3b 100644 --- a/sys/dev/bktr/bktr_os.c +++ b/sys/dev/bktr/bktr_os.c @@ -889,10 +889,11 @@ vm_offset_t vm_page_alloc_contig(vm_offset_t, vm_offset_t, #if defined(__OpenBSD__) static int bktr_probe(struct device *, void *, void *); +static void bktr_attach(struct device *, struct device *, void *); #else -static int bktr_probe(struct device *, struct cfdata *, void *); +static int bktr_probe(device_t, struct cfdata *, void *); +static void bktr_attach(device_t, device_t, void *); #endif -static void bktr_attach(struct device *, struct device *, void *); struct cfattach bktr_ca = { sizeof(struct bktr_softc), bktr_probe, bktr_attach @@ -908,10 +909,11 @@ struct cfdriver bktr_cd = { int bktr_probe(parent, match, aux) - struct device *parent; #if defined(__OpenBSD__) + struct device *parent; void *match; #else + device_t parent; struct cfdata *match; #endif void *aux; @@ -933,7 +935,15 @@ bktr_probe(parent, match, aux) * the attach routine. */ static void -bktr_attach(struct device *parent, struct device *self, void *aux) +bktr_attach(parent, self, aux) +#if defined(__OpenBSD__) + struct device *parent; + struct device *self; +#else + device_t parent; + device_t self; +#endif + void *aux; { bktr_ptr_t bktr; u_long latency; diff --git a/sys/dev/bktr/bktr_reg.h b/sys/dev/bktr/bktr_reg.h index 4a3af1a490b0..65a6dbc359a0 100644 --- a/sys/dev/bktr/bktr_reg.h +++ b/sys/dev/bktr/bktr_reg.h @@ -35,7 +35,7 @@ */ #ifdef __NetBSD__ -#include <machine/bus.h> /* struct device */ +#include <machine/bus.h> /* device_t */ #include <sys/device.h> #include <sys/select.h> /* struct selinfo */ # ifdef DEBUG diff --git a/sys/dev/bxe/bxe.c b/sys/dev/bxe/bxe.c index e832389ba610..a5c13ea600f3 100644 --- a/sys/dev/bxe/bxe.c +++ b/sys/dev/bxe/bxe.c @@ -5624,7 +5624,8 @@ bxe_tx_mq_start_locked(struct bxe_softc *sc, if (!sc->link_vars.link_up || (if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != IFF_DRV_RUNNING) { - rc = drbr_enqueue(ifp, tx_br, m); + if (m != NULL) + rc = drbr_enqueue(ifp, tx_br, m); goto bxe_tx_mq_start_locked_exit; } @@ -13331,7 +13332,7 @@ bxe_get_shmem_info(struct bxe_softc *sc) /* get the port feature config */ sc->port.config = - SHMEM_RD(sc, dev_info.port_feature_config[port].config), + SHMEM_RD(sc, dev_info.port_feature_config[port].config); /* get the link params */ sc->link_params.speed_cap_mask[0] = diff --git a/sys/dev/cfe/cfe_api.c b/sys/dev/cfe/cfe_api.c index bf3fa66e895e..1b3610af875c 100644 --- a/sys/dev/cfe/cfe_api.c +++ b/sys/dev/cfe/cfe_api.c @@ -1,5 +1,4 @@ -/* $NetBSD: cfe_api.c,v 1.5 2005/12/11 12:18:07 christos Exp $ */ -/* from: SiByte Id: cfe_api.c,v 1.16 2002/07/09 23:29:11 cgd Exp $ */ +/* from: Broadcom Id: cfe_api.c,v 1.18 2006/08/24 02:13:56 binh Exp $ */ /*- * Copyright 2000, 2001, 2002 @@ -177,6 +176,27 @@ cfe_enumenv(int idx, char *name, int namelen, char *val, int vallen) } #endif /* CFE_API_enumenv || CFE_API_ALL */ +#if defined(CFE_API_enumdev) || defined(CFE_API_ALL) +int +cfe_enumdev(int idx, char *name, int namelen) +{ + cfe_xiocb_t xiocb; + + xiocb.xiocb_fcode = CFE_CMD_DEV_ENUM; + xiocb.xiocb_status = 0; + xiocb.xiocb_handle = 0; + xiocb.xiocb_flags = 0; + xiocb.xiocb_psize = sizeof(xiocb_envbuf_t); + xiocb.plist.xiocb_envbuf.enum_idx = idx; + xiocb.plist.xiocb_envbuf.name_ptr = XPTR_FROM_NATIVE(name); + xiocb.plist.xiocb_envbuf.name_length = namelen; + + cfe_iocb_dispatch(&xiocb); + + return xiocb.xiocb_status; +} +#endif /* CFE_API_enumdev || CFE_API_ALL */ + #if defined(CFE_API_enummem) || defined(CFE_API_ALL) int cfe_enummem(int idx, int flags, cfe_xuint_t *start, cfe_xuint_t *length, diff --git a/sys/dev/cfe/cfe_api.h b/sys/dev/cfe/cfe_api.h index 943459dd7164..ade7a81c1842 100644 --- a/sys/dev/cfe/cfe_api.h +++ b/sys/dev/cfe/cfe_api.h @@ -1,5 +1,4 @@ -/* $NetBSD: cfe_api.h,v 1.3 2003/02/07 17:38:48 cgd Exp $ */ -/* from: SiByte Id: cfe_api.h,v 1.29 2002/07/09 23:29:11 cgd Exp $ */ +/* from: Broadcom Id: cfe_api.h,v 1.31 2006/08/24 02:13:56 binh Exp $ */ /*- * Copyright 2000, 2001, 2002 @@ -154,6 +153,7 @@ int64_t cfe_getticks(void); #define cfe_cpu_start(a,b,c,d,e) __cfe_cpu_start(a,b,c,d,e) #define cfe_cpu_stop(a) __cfe_cpu_stop(a) #define cfe_enumenv(a,b,d,e,f) __cfe_enumenv(a,b,d,e,f) +#define cfe_enumdev(a,b,c) __cfe_enumdev(a,b,c) #define cfe_enummem(a,b,c,d,e) __cfe_enummem(a,b,c,d,e) #define cfe_exit(a,b) __cfe_exit(a,b) #define cfe_flushcache(a) __cfe_cacheflush(a) @@ -176,6 +176,7 @@ int cfe_close(int handle); int cfe_cpu_start(int cpu, void (*fn)(void), long sp, long gp, long a1); int cfe_cpu_stop(int cpu); int cfe_enumenv(int idx, char *name, int namelen, char *val, int vallen); +int cfe_enumdev(int idx, char *name, int namelen); int cfe_enummem(int idx, int flags, uint64_t *start, uint64_t *length, uint64_t *type); int cfe_exit(int warm,int status); diff --git a/sys/dev/cfe/cfe_api_int.h b/sys/dev/cfe/cfe_api_int.h index dfdde478f692..b15a17697087 100644 --- a/sys/dev/cfe/cfe_api_int.h +++ b/sys/dev/cfe/cfe_api_int.h @@ -1,5 +1,4 @@ -/* $NetBSD: cfe_api_int.h,v 1.2 2003/02/07 17:38:48 cgd Exp $ */ -/* from: SiByte Id: cfe_api_int.h,v 1.21 2002/07/09 23:29:11 cgd Exp $ */ +/* from: Broadcom Id: cfe_api_int.h,v 1.22 2003/02/07 17:27:56 cgd Exp $ */ /*- * Copyright 2000, 2001, 2002 diff --git a/sys/dev/cfe/cfe_error.h b/sys/dev/cfe/cfe_error.h index 6b57d14c6423..d32f1d9065dd 100644 --- a/sys/dev/cfe/cfe_error.h +++ b/sys/dev/cfe/cfe_error.h @@ -1,5 +1,4 @@ -/* $NetBSD: cfe_error.h,v 1.2 2003/02/07 17:38:48 cgd Exp $ */ -/* from: SiByte Id: cfe_error.h,v 1.2 2002/07/09 19:37:52 cgd Exp $ */ +/* from: Broadcom Id: cfe_error.h,v 1.3 2003/02/07 17:27:56 cgd Exp $ */ /*- * Copyright 2000, 2001, 2002 diff --git a/sys/dev/cfe/cfe_ioctl.h b/sys/dev/cfe/cfe_ioctl.h index a45ba1d004b8..c02e79d1f1e8 100644 --- a/sys/dev/cfe/cfe_ioctl.h +++ b/sys/dev/cfe/cfe_ioctl.h @@ -1,14 +1,13 @@ -/* $NetBSD: cfe_ioctl.h,v 1.2 2003/02/07 17:52:08 cgd Exp $ */ - /*- - * Copyright 2000, 2001 + * Copyright 2000, 2001, 2002, 2003 * Broadcom Corporation. All rights reserved. * - * This software is furnished under license and may be used and copied only - * in accordance with the following terms and conditions. Subject to these - * conditions, you may download, copy, install, use, modify and distribute - * modified or unmodified copies of this software in source and/or binary - * form. No title or ownership is transferred hereby. + * This software is furnished under license and may be used and + * copied only in accordance with the following terms and + * conditions. Subject to these conditions, you may download, + * copy, install, use, modify and distribute modified or unmodified + * copies of this software in source and/or binary form. No title + * or ownership is transferred hereby. * * 1) Any source code used, modified or distributed must reproduce and * retain this copyright notice and list of conditions as they appear in @@ -41,7 +40,7 @@ * * IOCTL function numbers and I/O data structures. * - * Author: Mitch Lichtenberg (mpl@broadcom.com) + * Author: Mitch Lichtenberg * ********************************************************************* */ @@ -53,23 +52,115 @@ #define IOCTL_NVRAM_GETINFO 1 /* return nvram_info_t */ #define IOCTL_NVRAM_ERASE 2 /* erase sector containing nvram_info_t area */ #define IOCTL_FLASH_ERASE_SECTOR 3 /* erase an arbitrary sector */ -#define IOCTL_FLASH_ERASE_ALL 4 /* Erase the entire flash */ +#define IOCTL_FLASH_ERASE_ALL 4 /* Erase the entire flash */ +#define IOCTL_FLASH_WRITE_ALL 5 /* write entire flash */ +#define IOCTL_FLASH_GETINFO 6 /* get flash device info */ +#define IOCTL_FLASH_GETSECTORS 7 /* get sector information */ +#define IOCTL_FLASH_ERASE_RANGE 8 /* erase range of bytes */ +#define IOCTL_NVRAM_UNLOCK 9 /* allow r/w beyond logical end of device */ +#define IOCTL_FLASH_PROTECT_RANGE 10 /* Protect a group of sectors */ +#define IOCTL_FLASH_UNPROTECT_RANGE 11 /* unprotect a group of sectors */ +#define IOCTL_FLASH_DATA_WIDTH_MODE 12 /* switch flash and gen bus to support 8 or 16-bit mode I/Os */ +#define IOCTL_FLASH_BURST_MODE 13 /* configure gen bus for burst mode */ + +typedef struct flash_range_s { + unsigned int range_base; + unsigned int range_length; +} flash_range_t; + +typedef struct flash_info_s { + unsigned long long flash_base; /* flash physical base address */ + unsigned int flash_size; /* available device size in bytes */ + unsigned int flash_type; /* type, from FLASH_TYPE below */ + unsigned int flash_flags; /* Various flags (FLASH_FLAG_xxx) */ +} flash_info_t; + +typedef struct flash_sector_s { + int flash_sector_idx; + int flash_sector_status; + unsigned int flash_sector_offset; + unsigned int flash_sector_size; +} flash_sector_t; + +#define FLASH_SECTOR_OK 0 +#define FLASH_SECTOR_INVALID -1 + +#define FLASH_TYPE_UNKNOWN 0 /* not sure what kind of flash */ +#define FLASH_TYPE_SRAM 1 /* not flash: it's SRAM */ +#define FLASH_TYPE_ROM 2 /* not flash: it's ROM */ +#define FLASH_TYPE_FLASH 3 /* it's flash memory of some sort */ + +#define FLASH_FLAG_NOERASE 1 /* Byte-range writes supported, + Erasing is not necessary */ typedef struct nvram_info_s { - int nvram_offset; /* offset of environment area */ - int nvram_size; /* size of environment area */ - int nvram_eraseflg; /* true if we need to erase first */ + int nvram_offset; /* offset of environment area */ + int nvram_size; /* size of environment area */ + int nvram_eraseflg; /* true if we need to erase first */ } nvram_info_t; /* ********************************************************************* * Ethernet stuff ********************************************************************* */ -#define IOCTL_ETHER_GETHWADDR 1 +#define IOCTL_ETHER_GETHWADDR 1 /* Get hardware address (6bytes) */ +#define IOCTL_ETHER_SETHWADDR 2 /* Set hardware address (6bytes) */ +#define IOCTL_ETHER_GETSPEED 3 /* Get Speed and Media (int) */ +#define IOCTL_ETHER_SETSPEED 4 /* Set Speed and Media (int) */ +#define IOCTL_ETHER_GETLINK 5 /* get link status (int) */ +#define IOCTL_ETHER_GETLOOPBACK 7 /* get loopback state */ +#define IOCTL_ETHER_SETLOOPBACK 8 /* set loopback state */ +#define IOCTL_ETHER_SETPACKETFIFO 9 /* set packet fifo mode (int) */ +#define IOCTL_ETHER_SETSTROBESIG 10 /* set strobe signal (int) */ + +#define ETHER_LOOPBACK_OFF 0 /* no loopback */ +#define ETHER_LOOPBACK_INT 1 /* Internal loopback */ +#define ETHER_LOOPBACK_EXT 2 /* External loopback (through PHY) */ + +#define ETHER_SPEED_AUTO 0 /* Auto detect */ +#define ETHER_SPEED_UNKNOWN 0 /* Speed not known (on link status) */ +#define ETHER_SPEED_10HDX 1 /* 10MB hdx and fdx */ +#define ETHER_SPEED_10FDX 2 +#define ETHER_SPEED_100HDX 3 /* 100MB hdx and fdx */ +#define ETHER_SPEED_100FDX 4 +#define ETHER_SPEED_1000HDX 5 /* 1000MB hdx and fdx */ +#define ETHER_SPEED_1000FDX 6 + +#define ETHER_FIFO_8 0 /* 8-bit packet fifo mode */ +#define ETHER_FIFO_16 1 /* 16-bit packet fifo mode */ +#define ETHER_ETHER 2 /* Standard ethernet mode */ + +#define ETHER_STROBE_GMII 0 /* GMII style strobe signal */ +#define ETHER_STROBE_ENCODED 1 /* Encoded */ +#define ETHER_STROBE_SOP 2 /* SOP flagged. Only in 8-bit mode*/ +#define ETHER_STROBE_EOP 3 /* EOP flagged. Only in 8-bit mode*/ + +/* ********************************************************************* + * Serial Ports + ********************************************************************* */ + +#define IOCTL_SERIAL_SETSPEED 1 /* get baud rate (int) */ +#define IOCTL_SERIAL_GETSPEED 2 /* set baud rate (int) */ +#define IOCTL_SERIAL_SETFLOW 3 /* Set Flow Control */ +#define IOCTL_SERIAL_GETFLOW 4 /* Get Flow Control */ + +#define SERIAL_FLOW_NONE 0 /* no flow control */ +#define SERIAL_FLOW_SOFTWARE 1 /* software flow control (not impl) */ +#define SERIAL_FLOW_HARDWARE 2 /* hardware flow control */ /* ********************************************************************* * Block device stuff ********************************************************************* */ -#define IOCTL_BLOCK_GETBLOCKSIZE 1 -#define IOCTL_BLOCK_GETTOTALBLOCKS 2 +#define IOCTL_BLOCK_GETBLOCKSIZE 1 /* get block size (int) */ +#define IOCTL_BLOCK_GETTOTALBLOCKS 2 /* get total bocks (long long) */ +#define IOCTL_BLOCK_GETDEVTYPE 3 /* get device type (struct) */ + +typedef struct blockdev_info_s { + unsigned long long blkdev_totalblocks; + unsigned int blkdev_blocksize; + unsigned int blkdev_devtype; +} blockdev_info_t; + +#define BLOCK_DEVTYPE_DISK 0 +#define BLOCK_DEVTYPE_CDROM 1 diff --git a/sys/dev/cxgbe/adapter.h b/sys/dev/cxgbe/adapter.h index d31124581efa..73af382dcd6a 100644 --- a/sys/dev/cxgbe/adapter.h +++ b/sys/dev/cxgbe/adapter.h @@ -195,6 +195,7 @@ enum { ADAP_SYSCTL_CTX = (1 << 4), /* TOM_INIT_DONE= (1 << 5), No longer used */ BUF_PACKING_OK = (1 << 6), + IS_VF = (1 << 7), CXGBE_BUSY = (1 << 9), @@ -429,6 +430,7 @@ enum {DOORBELL_UDB, DOORBELL_WCWR, DOORBELL_UDBWC, DOORBELL_KDB}; struct sge_eq { unsigned int flags; /* MUST be first */ unsigned int cntxt_id; /* SGE context id for the eq */ + unsigned int abs_id; /* absolute SGE id for the eq */ struct mtx eq_lock; struct tx_desc *desc; /* KVA of descriptor ring */ @@ -737,8 +739,10 @@ struct sge { struct sge_nm_txq *nm_txq; /* netmap tx queues */ struct sge_nm_rxq *nm_rxq; /* netmap rx queues */ - uint16_t iq_start; - int eq_start; + uint16_t iq_start; /* first cntxt_id */ + uint16_t iq_base; /* first abs_id */ + int eq_start; /* first cntxt_id */ + int eq_base; /* first abs_id */ struct sge_iq **iqmap; /* iq->cntxt_id to iq mapping */ struct sge_eq **eqmap; /* eq->cntxt_id to eq mapping */ @@ -781,6 +785,8 @@ struct adapter { struct sge_rxq *rxq; struct sge_nm_rxq *nm_rxq; } __aligned(CACHE_LINE_SIZE) *irq; + int sge_gts_reg; + int sge_kdoorbell_reg; bus_dma_tag_t dmat; /* Parent DMA tag */ diff --git a/sys/dev/cxgbe/common/common.h b/sys/dev/cxgbe/common/common.h index cc6edf32a3c7..1cbdf8ee059a 100644 --- a/sys/dev/cxgbe/common/common.h +++ b/sys/dev/cxgbe/common/common.h @@ -215,6 +215,8 @@ struct sge_params { int pad_boundary; int pack_boundary; int fl_pktshift; + u32 sge_control; + u32 sge_fl_buffer_size[SGE_FLBUF_SIZES]; }; struct tp_params { diff --git a/sys/dev/cxgbe/common/t4_hw.c b/sys/dev/cxgbe/common/t4_hw.c index 7ecdb81a44b8..ca6d2069a5f0 100644 --- a/sys/dev/cxgbe/common/t4_hw.c +++ b/sys/dev/cxgbe/common/t4_hw.c @@ -289,6 +289,14 @@ int t4_wr_mbox_meat_timeout(struct adapter *adap, int mbox, const void *cmd, if ((size & 15) || size > MBOX_LEN) return -EINVAL; + if (adap->flags & IS_VF) { + if (is_t6(adap)) + data_reg = FW_T6VF_MBDATA_BASE_ADDR; + else + data_reg = FW_T4VF_MBDATA_BASE_ADDR; + ctl_reg = VF_CIM_REG(A_CIM_VF_EXT_MAILBOX_CTRL); + } + /* * If we have a negative timeout, that implies that we can't sleep. */ @@ -343,6 +351,22 @@ int t4_wr_mbox_meat_timeout(struct adapter *adap, int mbox, const void *cmd, for (i = 0; i < size; i += 8, p++) t4_write_reg64(adap, data_reg + i, be64_to_cpu(*p)); + if (adap->flags & IS_VF) { + /* + * For the VFs, the Mailbox Data "registers" are + * actually backed by T4's "MA" interface rather than + * PL Registers (as is the case for the PFs). Because + * these are in different coherency domains, the write + * to the VF's PL-register-backed Mailbox Control can + * race in front of the writes to the MA-backed VF + * Mailbox Data "registers". So we need to do a + * read-back on at least one byte of the VF Mailbox + * Data registers before doing the write to the VF + * Mailbox Control register. + */ + t4_read_reg(adap, data_reg); + } + CH_DUMP_MBOX(adap, mbox, data_reg); t4_write_reg(adap, ctl_reg, F_MBMSGVALID | V_MBOWNER(X_MBOWNER_FW)); @@ -355,10 +379,13 @@ int t4_wr_mbox_meat_timeout(struct adapter *adap, int mbox, const void *cmd, * Loop waiting for the reply; bail out if we time out or the firmware * reports an error. */ - for (i = 0; - !((pcie_fw = t4_read_reg(adap, A_PCIE_FW)) & F_PCIE_FW_ERR) && - i < timeout; - i += ms) { + pcie_fw = 0; + for (i = 0; i < timeout; i += ms) { + if (!(adap->flags & IS_VF)) { + pcie_fw = t4_read_reg(adap, A_PCIE_FW); + if (pcie_fw & F_PCIE_FW_ERR) + break; + } if (sleep_ok) { ms = delay[delay_idx]; /* last element may repeat */ if (delay_idx < ARRAY_SIZE(delay) - 1) @@ -698,10 +725,14 @@ unsigned int t4_get_regs_len(struct adapter *adapter) switch (chip_version) { case CHELSIO_T4: + if (adapter->flags & IS_VF) + return FW_T4VF_REGMAP_SIZE; return T4_REGMAP_SIZE; case CHELSIO_T5: case CHELSIO_T6: + if (adapter->flags & IS_VF) + return FW_T4VF_REGMAP_SIZE; return T5_REGMAP_SIZE; } @@ -1180,6 +1211,18 @@ void t4_get_regs(struct adapter *adap, u8 *buf, size_t buf_size) 0x27e00, 0x27e04, }; + static const unsigned int t4vf_reg_ranges[] = { + VF_SGE_REG(A_SGE_VF_KDOORBELL), VF_SGE_REG(A_SGE_VF_GTS), + VF_MPS_REG(A_MPS_VF_CTL), + VF_MPS_REG(A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H), + VF_PL_REG(A_PL_VF_WHOAMI), VF_PL_REG(A_PL_VF_WHOAMI), + VF_CIM_REG(A_CIM_VF_EXT_MAILBOX_CTRL), + VF_CIM_REG(A_CIM_VF_EXT_MAILBOX_STATUS), + FW_T4VF_MBDATA_BASE_ADDR, + FW_T4VF_MBDATA_BASE_ADDR + + ((NUM_CIM_PF_MAILBOX_DATA_INSTANCES - 1) * 4), + }; + static const unsigned int t5_reg_ranges[] = { 0x1008, 0x10c0, 0x10cc, 0x10f8, @@ -1955,6 +1998,18 @@ void t4_get_regs(struct adapter *adap, u8 *buf, size_t buf_size) 0x51300, 0x51308, }; + static const unsigned int t5vf_reg_ranges[] = { + VF_SGE_REG(A_SGE_VF_KDOORBELL), VF_SGE_REG(A_SGE_VF_GTS), + VF_MPS_REG(A_MPS_VF_CTL), + VF_MPS_REG(A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H), + VF_PL_REG(A_PL_VF_WHOAMI), VF_PL_REG(A_PL_VF_REVISION), + VF_CIM_REG(A_CIM_VF_EXT_MAILBOX_CTRL), + VF_CIM_REG(A_CIM_VF_EXT_MAILBOX_STATUS), + FW_T4VF_MBDATA_BASE_ADDR, + FW_T4VF_MBDATA_BASE_ADDR + + ((NUM_CIM_PF_MAILBOX_DATA_INSTANCES - 1) * 4), + }; + static const unsigned int t6_reg_ranges[] = { 0x1008, 0x101c, 0x1024, 0x10a8, @@ -2532,6 +2587,18 @@ void t4_get_regs(struct adapter *adap, u8 *buf, size_t buf_size) 0x51300, 0x51324, }; + static const unsigned int t6vf_reg_ranges[] = { + VF_SGE_REG(A_SGE_VF_KDOORBELL), VF_SGE_REG(A_SGE_VF_GTS), + VF_MPS_REG(A_MPS_VF_CTL), + VF_MPS_REG(A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H), + VF_PL_REG(A_PL_VF_WHOAMI), VF_PL_REG(A_PL_VF_REVISION), + VF_CIM_REG(A_CIM_VF_EXT_MAILBOX_CTRL), + VF_CIM_REG(A_CIM_VF_EXT_MAILBOX_STATUS), + FW_T6VF_MBDATA_BASE_ADDR, + FW_T6VF_MBDATA_BASE_ADDR + + ((NUM_CIM_PF_MAILBOX_DATA_INSTANCES - 1) * 4), + }; + u32 *buf_end = (u32 *)(buf + buf_size); const unsigned int *reg_ranges; int reg_ranges_size, range; @@ -2543,18 +2610,33 @@ void t4_get_regs(struct adapter *adap, u8 *buf, size_t buf_size) */ switch (chip_version) { case CHELSIO_T4: - reg_ranges = t4_reg_ranges; - reg_ranges_size = ARRAY_SIZE(t4_reg_ranges); + if (adap->flags & IS_VF) { + reg_ranges = t4vf_reg_ranges; + reg_ranges_size = ARRAY_SIZE(t4vf_reg_ranges); + } else { + reg_ranges = t4_reg_ranges; + reg_ranges_size = ARRAY_SIZE(t4_reg_ranges); + } break; case CHELSIO_T5: - reg_ranges = t5_reg_ranges; - reg_ranges_size = ARRAY_SIZE(t5_reg_ranges); + if (adap->flags & IS_VF) { + reg_ranges = t5vf_reg_ranges; + reg_ranges_size = ARRAY_SIZE(t5vf_reg_ranges); + } else { + reg_ranges = t5_reg_ranges; + reg_ranges_size = ARRAY_SIZE(t5_reg_ranges); + } break; case CHELSIO_T6: - reg_ranges = t6_reg_ranges; - reg_ranges_size = ARRAY_SIZE(t6_reg_ranges); + if (adap->flags & IS_VF) { + reg_ranges = t6vf_reg_ranges; + reg_ranges_size = ARRAY_SIZE(t6vf_reg_ranges); + } else { + reg_ranges = t6_reg_ranges; + reg_ranges_size = ARRAY_SIZE(t6_reg_ranges); + } break; default: @@ -7644,6 +7726,7 @@ int t4_init_sge_params(struct adapter *adapter) { u32 r; struct sge_params *sp = &adapter->params.sge; + unsigned i; r = t4_read_reg(adapter, A_SGE_INGRESS_RX_THRESHOLD); sp->counter_val[0] = G_THRESHOLD_0(r); @@ -7686,6 +7769,7 @@ int t4_init_sge_params(struct adapter *adapter) sp->page_shift = (r & M_HOSTPAGESIZEPF0) + 10; r = t4_read_reg(adapter, A_SGE_CONTROL); + sp->sge_control = r; sp->spg_len = r & F_EGRSTATUSPAGESIZE ? 128 : 64; sp->fl_pktshift = G_PKTSHIFT(r); sp->pad_boundary = 1 << (G_INGPADBOUNDARY(r) + 5); @@ -7698,6 +7782,9 @@ int t4_init_sge_params(struct adapter *adapter) else sp->pack_boundary = 1 << (G_INGPACKBOUNDARY(r) + 5); } + for (i = 0; i < SGE_FLBUF_SIZES; i++) + sp->sge_fl_buffer_size[i] = t4_read_reg(adapter, + A_SGE_FL_BUFFER_SIZE0 + (4 * i)); return 0; } diff --git a/sys/dev/cxgbe/iw_cxgbe/cq.c b/sys/dev/cxgbe/iw_cxgbe/cq.c index b40ffc7fb6c1..5c040e97377d 100644 --- a/sys/dev/cxgbe/iw_cxgbe/cq.c +++ b/sys/dev/cxgbe/iw_cxgbe/cq.c @@ -172,7 +172,7 @@ create_cq(struct c4iw_rdev *rdev, struct t4_cq *cq, cq->gen = 1; cq->gts = (void *)((unsigned long)rman_get_virtual(sc->regs_res) + - MYPF_REG(SGE_PF_GTS)); + sc->sge_gts_reg); cq->rdev = rdev; if (user) { diff --git a/sys/dev/cxgbe/iw_cxgbe/qp.c b/sys/dev/cxgbe/iw_cxgbe/qp.c index d1be854c3471..5311b2c8577a 100644 --- a/sys/dev/cxgbe/iw_cxgbe/qp.c +++ b/sys/dev/cxgbe/iw_cxgbe/qp.c @@ -182,9 +182,9 @@ static int create_qp(struct c4iw_rdev *rdev, struct t4_wq *wq, pci_unmap_addr_set(&wq->rq, mapping, wq->rq.dma_addr); wq->db = (void *)((unsigned long)rman_get_virtual(sc->regs_res) + - MYPF_REG(SGE_PF_KDOORBELL)); + sc->sge_kdoorbell_reg); wq->gts = (void *)((unsigned long)rman_get_virtual(rdev->adap->regs_res) - + MYPF_REG(SGE_PF_GTS)); + + sc->sge_gts_reg); if (user) { wq->sq.udb = (u64)((char*)rman_get_virtual(rdev->adap->udbs_res) + (wq->sq.qid << rdev->qpshift)); diff --git a/sys/dev/cxgbe/offload.h b/sys/dev/cxgbe/offload.h index 22612d5156a1..cb0006c00a20 100644 --- a/sys/dev/cxgbe/offload.h +++ b/sys/dev/cxgbe/offload.h @@ -147,6 +147,7 @@ struct tom_tunables { int ddp; int rx_coalesce; int tx_align; + int tx_zcopy; }; #ifdef TCP_OFFLOAD diff --git a/sys/dev/cxgbe/t4_if.m b/sys/dev/cxgbe/t4_if.m index 430b4801dfa7..59e86bdab075 100644 --- a/sys/dev/cxgbe/t4_if.m +++ b/sys/dev/cxgbe/t4_if.m @@ -55,11 +55,11 @@ METHOD int detach_child { device_t dev; }; -# Called by a driver to query the PF4 driver for the unit number to use -# for a given port. If the port is not enabled on the adapter, this -# will fail. -METHOD int read_port_unit { +# Called by a driver to query the PF4 driver for the child device +# associated with a given port. If the port is not enabled on the adapter, +# this will fail. +METHOD int read_port_device { device_t dev; int port; - int *unit; + device_t *child; }; diff --git a/sys/dev/cxgbe/t4_iov.c b/sys/dev/cxgbe/t4_iov.c index 9ac1de5c8e4a..d0c31e82b330 100644 --- a/sys/dev/cxgbe/t4_iov.c +++ b/sys/dev/cxgbe/t4_iov.c @@ -104,6 +104,7 @@ t4iov_probe(device_t dev) for (i = 0; i < nitems(t4iov_pciids); i++) { if (d == t4iov_pciids[i].device) { device_set_desc(dev, t4iov_pciids[i].desc); + device_quiet(dev); return (BUS_PROBE_DEFAULT); } } @@ -120,6 +121,7 @@ t5iov_probe(device_t dev) for (i = 0; i < nitems(t5iov_pciids); i++) { if (d == t5iov_pciids[i].device) { device_set_desc(dev, t5iov_pciids[i].desc); + device_quiet(dev); return (BUS_PROBE_DEFAULT); } } @@ -148,25 +150,27 @@ t4iov_attach_child(device_t dev) #ifdef PCI_IOV nvlist_t *pf_schema, *vf_schema; #endif - int error, unit; + device_t pdev; + int error; sc = device_get_softc(dev); MPASS(!sc->sc_attached); /* * PF0-3 are associated with a specific port on the NIC (PF0 - * with port 0, etc.). Ask the PF4 driver for the unit number - * for this function's associated port to determine if the port - * is present. + * with port 0, etc.). Ask the PF4 driver for the device for + * this function's associated port to determine if the port is + * present. */ - error = T4_READ_PORT_UNIT(sc->sc_main, pci_get_function(dev), &unit); + error = T4_READ_PORT_DEVICE(sc->sc_main, pci_get_function(dev), &pdev); if (error) return (0); #ifdef PCI_IOV pf_schema = pci_iov_schema_alloc_node(); vf_schema = pci_iov_schema_alloc_node(); - error = pci_iov_attach(dev, pf_schema, vf_schema); + error = pci_iov_attach_name(dev, pf_schema, vf_schema, "%s", + device_get_nameunit(pdev)); if (error) { device_printf(dev, "Failed to initialize SR-IOV: %d\n", error); return (0); diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c index 773d50ac033b..6b7edaf215d2 100644 --- a/sys/dev/cxgbe/t4_main.c +++ b/sys/dev/cxgbe/t4_main.c @@ -83,14 +83,14 @@ static int t4_probe(device_t); static int t4_attach(device_t); static int t4_detach(device_t); static int t4_ready(device_t); -static int t4_read_port_unit(device_t, int, int *); +static int t4_read_port_device(device_t, int, device_t *); static device_method_t t4_methods[] = { DEVMETHOD(device_probe, t4_probe), DEVMETHOD(device_attach, t4_attach), DEVMETHOD(device_detach, t4_detach), DEVMETHOD(t4_is_main_ready, t4_ready), - DEVMETHOD(t4_read_port_unit, t4_read_port_unit), + DEVMETHOD(t4_read_port_device, t4_read_port_device), DEVMETHOD_END }; @@ -134,14 +134,9 @@ static driver_t vcxgbe_driver = { }; static d_ioctl_t t4_ioctl; -static d_open_t t4_open; -static d_close_t t4_close; static struct cdevsw t4_cdevsw = { .d_version = D_VERSION, - .d_flags = 0, - .d_open = t4_open, - .d_close = t4_close, .d_ioctl = t4_ioctl, .d_name = "t4nex", }; @@ -154,7 +149,7 @@ static device_method_t t5_methods[] = { DEVMETHOD(device_detach, t4_detach), DEVMETHOD(t4_is_main_ready, t4_ready), - DEVMETHOD(t4_read_port_unit, t4_read_port_unit), + DEVMETHOD(t4_read_port_device, t4_read_port_device), DEVMETHOD_END }; @@ -179,15 +174,6 @@ static driver_t vcxl_driver = { sizeof(struct vi_info) }; -static struct cdevsw t5_cdevsw = { - .d_version = D_VERSION, - .d_flags = 0, - .d_open = t4_open, - .d_close = t4_close, - .d_ioctl = t4_ioctl, - .d_name = "t5nex", -}; - /* ifnet + media interface */ static void cxgbe_init(void *); static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); @@ -681,6 +667,7 @@ t4_attach(device_t dev) { struct adapter *sc; int rc = 0, i, j, n10g, n1g, rqidx, tqidx; + struct make_dev_args mda; struct intrs_and_queues iaq; struct sge *s; uint8_t *buf; @@ -710,6 +697,8 @@ t4_attach(device_t dev) sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5); } + sc->sge_gts_reg = MYPF_REG(A_SGE_PF_GTS); + sc->sge_kdoorbell_reg = MYPF_REG(A_SGE_PF_KDOORBELL); sc->traceq = -1; mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", @@ -760,13 +749,16 @@ t4_attach(device_t dev) setup_memwin(sc); if (t4_init_devlog_params(sc, 0) == 0) fixup_devlog_params(sc); - sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, - device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", - device_get_nameunit(dev)); - if (sc->cdev == NULL) - device_printf(dev, "failed to create nexus char device.\n"); - else - sc->cdev->si_drv1 = sc; + make_dev_args_init(&mda); + mda.mda_devsw = &t4_cdevsw; + mda.mda_uid = UID_ROOT; + mda.mda_gid = GID_WHEEL; + mda.mda_mode = 0600; + mda.mda_si_drv1 = sc; + rc = make_dev_s(&mda, &sc->cdev, "%s", device_get_nameunit(dev)); + if (rc != 0) + device_printf(dev, "failed to create nexus char device: %d.\n", + rc); /* Go no further if recovery mode has been requested. */ if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { @@ -1102,7 +1094,7 @@ t4_ready(device_t dev) } static int -t4_read_port_unit(device_t dev, int port, int *unit) +t4_read_port_device(device_t dev, int port, device_t *child) { struct adapter *sc; struct port_info *pi; @@ -1113,7 +1105,7 @@ t4_read_port_unit(device_t dev, int port, int *unit) pi = sc->port[port]; if (pi == NULL || pi->dev == NULL) return (ENXIO); - *unit = device_get_unit(pi->dev); + *child = pi->dev; return (0); } @@ -3340,6 +3332,8 @@ get_params__post_init(struct adapter *sc) sc->vres.iscsi.size = val[1] - val[0] + 1; } + t4_init_sge_params(sc); + /* * We've got the params we wanted to query via the firmware. Now grab * some others directly from the chip. @@ -4372,7 +4366,7 @@ t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, "failed to setup interrupt for rid %d, name %s: %d\n", rid, name, rc); } else if (name) - bus_describe_intr(sc->dev, irq->res, irq->tag, name); + bus_describe_intr(sc->dev, irq->res, irq->tag, "%s", name); return (rc); } @@ -4864,6 +4858,11 @@ t4_sysctls(struct adapter *sc) SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align", CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload"); + sc->tt.tx_zcopy = 0; + SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_zcopy", + CTLFLAG_RW, &sc->tt.tx_zcopy, 0, + "Enable zero-copy aio_write(2)"); + SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "timer_tick", CTLTYPE_STRING | CTLFLAG_RD, sc, 0, sysctl_tp_tick, "A", "TP timer tick (us)"); @@ -7909,11 +7908,6 @@ set_filter(struct adapter *sc, struct t4_filter *t) goto done; } - if (!(sc->flags & FULL_INIT_DONE)) { - rc = EAGAIN; - goto done; - } - if (t->idx >= nfilters) { rc = EINVAL; goto done; @@ -7947,6 +7941,10 @@ set_filter(struct adapter *sc, struct t4_filter *t) goto done; } + if (!(sc->flags & FULL_INIT_DONE) && + ((rc = adapter_full_init(sc)) != 0)) + goto done; + if (sc->tids.ftid_tab == NULL) { KASSERT(sc->tids.ftids_in_use == 0, ("%s: no memory allocated but filters_in_use > 0", @@ -8723,18 +8721,6 @@ t4_iterate(void (*func)(struct adapter *, void *), void *arg) } static int -t4_open(struct cdev *dev, int flags, int type, struct thread *td) -{ - return (0); -} - -static int -t4_close(struct cdev *dev, int flags, int type, struct thread *td) -{ - return (0); -} - -static int t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, struct thread *td) { @@ -8779,7 +8765,7 @@ t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, } case CHELSIO_T4_REGDUMP: { struct t4_regdump *regs = (struct t4_regdump *)data; - int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; + int reglen = t4_get_regs_len(sc); uint8_t *buf; if (regs->len < reglen) { @@ -8905,7 +8891,7 @@ t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, rc = t4_set_tracer(sc, (struct t4_tracer *)data); break; default: - rc = EINVAL; + rc = ENOTTY; } return (rc); diff --git a/sys/dev/cxgbe/t4_netmap.c b/sys/dev/cxgbe/t4_netmap.c index f6c96e13fb78..d7356e0bdb43 100644 --- a/sys/dev/cxgbe/t4_netmap.c +++ b/sys/dev/cxgbe/t4_netmap.c @@ -204,7 +204,7 @@ alloc_nm_rxq_hwq(struct vi_info *vi, struct sge_nm_rxq *nm_rxq, int cong) } } - t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), + t4_write_reg(sc, sc->sge_gts_reg, V_INGRESSQID(nm_rxq->iq_cntxt_id) | V_SEINTARM(V_QINTR_TIMER_IDX(holdoff_tmr_idx))); @@ -364,7 +364,7 @@ cxgbe_netmap_on(struct adapter *sc, struct vi_info *vi, struct ifnet *ifp, MPASS((j & 7) == 0); j /= 8; /* driver pidx to hardware pidx */ wmb(); - t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), + t4_write_reg(sc, sc->sge_kdoorbell_reg, nm_rxq->fl_db_val | V_PIDX(j)); atomic_cmpset_int(&irq->nm_state, NM_OFF, NM_ON); @@ -537,7 +537,7 @@ ring_nm_txq_db(struct adapter *sc, struct sge_nm_txq *nm_txq) break; case DOORBELL_KDB: - t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), + t4_write_reg(sc, sc->sge_kdoorbell_reg, V_QID(nm_txq->cntxt_id) | V_PIDX(n)); break; } @@ -818,7 +818,7 @@ cxgbe_netmap_rxsync(struct netmap_kring *kring, int flags) } if (++dbinc == 8 && n >= 32) { wmb(); - t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), + t4_write_reg(sc, sc->sge_kdoorbell_reg, nm_rxq->fl_db_val | V_PIDX(dbinc)); dbinc = 0; } @@ -827,7 +827,7 @@ cxgbe_netmap_rxsync(struct netmap_kring *kring, int flags) if (dbinc > 0) { wmb(); - t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), + t4_write_reg(sc, sc->sge_kdoorbell_reg, nm_rxq->fl_db_val | V_PIDX(dbinc)); } } @@ -981,14 +981,14 @@ t4_nm_intr(void *arg) fl_credits /= 8; IDXINCR(nm_rxq->fl_pidx, fl_credits * 8, nm_rxq->fl_sidx); - t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), + t4_write_reg(sc, sc->sge_kdoorbell_reg, nm_rxq->fl_db_val | V_PIDX(fl_credits)); fl_credits = fl_cidx & 7; } else if (!black_hole) { netmap_rx_irq(ifp, nm_rxq->nid, &work); MPASS(work != 0); } - t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), + t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(n) | V_INGRESSQID(nm_rxq->iq_cntxt_id) | V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX))); n = 0; @@ -999,12 +999,12 @@ t4_nm_intr(void *arg) if (black_hole) { fl_credits /= 8; IDXINCR(nm_rxq->fl_pidx, fl_credits * 8, nm_rxq->fl_sidx); - t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), + t4_write_reg(sc, sc->sge_kdoorbell_reg, nm_rxq->fl_db_val | V_PIDX(fl_credits)); } else netmap_rx_irq(ifp, nm_rxq->nid, &work); - t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(n) | + t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(n) | V_INGRESSQID((u32)nm_rxq->iq_cntxt_id) | V_SEINTARM(V_QINTR_TIMER_IDX(holdoff_tmr_idx))); } diff --git a/sys/dev/cxgbe/t4_sge.c b/sys/dev/cxgbe/t4_sge.c index cfccdbc42d53..bd949b097fa0 100644 --- a/sys/dev/cxgbe/t4_sge.c +++ b/sys/dev/cxgbe/t4_sge.c @@ -81,7 +81,7 @@ __FBSDID("$FreeBSD$"); * Ethernet frames are DMA'd at this byte offset into the freelist buffer. * 0-7 are valid values. */ -int fl_pktshift = 2; +static int fl_pktshift = 2; TUNABLE_INT("hw.cxgbe.fl_pktshift", &fl_pktshift); /* @@ -98,7 +98,7 @@ TUNABLE_INT("hw.cxgbe.fl_pad", &fl_pad); * -1: driver should figure out a good value. * 64 or 128 are the only other valid values. */ -int spg_len = -1; +static int spg_len = -1; TUNABLE_INT("hw.cxgbe.spg_len", &spg_len); /* @@ -590,7 +590,7 @@ t4_tweak_chip_settings(struct adapter *sc) /* * SGE wants the buffer to be at least 64B and then a multiple of 16. If - * padding is is use the buffer's start and end need to be aligned to the pad + * padding is in use, the buffer's start and end need to be aligned to the pad * boundary as well. We'll just make sure that the size is a multiple of the * boundary here, it is up to the buffer allocation code to make sure the start * of the buffer is aligned as well. @@ -625,11 +625,9 @@ t4_read_chip_settings(struct adapter *sc) struct sw_zone_info *swz, *safe_swz; struct hw_buf_info *hwb; - t4_init_sge_params(sc); - m = F_RXPKTCPLMODE; v = F_RXPKTCPLMODE; - r = t4_read_reg(sc, A_SGE_CONTROL); + r = sc->params.sge.sge_control; if ((r & m) != v) { device_printf(sc->dev, "invalid SGE_CONTROL(0x%x)\n", r); rc = EINVAL; @@ -647,7 +645,7 @@ t4_read_chip_settings(struct adapter *sc) /* Filter out unusable hw buffer sizes entirely (mark with -2). */ hwb = &s->hw_buf_info[0]; for (i = 0; i < nitems(s->hw_buf_info); i++, hwb++) { - r = t4_read_reg(sc, A_SGE_FL_BUFFER_SIZE0 + (4 * i)); + r = sc->params.sge.sge_fl_buffer_size[i]; hwb->size = r; hwb->zidx = hwsz_ok(sc, r) ? -1 : -2; hwb->next = -1; @@ -1444,7 +1442,8 @@ service_iq(struct sge_iq *iq, int budget) break; } - q = sc->sge.iqmap[lq - sc->sge.iq_start]; + q = sc->sge.iqmap[lq - sc->sge.iq_start - + sc->sge.iq_base]; if (atomic_cmpset_int(&q->state, IQS_IDLE, IQS_BUSY)) { if (service_iq(q, q->qsize / 16) == 0) { @@ -1474,7 +1473,7 @@ service_iq(struct sge_iq *iq, int budget) d = &iq->desc[0]; } if (__predict_false(++ndescs == limit)) { - t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), + t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) | V_INGRESSQID(iq->cntxt_id) | V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX))); @@ -1529,7 +1528,7 @@ process_iql: } #endif - t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) | + t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) | V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params)); if (iq->flags & IQ_HAS_FL) { @@ -2793,7 +2792,7 @@ alloc_iq_fl(struct vi_info *vi, struct sge_iq *iq, struct sge_fl *fl, /* Enable IQ interrupts */ atomic_store_rel_int(&iq->state, IQS_IDLE); - t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_SEINTARM(iq->intr_params) | + t4_write_reg(sc, sc->sge_gts_reg, V_SEINTARM(iq->intr_params) | V_INGRESSQID(iq->cntxt_id)); return (0); @@ -2972,6 +2971,7 @@ alloc_rxq(struct vi_info *vi, struct sge_rxq *rxq, int intr_idx, int idx, struct sysctl_oid *oid) { int rc; + struct adapter *sc = vi->pi->adapter; struct sysctl_oid_list *children; char name[16]; @@ -2980,12 +2980,20 @@ alloc_rxq(struct vi_info *vi, struct sge_rxq *rxq, int intr_idx, int idx, if (rc != 0) return (rc); + if (idx == 0) + sc->sge.iq_base = rxq->iq.abs_id - rxq->iq.cntxt_id; + else + KASSERT(rxq->iq.cntxt_id + sc->sge.iq_base == rxq->iq.abs_id, + ("iq_base mismatch")); + KASSERT(sc->sge.iq_base == 0 || sc->flags & IS_VF, + ("PF with non-zero iq_base")); + /* * The freelist is just barely above the starvation threshold right now, * fill it up a bit more. */ FL_LOCK(&rxq->fl); - refill_fl(vi->pi->adapter, &rxq->fl, 128); + refill_fl(sc, &rxq->fl, 128); FL_UNLOCK(&rxq->fl); #if defined(INET) || defined(INET6) @@ -3317,6 +3325,7 @@ eth_eq_alloc(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq) eq->flags |= EQ_ALLOCATED; eq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(be32toh(c.eqid_pkd)); + eq->abs_id = G_FW_EQ_ETH_CMD_PHYSEQID(be32toh(c.physeqid_pkd)); cntxt_id = eq->cntxt_id - sc->sge.eq_start; if (cntxt_id >= sc->sge.neq) panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, @@ -3557,6 +3566,14 @@ alloc_txq(struct vi_info *vi, struct sge_txq *txq, int idx, /* Can't fail after this point. */ + if (idx == 0) + sc->sge.eq_base = eq->abs_id - eq->cntxt_id; + else + KASSERT(eq->cntxt_id + sc->sge.eq_base == eq->abs_id, + ("eq_base mismatch")); + KASSERT(sc->sge.eq_base == 0 || sc->flags & IS_VF, + ("PF with non-zero eq_base")); + TASK_INIT(&txq->tx_reclaim_task, 0, tx_reclaim, eq); txq->ifp = vi->ifp; txq->gl = sglist_alloc(TX_SGL_SEGS, M_WAITOK); @@ -3572,6 +3589,8 @@ alloc_txq(struct vi_info *vi, struct sge_txq *txq, int idx, NULL, "tx queue"); children = SYSCTL_CHILDREN(oid); + SYSCTL_ADD_UINT(&vi->ctx, children, OID_AUTO, "abs_id", CTLFLAG_RD, + &eq->abs_id, 0, "absolute id of the queue"); SYSCTL_ADD_UINT(&vi->ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD, &eq->cntxt_id, 0, "SGE context id of the queue"); SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "cidx", @@ -3676,7 +3695,7 @@ ring_fl_db(struct adapter *sc, struct sge_fl *fl) if (fl->udb) *fl->udb = htole32(v); else - t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), v); + t4_write_reg(sc, sc->sge_kdoorbell_reg, v); IDXINCR(fl->dbidx, n, fl->sidx); } @@ -4409,7 +4428,7 @@ ring_eq_db(struct adapter *sc, struct sge_eq *eq, u_int n) break; case DOORBELL_KDB: - t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), + t4_write_reg(sc, sc->sge_kdoorbell_reg, V_QID(eq->cntxt_id) | V_PIDX(n)); break; } @@ -4755,7 +4774,7 @@ handle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss, KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, rss->opcode)); - eq = s->eqmap[qid - s->eq_start]; + eq = s->eqmap[qid - s->eq_start - s->eq_base]; (*h[eq->flags & EQ_TYPEMASK])(sc, eq); return (0); diff --git a/sys/dev/cxgbe/tom/t4_cpl_io.c b/sys/dev/cxgbe/tom/t4_cpl_io.c index f7ef499f4488..c7e0661862e0 100644 --- a/sys/dev/cxgbe/tom/t4_cpl_io.c +++ b/sys/dev/cxgbe/tom/t4_cpl_io.c @@ -32,15 +32,18 @@ __FBSDID("$FreeBSD$"); #ifdef TCP_OFFLOAD #include <sys/param.h> -#include <sys/types.h> +#include <sys/aio.h> +#include <sys/file.h> #include <sys/kernel.h> #include <sys/ktr.h> #include <sys/module.h> +#include <sys/proc.h> #include <sys/protosw.h> #include <sys/domain.h> #include <sys/socket.h> #include <sys/socketvar.h> #include <sys/sglist.h> +#include <sys/taskqueue.h> #include <netinet/in.h> #include <netinet/in_pcb.h> #include <netinet/ip.h> @@ -51,6 +54,14 @@ __FBSDID("$FreeBSD$"); #include <netinet/tcp_var.h> #include <netinet/toecore.h> +#include <security/mac/mac_framework.h> + +#include <vm/vm.h> +#include <vm/vm_extern.h> +#include <vm/pmap.h> +#include <vm/vm_map.h> +#include <vm/vm_page.h> + #include "common/common.h" #include "common/t4_msg.h" #include "common/t4_regs.h" @@ -71,6 +82,34 @@ VNET_DECLARE(int, tcp_autorcvbuf_inc); VNET_DECLARE(int, tcp_autorcvbuf_max); #define V_tcp_autorcvbuf_max VNET(tcp_autorcvbuf_max) +#define IS_AIOTX_MBUF(m) \ + ((m)->m_flags & M_EXT && (m)->m_ext.ext_flags & EXT_FLAG_AIOTX) + +static void t4_aiotx_cancel(struct kaiocb *job); +static void t4_aiotx_queue_toep(struct toepcb *toep); + +static size_t +aiotx_mbuf_pgoff(struct mbuf *m) +{ + struct aiotx_buffer *ab; + + MPASS(IS_AIOTX_MBUF(m)); + ab = m->m_ext.ext_arg1; + return ((ab->ps.offset + (uintptr_t)m->m_ext.ext_arg2) % PAGE_SIZE); +} + +static vm_page_t * +aiotx_mbuf_pages(struct mbuf *m) +{ + struct aiotx_buffer *ab; + int npages; + + MPASS(IS_AIOTX_MBUF(m)); + ab = m->m_ext.ext_arg1; + npages = (ab->ps.offset + (uintptr_t)m->m_ext.ext_arg2) / PAGE_SIZE; + return (ab->ps.pages + npages); +} + void send_flowc_wr(struct toepcb *toep, struct flowc_tx_params *ftxp) { @@ -519,7 +558,11 @@ write_tx_sgl(void *dst, struct mbuf *start, struct mbuf *stop, int nsegs, int n) i = -1; for (m = start; m != stop; m = m->m_next) { - rc = sglist_append(&sg, mtod(m, void *), m->m_len); + if (IS_AIOTX_MBUF(m)) + rc = sglist_append_vmpages(&sg, aiotx_mbuf_pages(m), + aiotx_mbuf_pgoff(m), m->m_len); + else + rc = sglist_append(&sg, mtod(m, void *), m->m_len); if (__predict_false(rc != 0)) panic("%s: sglist_append %d", __func__, rc); @@ -579,6 +622,7 @@ t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop) struct sockbuf *sb = &so->so_snd; int tx_credits, shove, compl, sowwakeup; struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx]; + bool aiotx_mbuf_seen; INP_WLOCK_ASSERT(inp); KASSERT(toep->flags & TPF_FLOWC_WR_SENT, @@ -589,6 +633,10 @@ t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop) toep->ulp_mode == ULP_MODE_RDMA, ("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep)); +#ifdef VERBOSE_TRACES + CTR4(KTR_CXGBE, "%s: tid %d toep flags %#x tp flags %#x drop %d", + __func__, toep->tid, toep->flags, tp->t_flags); +#endif if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) return; @@ -618,8 +666,15 @@ t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop) plen = 0; nsegs = 0; max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */ + aiotx_mbuf_seen = false; for (m = sndptr; m != NULL; m = m->m_next) { - int n = sglist_count(mtod(m, void *), m->m_len); + int n; + + if (IS_AIOTX_MBUF(m)) + n = sglist_count_vmpages(aiotx_mbuf_pages(m), + aiotx_mbuf_pgoff(m), m->m_len); + else + n = sglist_count(mtod(m, void *), m->m_len); nsegs += n; plen += m->m_len; @@ -631,9 +686,13 @@ t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop) if (plen == 0) { /* Too few credits */ toep->flags |= TPF_TX_SUSPENDED; - if (sowwakeup) + if (sowwakeup) { + if (!TAILQ_EMPTY( + &toep->aiotx_jobq)) + t4_aiotx_queue_toep( + toep); sowwakeup_locked(so); - else + } else SOCKBUF_UNLOCK(sb); SOCKBUF_UNLOCK_ASSERT(sb); return; @@ -641,6 +700,8 @@ t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop) break; } + if (IS_AIOTX_MBUF(m)) + aiotx_mbuf_seen = true; if (max_nsegs_1mbuf < n) max_nsegs_1mbuf = n; sb_sndptr = m; /* new sb->sb_sndptr if all goes well */ @@ -670,9 +731,11 @@ t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop) else sowwakeup = 1; /* room available */ } - if (sowwakeup) + if (sowwakeup) { + if (!TAILQ_EMPTY(&toep->aiotx_jobq)) + t4_aiotx_queue_toep(toep); sowwakeup_locked(so); - else + } else SOCKBUF_UNLOCK(sb); SOCKBUF_UNLOCK_ASSERT(sb); @@ -687,7 +750,7 @@ t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop) panic("%s: excess tx.", __func__); shove = m == NULL && !(tp->t_flags & TF_MORETOCOME); - if (plen <= max_imm) { + if (plen <= max_imm && !aiotx_mbuf_seen) { /* Immediate data tx */ @@ -1616,6 +1679,9 @@ do_fw4_ack(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) } } +#ifdef VERBOSE_TRACES + CTR3(KTR_CXGBE, "%s: tid %d credits %u", __func__, tid, credits); +#endif so = inp->inp_socket; txsd = &toep->txsd[toep->txsd_cidx]; plen = 0; @@ -1642,6 +1708,10 @@ do_fw4_ack(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) if (toep->flags & TPF_TX_SUSPENDED && toep->tx_credits >= toep->tx_total / 4) { +#ifdef VERBOSE_TRACES + CTR2(KTR_CXGBE, "%s: tid %d calling t4_push_frames", __func__, + tid); +#endif toep->flags &= ~TPF_TX_SUSPENDED; if (toep->ulp_mode == ULP_MODE_ISCSI) t4_push_pdus(sc, toep, plen); @@ -1668,7 +1738,13 @@ do_fw4_ack(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) sowwakeup_locked(so); /* unlocks so_snd */ rqdrop_locked(&toep->ulp_pdu_reclaimq, plen); } else { +#ifdef VERBOSE_TRACES + CTR3(KTR_CXGBE, "%s: tid %d dropped %d bytes", __func__, + tid, plen); +#endif sbdrop_locked(sb, plen); + if (!TAILQ_EMPTY(&toep->aiotx_jobq)) + t4_aiotx_queue_toep(toep); sowwakeup_locked(so); /* unlocks so_snd */ } SOCKBUF_UNLOCK_ASSERT(sb); @@ -1768,4 +1844,397 @@ t4_uninit_cpl_io_handlers(void) t4_register_cpl_handler(CPL_RX_DATA, do_rx_data); t4_register_cpl_handler(CPL_FW4_ACK, do_fw4_ack); } + +/* + * Use the 'backend3' field in AIO jobs to store the amount of data + * sent by the AIO job so far and the 'backend4' field to hold an + * error that should be reported when the job is completed. + */ +#define aio_sent backend3 +#define aio_error backend4 + +#define jobtotid(job) \ + (((struct toepcb *)(so_sototcpcb((job)->fd_file->f_data)->t_toe))->tid) + +static void +free_aiotx_buffer(struct aiotx_buffer *ab) +{ + struct kaiocb *job; + long status; + int error; + + if (refcount_release(&ab->refcount) == 0) + return; + + job = ab->job; + error = job->aio_error; + status = job->aio_sent; + vm_page_unhold_pages(ab->ps.pages, ab->ps.npages); + free(ab, M_CXGBE); +#ifdef VERBOSE_TRACES + CTR5(KTR_CXGBE, "%s: tid %d completed %p len %ld, error %d", __func__, + jobtotid(job), job, status, error); +#endif + if (error == ECANCELED && status != 0) + error = 0; + if (error == ECANCELED) + aio_cancel(job); + else if (error) + aio_complete(job, -1, error); + else + aio_complete(job, status, 0); +} + +static void +t4_aiotx_mbuf_free(struct mbuf *m, void *buffer, void *arg) +{ + struct aiotx_buffer *ab = buffer; + +#ifdef VERBOSE_TRACES + CTR3(KTR_CXGBE, "%s: completed %d bytes for tid %d", __func__, + m->m_len, jobtotid(ab->job)); +#endif + free_aiotx_buffer(ab); +} + +/* + * Hold the buffer backing an AIO request and return an AIO transmit + * buffer. + */ +static int +hold_aio(struct kaiocb *job) +{ + struct aiotx_buffer *ab; + struct vmspace *vm; + vm_map_t map; + vm_offset_t start, end, pgoff; + int n; + + MPASS(job->backend1 == NULL); + + /* + * The AIO subsystem will cancel and drain all requests before + * permitting a process to exit or exec, so p_vmspace should + * be stable here. + */ + vm = job->userproc->p_vmspace; + map = &vm->vm_map; + start = (uintptr_t)job->uaiocb.aio_buf; + pgoff = start & PAGE_MASK; + end = round_page(start + job->uaiocb.aio_nbytes); + start = trunc_page(start); + n = atop(end - start); + + ab = malloc(sizeof(*ab) + n * sizeof(vm_page_t), M_CXGBE, M_WAITOK | + M_ZERO); + refcount_init(&ab->refcount, 1); + ab->ps.pages = (vm_page_t *)(ab + 1); + ab->ps.npages = vm_fault_quick_hold_pages(map, start, end - start, + VM_PROT_WRITE, ab->ps.pages, n); + if (ab->ps.npages < 0) { + free(ab, M_CXGBE); + return (EFAULT); + } + + KASSERT(ab->ps.npages == n, + ("hold_aio: page count mismatch: %d vs %d", ab->ps.npages, n)); + + ab->ps.offset = pgoff; + ab->ps.len = job->uaiocb.aio_nbytes; + ab->job = job; + job->backend1 = ab; +#ifdef VERBOSE_TRACES + CTR5(KTR_CXGBE, "%s: tid %d, new pageset %p for job %p, npages %d", + __func__, jobtotid(job), &ab->ps, job, ab->ps.npages); +#endif + return (0); +} + +static void +t4_aiotx_process_job(struct toepcb *toep, struct socket *so, struct kaiocb *job) +{ + struct adapter *sc; + struct sockbuf *sb; + struct file *fp; + struct aiotx_buffer *ab; + struct inpcb *inp; + struct tcpcb *tp; + struct mbuf *m; + int error; + bool moretocome, sendmore; + + sc = td_adapter(toep->td); + sb = &so->so_snd; + SOCKBUF_UNLOCK(sb); + fp = job->fd_file; + ab = job->backend1; + m = NULL; + +#ifdef MAC + error = mac_socket_check_send(fp->f_cred, so); + if (error != 0) + goto out; +#endif + + if (ab == NULL) { + error = hold_aio(job); + if (error != 0) + goto out; + ab = job->backend1; + } + + /* Inline sosend_generic(). */ + + job->msgsnd = 1; + + error = sblock(sb, SBL_WAIT); + MPASS(error == 0); + +sendanother: + m = m_get(M_WAITOK, MT_DATA); + + SOCKBUF_LOCK(sb); + if (so->so_snd.sb_state & SBS_CANTSENDMORE) { + SOCKBUF_UNLOCK(sb); + sbunlock(sb); + if ((so->so_options & SO_NOSIGPIPE) == 0) { + PROC_LOCK(job->userproc); + kern_psignal(job->userproc, SIGPIPE); + PROC_UNLOCK(job->userproc); + } + error = EPIPE; + goto out; + } + if (so->so_error) { + error = so->so_error; + so->so_error = 0; + SOCKBUF_UNLOCK(sb); + sbunlock(sb); + goto out; + } + if ((so->so_state & SS_ISCONNECTED) == 0) { + SOCKBUF_UNLOCK(sb); + sbunlock(sb); + error = ENOTCONN; + goto out; + } + if (sbspace(sb) < sb->sb_lowat) { + MPASS(job->aio_sent == 0 || !(so->so_state & SS_NBIO)); + + /* + * Don't block if there is too little room in the socket + * buffer. Instead, requeue the request. + */ + if (!aio_set_cancel_function(job, t4_aiotx_cancel)) { + SOCKBUF_UNLOCK(sb); + sbunlock(sb); + error = ECANCELED; + goto out; + } + TAILQ_INSERT_HEAD(&toep->aiotx_jobq, job, list); + SOCKBUF_UNLOCK(sb); + sbunlock(sb); + goto out; + } + + /* + * Write as much data as the socket permits, but no more than a + * a single sndbuf at a time. + */ + m->m_len = sbspace(sb); + if (m->m_len > ab->ps.len - job->aio_sent) { + m->m_len = ab->ps.len - job->aio_sent; + moretocome = false; + } else + moretocome = true; + if (m->m_len > sc->tt.sndbuf) { + m->m_len = sc->tt.sndbuf; + sendmore = true; + } else + sendmore = false; + + if (!TAILQ_EMPTY(&toep->aiotx_jobq)) + moretocome = true; + SOCKBUF_UNLOCK(sb); + MPASS(m->m_len != 0); + + /* Inlined tcp_usr_send(). */ + + inp = toep->inp; + INP_WLOCK(inp); + if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { + INP_WUNLOCK(inp); + sbunlock(sb); + error = ECONNRESET; + goto out; + } + + refcount_acquire(&ab->refcount); + m_extadd(m, NULL, ab->ps.len, t4_aiotx_mbuf_free, ab, + (void *)(uintptr_t)job->aio_sent, 0, EXT_NET_DRV); + m->m_ext.ext_flags |= EXT_FLAG_AIOTX; + job->aio_sent += m->m_len; + + sbappendstream(sb, m, 0); + m = NULL; + + if (!(inp->inp_flags & INP_DROPPED)) { + tp = intotcpcb(inp); + if (moretocome) + tp->t_flags |= TF_MORETOCOME; + error = tp->t_fb->tfb_tcp_output(tp); + if (moretocome) + tp->t_flags &= ~TF_MORETOCOME; + } + + INP_WUNLOCK(inp); + if (sendmore) + goto sendanother; + sbunlock(sb); + + if (error) + goto out; + + /* + * If this is a non-blocking socket and the request has not + * been fully completed, requeue it until the socket is ready + * again. + */ + if (job->aio_sent < job->uaiocb.aio_nbytes && + !(so->so_state & SS_NBIO)) { + SOCKBUF_LOCK(sb); + if (!aio_set_cancel_function(job, t4_aiotx_cancel)) { + SOCKBUF_UNLOCK(sb); + error = ECANCELED; + goto out; + } + TAILQ_INSERT_HEAD(&toep->aiotx_jobq, job, list); + return; + } + + /* + * If the request will not be requeued, drop a reference on + * the the aiotx buffer. Any mbufs in flight should still + * contain a reference, but this drops the reference that the + * job owns while it is waiting to queue mbufs to the socket. + */ + free_aiotx_buffer(ab); + +out: + if (error) { + if (ab != NULL) { + job->aio_error = error; + free_aiotx_buffer(ab); + } else { + MPASS(job->aio_sent == 0); + aio_complete(job, -1, error); + } + } + if (m != NULL) + m_free(m); + SOCKBUF_LOCK(sb); +} + +static void +t4_aiotx_task(void *context, int pending) +{ + struct toepcb *toep = context; + struct inpcb *inp = toep->inp; + struct socket *so = inp->inp_socket; + struct kaiocb *job; + + CURVNET_SET(so->so_vnet); + SOCKBUF_LOCK(&so->so_snd); + while (!TAILQ_EMPTY(&toep->aiotx_jobq) && sowriteable(so)) { + job = TAILQ_FIRST(&toep->aiotx_jobq); + TAILQ_REMOVE(&toep->aiotx_jobq, job, list); + if (!aio_clear_cancel_function(job)) + continue; + + t4_aiotx_process_job(toep, so, job); + } + toep->aiotx_task_active = false; + SOCKBUF_UNLOCK(&so->so_snd); + CURVNET_RESTORE(); + + free_toepcb(toep); +} + +static void +t4_aiotx_queue_toep(struct toepcb *toep) +{ + + SOCKBUF_LOCK_ASSERT(&toep->inp->inp_socket->so_snd); +#ifdef VERBOSE_TRACES + CTR3(KTR_CXGBE, "%s: queueing aiotx task for tid %d, active = %s", + __func__, toep->tid, toep->aiotx_task_active ? "true" : "false"); +#endif + if (toep->aiotx_task_active) + return; + toep->aiotx_task_active = true; + hold_toepcb(toep); + soaio_enqueue(&toep->aiotx_task); +} + +static void +t4_aiotx_cancel(struct kaiocb *job) +{ + struct aiotx_buffer *ab; + struct socket *so; + struct sockbuf *sb; + struct tcpcb *tp; + struct toepcb *toep; + + so = job->fd_file->f_data; + tp = so_sototcpcb(so); + toep = tp->t_toe; + MPASS(job->uaiocb.aio_lio_opcode == LIO_WRITE); + sb = &so->so_snd; + + SOCKBUF_LOCK(sb); + if (!aio_cancel_cleared(job)) + TAILQ_REMOVE(&toep->aiotx_jobq, job, list); + SOCKBUF_UNLOCK(sb); + + ab = job->backend1; + if (ab != NULL) + free_aiotx_buffer(ab); + else + aio_cancel(job); +} + +int +t4_aio_queue_aiotx(struct socket *so, struct kaiocb *job) +{ + struct tcpcb *tp = so_sototcpcb(so); + struct toepcb *toep = tp->t_toe; + struct adapter *sc = td_adapter(toep->td); + + /* This only handles writes. */ + if (job->uaiocb.aio_lio_opcode != LIO_WRITE) + return (EOPNOTSUPP); + + if (!sc->tt.tx_zcopy) + return (EOPNOTSUPP); + + SOCKBUF_LOCK(&so->so_snd); +#ifdef VERBOSE_TRACES + CTR2(KTR_CXGBE, "%s: queueing %p", __func__, job); +#endif + if (!aio_set_cancel_function(job, t4_aiotx_cancel)) + panic("new job was cancelled"); + TAILQ_INSERT_TAIL(&toep->aiotx_jobq, job, list); + if (sowriteable(so)) + t4_aiotx_queue_toep(toep); + SOCKBUF_UNLOCK(&so->so_snd); + return (0); +} + +void +aiotx_init_toep(struct toepcb *toep) +{ + + TAILQ_INIT(&toep->aiotx_jobq); + TASK_INIT(&toep->aiotx_task, 0, t4_aiotx_task, toep); +} #endif diff --git a/sys/dev/cxgbe/tom/t4_ddp.c b/sys/dev/cxgbe/tom/t4_ddp.c index ff73c8bb3cad..d70fb5911d39 100644 --- a/sys/dev/cxgbe/tom/t4_ddp.c +++ b/sys/dev/cxgbe/tom/t4_ddp.c @@ -88,9 +88,6 @@ static void t4_aio_cancel_queued(struct kaiocb *job); #define PPOD_SZ(n) ((n) * sizeof(struct pagepod)) #define PPOD_SIZE (PPOD_SZ(1)) -/* XXX: must match A_ULP_RX_TDDP_PSZ */ -static int t4_ddp_pgsz[] = {4096, 4096 << 2, 4096 << 4, 4096 << 6}; - static TAILQ_HEAD(, pageset) ddp_orphan_pagesets; static struct mtx ddp_orphan_pagesets_lock; static struct task ddp_orphan_task; @@ -908,13 +905,13 @@ alloc_page_pods(struct tom_data *td, struct pageset *ps) } hcf = calculate_hcf(hcf, seglen); - if (hcf < t4_ddp_pgsz[1]) { + if (hcf < td->ddp_pgsz[1]) { idx = 0; goto have_pgsz; /* give up, short circuit */ } } - if (hcf % t4_ddp_pgsz[0] != 0) { + if (hcf % td->ddp_pgsz[0] != 0) { /* hmmm. This could only happen when PAGE_SIZE < 4K */ KASSERT(PAGE_SIZE < 4096, ("%s: PAGE_SIZE %d, hcf %d", __func__, PAGE_SIZE, hcf)); @@ -923,17 +920,17 @@ alloc_page_pods(struct tom_data *td, struct pageset *ps) return (0); } - for (idx = nitems(t4_ddp_pgsz) - 1; idx > 0; idx--) { - if (hcf % t4_ddp_pgsz[idx] == 0) + for (idx = nitems(td->ddp_pgsz) - 1; idx > 0; idx--) { + if (hcf % td->ddp_pgsz[idx] == 0) break; } have_pgsz: MPASS(idx <= M_PPOD_PGSZ); - nppods = pages_to_nppods(ps->npages, t4_ddp_pgsz[idx]); + nppods = pages_to_nppods(ps->npages, td->ddp_pgsz[idx]); if (alloc_ppods(td, nppods, &ppod_addr) != 0) { CTR4(KTR_CXGBE, "%s: no pods, nppods %d, npages %d, pgsz %d", - __func__, nppods, ps->npages, t4_ddp_pgsz[idx]); + __func__, nppods, ps->npages, td->ddp_pgsz[idx]); return (0); } @@ -944,7 +941,7 @@ have_pgsz: CTR5(KTR_CXGBE, "New page pods. " "ps %p, ddp_pgsz %d, ppod 0x%x, npages %d, nppods %d", - ps, t4_ddp_pgsz[idx], ppod, ps->npages, ps->nppods); + ps, td->ddp_pgsz[idx], ppod, ps->npages, ps->nppods); return (1); } @@ -958,6 +955,7 @@ write_page_pods(struct adapter *sc, struct toepcb *toep, struct pageset *ps) struct ulp_mem_io *ulpmc; struct ulptx_idata *ulpsc; struct pagepod *ppod; + struct tom_data *td = sc->tom_softc; int i, j, k, n, chunk, len, ddp_pgsz, idx; u_int ppod_addr; uint32_t cmd; @@ -970,7 +968,7 @@ write_page_pods(struct adapter *sc, struct toepcb *toep, struct pageset *ps) cmd |= htobe32(F_ULP_MEMIO_ORDER); else cmd |= htobe32(F_T5_ULP_MEMIO_IMM); - ddp_pgsz = t4_ddp_pgsz[G_PPOD_PGSZ(ps->tag)]; + ddp_pgsz = td->ddp_pgsz[G_PPOD_PGSZ(ps->tag)]; ppod_addr = ps->ppod_addr; for (i = 0; i < ps->nppods; ppod_addr += chunk) { @@ -1069,10 +1067,27 @@ prep_pageset(struct adapter *sc, struct toepcb *toep, struct pageset *ps) void t4_init_ddp(struct adapter *sc, struct tom_data *td) { + int i; + uint32_t r; + + r = t4_read_reg(sc, A_ULP_RX_TDDP_PSZ); + td->ddp_pgsz[0] = 4096 << G_HPZ0(r); + td->ddp_pgsz[1] = 4096 << G_HPZ1(r); + td->ddp_pgsz[2] = 4096 << G_HPZ2(r); + td->ddp_pgsz[3] = 4096 << G_HPZ3(r); + + /* + * The SGL -> page pod algorithm requires the sizes to be in increasing + * order. + */ + for (i = 1; i < nitems(td->ddp_pgsz); i++) { + if (td->ddp_pgsz[i] <= td->ddp_pgsz[i - 1]) + return; + } td->ppod_start = sc->vres.ddp.start; td->ppod_arena = vmem_create("DDP page pods", sc->vres.ddp.start, - sc->vres.ddp.size, 1, 32, M_FIRSTFIT | M_NOWAIT); + sc->vres.ddp.size, PPOD_SIZE, 512, M_FIRSTFIT | M_NOWAIT); } void diff --git a/sys/dev/cxgbe/tom/t4_tom.c b/sys/dev/cxgbe/tom/t4_tom.c index 2a8082e92865..56006d0ac891 100644 --- a/sys/dev/cxgbe/tom/t4_tom.c +++ b/sys/dev/cxgbe/tom/t4_tom.c @@ -68,11 +68,11 @@ __FBSDID("$FreeBSD$"); #include "tom/t4_tom_l2t.h" #include "tom/t4_tom.h" -static struct protosw ddp_protosw; -static struct pr_usrreqs ddp_usrreqs; +static struct protosw toe_protosw; +static struct pr_usrreqs toe_usrreqs; -static struct protosw ddp6_protosw; -static struct pr_usrreqs ddp6_usrreqs; +static struct protosw toe6_protosw; +static struct pr_usrreqs toe6_usrreqs; /* Module ops */ static int t4_tom_mod_load(void); @@ -167,6 +167,7 @@ alloc_toepcb(struct vi_info *vi, int txqid, int rxqid, int flags) toep->txsd_avail = txsd_total; toep->txsd_pidx = 0; toep->txsd_cidx = 0; + aiotx_init_toep(toep); ddp_init_toep(toep); return (toep); @@ -217,12 +218,10 @@ offload_socket(struct socket *so, struct toepcb *toep) sb = &so->so_rcv; SOCKBUF_LOCK(sb); sb->sb_flags |= SB_NOCOALESCE; - if (toep->ulp_mode == ULP_MODE_TCPDDP) { - if (inp->inp_vflag & INP_IPV6) - so->so_proto = &ddp6_protosw; - else - so->so_proto = &ddp_protosw; - } + if (inp->inp_vflag & INP_IPV6) + so->so_proto = &toe6_protosw; + else + so->so_proto = &toe_protosw; SOCKBUF_UNLOCK(sb); /* Update TCP PCB */ @@ -1120,6 +1119,22 @@ t4_tom_ifaddr_event(void *arg __unused, struct ifnet *ifp) } static int +t4_aio_queue_tom(struct socket *so, struct kaiocb *job) +{ + struct tcpcb *tp = so_sototcpcb(so); + struct toepcb *toep = tp->t_toe; + int error; + + if (toep->ulp_mode == ULP_MODE_TCPDDP) { + error = t4_aio_queue_ddp(so, job); + if (error != EOPNOTSUPP) + return (error); + } + + return (t4_aio_queue_aiotx(so, job)); +} + +static int t4_tom_mod_load(void) { int rc; @@ -1137,18 +1152,18 @@ t4_tom_mod_load(void) tcp_protosw = pffindproto(PF_INET, IPPROTO_TCP, SOCK_STREAM); if (tcp_protosw == NULL) return (ENOPROTOOPT); - bcopy(tcp_protosw, &ddp_protosw, sizeof(ddp_protosw)); - bcopy(tcp_protosw->pr_usrreqs, &ddp_usrreqs, sizeof(ddp_usrreqs)); - ddp_usrreqs.pru_aio_queue = t4_aio_queue_ddp; - ddp_protosw.pr_usrreqs = &ddp_usrreqs; + bcopy(tcp_protosw, &toe_protosw, sizeof(toe_protosw)); + bcopy(tcp_protosw->pr_usrreqs, &toe_usrreqs, sizeof(toe_usrreqs)); + toe_usrreqs.pru_aio_queue = t4_aio_queue_tom; + toe_protosw.pr_usrreqs = &toe_usrreqs; tcp6_protosw = pffindproto(PF_INET6, IPPROTO_TCP, SOCK_STREAM); if (tcp6_protosw == NULL) return (ENOPROTOOPT); - bcopy(tcp6_protosw, &ddp6_protosw, sizeof(ddp6_protosw)); - bcopy(tcp6_protosw->pr_usrreqs, &ddp6_usrreqs, sizeof(ddp6_usrreqs)); - ddp6_usrreqs.pru_aio_queue = t4_aio_queue_ddp; - ddp6_protosw.pr_usrreqs = &ddp6_usrreqs; + bcopy(tcp6_protosw, &toe6_protosw, sizeof(toe6_protosw)); + bcopy(tcp6_protosw->pr_usrreqs, &toe6_usrreqs, sizeof(toe6_usrreqs)); + toe6_usrreqs.pru_aio_queue = t4_aio_queue_tom; + toe6_protosw.pr_usrreqs = &toe6_usrreqs; TIMEOUT_TASK_INIT(taskqueue_thread, &clip_task, 0, t4_clip_task, NULL); ifaddr_evhandler = EVENTHANDLER_REGISTER(ifaddr_event, diff --git a/sys/dev/cxgbe/tom/t4_tom.h b/sys/dev/cxgbe/tom/t4_tom.h index f114a3dfb429..313fb7b510d6 100644 --- a/sys/dev/cxgbe/tom/t4_tom.h +++ b/sys/dev/cxgbe/tom/t4_tom.h @@ -102,6 +102,8 @@ TAILQ_HEAD(pagesetq, pageset); #define PS_WIRED 0x0001 /* Pages wired rather than held. */ #define PS_PPODS_WRITTEN 0x0002 /* Page pods written to the card. */ +#define EXT_FLAG_AIOTX EXT_FLAG_VENDOR1 + struct ddp_buffer { struct pageset *ps; @@ -109,6 +111,12 @@ struct ddp_buffer { int cancel_pending; }; +struct aiotx_buffer { + struct pageset ps; + struct kaiocb *job; + int refcount; +}; + struct toepcb { TAILQ_ENTRY(toepcb) link; /* toep_list */ u_int flags; /* miscellaneous flags */ @@ -151,6 +159,10 @@ struct toepcb { struct kaiocb *ddp_queueing; struct mtx ddp_lock; + TAILQ_HEAD(, kaiocb) aiotx_jobq; + struct task aiotx_task; + bool aiotx_task_active; + /* Tx software descriptor */ uint8_t txsd_total; uint8_t txsd_pidx; @@ -228,6 +240,7 @@ struct tom_data { int lctx_count; /* # of lctx in the hash table */ u_int ppod_start; + u_int ddp_pgsz[4]; vmem_t *ppod_arena; struct mtx clip_table_lock; @@ -313,6 +326,8 @@ int do_abort_rpl_synqe(struct sge_iq *, const struct rss_header *, void t4_offload_socket(struct toedev *, void *, struct socket *); /* t4_cpl_io.c */ +void aiotx_init_toep(struct toepcb *); +int t4_aio_queue_aiotx(struct socket *, struct kaiocb *); void t4_init_cpl_io_handlers(void); void t4_uninit_cpl_io_handlers(void); void send_abort_rpl(struct adapter *, struct sge_wrq *, int , int); diff --git a/sys/dev/e1000/e1000_api.c b/sys/dev/e1000/e1000_api.c index 28379cc572d3..52e260950d24 100644 --- a/sys/dev/e1000/e1000_api.c +++ b/sys/dev/e1000/e1000_api.c @@ -304,6 +304,10 @@ s32 e1000_set_mac_type(struct e1000_hw *hw) case E1000_DEV_ID_PCH_SPT_I219_LM2: case E1000_DEV_ID_PCH_SPT_I219_V2: case E1000_DEV_ID_PCH_LBG_I219_LM3: + case E1000_DEV_ID_PCH_SPT_I219_LM4: + case E1000_DEV_ID_PCH_SPT_I219_V4: + case E1000_DEV_ID_PCH_SPT_I219_LM5: + case E1000_DEV_ID_PCH_SPT_I219_V5: mac->type = e1000_pch_spt; break; case E1000_DEV_ID_82575EB_COPPER: diff --git a/sys/dev/e1000/e1000_hw.h b/sys/dev/e1000/e1000_hw.h index 1792e14ef38a..e1464a7b655a 100644 --- a/sys/dev/e1000/e1000_hw.h +++ b/sys/dev/e1000/e1000_hw.h @@ -142,6 +142,10 @@ struct e1000_hw; #define E1000_DEV_ID_PCH_SPT_I219_LM2 0x15B7 /* Sunrise Point-H PCH */ #define E1000_DEV_ID_PCH_SPT_I219_V2 0x15B8 /* Sunrise Point-H PCH */ #define E1000_DEV_ID_PCH_LBG_I219_LM3 0x15B9 /* LEWISBURG PCH */ +#define E1000_DEV_ID_PCH_SPT_I219_LM4 0x15D7 +#define E1000_DEV_ID_PCH_SPT_I219_V4 0x15D8 +#define E1000_DEV_ID_PCH_SPT_I219_LM5 0x15E3 +#define E1000_DEV_ID_PCH_SPT_I219_V5 0x15D6 #define E1000_DEV_ID_82576 0x10C9 #define E1000_DEV_ID_82576_FIBER 0x10E6 #define E1000_DEV_ID_82576_SERDES 0x10E7 @@ -957,9 +961,13 @@ struct e1000_dev_spec_ich8lan { E1000_MUTEX nvm_mutex; E1000_MUTEX swflag_mutex; bool nvm_k1_enabled; + bool disable_k1_off; bool eee_disable; u16 eee_lp_ability; enum e1000_ulp_state ulp_state; + bool ulp_capability_disabled; + bool during_suspend_flow; + bool during_dpg_exit; }; struct e1000_dev_spec_82575 { diff --git a/sys/dev/e1000/e1000_ich8lan.c b/sys/dev/e1000/e1000_ich8lan.c index ae97a8c0d389..4c50ce296385 100644 --- a/sys/dev/e1000/e1000_ich8lan.c +++ b/sys/dev/e1000/e1000_ich8lan.c @@ -288,7 +288,7 @@ static void e1000_toggle_lanphypc_pch_lpt(struct e1000_hw *hw) mac_reg &= ~E1000_CTRL_LANPHYPC_VALUE; E1000_WRITE_REG(hw, E1000_CTRL, mac_reg); E1000_WRITE_FLUSH(hw); - usec_delay(10); + msec_delay(1); mac_reg &= ~E1000_CTRL_LANPHYPC_OVERRIDE; E1000_WRITE_REG(hw, E1000_CTRL, mac_reg); E1000_WRITE_FLUSH(hw); @@ -1625,7 +1625,17 @@ static s32 e1000_check_for_copper_link_ich8lan(struct e1000_hw *hw) hw->phy.ops.write_reg_locked(hw, I217_PLL_CLOCK_GATE_REG, phy_reg); - } + + if (speed == SPEED_1000) { + hw->phy.ops.read_reg_locked(hw, HV_PM_CTRL, + &phy_reg); + + phy_reg |= HV_PM_CTRL_K1_CLK_REQ; + + hw->phy.ops.write_reg_locked(hw, HV_PM_CTRL, + phy_reg); + } + } hw->phy.ops.release(hw); if (ret_val) @@ -1718,7 +1728,8 @@ static s32 e1000_check_for_copper_link_ich8lan(struct e1000_hw *hw) u32 pcieanacfg = E1000_READ_REG(hw, E1000_PCIEANACFG); u32 fextnvm6 = E1000_READ_REG(hw, E1000_FEXTNVM6); - if (pcieanacfg & E1000_FEXTNVM6_K1_OFF_ENABLE) + if ((pcieanacfg & E1000_FEXTNVM6_K1_OFF_ENABLE) && + (hw->dev_spec.ich8lan.disable_k1_off == FALSE)) fextnvm6 |= E1000_FEXTNVM6_K1_OFF_ENABLE; else fextnvm6 &= ~E1000_FEXTNVM6_K1_OFF_ENABLE; diff --git a/sys/dev/e1000/e1000_ich8lan.h b/sys/dev/e1000/e1000_ich8lan.h index edc1dd14ccc9..6d812911e8b8 100644 --- a/sys/dev/e1000/e1000_ich8lan.h +++ b/sys/dev/e1000/e1000_ich8lan.h @@ -239,7 +239,7 @@ /* PHY Power Management Control */ #define HV_PM_CTRL PHY_REG(770, 17) -#define HV_PM_CTRL_PLL_STOP_IN_K1_GIGA 0x100 +#define HV_PM_CTRL_K1_CLK_REQ 0x200 #define HV_PM_CTRL_K1_ENABLE 0x4000 #define I217_PLL_CLOCK_GATE_REG PHY_REG(772, 28) diff --git a/sys/dev/e1000/e1000_osdep.h b/sys/dev/e1000/e1000_osdep.h index c0bf4182ec62..c7c23e582ca9 100644 --- a/sys/dev/e1000/e1000_osdep.h +++ b/sys/dev/e1000/e1000_osdep.h @@ -134,7 +134,7 @@ struct e1000_osdep bus_space_handle_t io_bus_space_handle; bus_space_tag_t flash_bus_space_tag; bus_space_handle_t flash_bus_space_handle; - struct device *dev; + device_t dev; }; #define E1000_REGISTER(hw, reg) (((hw)->mac.type >= e1000_82543) \ diff --git a/sys/dev/e1000/e1000_phy.c b/sys/dev/e1000/e1000_phy.c index 847d3155e2af..9684b43f5503 100644 --- a/sys/dev/e1000/e1000_phy.c +++ b/sys/dev/e1000/e1000_phy.c @@ -4146,12 +4146,13 @@ s32 e1000_read_phy_reg_mphy(struct e1000_hw *hw, u32 address, u32 *data) *data = E1000_READ_REG(hw, E1000_MPHY_DATA); /* Disable access to mPHY if it was originally disabled */ - if (locked) { + if (locked) ready = e1000_is_mphy_ready(hw); - if (!ready) - return -E1000_ERR_PHY; - } - E1000_WRITE_REG(hw, E1000_MPHY_ADDR_CTRL, E1000_MPHY_DIS_ACCESS); + if (!ready) + return -E1000_ERR_PHY; + E1000_WRITE_REG(hw, E1000_MPHY_ADDR_CTRL, + E1000_MPHY_DIS_ACCESS); + return E1000_SUCCESS; } @@ -4210,12 +4211,13 @@ s32 e1000_write_phy_reg_mphy(struct e1000_hw *hw, u32 address, u32 data, E1000_WRITE_REG(hw, E1000_MPHY_DATA, data); /* Disable access to mPHY if it was originally disabled */ - if (locked) { + if (locked) ready = e1000_is_mphy_ready(hw); - if (!ready) - return -E1000_ERR_PHY; - } - E1000_WRITE_REG(hw, E1000_MPHY_ADDR_CTRL, E1000_MPHY_DIS_ACCESS); + if (!ready) + return -E1000_ERR_PHY; + E1000_WRITE_REG(hw, E1000_MPHY_ADDR_CTRL, + E1000_MPHY_DIS_ACCESS); + return E1000_SUCCESS; } diff --git a/sys/dev/e1000/if_em.c b/sys/dev/e1000/if_em.c index ddd651d82d18..7e2690eae084 100644 --- a/sys/dev/e1000/if_em.c +++ b/sys/dev/e1000/if_em.c @@ -193,6 +193,12 @@ static em_vendor_info_t em_vendor_info_array[] = { 0x8086, E1000_DEV_ID_PCH_SPT_I219_V2, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_PCH_LBG_I219_LM3, PCI_ANY_ID, PCI_ANY_ID, 0}, + { 0x8086, E1000_DEV_ID_PCH_SPT_I219_LM4, + PCI_ANY_ID, PCI_ANY_ID, 0}, + { 0x8086, E1000_DEV_ID_PCH_SPT_I219_V4, PCI_ANY_ID, PCI_ANY_ID, 0}, + { 0x8086, E1000_DEV_ID_PCH_SPT_I219_LM5, + PCI_ANY_ID, PCI_ANY_ID, 0}, + { 0x8086, E1000_DEV_ID_PCH_SPT_I219_V5, PCI_ANY_ID, PCI_ANY_ID, 0}, /* required last entry */ { 0, 0, 0, 0, 0} }; @@ -4392,6 +4398,7 @@ em_setup_receive_ring(struct rx_ring *rxr) addr = PNMB(na, slot + si, &paddr); netmap_load_map(na, rxr->rxtag, rxbuf->map, addr); + rxbuf->paddr = paddr; em_setup_rxdesc(&rxr->rx_base[j], rxbuf); continue; } diff --git a/sys/dev/e1000/if_em.h b/sys/dev/e1000/if_em.h index c8c98cde6c02..2a2bf2ccb7ab 100644 --- a/sys/dev/e1000/if_em.h +++ b/sys/dev/e1000/if_em.h @@ -394,7 +394,7 @@ struct adapter { /* FreeBSD operating-system-specific structures. */ struct e1000_osdep osdep; - struct device *dev; + device_t dev; struct cdev *led_dev; struct resource *memory; diff --git a/sys/dev/e1000/if_igb.h b/sys/dev/e1000/if_igb.h index a30fc993230c..e2f2219cc94c 100644 --- a/sys/dev/e1000/if_igb.h +++ b/sys/dev/e1000/if_igb.h @@ -429,7 +429,7 @@ struct adapter { struct e1000_hw hw; struct e1000_osdep osdep; - struct device *dev; + device_t dev; struct cdev *led_dev; struct resource *pci_mem; diff --git a/sys/dev/e1000/if_lem.h b/sys/dev/e1000/if_lem.h index 2bdc21a2c335..4a27c34bd772 100644 --- a/sys/dev/e1000/if_lem.h +++ b/sys/dev/e1000/if_lem.h @@ -298,7 +298,7 @@ struct adapter { /* FreeBSD operating-system-specific structures. */ struct e1000_osdep osdep; - struct device *dev; + device_t dev; struct cdev *led_dev; struct resource *memory; diff --git a/sys/dev/etherswitch/arswitch/arswitch.c b/sys/dev/etherswitch/arswitch/arswitch.c index 8396f8aee501..09055632da0e 100644 --- a/sys/dev/etherswitch/arswitch/arswitch.c +++ b/sys/dev/etherswitch/arswitch/arswitch.c @@ -73,9 +73,13 @@ #include "miibus_if.h" #include "etherswitch_if.h" -#if defined(DEBUG) -static SYSCTL_NODE(_debug, OID_AUTO, arswitch, CTLFLAG_RD, 0, "arswitch"); -#endif +/* Map ETHERSWITCH_PORT_LED_* to Atheros pattern codes */ +static int led_pattern_table[] = { + [ETHERSWITCH_PORT_LED_DEFAULT] = 0x3, + [ETHERSWITCH_PORT_LED_ON] = 0x2, + [ETHERSWITCH_PORT_LED_OFF] = 0x0, + [ETHERSWITCH_PORT_LED_BLINK] = 0x1 +}; static inline int arswitch_portforphy(int phy); static void arswitch_tick(void *arg); @@ -85,6 +89,8 @@ static int ar8xxx_port_vlan_setup(struct arswitch_softc *sc, etherswitch_port_t *p); static int ar8xxx_port_vlan_get(struct arswitch_softc *sc, etherswitch_port_t *p); +static int arswitch_setled(struct arswitch_softc *sc, int phy, int led, + int style); static int arswitch_probe(device_t dev) @@ -146,7 +152,7 @@ arswitch_probe(device_t dev) done: - DPRINTF(dev, "chipname=%s, id=%08x\n", chipname, id); + DPRINTF(sc, ARSWITCH_DBG_ANY, "chipname=%s, id=%08x\n", chipname, id); if (chipname != NULL) { snprintf(desc, sizeof(desc), "Atheros %s Ethernet Switch (ver %d rev %d)", @@ -188,9 +194,23 @@ arswitch_attach_phys(struct arswitch_softc *sc) device_printf(sc->sc_dev, "attaching PHY %d failed\n", phy); + return (err); + } + + if (AR8X16_IS_SWITCH(sc, AR8327)) { + int led; + char ledname[IFNAMSIZ+4]; + + for (led = 0; led < 3; led++) { + sprintf(ledname, "%s%dled%d", name, + arswitch_portforphy(phy), led+1); + sc->dev_led[phy][led].sc = sc; + sc->dev_led[phy][led].phy = phy; + sc->dev_led[phy][led].lednum = led; + } } } - return (err); + return (0); } static int @@ -285,12 +305,12 @@ ar8xxx_atu_flush(struct arswitch_softc *sc) static int arswitch_attach(device_t dev) { - struct arswitch_softc *sc; + struct arswitch_softc *sc = device_get_softc(dev); + struct sysctl_ctx_list *ctx; + struct sysctl_oid *tree; int err = 0; int port; - sc = device_get_softc(dev); - /* sc->sc_switchtype is already decided in arswitch_probe() */ sc->sc_dev = dev; mtx_init(&sc->sc_mtx, "arswitch", NULL, MTX_DEF); @@ -298,6 +318,13 @@ arswitch_attach(device_t dev) strlcpy(sc->info.es_name, device_get_desc(dev), sizeof(sc->info.es_name)); + /* Debugging */ + ctx = device_get_sysctl_ctx(sc->sc_dev); + tree = device_get_sysctl_tree(sc->sc_dev); + SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, + "debug", CTLFLAG_RW, &sc->sc_debug, 0, + "control debugging printfs"); + /* Default HAL methods */ sc->hal.arswitch_port_init = ar8xxx_port_init; sc->hal.arswitch_port_vlan_setup = ar8xxx_port_vlan_setup; @@ -339,7 +366,8 @@ arswitch_attach(device_t dev) else if (AR8X16_IS_SWITCH(sc, AR8327)) ar8327_attach(sc); else { - DPRINTF(dev, "%s: unknown switch (%d)?\n", __func__, sc->sc_switchtype); + DPRINTF(sc, ARSWITCH_DBG_ANY, + "%s: unknown switch (%d)?\n", __func__, sc->sc_switchtype); return (ENXIO); } @@ -369,19 +397,24 @@ arswitch_attach(device_t dev) /* Reset the switch. */ if (arswitch_reset(dev)) { - DPRINTF(dev, "%s: arswitch_reset: failed\n", __func__); + DPRINTF(sc, ARSWITCH_DBG_ANY, + "%s: arswitch_reset: failed\n", __func__); return (ENXIO); } err = sc->hal.arswitch_hw_setup(sc); - DPRINTF(dev, "%s: hw_setup: err=%d\n", __func__, err); - if (err != 0) + if (err != 0) { + DPRINTF(sc, ARSWITCH_DBG_ANY, + "%s: hw_setup: err=%d\n", __func__, err); return (err); + } err = sc->hal.arswitch_hw_global_setup(sc); - DPRINTF(dev, "%s: hw_global_setup: err=%d\n", __func__, err); - if (err != 0) + if (err != 0) { + DPRINTF(sc, ARSWITCH_DBG_ANY, + "%s: hw_global_setup: err=%d\n", __func__, err); return (err); + } /* Initialize the switch ports. */ for (port = 0; port <= sc->numphys; port++) { @@ -392,22 +425,28 @@ arswitch_attach(device_t dev) * Attach the PHYs and complete the bus enumeration. */ err = arswitch_attach_phys(sc); - DPRINTF(dev, "%s: attach_phys: err=%d\n", __func__, err); - if (err != 0) + if (err != 0) { + DPRINTF(sc, ARSWITCH_DBG_ANY, + "%s: attach_phys: err=%d\n", __func__, err); return (err); + } /* Default to ingress filters off. */ err = arswitch_set_vlan_mode(sc, 0); - DPRINTF(dev, "%s: set_vlan_mode: err=%d\n", __func__, err); - if (err != 0) + if (err != 0) { + DPRINTF(sc, ARSWITCH_DBG_ANY, + "%s: set_vlan_mode: err=%d\n", __func__, err); return (err); + } bus_generic_probe(dev); bus_enumerate_hinted_children(dev); err = bus_generic_attach(dev); - DPRINTF(dev, "%s: bus_generic_attach: err=%d\n", __func__, err); - if (err != 0) + if (err != 0) { + DPRINTF(sc, ARSWITCH_DBG_ANY, + "%s: bus_generic_attach: err=%d\n", __func__, err); return (err); + } callout_init_mtx(&sc->callout_tick, &sc->sc_mtx, 0); @@ -536,10 +575,11 @@ arswitch_miipollstat(struct arswitch_softc *sc) else portstatus = arswitch_readreg(sc->sc_dev, AR8X16_REG_PORT_STS(arswitch_portforphy(i))); -#if 0 - DPRINTF(sc->sc_dev, "p[%d]=%b\n", +#if 1 + DPRINTF(sc, ARSWITCH_DBG_POLL, "p[%d]=0x%08x (%b)\n", i, portstatus, + portstatus, "\20\3TXMAC\4RXMAC\5TXFLOW\6RXFLOW\7" "DUPLEX\11LINK_UP\12LINK_AUTO\13LINK_PAUSE"); #endif @@ -683,6 +723,38 @@ arswitch_getport(device_t dev, etherswitch_port_t *p) } else { return (ENXIO); } + + if (!arswitch_is_cpuport(sc, p->es_port) && + AR8X16_IS_SWITCH(sc, AR8327)) { + int led; + p->es_nleds = 3; + + for (led = 0; led < p->es_nleds; led++) + { + int style; + uint32_t val; + + /* Find the right style enum for our pattern */ + val = arswitch_readreg(dev, + ar8327_led_mapping[p->es_port-1][led].reg); + val = (val>>ar8327_led_mapping[p->es_port-1][led].shift)&0x03; + + for (style = 0; style < ETHERSWITCH_PORT_LED_MAX; style++) + { + if (led_pattern_table[style] == val) break; + } + + /* can't happen */ + if (style == ETHERSWITCH_PORT_LED_MAX) + style = ETHERSWITCH_PORT_LED_DEFAULT; + + p->es_led[led] = style; + } + } else + { + p->es_nleds = 0; + } + return (0); } @@ -727,7 +799,7 @@ ar8xxx_port_vlan_setup(struct arswitch_softc *sc, etherswitch_port_t *p) static int arswitch_setport(device_t dev, etherswitch_port_t *p) { - int err; + int err, i; struct arswitch_softc *sc; struct ifmedia *ifm; struct mii_data *mii; @@ -744,9 +816,20 @@ arswitch_setport(device_t dev, etherswitch_port_t *p) return (err); } - /* Do not allow media changes on CPU port. */ + /* Do not allow media or led changes on CPU port. */ if (arswitch_is_cpuport(sc, p->es_port)) return (0); + + if (AR8X16_IS_SWITCH(sc, AR8327)) + { + for (i = 0; i < 3; i++) + { + int err; + err = arswitch_setled(sc, p->es_port-1, i, p->es_led[i]); + if (err) + return (err); + } + } mii = arswitch_miiforport(sc, p->es_port); if (mii == NULL) @@ -758,11 +841,29 @@ arswitch_setport(device_t dev, etherswitch_port_t *p) return (ifmedia_ioctl(ifp, &p->es_ifr, ifm, SIOCSIFMEDIA)); } +static int +arswitch_setled(struct arswitch_softc *sc, int phy, int led, int style) +{ + int shift; + + if (phy < 0 || phy > sc->numphys) + return EINVAL; + + if (style < 0 || style > ETHERSWITCH_PORT_LED_MAX) + return (EINVAL); + + shift = ar8327_led_mapping[phy][led].shift; + return (arswitch_modifyreg(sc->sc_dev, + ar8327_led_mapping[phy][led].reg, + 0x03 << shift, led_pattern_table[style] << shift)); +} + static void arswitch_statchg(device_t dev) { + struct arswitch_softc *sc = device_get_softc(dev); - DPRINTF(dev, "%s\n", __func__); + DPRINTF(sc, ARSWITCH_DBG_POLL, "%s\n", __func__); } static int @@ -783,7 +884,7 @@ arswitch_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr) struct arswitch_softc *sc = ifp->if_softc; struct mii_data *mii = arswitch_miiforport(sc, ifp->if_dunit); - DPRINTF(sc->sc_dev, "%s\n", __func__); + DPRINTF(sc, ARSWITCH_DBG_POLL, "%s\n", __func__); if (mii == NULL) return; diff --git a/sys/dev/etherswitch/arswitch/arswitch_7240.c b/sys/dev/etherswitch/arswitch/arswitch_7240.c index 3d2543a7017f..01ea9b3e9c1b 100644 --- a/sys/dev/etherswitch/arswitch/arswitch_7240.c +++ b/sys/dev/etherswitch/arswitch/arswitch_7240.c @@ -97,6 +97,8 @@ ar7240_hw_global_setup(struct arswitch_softc *sc) AR7240_GLOBAL_CTRL_MTU_MASK, SM(1536, AR7240_GLOBAL_CTRL_MTU_MASK)); + /* XXX ARP? Frame Age enable? */ + /* Service Tag */ arswitch_modifyreg(sc->sc_dev, AR8X16_REG_SERVICE_TAG, AR8X16_SERVICE_TAG_MASK, 0); diff --git a/sys/dev/etherswitch/arswitch/arswitch_8316.c b/sys/dev/etherswitch/arswitch/arswitch_8316.c index d646e182f80c..54d6618bd805 100644 --- a/sys/dev/etherswitch/arswitch/arswitch_8316.c +++ b/sys/dev/etherswitch/arswitch/arswitch_8316.c @@ -137,6 +137,7 @@ ar8316_hw_global_setup(struct arswitch_softc *sc) arswitch_writereg(sc->sc_dev, AR8X16_REG_TAG_PRIO, 0xfa50); /* Enable ARP frame acknowledge. */ + /* XXX TODO: aging? */ arswitch_modifyreg(sc->sc_dev, AR8X16_REG_AT_CTRL, 0, AR8X16_AT_CTRL_ARP_EN); diff --git a/sys/dev/etherswitch/arswitch/arswitch_8327.c b/sys/dev/etherswitch/arswitch/arswitch_8327.c index 92e44fc9acb4..013688f1cce4 100644 --- a/sys/dev/etherswitch/arswitch/arswitch_8327.c +++ b/sys/dev/etherswitch/arswitch/arswitch_8327.c @@ -75,6 +75,36 @@ * lead to traffic storms/loops. */ +/* Map port+led to register+shift */ +struct ar8327_led_mapping ar8327_led_mapping[AR8327_NUM_PHYS][ETHERSWITCH_PORT_MAX_LEDS] = +{ + { /* PHY0 */ + {AR8327_REG_LED_CTRL0, 14 }, + {AR8327_REG_LED_CTRL1, 14 }, + {AR8327_REG_LED_CTRL2, 14 } + }, + { /* PHY1 */ + {AR8327_REG_LED_CTRL3, 8 }, + {AR8327_REG_LED_CTRL3, 10 }, + {AR8327_REG_LED_CTRL3, 12 } + }, + { /* PHY2 */ + {AR8327_REG_LED_CTRL3, 14 }, + {AR8327_REG_LED_CTRL3, 16 }, + {AR8327_REG_LED_CTRL3, 18 } + }, + { /* PHY3 */ + {AR8327_REG_LED_CTRL3, 20 }, + {AR8327_REG_LED_CTRL3, 22 }, + {AR8327_REG_LED_CTRL3, 24 } + }, + { /* PHY4 */ + {AR8327_REG_LED_CTRL0, 30 }, + {AR8327_REG_LED_CTRL1, 30 }, + {AR8327_REG_LED_CTRL2, 30 } + } +}; + static int ar8327_vlan_op(struct arswitch_softc *sc, uint32_t op, uint32_t vid, uint32_t data) @@ -1056,7 +1086,7 @@ ar8327_get_dot1q_vlan(struct arswitch_softc *sc, uint32_t *ports, } reg = arswitch_readreg(sc->sc_dev, AR8327_REG_VTU_FUNC0); - DPRINTF(sc->sc_dev, "%s: %d: reg=0x%08x\n", __func__, vid, reg); + DPRINTF(sc, ARSWITCH_DBG_REGIO, "%s: %d: reg=0x%08x\n", __func__, vid, reg); /* * If any of the bits are set, update the port mask. @@ -1088,7 +1118,7 @@ ar8327_set_dot1q_vlan(struct arswitch_softc *sc, uint32_t ports, op = AR8327_VTU_FUNC1_OP_LOAD; vid &= 0xfff; - DPRINTF(sc->sc_dev, + DPRINTF(sc, ARSWITCH_DBG_VLAN, "%s: vid: %d, ports=0x%08x, untagged_ports=0x%08x\n", __func__, vid, diff --git a/sys/dev/etherswitch/arswitch/arswitch_8327.h b/sys/dev/etherswitch/arswitch/arswitch_8327.h index 1f35d96639d1..e2a5f5ec7fdd 100644 --- a/sys/dev/etherswitch/arswitch/arswitch_8327.h +++ b/sys/dev/etherswitch/arswitch/arswitch_8327.h @@ -85,6 +85,11 @@ struct ar8327_port_cfg { uint32_t rxpause; }; +extern struct ar8327_led_mapping { + int reg; + int shift; +} ar8327_led_mapping[AR8327_NUM_PHYS][ETHERSWITCH_PORT_MAX_LEDS]; + extern void ar8327_attach(struct arswitch_softc *sc); #endif /* __ARSWITCH_8327_H__ */ diff --git a/sys/dev/etherswitch/arswitch/arswitch_phy.c b/sys/dev/etherswitch/arswitch/arswitch_phy.c index ceeb307efb6d..cb9d0a0a40b5 100644 --- a/sys/dev/etherswitch/arswitch/arswitch_phy.c +++ b/sys/dev/etherswitch/arswitch/arswitch_phy.c @@ -62,10 +62,6 @@ #include "miibus_if.h" #include "etherswitch_if.h" -#if defined(DEBUG) -static SYSCTL_NODE(_debug, OID_AUTO, arswitch, CTLFLAG_RD, 0, "arswitch"); -#endif - /* * Access PHYs integrated into the switch by going direct * to the PHY space itself, rather than through the switch @@ -81,6 +77,9 @@ arswitch_readphy_external(device_t dev, int phy, int reg) ARSWITCH_LOCK(sc); ret = (MDIO_READREG(device_get_parent(dev), phy, reg)); + DPRINTF(sc, ARSWITCH_DBG_PHYIO, + "%s: phy=0x%08x, reg=0x%08x, ret=0x%08x\n", + __func__, phy, reg, ret); ARSWITCH_UNLOCK(sc); return (ret); @@ -96,6 +95,9 @@ arswitch_writephy_external(device_t dev, int phy, int reg, int data) ARSWITCH_LOCK(sc); (void) MDIO_WRITEREG(device_get_parent(dev), phy, reg, data); + DPRINTF(sc, ARSWITCH_DBG_PHYIO, + "%s: phy=0x%08x, reg=0x%08x, data=0x%08x\n", + __func__, phy, reg, data); ARSWITCH_UNLOCK(sc); return (0); @@ -141,7 +143,9 @@ arswitch_readphy_internal(device_t dev, int phy, int reg) break; } if (timeout < 0) { - DPRINTF(dev, "arswitch_readphy(): phy=%d.%02x; timeout=%d\n", phy, reg, timeout); + DPRINTF(sc, ARSWITCH_DBG_ANY, + "arswitch_readphy(): phy=%d.%02x; timeout=%d\n", + phy, reg, timeout); goto fail; } data = arswitch_readreg_lsb(dev, a) & diff --git a/sys/dev/etherswitch/arswitch/arswitch_reg.c b/sys/dev/etherswitch/arswitch/arswitch_reg.c index 420e51644132..fb6baeb95441 100644 --- a/sys/dev/etherswitch/arswitch/arswitch_reg.c +++ b/sys/dev/etherswitch/arswitch/arswitch_reg.c @@ -232,6 +232,9 @@ arswitch_modifyreg(device_t dev, int addr, int mask, int set) int value; uint16_t phy, reg; + ARSWITCH_LOCK_ASSERT((struct arswitch_softc *)device_get_softc(dev), + MA_OWNED); + arswitch_split_setpage(dev, addr, &phy, ®); value = arswitch_reg_read32(dev, 0x10 | phy, reg); @@ -243,9 +246,12 @@ arswitch_modifyreg(device_t dev, int addr, int mask, int set) int arswitch_waitreg(device_t dev, int addr, int mask, int val, int timeout) { + struct arswitch_softc *sc = device_get_softc(dev); int err, v; uint16_t phy, reg; + ARSWITCH_LOCK_ASSERT(sc, MA_OWNED); + arswitch_split_setpage(dev, addr, &phy, ®); err = -1; @@ -261,5 +267,10 @@ arswitch_waitreg(device_t dev, int addr, int mask, int val, int timeout) DELAY(1); timeout--; } + if (err != 0) { + DPRINTF(sc, ARSWITCH_DBG_ANY, + "%s: waitreg failed; addr=0x%08x, mask=0x%08x, val=0x%08x\n", + __func__, addr, mask, val); + } return (err); } diff --git a/sys/dev/etherswitch/arswitch/arswitchvar.h b/sys/dev/etherswitch/arswitch/arswitchvar.h index a322a4fa9aac..19731f3b9413 100644 --- a/sys/dev/etherswitch/arswitch/arswitchvar.h +++ b/sys/dev/etherswitch/arswitch/arswitchvar.h @@ -48,6 +48,15 @@ typedef enum { #define ARSWITCH_NUM_PORTS MAX(AR8327_NUM_PORTS, AR8X16_NUM_PORTS) #define ARSWITCH_NUM_PHYS MAX(AR8327_NUM_PHYS, AR8X16_NUM_PHYS) +#define ARSWITCH_NUM_LEDS 3 + +struct arswitch_dev_led { + struct arswitch_softc *sc; + struct cdev *led; + int phy; + int lednum; +}; + struct arswitch_softc { struct mtx sc_mtx; /* serialize access to softc */ device_t sc_dev; @@ -66,9 +75,12 @@ struct arswitch_softc { char *ifname[ARSWITCH_NUM_PHYS]; device_t miibus[ARSWITCH_NUM_PHYS]; struct ifnet *ifp[ARSWITCH_NUM_PHYS]; + struct arswitch_dev_led dev_led[ARSWITCH_NUM_PHYS][ARSWITCH_NUM_LEDS]; struct callout callout_tick; etherswitch_info_t info; + uint32_t sc_debug; + /* VLANs support */ int vid[AR8X16_MAX_VLANS]; uint32_t vlan_mode; @@ -132,18 +144,27 @@ struct arswitch_softc { #define ARSWITCH_TRYLOCK(_sc) \ mtx_trylock(&(_sc)->sc_mtx) -#if defined(DEBUG) -#define DPRINTF(dev, args...) device_printf(dev, args) +#define ARSWITCH_DBG_RESET 0x00000001 +#define ARSWITCH_DBG_REGIO 0x00000002 +#define ARSWITCH_DBG_PHYIO 0x00000004 +#define ARSWITCH_DBG_POLL 0x00000008 +#define ARSWITCH_DBG_VLAN 0x00000010 +#define ARSWITCH_DBG_ANY 0xffffffff + +#if 1 +#define DPRINTF(sc, dbg, args...) \ + do { \ + if (((sc)->sc_debug & (dbg)) || \ + ((sc)->sc_debug == ARSWITCH_DBG_ANY)) { \ + device_printf((sc)->sc_dev, args); \ + } \ + } while (0) #define DEVERR(dev, err, fmt, args...) do { \ if (err != 0) device_printf(dev, fmt, err, args); \ } while (0) -#define DEBUG_INCRVAR(var) do { \ - var++; \ - } while (0) #else -#define DPRINTF(dev, args...) +#define DPRINTF(dev, dbg, args...) #define DEVERR(dev, err, fmt, args...) -#define DEBUG_INCRVAR(var) #endif #endif /* __ARSWITCHVAR_H__ */ diff --git a/sys/dev/etherswitch/etherswitch.h b/sys/dev/etherswitch/etherswitch.h index 26190198b529..0076177ebf99 100644 --- a/sys/dev/etherswitch/etherswitch.h +++ b/sys/dev/etherswitch/etherswitch.h @@ -14,7 +14,7 @@ extern driver_t etherswitch_driver; struct etherswitch_reg { uint16_t reg; - uint16_t val; + uint32_t val; }; typedef struct etherswitch_reg etherswitch_reg_t; @@ -64,10 +64,23 @@ typedef struct etherswitch_conf etherswitch_conf_t; #define ETHERSWITCH_PORT_FLAGS_BITS \ "\020\1CPUPORT\2STRIPTAG\3ADDTAG\4FIRSTLOCK\5DROPUNTAGGED\6QinQ\7INGRESS" +#define ETHERSWITCH_PORT_MAX_LEDS 3 + +enum etherswitch_port_led { + ETHERSWITCH_PORT_LED_DEFAULT, + ETHERSWITCH_PORT_LED_ON, + ETHERSWITCH_PORT_LED_OFF, + ETHERSWITCH_PORT_LED_BLINK, + ETHERSWITCH_PORT_LED_MAX +}; +typedef enum etherswitch_port_led etherswitch_port_led_t; + struct etherswitch_port { int es_port; int es_pvid; + int es_nleds; uint32_t es_flags; + etherswitch_port_led_t es_led[ETHERSWITCH_PORT_MAX_LEDS]; union { struct ifreq es_uifr; struct ifmediareq es_uifmr; diff --git a/sys/dev/etherswitch/ip17x/ip17x.c b/sys/dev/etherswitch/ip17x/ip17x.c index 43cdddb27ff2..e39663498679 100644 --- a/sys/dev/etherswitch/ip17x/ip17x.c +++ b/sys/dev/etherswitch/ip17x/ip17x.c @@ -84,7 +84,7 @@ ip17x_probe(device_t dev) phy_id1 = MDIO_READREG(device_get_parent(dev), 0, MII_PHYIDR1); phy_id2 = MDIO_READREG(device_get_parent(dev), 0, MII_PHYIDR2); - oui = MII_OUI(phy_id1, phy_id2), + oui = MII_OUI(phy_id1, phy_id2); model = MII_MODEL(phy_id2); /* We only care about IC+ devices. */ if (oui != IP17X_OUI) { diff --git a/sys/dev/fdt/fdt_common.c b/sys/dev/fdt/fdt_common.c index 4e0d6e206c5b..2f4555f81a4d 100644 --- a/sys/dev/fdt/fdt_common.c +++ b/sys/dev/fdt/fdt_common.c @@ -38,6 +38,7 @@ __FBSDID("$FreeBSD$"); #include <sys/module.h> #include <sys/bus.h> #include <sys/limits.h> +#include <sys/sysctl.h> #include <machine/resource.h> @@ -60,6 +61,8 @@ __FBSDID("$FreeBSD$"); #define FDT_REG_CELLS 4 +SYSCTL_NODE(_hw, OID_AUTO, fdt, CTLFLAG_RD, 0, "Flattened Device Tree"); + vm_paddr_t fdt_immr_pa; vm_offset_t fdt_immr_va; vm_offset_t fdt_immr_size; diff --git a/sys/dev/fdt/fdt_common.h b/sys/dev/fdt/fdt_common.h index 94f84ff3fe49..afddc310c127 100644 --- a/sys/dev/fdt/fdt_common.h +++ b/sys/dev/fdt/fdt_common.h @@ -32,6 +32,7 @@ #ifndef _FDT_COMMON_H_ #define _FDT_COMMON_H_ +#include <sys/sysctl.h> #include <sys/slicer.h> #include <contrib/libfdt/libfdt_env.h> #include <dev/ofw/ofw_bus.h> @@ -80,6 +81,8 @@ extern struct fdt_pm_mask_entry fdt_pm_mask_table[]; extern u_char fdt_static_dtb; #endif +SYSCTL_DECL(_hw_fdt); + int fdt_addrsize_cells(phandle_t, int *, int *); u_long fdt_data_get(void *, int); int fdt_data_to_res(pcell_t *, int, int, u_long *, u_long *); diff --git a/sys/dev/filemon/filemon.c b/sys/dev/filemon/filemon.c index 919af9d6dbe6..26e1bc38d071 100644 --- a/sys/dev/filemon/filemon.c +++ b/sys/dev/filemon/filemon.c @@ -137,6 +137,8 @@ filemon_proc_get(struct proc *p) { struct filemon *filemon; + if (p->p_filemon == NULL) + return (NULL); PROC_LOCK(p); filemon = filemon_acquire(p->p_filemon); PROC_UNLOCK(p); diff --git a/sys/dev/gpio/gpioled.c b/sys/dev/gpio/gpioled.c index bd77f5386d53..2c579e404bc3 100644 --- a/sys/dev/gpio/gpioled.c +++ b/sys/dev/gpio/gpioled.c @@ -66,6 +66,7 @@ struct gpioled_softc device_t sc_busdev; struct mtx sc_mtx; struct cdev *sc_leddev; + int sc_invert; }; static void gpioled_control(void *, int); @@ -82,6 +83,8 @@ gpioled_control(void *priv, int onoff) GPIOLED_LOCK(sc); if (GPIOBUS_PIN_SETFLAGS(sc->sc_busdev, sc->sc_dev, GPIOLED_PIN, GPIO_PIN_OUTPUT) == 0) { + if (sc->sc_invert) + onoff = !onoff; GPIOBUS_PIN_SET(sc->sc_busdev, sc->sc_dev, GPIOLED_PIN, onoff ? GPIO_PIN_HIGH : GPIO_PIN_LOW); } @@ -199,6 +202,8 @@ gpioled_attach(device_t dev) if (resource_string_value(device_get_name(dev), device_get_unit(dev), "name", &name)) name = NULL; + resource_int_value(device_get_name(dev), + device_get_unit(dev), "invert", &sc->sc_invert); #endif sc->sc_leddev = led_create_state(gpioled_control, sc, name ? name : diff --git a/sys/dev/hwpmc/hwpmc_core.c b/sys/dev/hwpmc/hwpmc_core.c index a54ac27cc391..c734c5ad3c1a 100644 --- a/sys/dev/hwpmc/hwpmc_core.c +++ b/sys/dev/hwpmc/hwpmc_core.c @@ -365,7 +365,7 @@ iaf_read_pmc(int cpu, int ri, pmc_value_t *v) if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) *v = iaf_perfctr_value_to_reload_count(tmp); else - *v = tmp; + *v = tmp & ((1ULL << core_iaf_width) - 1); PMCDBG4(MDP,REA,1, "iaf-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri, IAF_RI_TO_MSR(ri), *v); diff --git a/sys/dev/hyperv/include/hyperv.h b/sys/dev/hyperv/include/hyperv.h index a502a648914f..f35685645935 100644 --- a/sys/dev/hyperv/include/hyperv.h +++ b/sys/dev/hyperv/include/hyperv.h @@ -28,43 +28,21 @@ * $FreeBSD$ */ -/** - * HyperV definitions for messages that are sent between instances of the - * Channel Management Library in separate partitions, or in some cases, - * back to itself. - */ - -#ifndef __HYPERV_H__ -#define __HYPERV_H__ +#ifndef _HYPERV_H_ +#define _HYPERV_H_ #include <sys/param.h> -#include <sys/mbuf.h> -#include <sys/queue.h> -#include <sys/malloc.h> -#include <sys/kthread.h> -#include <sys/taskqueue.h> -#include <sys/systm.h> -#include <sys/lock.h> -#include <sys/sema.h> -#include <sys/smp.h> -#include <sys/mutex.h> -#include <sys/bus.h> -#include <sys/sysctl.h> + #include <vm/vm.h> -#include <vm/vm_param.h> #include <vm/pmap.h> -#include <amd64/include/xen/synch_bitops.h> -#include <amd64/include/atomic.h> -#include <dev/hyperv/include/hyperv_busdma.h> - struct hyperv_guid { uint8_t hv_guid[16]; } __packed; #define HYPERV_GUID_STRLEN 40 -int hyperv_guid2str(const struct hyperv_guid *, char *, size_t); +int hyperv_guid2str(const struct hyperv_guid *, char *, size_t); /** * @brief Get physical address from virtual @@ -77,4 +55,4 @@ hv_get_phys_addr(void *virt) return (ret); } -#endif /* __HYPERV_H__ */ +#endif /* _HYPERV_H_ */ diff --git a/sys/dev/hyperv/include/hyperv_busdma.h b/sys/dev/hyperv/include/hyperv_busdma.h index a27d2dba7bf4..ff01b3e27a95 100644 --- a/sys/dev/hyperv/include/hyperv_busdma.h +++ b/sys/dev/hyperv/include/hyperv_busdma.h @@ -39,11 +39,11 @@ struct hyperv_dma { bus_dmamap_t hv_dmap; }; -void hyperv_dma_map_paddr(void *arg, bus_dma_segment_t *segs, int nseg, - int error); -void *hyperv_dmamem_alloc(bus_dma_tag_t parent_dtag, bus_size_t alignment, - bus_addr_t boundary, bus_size_t size, struct hyperv_dma *dma, - int flags); -void hyperv_dmamem_free(struct hyperv_dma *dma, void *ptr); +void hyperv_dma_map_paddr(void *arg, bus_dma_segment_t *segs, + int nseg, int error); +void *hyperv_dmamem_alloc(bus_dma_tag_t parent_dtag, + bus_size_t alignment, bus_addr_t boundary, bus_size_t size, + struct hyperv_dma *dma, int flags); +void hyperv_dmamem_free(struct hyperv_dma *dma, void *ptr); #endif /* !_HYPERV_BUSDMA_H_ */ diff --git a/sys/dev/hyperv/include/vmbus.h b/sys/dev/hyperv/include/vmbus.h index 7c06580dcf4d..5843f0345e0a 100644 --- a/sys/dev/hyperv/include/vmbus.h +++ b/sys/dev/hyperv/include/vmbus.h @@ -30,6 +30,7 @@ #define _VMBUS_H_ #include <sys/param.h> +#include <sys/bus.h> /* * VMBUS version is 32 bit, upper 16 bit for major_number and lower @@ -82,12 +83,18 @@ struct vmbus_chanpkt_hdr { #define VMBUS_CHANPKT_TYPE_GPA 0x0009 #define VMBUS_CHANPKT_TYPE_COMP 0x000b +#define VMBUS_CHANPKT_FLAG_NONE 0 #define VMBUS_CHANPKT_FLAG_RC 0x0001 /* report completion */ #define VMBUS_CHANPKT_CONST_DATA(pkt) \ (const void *)((const uint8_t *)(pkt) + \ VMBUS_CHANPKT_GETLEN((pkt)->cph_hlen)) +/* Include padding */ +#define VMBUS_CHANPKT_DATALEN(pkt) \ + (VMBUS_CHANPKT_GETLEN((pkt)->cph_tlen) -\ + VMBUS_CHANPKT_GETLEN((pkt)->cph_hlen)) + struct vmbus_rxbuf_desc { uint32_t rb_len; uint32_t rb_ofs; @@ -101,9 +108,6 @@ struct vmbus_chanpkt_rxbuf { struct vmbus_rxbuf_desc cp_rxbuf[]; } __packed; -#define VMBUS_CHAN_SGLIST_MAX 32 -#define VMBUS_CHAN_PRPLIST_MAX 32 - struct vmbus_channel; struct hyperv_guid; @@ -115,45 +119,47 @@ vmbus_get_channel(device_t dev) return device_get_ivars(dev); } -int vmbus_chan_open(struct vmbus_channel *chan, - int txbr_size, int rxbr_size, const void *udata, int udlen, - vmbus_chan_callback_t cb, void *cbarg); -void vmbus_chan_close(struct vmbus_channel *chan); +int vmbus_chan_open(struct vmbus_channel *chan, + int txbr_size, int rxbr_size, const void *udata, int udlen, + vmbus_chan_callback_t cb, void *cbarg); +void vmbus_chan_close(struct vmbus_channel *chan); -int vmbus_chan_gpadl_connect(struct vmbus_channel *chan, - bus_addr_t paddr, int size, uint32_t *gpadl); -int vmbus_chan_gpadl_disconnect(struct vmbus_channel *chan, - uint32_t gpadl); +int vmbus_chan_gpadl_connect(struct vmbus_channel *chan, + bus_addr_t paddr, int size, uint32_t *gpadl); +int vmbus_chan_gpadl_disconnect(struct vmbus_channel *chan, + uint32_t gpadl); -void vmbus_chan_cpu_set(struct vmbus_channel *chan, int cpu); -void vmbus_chan_cpu_rr(struct vmbus_channel *chan); +void vmbus_chan_cpu_set(struct vmbus_channel *chan, int cpu); +void vmbus_chan_cpu_rr(struct vmbus_channel *chan); struct vmbus_channel * - vmbus_chan_cpu2chan(struct vmbus_channel *chan, int cpu); -void vmbus_chan_set_readbatch(struct vmbus_channel *chan, bool on); + vmbus_chan_cpu2chan(struct vmbus_channel *chan, int cpu); +void vmbus_chan_set_readbatch(struct vmbus_channel *chan, bool on); struct vmbus_channel ** - vmbus_subchan_get(struct vmbus_channel *pri_chan, int subchan_cnt); -void vmbus_subchan_rel(struct vmbus_channel **subchan, int subchan_cnt); -void vmbus_subchan_drain(struct vmbus_channel *pri_chan); - -int vmbus_chan_recv(struct vmbus_channel *chan, void *data, int *dlen, - uint64_t *xactid); -int vmbus_chan_recv_pkt(struct vmbus_channel *chan, - struct vmbus_chanpkt_hdr *pkt, int *pktlen); - -int vmbus_chan_send(struct vmbus_channel *chan, uint16_t type, - uint16_t flags, void *data, int dlen, uint64_t xactid); -int vmbus_chan_send_sglist(struct vmbus_channel *chan, - struct vmbus_gpa sg[], int sglen, void *data, int dlen, - uint64_t xactid); -int vmbus_chan_send_prplist(struct vmbus_channel *chan, - struct vmbus_gpa_range *prp, int prp_cnt, void *data, int dlen, - uint64_t xactid); - -uint32_t vmbus_chan_id(const struct vmbus_channel *chan); -uint32_t vmbus_chan_subidx(const struct vmbus_channel *chan); -bool vmbus_chan_is_primary(const struct vmbus_channel *chan); + vmbus_subchan_get(struct vmbus_channel *pri_chan, + int subchan_cnt); +void vmbus_subchan_rel(struct vmbus_channel **subchan, + int subchan_cnt); +void vmbus_subchan_drain(struct vmbus_channel *pri_chan); + +int vmbus_chan_recv(struct vmbus_channel *chan, void *data, int *dlen, + uint64_t *xactid); +int vmbus_chan_recv_pkt(struct vmbus_channel *chan, + struct vmbus_chanpkt_hdr *pkt, int *pktlen); + +int vmbus_chan_send(struct vmbus_channel *chan, uint16_t type, + uint16_t flags, void *data, int dlen, uint64_t xactid); +int vmbus_chan_send_sglist(struct vmbus_channel *chan, + struct vmbus_gpa sg[], int sglen, void *data, int dlen, + uint64_t xactid); +int vmbus_chan_send_prplist(struct vmbus_channel *chan, + struct vmbus_gpa_range *prp, int prp_cnt, void *data, + int dlen, uint64_t xactid); + +uint32_t vmbus_chan_id(const struct vmbus_channel *chan); +uint32_t vmbus_chan_subidx(const struct vmbus_channel *chan); +bool vmbus_chan_is_primary(const struct vmbus_channel *chan); const struct hyperv_guid * - vmbus_chan_guid_inst(const struct vmbus_channel *chan); + vmbus_chan_guid_inst(const struct vmbus_channel *chan); #endif /* !_VMBUS_H_ */ diff --git a/sys/dev/hyperv/include/vmbus_xact.h b/sys/dev/hyperv/include/vmbus_xact.h new file mode 100644 index 000000000000..c2919aa8c1f2 --- /dev/null +++ b/sys/dev/hyperv/include/vmbus_xact.h @@ -0,0 +1,59 @@ +/*- + * Copyright (c) 2016 Microsoft Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _VMBUS_XACT_H_ +#define _VMBUS_XACT_H_ + +#include <sys/param.h> +#include <sys/bus.h> + +struct vmbus_xact; +struct vmbus_xact_ctx; + +struct vmbus_xact_ctx *vmbus_xact_ctx_create(bus_dma_tag_t dtag, + size_t req_size, size_t resp_size, + size_t priv_size); +void vmbus_xact_ctx_destroy(struct vmbus_xact_ctx *ctx); +struct vmbus_xact *vmbus_xact_get(struct vmbus_xact_ctx *ctx, + size_t req_len); +void vmbus_xact_put(struct vmbus_xact *xact); + +void *vmbus_xact_req_data(const struct vmbus_xact *xact); +bus_addr_t vmbus_xact_req_paddr(const struct vmbus_xact *xact); +void *vmbus_xact_priv(const struct vmbus_xact *xact, + size_t priv_len); +void vmbus_xact_activate(struct vmbus_xact *xact); +void vmbus_xact_deactivate(struct vmbus_xact *xact); +const void *vmbus_xact_wait(struct vmbus_xact *xact, + size_t *resp_len); +void vmbus_xact_wakeup(struct vmbus_xact *xact, + const void *data, size_t dlen); +void vmbus_xact_ctx_wakeup(struct vmbus_xact_ctx *ctx, + const void *data, size_t dlen); + +#endif /* !_VMBUS_XACT_H_ */ diff --git a/sys/dev/hyperv/netvsc/hv_net_vsc.c b/sys/dev/hyperv/netvsc/hv_net_vsc.c index eb8a391480d3..38242ceba10f 100644 --- a/sys/dev/hyperv/netvsc/hv_net_vsc.c +++ b/sys/dev/hyperv/netvsc/hv_net_vsc.c @@ -45,9 +45,11 @@ #include <machine/atomic.h> #include <dev/hyperv/include/hyperv.h> -#include "hv_net_vsc.h" -#include "hv_rndis.h" -#include "hv_rndis_filter.h" +#include <dev/hyperv/include/vmbus_xact.h> +#include <dev/hyperv/netvsc/hv_net_vsc.h> +#include <dev/hyperv/netvsc/hv_rndis.h> +#include <dev/hyperv/netvsc/hv_rndis_filter.h> +#include <dev/hyperv/netvsc/if_hnreg.h> MALLOC_DEFINE(M_NETVSC, "netvsc", "Hyper-V netvsc driver"); @@ -68,6 +70,12 @@ static void hv_nv_on_receive_completion(struct vmbus_channel *chan, static void hv_nv_on_receive(netvsc_dev *net_dev, struct hn_rx_ring *rxr, struct vmbus_channel *chan, const struct vmbus_chanpkt_hdr *pkt); +static void hn_nvs_sent_none(struct hn_send_ctx *sndc, + struct netvsc_dev_ *net_dev, struct vmbus_channel *chan, + const struct nvsp_msg_ *msg, int); + +static struct hn_send_ctx hn_send_ctx_none = + HN_SEND_CTX_INITIALIZER(hn_nvs_sent_none, NULL); /* * @@ -110,7 +118,7 @@ hv_nv_get_next_send_section(netvsc_dev *net_dev) unsigned long bitsmap_words = net_dev->bitsmap_words; unsigned long *bitsmap = net_dev->send_section_bitsmap; unsigned long idx; - int ret = NVSP_1_CHIMNEY_SEND_INVALID_SECTION_INDEX; + int ret = HN_NVS_CHIM_IDX_INVALID; int i; for (i = 0; i < bitsmap_words; i++) { @@ -141,9 +149,14 @@ hv_nv_get_next_send_section(netvsc_dev *net_dev) static int hv_nv_init_rx_buffer_with_net_vsp(struct hn_softc *sc) { + struct vmbus_xact *xact; + struct hn_nvs_rxbuf_conn *conn; + const struct hn_nvs_rxbuf_connresp *resp; + size_t resp_len; + struct hn_send_ctx sndc; netvsc_dev *net_dev; - nvsp_msg *init_pkt; - int ret = 0; + uint32_t status; + int error; net_dev = hv_nv_get_outbound_net_device(sc); if (!net_dev) { @@ -155,7 +168,7 @@ hv_nv_init_rx_buffer_with_net_vsp(struct hn_softc *sc) BUS_DMA_WAITOK | BUS_DMA_ZERO); if (net_dev->rx_buf == NULL) { device_printf(sc->hn_dev, "allocate rxbuf failed\n"); - return ENOMEM; + return (ENOMEM); } /* @@ -165,73 +178,75 @@ hv_nv_init_rx_buffer_with_net_vsp(struct hn_softc *sc) * Only primary channel has RXBUF connected to it. Sub-channels * just share this RXBUF. */ - ret = vmbus_chan_gpadl_connect(sc->hn_prichan, + error = vmbus_chan_gpadl_connect(sc->hn_prichan, net_dev->rxbuf_dma.hv_paddr, net_dev->rx_buf_size, &net_dev->rx_buf_gpadl_handle); - if (ret != 0) { - device_printf(sc->hn_dev, "rxbuf gpadl connect failed: %d\n", - ret); + if (error) { + if_printf(sc->hn_ifp, "rxbuf gpadl connect failed: %d\n", + error); goto cleanup; } - - /* sema_wait(&ext->channel_init_sema); KYS CHECK */ - /* Notify the NetVsp of the gpadl handle */ - init_pkt = &net_dev->channel_init_packet; - - memset(init_pkt, 0, sizeof(nvsp_msg)); - - init_pkt->hdr.msg_type = nvsp_msg_1_type_send_rx_buf; - init_pkt->msgs.vers_1_msgs.send_rx_buf.gpadl_handle = - net_dev->rx_buf_gpadl_handle; - init_pkt->msgs.vers_1_msgs.send_rx_buf.id = - NETVSC_RECEIVE_BUFFER_ID; - - /* Send the gpadl notification request */ + /* + * Connect RXBUF to NVS. + */ - ret = vmbus_chan_send(sc->hn_prichan, - VMBUS_CHANPKT_TYPE_INBAND, VMBUS_CHANPKT_FLAG_RC, - init_pkt, sizeof(nvsp_msg), (uint64_t)(uintptr_t)init_pkt); - if (ret != 0) { + xact = vmbus_xact_get(sc->hn_xact, sizeof(*conn)); + if (xact == NULL) { + if_printf(sc->hn_ifp, "no xact for nvs rxbuf conn\n"); + error = ENXIO; goto cleanup; } - sema_wait(&net_dev->channel_init_sema); - - /* Check the response */ - if (init_pkt->msgs.vers_1_msgs.send_rx_buf_complete.status - != nvsp_status_success) { - ret = EINVAL; + conn = vmbus_xact_req_data(xact); + conn->nvs_type = HN_NVS_TYPE_RXBUF_CONN; + conn->nvs_gpadl = net_dev->rx_buf_gpadl_handle; + conn->nvs_sig = HN_NVS_RXBUF_SIG; + + hn_send_ctx_init_simple(&sndc, hn_nvs_sent_xact, xact); + vmbus_xact_activate(xact); + + error = hn_nvs_send(sc->hn_prichan, VMBUS_CHANPKT_FLAG_RC, + conn, sizeof(*conn), &sndc); + if (error != 0) { + if_printf(sc->hn_ifp, "send nvs rxbuf conn failed: %d\n", + error); + vmbus_xact_deactivate(xact); + vmbus_xact_put(xact); goto cleanup; } - net_dev->rx_section_count = - init_pkt->msgs.vers_1_msgs.send_rx_buf_complete.num_sections; - - net_dev->rx_sections = malloc(net_dev->rx_section_count * - sizeof(nvsp_1_rx_buf_section), M_NETVSC, M_WAITOK); - memcpy(net_dev->rx_sections, - init_pkt->msgs.vers_1_msgs.send_rx_buf_complete.sections, - net_dev->rx_section_count * sizeof(nvsp_1_rx_buf_section)); + resp = vmbus_xact_wait(xact, &resp_len); + if (resp_len < sizeof(*resp)) { + if_printf(sc->hn_ifp, "invalid rxbuf conn resp length %zu\n", + resp_len); + vmbus_xact_put(xact); + error = EINVAL; + goto cleanup; + } + if (resp->nvs_type != HN_NVS_TYPE_RXBUF_CONNRESP) { + if_printf(sc->hn_ifp, "not rxbuf conn resp, type %u\n", + resp->nvs_type); + vmbus_xact_put(xact); + error = EINVAL; + goto cleanup; + } + status = resp->nvs_status; + vmbus_xact_put(xact); - /* - * For first release, there should only be 1 section that represents - * the entire receive buffer - */ - if (net_dev->rx_section_count != 1 - || net_dev->rx_sections->offset != 0) { - ret = EINVAL; + if (status != HN_NVS_STATUS_OK) { + if_printf(sc->hn_ifp, "rxbuf conn failed: %x\n", status); + error = EIO; goto cleanup; } + net_dev->rx_section_count = 1; - goto exit; + return (0); cleanup: hv_nv_destroy_rx_buffer(net_dev); - -exit: - return (ret); + return (error); } /* @@ -240,9 +255,14 @@ exit: static int hv_nv_init_send_buffer_with_net_vsp(struct hn_softc *sc) { + struct hn_send_ctx sndc; + struct vmbus_xact *xact; + struct hn_nvs_chim_conn *chim; + const struct hn_nvs_chim_connresp *resp; + size_t resp_len; + uint32_t status, sectsz; netvsc_dev *net_dev; - nvsp_msg *init_pkt; - int ret = 0; + int error; net_dev = hv_nv_get_outbound_net_device(sc); if (!net_dev) { @@ -254,7 +274,7 @@ hv_nv_init_send_buffer_with_net_vsp(struct hn_softc *sc) BUS_DMA_WAITOK | BUS_DMA_ZERO); if (net_dev->send_buf == NULL) { device_printf(sc->hn_dev, "allocate chimney txbuf failed\n"); - return ENOMEM; + return (ENOMEM); } /* @@ -264,47 +284,76 @@ hv_nv_init_send_buffer_with_net_vsp(struct hn_softc *sc) * Only primary channel has chimney sending buffer connected to it. * Sub-channels just share this chimney sending buffer. */ - ret = vmbus_chan_gpadl_connect(sc->hn_prichan, + error = vmbus_chan_gpadl_connect(sc->hn_prichan, net_dev->txbuf_dma.hv_paddr, net_dev->send_buf_size, &net_dev->send_buf_gpadl_handle); - if (ret != 0) { - device_printf(sc->hn_dev, "chimney sending buffer gpadl " - "connect failed: %d\n", ret); + if (error) { + if_printf(sc->hn_ifp, "chimney sending buffer gpadl " + "connect failed: %d\n", error); goto cleanup; } - /* Notify the NetVsp of the gpadl handle */ - - init_pkt = &net_dev->channel_init_packet; - - memset(init_pkt, 0, sizeof(nvsp_msg)); + /* + * Connect chimney sending buffer to NVS + */ - init_pkt->hdr.msg_type = nvsp_msg_1_type_send_send_buf; - init_pkt->msgs.vers_1_msgs.send_rx_buf.gpadl_handle = - net_dev->send_buf_gpadl_handle; - init_pkt->msgs.vers_1_msgs.send_rx_buf.id = - NETVSC_SEND_BUFFER_ID; + xact = vmbus_xact_get(sc->hn_xact, sizeof(*chim)); + if (xact == NULL) { + if_printf(sc->hn_ifp, "no xact for nvs chim conn\n"); + error = ENXIO; + goto cleanup; + } - /* Send the gpadl notification request */ + chim = vmbus_xact_req_data(xact); + chim->nvs_type = HN_NVS_TYPE_CHIM_CONN; + chim->nvs_gpadl = net_dev->send_buf_gpadl_handle; + chim->nvs_sig = HN_NVS_CHIM_SIG; + + hn_send_ctx_init_simple(&sndc, hn_nvs_sent_xact, xact); + vmbus_xact_activate(xact); + + error = hn_nvs_send(sc->hn_prichan, VMBUS_CHANPKT_FLAG_RC, + chim, sizeof(*chim), &sndc); + if (error) { + if_printf(sc->hn_ifp, "send nvs chim conn failed: %d\n", + error); + vmbus_xact_deactivate(xact); + vmbus_xact_put(xact); + goto cleanup; + } - ret = vmbus_chan_send(sc->hn_prichan, - VMBUS_CHANPKT_TYPE_INBAND, VMBUS_CHANPKT_FLAG_RC, - init_pkt, sizeof(nvsp_msg), (uint64_t)init_pkt); - if (ret != 0) { + resp = vmbus_xact_wait(xact, &resp_len); + if (resp_len < sizeof(*resp)) { + if_printf(sc->hn_ifp, "invalid chim conn resp length %zu\n", + resp_len); + vmbus_xact_put(xact); + error = EINVAL; + goto cleanup; + } + if (resp->nvs_type != HN_NVS_TYPE_CHIM_CONNRESP) { + if_printf(sc->hn_ifp, "not chim conn resp, type %u\n", + resp->nvs_type); + vmbus_xact_put(xact); + error = EINVAL; goto cleanup; } - sema_wait(&net_dev->channel_init_sema); + status = resp->nvs_status; + sectsz = resp->nvs_sectsz; + vmbus_xact_put(xact); - /* Check the response */ - if (init_pkt->msgs.vers_1_msgs.send_send_buf_complete.status - != nvsp_status_success) { - ret = EINVAL; + if (status != HN_NVS_STATUS_OK) { + if_printf(sc->hn_ifp, "chim conn failed: %x\n", status); + error = EIO; goto cleanup; } + if (sectsz == 0) { + if_printf(sc->hn_ifp, "zero chimney sending buffer " + "section size\n"); + return 0; + } - net_dev->send_section_size = - init_pkt->msgs.vers_1_msgs.send_send_buf_complete.section_size; + net_dev->send_section_size = sectsz; net_dev->send_section_count = net_dev->send_buf_size / net_dev->send_section_size; net_dev->bitsmap_words = howmany(net_dev->send_section_count, @@ -313,13 +362,15 @@ hv_nv_init_send_buffer_with_net_vsp(struct hn_softc *sc) malloc(net_dev->bitsmap_words * sizeof(long), M_NETVSC, M_WAITOK | M_ZERO); - goto exit; + if (bootverbose) { + if_printf(sc->hn_ifp, "chimney sending buffer %u/%u\n", + net_dev->send_section_size, net_dev->send_section_count); + } + return 0; cleanup: hv_nv_destroy_send_buffer(net_dev); - -exit: - return (ret); + return (error); } /* @@ -328,35 +379,28 @@ exit: static int hv_nv_destroy_rx_buffer(netvsc_dev *net_dev) { - nvsp_msg *revoke_pkt; int ret = 0; - /* - * If we got a section count, it means we received a - * send_rx_buf_complete msg - * (ie sent nvsp_msg_1_type_send_rx_buf msg) therefore, - * we need to send a revoke msg here - */ if (net_dev->rx_section_count) { - /* Send the revoke receive buffer */ - revoke_pkt = &net_dev->revoke_packet; - memset(revoke_pkt, 0, sizeof(nvsp_msg)); - - revoke_pkt->hdr.msg_type = nvsp_msg_1_type_revoke_rx_buf; - revoke_pkt->msgs.vers_1_msgs.revoke_rx_buf.id = - NETVSC_RECEIVE_BUFFER_ID; - - ret = vmbus_chan_send(net_dev->sc->hn_prichan, - VMBUS_CHANPKT_TYPE_INBAND, 0, revoke_pkt, sizeof(nvsp_msg), - (uint64_t)(uintptr_t)revoke_pkt); + struct hn_nvs_rxbuf_disconn disconn; /* - * If we failed here, we might as well return and have a leak - * rather than continue and a bugchk + * Disconnect RXBUF from NVS. */ + memset(&disconn, 0, sizeof(disconn)); + disconn.nvs_type = HN_NVS_TYPE_RXBUF_DISCONN; + disconn.nvs_sig = HN_NVS_RXBUF_SIG; + + /* NOTE: No response. */ + ret = hn_nvs_send(net_dev->sc->hn_prichan, + VMBUS_CHANPKT_FLAG_NONE, &disconn, sizeof(disconn), + &hn_send_ctx_none); if (ret != 0) { + if_printf(net_dev->sc->hn_ifp, + "send rxbuf disconn failed: %d\n", ret); return (ret); } + net_dev->rx_section_count = 0; } /* Tear down the gpadl on the vsp end */ @@ -379,12 +423,6 @@ hv_nv_destroy_rx_buffer(netvsc_dev *net_dev) net_dev->rx_buf = NULL; } - if (net_dev->rx_sections) { - free(net_dev->rx_sections, M_NETVSC); - net_dev->rx_sections = NULL; - net_dev->rx_section_count = 0; - } - return (ret); } @@ -394,34 +432,25 @@ hv_nv_destroy_rx_buffer(netvsc_dev *net_dev) static int hv_nv_destroy_send_buffer(netvsc_dev *net_dev) { - nvsp_msg *revoke_pkt; int ret = 0; - /* - * If we got a section count, it means we received a - * send_rx_buf_complete msg - * (ie sent nvsp_msg_1_type_send_rx_buf msg) therefore, - * we need to send a revoke msg here - */ if (net_dev->send_section_size) { - /* Send the revoke send buffer */ - revoke_pkt = &net_dev->revoke_packet; - memset(revoke_pkt, 0, sizeof(nvsp_msg)); - - revoke_pkt->hdr.msg_type = - nvsp_msg_1_type_revoke_send_buf; - revoke_pkt->msgs.vers_1_msgs.revoke_send_buf.id = - NETVSC_SEND_BUFFER_ID; - - ret = vmbus_chan_send(net_dev->sc->hn_prichan, - VMBUS_CHANPKT_TYPE_INBAND, 0, - revoke_pkt, sizeof(nvsp_msg), - (uint64_t)(uintptr_t)revoke_pkt); + struct hn_nvs_chim_disconn disconn; + /* - * If we failed here, we might as well return and have a leak - * rather than continue and a bugchk + * Disconnect chimney sending buffer from NVS. */ + memset(&disconn, 0, sizeof(disconn)); + disconn.nvs_type = HN_NVS_TYPE_CHIM_DISCONN; + disconn.nvs_sig = HN_NVS_CHIM_SIG; + + /* NOTE: No response. */ + ret = hn_nvs_send(net_dev->sc->hn_prichan, + VMBUS_CHANPKT_FLAG_NONE, &disconn, sizeof(disconn), + &hn_send_ctx_none); if (ret != 0) { + if_printf(net_dev->sc->hn_ifp, + "send chim disconn failed: %d\n", ret); return (ret); } } @@ -454,43 +483,63 @@ hv_nv_destroy_send_buffer(netvsc_dev *net_dev) return (ret); } - -/* - * Attempt to negotiate the caller-specified NVSP version - * - * For NVSP v2, Server 2008 R2 does not set - * init_pkt->msgs.init_msgs.init_compl.negotiated_prot_vers - * to the negotiated version, so we cannot rely on that. - */ static int hv_nv_negotiate_nvsp_protocol(struct hn_softc *sc, netvsc_dev *net_dev, - uint32_t nvsp_ver) + uint32_t nvs_ver) { - nvsp_msg *init_pkt; - int ret; - - init_pkt = &net_dev->channel_init_packet; - memset(init_pkt, 0, sizeof(nvsp_msg)); - init_pkt->hdr.msg_type = nvsp_msg_type_init; + struct hn_send_ctx sndc; + struct vmbus_xact *xact; + struct hn_nvs_init *init; + const struct hn_nvs_init_resp *resp; + size_t resp_len; + uint32_t status; + int error; + + xact = vmbus_xact_get(sc->hn_xact, sizeof(*init)); + if (xact == NULL) { + if_printf(sc->hn_ifp, "no xact for nvs init\n"); + return (ENXIO); + } - /* - * Specify parameter as the only acceptable protocol version - */ - init_pkt->msgs.init_msgs.init.p1.protocol_version = nvsp_ver; - init_pkt->msgs.init_msgs.init.protocol_version_2 = nvsp_ver; + init = vmbus_xact_req_data(xact); + init->nvs_type = HN_NVS_TYPE_INIT; + init->nvs_ver_min = nvs_ver; + init->nvs_ver_max = nvs_ver; + + vmbus_xact_activate(xact); + hn_send_ctx_init_simple(&sndc, hn_nvs_sent_xact, xact); + + error = hn_nvs_send(sc->hn_prichan, VMBUS_CHANPKT_FLAG_RC, + init, sizeof(*init), &sndc); + if (error) { + if_printf(sc->hn_ifp, "send nvs init failed: %d\n", error); + vmbus_xact_deactivate(xact); + vmbus_xact_put(xact); + return (error); + } - /* Send the init request */ - ret = vmbus_chan_send(sc->hn_prichan, - VMBUS_CHANPKT_TYPE_INBAND, VMBUS_CHANPKT_FLAG_RC, - init_pkt, sizeof(nvsp_msg), (uint64_t)(uintptr_t)init_pkt); - if (ret != 0) - return (-1); + resp = vmbus_xact_wait(xact, &resp_len); + if (resp_len < sizeof(*resp)) { + if_printf(sc->hn_ifp, "invalid init resp length %zu\n", + resp_len); + vmbus_xact_put(xact); + return (EINVAL); + } + if (resp->nvs_type != HN_NVS_TYPE_INIT_RESP) { + if_printf(sc->hn_ifp, "not init resp, type %u\n", + resp->nvs_type); + vmbus_xact_put(xact); + return (EINVAL); + } - sema_wait(&net_dev->channel_init_sema); + status = resp->nvs_status; + vmbus_xact_put(xact); - if (init_pkt->msgs.init_msgs.init_compl.status != nvsp_status_success) + if (status != HN_NVS_STATUS_OK) { + if_printf(sc->hn_ifp, "nvs init failed for ver 0x%x\n", + nvs_ver); return (EINVAL); - + } return (0); } @@ -502,33 +551,20 @@ hv_nv_negotiate_nvsp_protocol(struct hn_softc *sc, netvsc_dev *net_dev, static int hv_nv_send_ndis_config(struct hn_softc *sc, uint32_t mtu) { - netvsc_dev *net_dev; - nvsp_msg *init_pkt; - int ret; - - net_dev = hv_nv_get_outbound_net_device(sc); - if (!net_dev) - return (-ENODEV); - - /* - * Set up configuration packet, write MTU - * Indicate we are capable of handling VLAN tags - */ - init_pkt = &net_dev->channel_init_packet; - memset(init_pkt, 0, sizeof(nvsp_msg)); - init_pkt->hdr.msg_type = nvsp_msg_2_type_send_ndis_config; - init_pkt->msgs.vers_2_msgs.send_ndis_config.mtu = mtu; - init_pkt-> - msgs.vers_2_msgs.send_ndis_config.capabilities.u1.u2.ieee8021q - = 1; - - /* Send the configuration packet */ - ret = vmbus_chan_send(sc->hn_prichan, VMBUS_CHANPKT_TYPE_INBAND, 0, - init_pkt, sizeof(nvsp_msg), (uint64_t)(uintptr_t)init_pkt); - if (ret != 0) - return (-EINVAL); - - return (0); + struct hn_nvs_ndis_conf conf; + int error; + + memset(&conf, 0, sizeof(conf)); + conf.nvs_type = HN_NVS_TYPE_NDIS_CONF; + conf.nvs_mtu = mtu; + conf.nvs_caps = HN_NVS_NDIS_CONF_VLAN; + + /* NOTE: No response. */ + error = hn_nvs_send(sc->hn_prichan, VMBUS_CHANPKT_FLAG_NONE, + &conf, sizeof(conf), &hn_send_ctx_none); + if (error) + if_printf(sc->hn_ifp, "send nvs ndis conf failed: %d\n", error); + return (error); } /* @@ -538,8 +574,6 @@ static int hv_nv_connect_to_vsp(struct hn_softc *sc) { netvsc_dev *net_dev; - nvsp_msg *init_pkt; - uint32_t ndis_version; uint32_t protocol_list[] = { NVSP_PROTOCOL_VERSION_1, NVSP_PROTOCOL_VERSION_2, NVSP_PROTOCOL_VERSION_4, @@ -549,6 +583,7 @@ hv_nv_connect_to_vsp(struct hn_softc *sc) int ret = 0; device_t dev = sc->hn_dev; struct ifnet *ifp = sc->hn_ifp; + struct hn_nvs_ndis_init ndis; net_dev = hv_nv_get_outbound_net_device(sc); @@ -581,37 +616,24 @@ hv_nv_connect_to_vsp(struct hn_softc *sc) ret = hv_nv_send_ndis_config(sc, ifp->if_mtu); /* - * Send the NDIS version + * Initialize NDIS. */ - init_pkt = &net_dev->channel_init_packet; - memset(init_pkt, 0, sizeof(nvsp_msg)); - - if (net_dev->nvsp_version <= NVSP_PROTOCOL_VERSION_4) { - ndis_version = NDIS_VERSION_6_1; - } else { - ndis_version = NDIS_VERSION_6_30; - } - - init_pkt->hdr.msg_type = nvsp_msg_1_type_send_ndis_vers; - init_pkt->msgs.vers_1_msgs.send_ndis_vers.ndis_major_vers = - (ndis_version & 0xFFFF0000) >> 16; - init_pkt->msgs.vers_1_msgs.send_ndis_vers.ndis_minor_vers = - ndis_version & 0xFFFF; - - /* Send the init request */ + memset(&ndis, 0, sizeof(ndis)); + ndis.nvs_type = HN_NVS_TYPE_NDIS_INIT; + ndis.nvs_ndis_major = NDIS_VERSION_MAJOR_6; + if (net_dev->nvsp_version <= NVSP_PROTOCOL_VERSION_4) + ndis.nvs_ndis_minor = NDIS_VERSION_MINOR_1; + else + ndis.nvs_ndis_minor = NDIS_VERSION_MINOR_30; - ret = vmbus_chan_send(sc->hn_prichan, VMBUS_CHANPKT_TYPE_INBAND, 0, - init_pkt, sizeof(nvsp_msg), (uint64_t)(uintptr_t)init_pkt); + /* NOTE: No response. */ + ret = hn_nvs_send(sc->hn_prichan, VMBUS_CHANPKT_FLAG_NONE, + &ndis, sizeof(ndis), &hn_send_ctx_none); if (ret != 0) { + if_printf(sc->hn_ifp, "send nvs ndis init failed: %d\n", ret); goto cleanup; } - /* - * TODO: BUGBUG - We have to wait for the above msg since the netvsp - * uses KMCL which acknowledges packet (completion packet) - * since our Vmbus always set the VMBUS_CHANPKT_FLAG_RC flag - */ - /* sema_wait(&NetVscChannel->channel_init_sema); */ /* Post the big receive buffer to NetVSP */ if (net_dev->nvsp_version <= NVSP_PROTOCOL_VERSION_2) @@ -731,6 +753,42 @@ hv_nv_on_device_remove(struct hn_softc *sc, boolean_t destroy_channel) return (0); } +void +hn_nvs_sent_xact(struct hn_send_ctx *sndc, + struct netvsc_dev_ *net_dev __unused, struct vmbus_channel *chan __unused, + const struct nvsp_msg_ *msg, int dlen) +{ + + vmbus_xact_wakeup(sndc->hn_cbarg, msg, dlen); +} + +static void +hn_nvs_sent_none(struct hn_send_ctx *sndc __unused, + struct netvsc_dev_ *net_dev __unused, struct vmbus_channel *chan __unused, + const struct nvsp_msg_ *msg __unused, int dlen __unused) +{ + /* EMPTY */ +} + +void +hn_chim_free(struct netvsc_dev_ *net_dev, uint32_t chim_idx) +{ + u_long mask; + uint32_t idx; + + idx = chim_idx / BITS_PER_LONG; + KASSERT(idx < net_dev->bitsmap_words, + ("invalid chimney index 0x%x", chim_idx)); + + mask = 1UL << (chim_idx % BITS_PER_LONG); + KASSERT(net_dev->send_section_bitsmap[idx] & mask, + ("index bitmap 0x%lx, chimney index %u, " + "bitmap idx %d, bitmask 0x%lx", + net_dev->send_section_bitsmap[idx], chim_idx, idx, mask)); + + atomic_clear_long(&net_dev->send_section_bitsmap[idx], mask); +} + /* * Net VSC on send completion */ @@ -738,59 +796,16 @@ static void hv_nv_on_send_completion(netvsc_dev *net_dev, struct vmbus_channel *chan, const struct vmbus_chanpkt_hdr *pkt) { - const nvsp_msg *nvsp_msg_pkt; - netvsc_packet *net_vsc_pkt; - - nvsp_msg_pkt = VMBUS_CHANPKT_CONST_DATA(pkt); - - if (nvsp_msg_pkt->hdr.msg_type == nvsp_msg_type_init_complete - || nvsp_msg_pkt->hdr.msg_type - == nvsp_msg_1_type_send_rx_buf_complete - || nvsp_msg_pkt->hdr.msg_type - == nvsp_msg_1_type_send_send_buf_complete - || nvsp_msg_pkt->hdr.msg_type - == nvsp_msg5_type_subchannel) { - /* Copy the response back */ - memcpy(&net_dev->channel_init_packet, nvsp_msg_pkt, - sizeof(nvsp_msg)); - sema_post(&net_dev->channel_init_sema); - } else if (nvsp_msg_pkt->hdr.msg_type == - nvsp_msg_1_type_send_rndis_pkt_complete) { - /* Get the send context */ - net_vsc_pkt = - (netvsc_packet *)(unsigned long)pkt->cph_xactid; - if (NULL != net_vsc_pkt) { - if (net_vsc_pkt->send_buf_section_idx != - NVSP_1_CHIMNEY_SEND_INVALID_SECTION_INDEX) { - u_long mask; - int idx; - - idx = net_vsc_pkt->send_buf_section_idx / - BITS_PER_LONG; - KASSERT(idx < net_dev->bitsmap_words, - ("invalid section index %u", - net_vsc_pkt->send_buf_section_idx)); - mask = 1UL << - (net_vsc_pkt->send_buf_section_idx % - BITS_PER_LONG); - - KASSERT(net_dev->send_section_bitsmap[idx] & - mask, - ("index bitmap 0x%lx, section index %u, " - "bitmap idx %d, bitmask 0x%lx", - net_dev->send_section_bitsmap[idx], - net_vsc_pkt->send_buf_section_idx, - idx, mask)); - atomic_clear_long( - &net_dev->send_section_bitsmap[idx], mask); - } - - /* Notify the layer above us */ - net_vsc_pkt->compl.send.on_send_completion(chan, - net_vsc_pkt->compl.send.send_completion_context); + struct hn_send_ctx *sndc; - } - } + sndc = (struct hn_send_ctx *)(uintptr_t)pkt->cph_xactid; + sndc->hn_cb(sndc, net_dev, chan, VMBUS_CHANPKT_CONST_DATA(pkt), + VMBUS_CHANPKT_DATALEN(pkt)); + /* + * NOTE: + * 'sndc' CAN NOT be accessed anymore, since it can be freed by + * its callback. + */ } /* @@ -799,32 +814,23 @@ hv_nv_on_send_completion(netvsc_dev *net_dev, struct vmbus_channel *chan, * Returns 0 on success, non-zero on failure. */ int -hv_nv_on_send(struct vmbus_channel *chan, netvsc_packet *pkt) +hv_nv_on_send(struct vmbus_channel *chan, uint32_t rndis_mtype, + struct hn_send_ctx *sndc, struct vmbus_gpa *gpa, int gpa_cnt) { - nvsp_msg send_msg; + struct hn_nvs_rndis rndis; int ret; - send_msg.hdr.msg_type = nvsp_msg_1_type_send_rndis_pkt; - if (pkt->is_data_pkt) { - /* 0 is RMC_DATA */ - send_msg.msgs.vers_1_msgs.send_rndis_pkt.chan_type = 0; - } else { - /* 1 is RMC_CONTROL */ - send_msg.msgs.vers_1_msgs.send_rndis_pkt.chan_type = 1; - } + rndis.nvs_type = HN_NVS_TYPE_RNDIS; + rndis.nvs_rndis_mtype = rndis_mtype; + rndis.nvs_chim_idx = sndc->hn_chim_idx; + rndis.nvs_chim_sz = sndc->hn_chim_sz; - send_msg.msgs.vers_1_msgs.send_rndis_pkt.send_buf_section_idx = - pkt->send_buf_section_idx; - send_msg.msgs.vers_1_msgs.send_rndis_pkt.send_buf_section_size = - pkt->send_buf_section_size; - - if (pkt->gpa_cnt) { - ret = vmbus_chan_send_sglist(chan, pkt->gpa, pkt->gpa_cnt, - &send_msg, sizeof(nvsp_msg), (uint64_t)(uintptr_t)pkt); + if (gpa_cnt) { + ret = hn_nvs_send_sglist(chan, gpa, gpa_cnt, + &rndis, sizeof(rndis), sndc); } else { - ret = vmbus_chan_send(chan, - VMBUS_CHANPKT_TYPE_INBAND, VMBUS_CHANPKT_FLAG_RC, - &send_msg, sizeof(nvsp_msg), (uint64_t)(uintptr_t)pkt); + ret = hn_nvs_send(chan, VMBUS_CHANPKT_FLAG_RC, + &rndis, sizeof(rndis), sndc); } return (ret); @@ -841,19 +847,18 @@ hv_nv_on_receive(netvsc_dev *net_dev, struct hn_rx_ring *rxr, struct vmbus_channel *chan, const struct vmbus_chanpkt_hdr *pkthdr) { const struct vmbus_chanpkt_rxbuf *pkt; - const nvsp_msg *nvsp_msg_pkt; + const struct hn_nvs_hdr *nvs_hdr; netvsc_packet vsc_pkt; netvsc_packet *net_vsc_pkt = &vsc_pkt; int count = 0; int i = 0; int status = nvsp_status_success; - nvsp_msg_pkt = VMBUS_CHANPKT_CONST_DATA(pkthdr); - - /* Make sure this is a valid nvsp packet */ - if (nvsp_msg_pkt->hdr.msg_type != nvsp_msg_1_type_send_rndis_pkt) { - if_printf(rxr->hn_ifp, "packet hdr type %u is invalid!\n", - nvsp_msg_pkt->hdr.msg_type); + /* Make sure that this is a RNDIS message. */ + nvs_hdr = VMBUS_CHANPKT_CONST_DATA(pkthdr); + if (__predict_false(nvs_hdr->nvs_type != HN_NVS_TYPE_RNDIS)) { + if_printf(rxr->hn_ifp, "nvs type %u, not RNDIS\n", + nvs_hdr->nvs_type); return; } diff --git a/sys/dev/hyperv/netvsc/hv_net_vsc.h b/sys/dev/hyperv/netvsc/hv_net_vsc.h index 623166875d84..c5dc3b2b9adb 100644 --- a/sys/dev/hyperv/netvsc/hv_net_vsc.h +++ b/sys/dev/hyperv/netvsc/hv_net_vsc.h @@ -44,6 +44,7 @@ #include <sys/malloc.h> #include <sys/queue.h> #include <sys/taskqueue.h> +#include <sys/sema.h> #include <sys/sx.h> #include <machine/bus.h> @@ -1059,7 +1060,6 @@ typedef struct netvsc_dev_ { uint32_t rx_buf_size; uint32_t rx_buf_gpadl_handle; uint32_t rx_section_count; - nvsp_1_rx_buf_section *rx_sections; /* Used for NetVSP initialization protocol */ struct sema channel_init_sema; @@ -1087,6 +1087,7 @@ struct vmbus_channel; typedef void (*pfn_on_send_rx_completion)(struct vmbus_channel *, void *); #define NETVSC_DEVICE_RING_BUFFER_SIZE (128 * PAGE_SIZE) +#define NETVSC_PACKET_MAXPAGE 32 #define NETVSC_VLAN_PRIO_MASK 0xe000 #define NETVSC_VLAN_PRIO_SHIFT 13 @@ -1110,33 +1111,10 @@ typedef void (*pfn_on_send_rx_completion)(struct vmbus_channel *, void *); #endif typedef struct netvsc_packet_ { - uint8_t is_data_pkt; /* One byte */ - uint16_t vlan_tci; - uint32_t status; - - /* Completion */ - union { - struct { - uint64_t rx_completion_tid; - void *rx_completion_context; - /* This is no longer used */ - pfn_on_send_rx_completion on_rx_completion; - } rx; - struct { - uint64_t send_completion_tid; - void *send_completion_context; - /* Still used in netvsc and filter code */ - pfn_on_send_rx_completion on_send_completion; - } send; - } compl; - uint32_t send_buf_section_idx; - uint32_t send_buf_section_size; - - void *rndis_mesg; + uint16_t vlan_tci; + uint32_t status; uint32_t tot_data_buf_len; void *data; - uint32_t gpa_cnt; - struct vmbus_gpa gpa[VMBUS_CHAN_SGLIST_MAX]; } netvsc_packet; typedef struct { @@ -1214,6 +1192,9 @@ struct hn_tx_ring { bus_dma_tag_t hn_tx_data_dtag; uint64_t hn_csum_assist; + int hn_gpa_cnt; + struct vmbus_gpa hn_gpa[NETVSC_PACKET_MAXPAGE]; + u_long hn_no_txdescs; u_long hn_send_failed; u_long hn_txdma_failed; @@ -1261,19 +1242,22 @@ typedef struct hn_softc { struct taskqueue *hn_tx_taskq; struct sysctl_oid *hn_tx_sysctl_tree; struct sysctl_oid *hn_rx_sysctl_tree; + struct vmbus_xact_ctx *hn_xact; } hn_softc_t; /* * Externs */ extern int hv_promisc_mode; +struct hn_send_ctx; void netvsc_linkstatus_callback(struct hn_softc *sc, uint32_t status); netvsc_dev *hv_nv_on_device_add(struct hn_softc *sc, void *additional_info, struct hn_rx_ring *rxr); int hv_nv_on_device_remove(struct hn_softc *sc, boolean_t destroy_channel); -int hv_nv_on_send(struct vmbus_channel *chan, netvsc_packet *pkt); +int hv_nv_on_send(struct vmbus_channel *chan, uint32_t rndis_mtype, + struct hn_send_ctx *sndc, struct vmbus_gpa *gpa, int gpa_cnt); int hv_nv_get_next_send_section(netvsc_dev *net_dev); void hv_nv_subchan_attach(struct vmbus_channel *chan, struct hn_rx_ring *rxr); diff --git a/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c b/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c index adabe8c4c32b..7cf9c54f60d9 100644 --- a/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c +++ b/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c @@ -69,6 +69,7 @@ __FBSDID("$FreeBSD$"); #include <sys/queue.h> #include <sys/lock.h> #include <sys/sx.h> +#include <sys/smp.h> #include <sys/sysctl.h> #include <sys/buf_ring.h> @@ -114,6 +115,7 @@ __FBSDID("$FreeBSD$"); #include <dev/hyperv/include/hyperv.h> #include <dev/hyperv/include/hyperv_busdma.h> +#include <dev/hyperv/include/vmbus_xact.h> #include "hv_net_vsc.h" #include "hv_rndis.h" @@ -123,6 +125,9 @@ __FBSDID("$FreeBSD$"); /* Short for Hyper-V network interface */ #define NETVSC_DEVNAME "hn" +#define HN_XACT_REQ_SIZE (2 * PAGE_SIZE) +#define HN_XACT_RESP_SIZE (2 * PAGE_SIZE) + /* * It looks like offset 0 of buf is reserved to hold the softc pointer. * The sc pointer evidently not needed, and is not presently populated. @@ -151,7 +156,7 @@ __FBSDID("$FreeBSD$"); #define HN_TX_DATA_MAXSIZE IP_MAXPACKET #define HN_TX_DATA_SEGSIZE PAGE_SIZE #define HN_TX_DATA_SEGCNT_MAX \ - (VMBUS_CHAN_SGLIST_MAX - HV_RF_NUM_TX_RESERVED_PAGE_BUFS) + (NETVSC_PACKET_MAXPAGE - HV_RF_NUM_TX_RESERVED_PAGE_BUFS) #define HN_DIRECT_TX_SIZE_DEF 128 @@ -165,7 +170,7 @@ struct hn_txdesc { struct hn_tx_ring *txr; int refs; uint32_t flags; /* HN_TXD_FLAG_ */ - netvsc_packet netvsc_pkt; /* XXX to be removed */ + struct hn_send_ctx send_ctx; bus_dmamap_t data_dmap; @@ -541,6 +546,11 @@ netvsc_attach(device_t dev) IFCAP_LRO; ifp->if_hwassist = sc->hn_tx_ring[0].hn_csum_assist | CSUM_TSO; + sc->hn_xact = vmbus_xact_ctx_create(bus_get_dma_tag(dev), + HN_XACT_REQ_SIZE, HN_XACT_RESP_SIZE, 0); + if (sc->hn_xact == NULL) + goto failed; + error = hv_rf_on_device_add(sc, &device_info, ring_cnt, &sc->hn_rx_ring[0]); if (error) @@ -642,6 +652,7 @@ netvsc_detach(device_t dev) if (sc->hn_tx_taskq != hn_tx_taskq) taskqueue_free(sc->hn_tx_taskq); + vmbus_xact_ctx_destroy(sc->hn_xact); return (0); } @@ -780,14 +791,15 @@ hn_txeof(struct hn_tx_ring *txr) } static void -hn_tx_done(struct vmbus_channel *chan, void *xpkt) +hn_tx_done(struct hn_send_ctx *sndc, struct netvsc_dev_ *net_dev, + struct vmbus_channel *chan, const struct nvsp_msg_ *msg __unused, + int dlen __unused) { - netvsc_packet *packet = xpkt; - struct hn_txdesc *txd; + struct hn_txdesc *txd = sndc->hn_cbarg; struct hn_tx_ring *txr; - txd = (struct hn_txdesc *)(uintptr_t) - packet->compl.send.send_completion_tid; + if (sndc->hn_chim_idx != HN_NVS_CHIM_IDX_INVALID) + hn_chim_free(net_dev, sndc->hn_chim_idx); txr = txd->txr; KASSERT(txr->hn_chan == chan, @@ -834,16 +846,14 @@ hn_encap(struct hn_tx_ring *txr, struct hn_txdesc *txd, struct mbuf **m_head0) bus_dma_segment_t segs[HN_TX_DATA_SEGCNT_MAX]; int error, nsegs, i; struct mbuf *m_head = *m_head0; - netvsc_packet *packet; rndis_msg *rndis_mesg; rndis_packet *rndis_pkt; rndis_per_packet_info *rppi; struct rndis_hash_value *hash_value; - uint32_t rndis_msg_size; + uint32_t rndis_msg_size, tot_data_buf_len, send_buf_section_idx; + int send_buf_section_size; - packet = &txd->netvsc_pkt; - packet->is_data_pkt = TRUE; - packet->tot_data_buf_len = m_head->m_pkthdr.len; + tot_data_buf_len = m_head->m_pkthdr.len; /* * extension points to the area reserved for the @@ -858,7 +868,7 @@ hn_encap(struct hn_tx_ring *txr, struct hn_txdesc *txd, struct mbuf **m_head0) rndis_pkt = &rndis_mesg->msg.packet; rndis_pkt->data_offset = sizeof(rndis_packet); - rndis_pkt->data_length = packet->tot_data_buf_len; + rndis_pkt->data_length = tot_data_buf_len; rndis_pkt->per_pkt_info_offset = sizeof(rndis_packet); rndis_msg_size = RNDIS_MESSAGE_SIZE(rndis_packet); @@ -966,21 +976,19 @@ hn_encap(struct hn_tx_ring *txr, struct hn_txdesc *txd, struct mbuf **m_head0) } } - rndis_mesg->msg_len = packet->tot_data_buf_len + rndis_msg_size; - packet->tot_data_buf_len = rndis_mesg->msg_len; + rndis_mesg->msg_len = tot_data_buf_len + rndis_msg_size; + tot_data_buf_len = rndis_mesg->msg_len; /* * Chimney send, if the packet could fit into one chimney buffer. */ - if (packet->tot_data_buf_len < txr->hn_tx_chimney_size) { + if (tot_data_buf_len < txr->hn_tx_chimney_size) { netvsc_dev *net_dev = txr->hn_sc->net_dev; - uint32_t send_buf_section_idx; txr->hn_tx_chimney_tried++; send_buf_section_idx = hv_nv_get_next_send_section(net_dev); - if (send_buf_section_idx != - NVSP_1_CHIMNEY_SEND_INVALID_SECTION_INDEX) { + if (send_buf_section_idx != HN_NVS_CHIM_IDX_INVALID) { uint8_t *dest = ((uint8_t *)net_dev->send_buf + (send_buf_section_idx * net_dev->send_section_size)); @@ -989,10 +997,8 @@ hn_encap(struct hn_tx_ring *txr, struct hn_txdesc *txd, struct mbuf **m_head0) dest += rndis_msg_size; m_copydata(m_head, 0, m_head->m_pkthdr.len, dest); - packet->send_buf_section_idx = send_buf_section_idx; - packet->send_buf_section_size = - packet->tot_data_buf_len; - packet->gpa_cnt = 0; + send_buf_section_size = tot_data_buf_len; + txr->hn_gpa_cnt = 0; txr->hn_tx_chimney++; goto done; } @@ -1018,19 +1024,19 @@ hn_encap(struct hn_tx_ring *txr, struct hn_txdesc *txd, struct mbuf **m_head0) } *m_head0 = m_head; - packet->gpa_cnt = nsegs + HV_RF_NUM_TX_RESERVED_PAGE_BUFS; + txr->hn_gpa_cnt = nsegs + HV_RF_NUM_TX_RESERVED_PAGE_BUFS; /* send packet with page buffer */ - packet->gpa[0].gpa_page = atop(txd->rndis_msg_paddr); - packet->gpa[0].gpa_ofs = txd->rndis_msg_paddr & PAGE_MASK; - packet->gpa[0].gpa_len = rndis_msg_size; + txr->hn_gpa[0].gpa_page = atop(txd->rndis_msg_paddr); + txr->hn_gpa[0].gpa_ofs = txd->rndis_msg_paddr & PAGE_MASK; + txr->hn_gpa[0].gpa_len = rndis_msg_size; /* * Fill the page buffers with mbuf info starting at index * HV_RF_NUM_TX_RESERVED_PAGE_BUFS. */ for (i = 0; i < nsegs; ++i) { - struct vmbus_gpa *gpa = &packet->gpa[ + struct vmbus_gpa *gpa = &txr->hn_gpa[ i + HV_RF_NUM_TX_RESERVED_PAGE_BUFS]; gpa->gpa_page = atop(segs[i].ds_addr); @@ -1038,16 +1044,14 @@ hn_encap(struct hn_tx_ring *txr, struct hn_txdesc *txd, struct mbuf **m_head0) gpa->gpa_len = segs[i].ds_len; } - packet->send_buf_section_idx = - NVSP_1_CHIMNEY_SEND_INVALID_SECTION_INDEX; - packet->send_buf_section_size = 0; + send_buf_section_idx = HN_NVS_CHIM_IDX_INVALID; + send_buf_section_size = 0; done: txd->m = m_head; /* Set the completion routine */ - packet->compl.send.on_send_completion = hn_tx_done; - packet->compl.send.send_completion_context = packet; - packet->compl.send.send_completion_tid = (uint64_t)(uintptr_t)txd; + hn_send_ctx_init(&txd->send_ctx, hn_tx_done, txd, + send_buf_section_idx, send_buf_section_size); return 0; } @@ -1067,7 +1071,8 @@ again: * Make sure that txd is not freed before ETHER_BPF_MTAP. */ hn_txdesc_hold(txd); - error = hv_nv_on_send(txr->hn_chan, &txd->netvsc_pkt); + error = hv_nv_on_send(txr->hn_chan, HN_NVS_RNDIS_MTYPE_DATA, + &txd->send_ctx, txr->hn_gpa, txr->hn_gpa_cnt); if (!error) { ETHER_BPF_MTAP(ifp, txd->m); if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); diff --git a/sys/dev/hyperv/netvsc/hv_rndis.h b/sys/dev/hyperv/netvsc/hv_rndis.h index da2b408c6494..6527668de9fe 100644 --- a/sys/dev/hyperv/netvsc/hv_rndis.h +++ b/sys/dev/hyperv/netvsc/hv_rndis.h @@ -41,6 +41,10 @@ #define NDIS_VERSION_6_1 0x00060001 #define NDIS_VERSION_6_30 0x0006001e +#define NDIS_VERSION_MAJOR_6 6 +#define NDIS_VERSION_MINOR_1 1 +#define NDIS_VERSION_MINOR_30 30 + #define NDIS_VERSION (NDIS_VERSION_5_1) /* diff --git a/sys/dev/hyperv/netvsc/hv_rndis_filter.c b/sys/dev/hyperv/netvsc/hv_rndis_filter.c index 7426b8926cdb..644b017c0516 100644 --- a/sys/dev/hyperv/netvsc/hv_rndis_filter.c +++ b/sys/dev/hyperv/netvsc/hv_rndis_filter.c @@ -46,10 +46,11 @@ __FBSDID("$FreeBSD$"); #include <vm/pmap.h> #include <dev/hyperv/include/hyperv.h> -#include <dev/hyperv/vmbus/hv_vmbus_priv.h> -#include "hv_net_vsc.h" -#include "hv_rndis.h" -#include "hv_rndis_filter.h" +#include <dev/hyperv/include/vmbus_xact.h> +#include <dev/hyperv/netvsc/hv_net_vsc.h> +#include <dev/hyperv/netvsc/hv_rndis.h> +#include <dev/hyperv/netvsc/hv_rndis_filter.h> +#include <dev/hyperv/netvsc/if_hnreg.h> struct hv_rf_recvinfo { const ndis_8021q_info *vlan_info; @@ -86,11 +87,17 @@ static int hv_rf_set_packet_filter(rndis_device *device, uint32_t new_filter); static int hv_rf_init_device(rndis_device *device); static int hv_rf_open_device(rndis_device *device); static int hv_rf_close_device(rndis_device *device); -static void hv_rf_on_send_request_completion(struct vmbus_channel *, void *context); -static void hv_rf_on_send_request_halt_completion(struct vmbus_channel *, void *context); int hv_rf_send_offload_request(struct hn_softc *sc, rndis_offload_params *offloads); + +static void hn_rndis_sent_halt(struct hn_send_ctx *sndc, + struct netvsc_dev_ *net_dev, struct vmbus_channel *chan, + const struct nvsp_msg_ *msg, int dlen); +static void hn_rndis_sent_cb(struct hn_send_ctx *sndc, + struct netvsc_dev_ *net_dev, struct vmbus_channel *chan, + const struct nvsp_msg_ *msg, int dlen); + /* * Set the Per-Packet-Info with the specified type */ @@ -239,66 +246,56 @@ static int hv_rf_send_request(rndis_device *device, rndis_request *request, uint32_t message_type) { - int ret; - netvsc_packet *packet; netvsc_dev *net_dev = device->net_dev; - int send_buf_section_idx; + uint32_t send_buf_section_idx, tot_data_buf_len; + struct vmbus_gpa gpa[2]; + int gpa_cnt, send_buf_section_size; + hn_sent_callback_t cb; /* Set up the packet to send it */ - packet = &request->pkt; - - packet->is_data_pkt = FALSE; - packet->tot_data_buf_len = request->request_msg.msg_len; - packet->gpa_cnt = 1; - - packet->gpa[0].gpa_page = - hv_get_phys_addr(&request->request_msg) >> PAGE_SHIFT; - packet->gpa[0].gpa_len = request->request_msg.msg_len; - packet->gpa[0].gpa_ofs = - (unsigned long)&request->request_msg & (PAGE_SIZE - 1); - - if (packet->gpa[0].gpa_ofs + packet->gpa[0].gpa_len > PAGE_SIZE) { - packet->gpa_cnt = 2; - packet->gpa[0].gpa_len = PAGE_SIZE - packet->gpa[0].gpa_ofs; - packet->gpa[1].gpa_page = - hv_get_phys_addr((char*)&request->request_msg + - packet->gpa[0].gpa_len) >> PAGE_SHIFT; - packet->gpa[1].gpa_ofs = 0; - packet->gpa[1].gpa_len = request->request_msg.msg_len - - packet->gpa[0].gpa_len; + tot_data_buf_len = request->request_msg.msg_len; + + gpa_cnt = 1; + gpa[0].gpa_page = hv_get_phys_addr(&request->request_msg) >> PAGE_SHIFT; + gpa[0].gpa_len = request->request_msg.msg_len; + gpa[0].gpa_ofs = (unsigned long)&request->request_msg & (PAGE_SIZE - 1); + + if (gpa[0].gpa_ofs + gpa[0].gpa_len > PAGE_SIZE) { + gpa_cnt = 2; + gpa[0].gpa_len = PAGE_SIZE - gpa[0].gpa_ofs; + gpa[1].gpa_page = + hv_get_phys_addr((char*)&request->request_msg + + gpa[0].gpa_len) >> PAGE_SHIFT; + gpa[1].gpa_ofs = 0; + gpa[1].gpa_len = request->request_msg.msg_len - gpa[0].gpa_len; } - packet->compl.send.send_completion_context = request; /* packet */ - if (message_type != REMOTE_NDIS_HALT_MSG) { - packet->compl.send.on_send_completion = - hv_rf_on_send_request_completion; - } else { - packet->compl.send.on_send_completion = - hv_rf_on_send_request_halt_completion; - } - packet->compl.send.send_completion_tid = (unsigned long)device; - if (packet->tot_data_buf_len < net_dev->send_section_size) { + if (message_type != REMOTE_NDIS_HALT_MSG) + cb = hn_rndis_sent_cb; + else + cb = hn_rndis_sent_halt; + + if (tot_data_buf_len < net_dev->send_section_size) { send_buf_section_idx = hv_nv_get_next_send_section(net_dev); - if (send_buf_section_idx != - NVSP_1_CHIMNEY_SEND_INVALID_SECTION_INDEX) { + if (send_buf_section_idx != HN_NVS_CHIM_IDX_INVALID) { char *dest = ((char *)net_dev->send_buf + send_buf_section_idx * net_dev->send_section_size); memcpy(dest, &request->request_msg, request->request_msg.msg_len); - packet->send_buf_section_idx = send_buf_section_idx; - packet->send_buf_section_size = packet->tot_data_buf_len; - packet->gpa_cnt = 0; + send_buf_section_size = tot_data_buf_len; + gpa_cnt = 0; goto sendit; } /* Failed to allocate chimney send buffer; move on */ } - packet->send_buf_section_idx = NVSP_1_CHIMNEY_SEND_INVALID_SECTION_INDEX; - packet->send_buf_section_size = 0; + send_buf_section_idx = HN_NVS_CHIM_IDX_INVALID; + send_buf_section_size = 0; sendit: - ret = hv_nv_on_send(device->net_dev->sc->hn_prichan, packet); - - return (ret); + hn_send_ctx_init(&request->send_ctx, cb, request, + send_buf_section_idx, send_buf_section_size); + return hv_nv_on_send(device->net_dev->sc->hn_prichan, + HN_NVS_RNDIS_MTYPE_CTRL, &request->send_ctx, gpa, gpa_cnt); } /* @@ -1060,15 +1057,20 @@ int hv_rf_on_device_add(struct hn_softc *sc, void *additl_info, int nchan, struct hn_rx_ring *rxr) { + struct hn_send_ctx sndc; int ret; netvsc_dev *net_dev; rndis_device *rndis_dev; - nvsp_msg *init_pkt; rndis_offload_params offloads; struct rndis_recv_scale_cap rsscaps; uint32_t rsscaps_size = sizeof(struct rndis_recv_scale_cap); netvsc_device_info *dev_info = (netvsc_device_info *)additl_info; device_t dev = sc->hn_dev; + struct hn_nvs_subch_req *req; + const struct hn_nvs_subch_resp *resp; + size_t resp_len; + struct vmbus_xact *xact; + uint32_t status, nsubch; rndis_dev = hv_get_rndis_device(); if (rndis_dev == NULL) { @@ -1156,35 +1158,64 @@ hv_rf_on_device_add(struct hn_softc *sc, void *additl_info, goto out; } - /* request host to create sub channels */ - init_pkt = &net_dev->channel_init_packet; - memset(init_pkt, 0, sizeof(nvsp_msg)); - - init_pkt->hdr.msg_type = nvsp_msg5_type_subchannel; - init_pkt->msgs.vers_5_msgs.subchannel_request.op = - NVSP_SUBCHANNE_ALLOCATE; - init_pkt->msgs.vers_5_msgs.subchannel_request.num_subchannels = - net_dev->num_channel - 1; - - ret = vmbus_chan_send(sc->hn_prichan, - VMBUS_CHANPKT_TYPE_INBAND, VMBUS_CHANPKT_FLAG_RC, - init_pkt, sizeof(nvsp_msg), (uint64_t)(uintptr_t)init_pkt); - if (ret != 0) { - device_printf(dev, "Fail to allocate subchannel\n"); + /* + * Ask NVS to allocate sub-channels. + */ + xact = vmbus_xact_get(sc->hn_xact, sizeof(*req)); + if (xact == NULL) { + if_printf(sc->hn_ifp, "no xact for nvs subch req\n"); + ret = ENXIO; goto out; } - sema_wait(&net_dev->channel_init_sema); + req = vmbus_xact_req_data(xact); + req->nvs_type = HN_NVS_TYPE_SUBCH_REQ; + req->nvs_op = HN_NVS_SUBCH_OP_ALLOC; + req->nvs_nsubch = net_dev->num_channel - 1; + + hn_send_ctx_init_simple(&sndc, hn_nvs_sent_xact, xact); + vmbus_xact_activate(xact); - if (init_pkt->msgs.vers_5_msgs.subchn_complete.status != - nvsp_status_success) { - ret = ENODEV; - device_printf(dev, "sub channel complete error\n"); + ret = hn_nvs_send(sc->hn_prichan, VMBUS_CHANPKT_FLAG_RC, + req, sizeof(*req), &sndc); + if (ret != 0) { + if_printf(sc->hn_ifp, "send nvs subch req failed: %d\n", ret); + vmbus_xact_deactivate(xact); + vmbus_xact_put(xact); goto out; } - net_dev->num_channel = 1 + - init_pkt->msgs.vers_5_msgs.subchn_complete.num_subchannels; + resp = vmbus_xact_wait(xact, &resp_len); + if (resp_len < sizeof(*resp)) { + if_printf(sc->hn_ifp, "invalid subch resp length %zu\n", + resp_len); + vmbus_xact_put(xact); + ret = EINVAL; + goto out; + } + if (resp->nvs_type != HN_NVS_TYPE_SUBCH_RESP) { + if_printf(sc->hn_ifp, "not subch resp, type %u\n", + resp->nvs_type); + vmbus_xact_put(xact); + ret = EINVAL; + goto out; + } + + status = resp->nvs_status; + nsubch = resp->nvs_nsubch; + vmbus_xact_put(xact); + + if (status != HN_NVS_STATUS_OK) { + if_printf(sc->hn_ifp, "subch req failed: %x\n", status); + ret = EIO; + goto out; + } + if (nsubch > net_dev->num_channel - 1) { + if_printf(sc->hn_ifp, "%u subchans are allocated, requested %u\n", + nsubch, net_dev->num_channel - 1); + nsubch = net_dev->num_channel - 1; + } + net_dev->num_channel = nsubch + 1; ret = hv_rf_set_rss_param(rndis_dev, net_dev->num_channel); @@ -1239,23 +1270,24 @@ hv_rf_on_close(struct hn_softc *sc) return (hv_rf_close_device((rndis_device *)net_dev->extension)); } -/* - * RNDIS filter on send request completion callback - */ -static void -hv_rf_on_send_request_completion(struct vmbus_channel *chan __unused, - void *context __unused) +static void +hn_rndis_sent_cb(struct hn_send_ctx *sndc, struct netvsc_dev_ *net_dev, + struct vmbus_channel *chan __unused, const struct nvsp_msg_ *msg __unused, + int dlen __unused) { + if (sndc->hn_chim_idx != HN_NVS_CHIM_IDX_INVALID) + hn_chim_free(net_dev, sndc->hn_chim_idx); } -/* - * RNDIS filter on send request (halt only) completion callback - */ -static void -hv_rf_on_send_request_halt_completion(struct vmbus_channel *chan __unused, - void *context) +static void +hn_rndis_sent_halt(struct hn_send_ctx *sndc, struct netvsc_dev_ *net_dev, + struct vmbus_channel *chan __unused, const struct nvsp_msg_ *msg __unused, + int dlen __unused) { - rndis_request *request = context; + rndis_request *request = sndc->hn_cbarg; + + if (sndc->hn_chim_idx != HN_NVS_CHIM_IDX_INVALID) + hn_chim_free(net_dev, sndc->hn_chim_idx); /* * Notify hv_rf_halt_device() about halt completion. diff --git a/sys/dev/hyperv/netvsc/hv_rndis_filter.h b/sys/dev/hyperv/netvsc/hv_rndis_filter.h index 2f940db33e89..ebfda20b94db 100644 --- a/sys/dev/hyperv/netvsc/hv_rndis_filter.h +++ b/sys/dev/hyperv/netvsc/hv_rndis_filter.h @@ -33,6 +33,7 @@ #include <sys/param.h> #include <net/ethernet.h> +#include <dev/hyperv/netvsc/if_hnvar.h> /* * Defines @@ -75,7 +76,7 @@ typedef struct rndis_request_ { uint8_t buf_resp[PAGE_SIZE]; /* Simplify allocation by having a netvsc packet inline */ - netvsc_packet pkt; + struct hn_send_ctx send_ctx; /* * The max request size is sizeof(rndis_msg) + PAGE_SIZE. diff --git a/sys/dev/hyperv/netvsc/if_hnreg.h b/sys/dev/hyperv/netvsc/if_hnreg.h new file mode 100644 index 000000000000..7d86e9e18d02 --- /dev/null +++ b/sys/dev/hyperv/netvsc/if_hnreg.h @@ -0,0 +1,199 @@ +/*- + * Copyright (c) 2016 Microsoft Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _IF_HNREG_H_ +#define _IF_HNREG_H_ + +#include <sys/param.h> +#include <sys/systm.h> + +#define HN_NVS_RXBUF_SIG 0xcafe +#define HN_NVS_CHIM_SIG 0xface + +#define HN_NVS_CHIM_IDX_INVALID 0xffffffff + +#define HN_NVS_RNDIS_MTYPE_DATA 0 +#define HN_NVS_RNDIS_MTYPE_CTRL 1 + +/* + * NVS message transacion status codes. + */ +#define HN_NVS_STATUS_OK 1 + +/* + * NVS request/response message types. + */ +#define HN_NVS_TYPE_INIT 1 +#define HN_NVS_TYPE_INIT_RESP 2 +#define HN_NVS_TYPE_NDIS_INIT 100 +#define HN_NVS_TYPE_RXBUF_CONN 101 +#define HN_NVS_TYPE_RXBUF_CONNRESP 102 +#define HN_NVS_TYPE_RXBUF_DISCONN 103 +#define HN_NVS_TYPE_CHIM_CONN 104 +#define HN_NVS_TYPE_CHIM_CONNRESP 105 +#define HN_NVS_TYPE_CHIM_DISCONN 106 +#define HN_NVS_TYPE_RNDIS 107 +#define HN_NVS_TYPE_NDIS_CONF 125 +#define HN_NVS_TYPE_SUBCH_REQ 133 +#define HN_NVS_TYPE_SUBCH_RESP 133 /* same as SUBCH_REQ */ + +/* + * Any size less than this one will _not_ work, e.g. hn_nvs_init + * only has 12B valid data, however, if only 12B data were sent, + * Hypervisor would never reply. + */ +#define HN_NVS_REQSIZE_MIN 32 + +/* NVS message common header */ +struct hn_nvs_hdr { + uint32_t nvs_type; +} __packed; + +struct hn_nvs_init { + uint32_t nvs_type; /* HN_NVS_TYPE_INIT */ + uint32_t nvs_ver_min; + uint32_t nvs_ver_max; + uint8_t nvs_rsvd[20]; +} __packed; +CTASSERT(sizeof(struct hn_nvs_init) >= HN_NVS_REQSIZE_MIN); + +struct hn_nvs_init_resp { + uint32_t nvs_type; /* HN_NVS_TYPE_INIT_RESP */ + uint32_t nvs_ver; /* deprecated */ + uint32_t nvs_rsvd; + uint32_t nvs_status; /* HN_NVS_STATUS_ */ +} __packed; + +/* No reponse */ +struct hn_nvs_ndis_conf { + uint32_t nvs_type; /* HN_NVS_TYPE_NDIS_CONF */ + uint32_t nvs_mtu; + uint32_t nvs_rsvd; + uint64_t nvs_caps; /* HN_NVS_NDIS_CONF_ */ + uint8_t nvs_rsvd1[12]; +} __packed; +CTASSERT(sizeof(struct hn_nvs_ndis_conf) >= HN_NVS_REQSIZE_MIN); + +#define HN_NVS_NDIS_CONF_SRIOV 0x0004 +#define HN_NVS_NDIS_CONF_VLAN 0x0008 + +/* No response */ +struct hn_nvs_ndis_init { + uint32_t nvs_type; /* HN_NVS_TYPE_NDIS_INIT */ + uint32_t nvs_ndis_major; /* NDIS_VERSION_MAJOR_ */ + uint32_t nvs_ndis_minor; /* NDIS_VERSION_MINOR_ */ + uint8_t nvs_rsvd[20]; +} __packed; +CTASSERT(sizeof(struct hn_nvs_ndis_init) >= HN_NVS_REQSIZE_MIN); + +struct hn_nvs_rxbuf_conn { + uint32_t nvs_type; /* HN_NVS_TYPE_RXBUF_CONN */ + uint32_t nvs_gpadl; /* RXBUF vmbus GPADL */ + uint16_t nvs_sig; /* HN_NVS_RXBUF_SIG */ + uint8_t nvs_rsvd[22]; +} __packed; +CTASSERT(sizeof(struct hn_nvs_rxbuf_conn) >= HN_NVS_REQSIZE_MIN); + +struct hn_nvs_rxbuf_sect { + uint32_t nvs_start; + uint32_t nvs_slotsz; + uint32_t nvs_slotcnt; + uint32_t nvs_end; +} __packed; + +struct hn_nvs_rxbuf_connresp { + uint32_t nvs_type; /* HN_NVS_TYPE_RXBUF_CONNRESP */ + uint32_t nvs_status; /* HN_NVS_STATUS_ */ + uint32_t nvs_nsect; /* # of elem in nvs_sect */ + struct hn_nvs_rxbuf_sect nvs_sect[]; +} __packed; + +/* No response */ +struct hn_nvs_rxbuf_disconn { + uint32_t nvs_type; /* HN_NVS_TYPE_RXBUF_DISCONN */ + uint16_t nvs_sig; /* HN_NVS_RXBUF_SIG */ + uint8_t nvs_rsvd[26]; +} __packed; +CTASSERT(sizeof(struct hn_nvs_rxbuf_disconn) >= HN_NVS_REQSIZE_MIN); + +struct hn_nvs_chim_conn { + uint32_t nvs_type; /* HN_NVS_TYPE_CHIM_CONN */ + uint32_t nvs_gpadl; /* chimney buf vmbus GPADL */ + uint16_t nvs_sig; /* NDIS_NVS_CHIM_SIG */ + uint8_t nvs_rsvd[22]; +} __packed; +CTASSERT(sizeof(struct hn_nvs_chim_conn) >= HN_NVS_REQSIZE_MIN); + +struct hn_nvs_chim_connresp { + uint32_t nvs_type; /* HN_NVS_TYPE_CHIM_CONNRESP */ + uint32_t nvs_status; /* HN_NVS_STATUS_ */ + uint32_t nvs_sectsz; /* section size */ +} __packed; + +/* No response */ +struct hn_nvs_chim_disconn { + uint32_t nvs_type; /* HN_NVS_TYPE_CHIM_DISCONN */ + uint16_t nvs_sig; /* HN_NVS_CHIM_SIG */ + uint8_t nvs_rsvd[26]; +} __packed; +CTASSERT(sizeof(struct hn_nvs_chim_disconn) >= HN_NVS_REQSIZE_MIN); + +#define HN_NVS_SUBCH_OP_ALLOC 1 + +struct hn_nvs_subch_req { + uint32_t nvs_type; /* HN_NVS_TYPE_SUBCH_REQ */ + uint32_t nvs_op; /* HN_NVS_SUBCH_OP_ */ + uint32_t nvs_nsubch; + uint8_t nvs_rsvd[20]; +} __packed; +CTASSERT(sizeof(struct hn_nvs_subch_req) >= HN_NVS_REQSIZE_MIN); + +struct hn_nvs_subch_resp { + uint32_t nvs_type; /* HN_NVS_TYPE_SUBCH_RESP */ + uint32_t nvs_status; /* HN_NVS_STATUS_ */ + uint32_t nvs_nsubch; +} __packed; + +struct hn_nvs_rndis { + uint32_t nvs_type; /* HN_NVS_TYPE_RNDIS */ + uint32_t nvs_rndis_mtype;/* HN_NVS_RNDIS_MTYPE_ */ + /* + * Chimney sending buffer index and size. + * + * NOTE: + * If nvs_chim_idx is set to HN_NVS_CHIM_IDX_INVALID + * and nvs_chim_sz is set to 0, then chimney sending + * buffer is _not_ used by this RNDIS message. + */ + uint32_t nvs_chim_idx; + uint32_t nvs_chim_sz; + uint8_t nvs_rsvd[16]; +} __packed; +CTASSERT(sizeof(struct hn_nvs_rndis) >= HN_NVS_REQSIZE_MIN); + +#endif /* !_IF_HNREG_H_ */ diff --git a/sys/dev/hyperv/netvsc/if_hnvar.h b/sys/dev/hyperv/netvsc/if_hnvar.h new file mode 100644 index 000000000000..d6aef5fb81ca --- /dev/null +++ b/sys/dev/hyperv/netvsc/if_hnvar.h @@ -0,0 +1,104 @@ +/*- + * Copyright (c) 2016 Microsoft Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _IF_HNVAR_H_ +#define _IF_HNVAR_H_ + +#include <sys/param.h> + +#include <dev/hyperv/include/vmbus.h> +#include <dev/hyperv/netvsc/if_hnreg.h> + +struct netvsc_dev_; +struct nvsp_msg_; + +struct vmbus_channel; +struct hn_send_ctx; + +typedef void (*hn_sent_callback_t) + (struct hn_send_ctx *, struct netvsc_dev_ *, + struct vmbus_channel *, const struct nvsp_msg_ *, int); + +struct hn_send_ctx { + hn_sent_callback_t hn_cb; + void *hn_cbarg; + uint32_t hn_chim_idx; + int hn_chim_sz; +}; + +#define HN_SEND_CTX_INITIALIZER(cb, cbarg) \ +{ \ + .hn_cb = cb, \ + .hn_cbarg = cbarg, \ + .hn_chim_idx = HN_NVS_CHIM_IDX_INVALID, \ + .hn_chim_sz = 0 \ +} + +static __inline void +hn_send_ctx_init(struct hn_send_ctx *sndc, hn_sent_callback_t cb, + void *cbarg, uint32_t chim_idx, int chim_sz) +{ + + sndc->hn_cb = cb; + sndc->hn_cbarg = cbarg; + sndc->hn_chim_idx = chim_idx; + sndc->hn_chim_sz = chim_sz; +} + +static __inline void +hn_send_ctx_init_simple(struct hn_send_ctx *sndc, hn_sent_callback_t cb, + void *cbarg) +{ + + hn_send_ctx_init(sndc, cb, cbarg, HN_NVS_CHIM_IDX_INVALID, 0); +} + +static __inline int +hn_nvs_send(struct vmbus_channel *chan, uint16_t flags, + void *nvs_msg, int nvs_msglen, struct hn_send_ctx *sndc) +{ + + return (vmbus_chan_send(chan, VMBUS_CHANPKT_TYPE_INBAND, flags, + nvs_msg, nvs_msglen, (uint64_t)(uintptr_t)sndc)); +} + +static __inline int +hn_nvs_send_sglist(struct vmbus_channel *chan, struct vmbus_gpa sg[], int sglen, + void *nvs_msg, int nvs_msglen, struct hn_send_ctx *sndc) +{ + + return (vmbus_chan_send_sglist(chan, sg, sglen, nvs_msg, nvs_msglen, + (uint64_t)(uintptr_t)sndc)); +} + +void hn_nvs_sent_xact(struct hn_send_ctx *sndc, + struct netvsc_dev_ *net_dev, struct vmbus_channel *chan, + const struct nvsp_msg_ *msg, int dlen); +void hn_chim_free(struct netvsc_dev_ *net_dev, uint32_t chim_idx); + +#endif /* !_IF_HNVAR_H_ */ diff --git a/sys/dev/hyperv/storvsc/hv_storvsc_drv_freebsd.c b/sys/dev/hyperv/storvsc/hv_storvsc_drv_freebsd.c index 4a03f7719cfa..b946f87c2c8b 100644 --- a/sys/dev/hyperv/storvsc/hv_storvsc_drv_freebsd.c +++ b/sys/dev/hyperv/storvsc/hv_storvsc_drv_freebsd.c @@ -40,6 +40,7 @@ __FBSDID("$FreeBSD$"); #include <sys/condvar.h> #include <sys/time.h> #include <sys/systm.h> +#include <sys/sysctl.h> #include <sys/sockio.h> #include <sys/mbuf.h> #include <sys/malloc.h> @@ -52,6 +53,7 @@ __FBSDID("$FreeBSD$"); #include <sys/bus.h> #include <sys/mutex.h> #include <sys/callout.h> +#include <sys/smp.h> #include <vm/vm.h> #include <vm/pmap.h> #include <vm/uma.h> @@ -86,7 +88,25 @@ __FBSDID("$FreeBSD$"); #define VSTOR_PKT_SIZE (sizeof(struct vstor_packet) - vmscsi_size_delta) -#define HV_ALIGN(x, a) roundup2(x, a) +/* + * 33 segments are needed to allow 128KB maxio, in case the data + * in the first page is _not_ PAGE_SIZE aligned, e.g. + * + * |<----------- 128KB ----------->| + * | | + * 0 2K 4K 8K 16K 124K 128K 130K + * | | | | | | | | + * +--+--+-----+-----+.......+-----+--+--+ + * | | | | | | | | | DATA + * | | | | | | | | | + * +--+--+-----+-----+.......------+--+--+ + * | | | | + * | 1| 31 | 1| ...... # of segments + */ +#define STORVSC_DATA_SEGCNT_MAX 33 +#define STORVSC_DATA_SEGSZ_MAX PAGE_SIZE +#define STORVSC_DATA_SIZE_MAX \ + ((STORVSC_DATA_SEGCNT_MAX - 1) * STORVSC_DATA_SEGSZ_MAX) struct storvsc_softc; @@ -101,7 +121,7 @@ struct hv_sgl_page_pool{ boolean_t is_init; } g_hv_sgl_page_pool; -#define STORVSC_MAX_SG_PAGE_CNT STORVSC_MAX_IO_REQUESTS * VMBUS_CHAN_PRPLIST_MAX +#define STORVSC_MAX_SG_PAGE_CNT STORVSC_MAX_IO_REQUESTS * STORVSC_DATA_SEGCNT_MAX enum storvsc_request_type { WRITE_TYPE, @@ -109,26 +129,41 @@ enum storvsc_request_type { UNKNOWN_TYPE }; -struct hvs_gpa_range { +SYSCTL_NODE(_hw, OID_AUTO, storvsc, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, + "Hyper-V storage interface"); + +static u_int hv_storvsc_use_pim_unmapped = 1; +SYSCTL_INT(_hw_storvsc, OID_AUTO, use_pim_unmapped, CTLFLAG_RDTUN, + &hv_storvsc_use_pim_unmapped, 0, + "Optimize storvsc by using unmapped I/O"); + +struct hv_storvsc_sysctl { + u_long data_bio_cnt; + u_long data_vaddr_cnt; + u_long data_sg_cnt; +}; + +struct storvsc_gpa_range { struct vmbus_gpa_range gpa_range; - uint64_t gpa_page[VMBUS_CHAN_PRPLIST_MAX]; + uint64_t gpa_page[STORVSC_DATA_SEGCNT_MAX]; } __packed; struct hv_storvsc_request { - LIST_ENTRY(hv_storvsc_request) link; - struct vstor_packet vstor_packet; - int prp_cnt; - struct hvs_gpa_range prp_list; - void *sense_data; - uint8_t sense_info_len; - uint8_t retries; - union ccb *ccb; - struct storvsc_softc *softc; - struct callout callout; - struct sema synch_sema; /*Synchronize the request/response if needed */ - struct sglist *bounce_sgl; - unsigned int bounce_sgl_count; - uint64_t not_aligned_seg_bits; + LIST_ENTRY(hv_storvsc_request) link; + struct vstor_packet vstor_packet; + int prp_cnt; + struct storvsc_gpa_range prp_list; + void *sense_data; + uint8_t sense_info_len; + uint8_t retries; + union ccb *ccb; + struct storvsc_softc *softc; + struct callout callout; + struct sema synch_sema; /*Synchronize the request/response if needed */ + struct sglist *bounce_sgl; + unsigned int bounce_sgl_count; + uint64_t not_aligned_seg_bits; + bus_dmamap_t data_dmap; }; struct storvsc_softc { @@ -147,6 +182,8 @@ struct storvsc_softc { struct hv_storvsc_request hs_init_req; struct hv_storvsc_request hs_reset_req; device_t hs_dev; + bus_dma_tag_t storvsc_req_dtag; + struct hv_storvsc_sysctl sysctl_data; struct vmbus_channel *hs_cpu2chan[MAXCPU]; }; @@ -881,6 +918,77 @@ storvsc_create_cpu2chan(struct storvsc_softc *sc) } } +static int +storvsc_init_requests(device_t dev) +{ + struct storvsc_softc *sc = device_get_softc(dev); + struct hv_storvsc_request *reqp; + int error, i; + + LIST_INIT(&sc->hs_free_list); + + error = bus_dma_tag_create( + bus_get_dma_tag(dev), /* parent */ + 1, /* alignment */ + PAGE_SIZE, /* boundary */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + STORVSC_DATA_SIZE_MAX, /* maxsize */ + STORVSC_DATA_SEGCNT_MAX, /* nsegments */ + STORVSC_DATA_SEGSZ_MAX, /* maxsegsize */ + 0, /* flags */ + NULL, /* lockfunc */ + NULL, /* lockfuncarg */ + &sc->storvsc_req_dtag); + if (error) { + device_printf(dev, "failed to create storvsc dma tag\n"); + return (error); + } + + for (i = 0; i < sc->hs_drv_props->drv_max_ios_per_target; ++i) { + reqp = malloc(sizeof(struct hv_storvsc_request), + M_DEVBUF, M_WAITOK|M_ZERO); + reqp->softc = sc; + error = bus_dmamap_create(sc->storvsc_req_dtag, 0, + &reqp->data_dmap); + if (error) { + device_printf(dev, "failed to allocate storvsc " + "data dmamap\n"); + goto cleanup; + } + LIST_INSERT_HEAD(&sc->hs_free_list, reqp, link); + } + return (0); + +cleanup: + while ((reqp = LIST_FIRST(&sc->hs_free_list)) != NULL) { + LIST_REMOVE(reqp, link); + bus_dmamap_destroy(sc->storvsc_req_dtag, reqp->data_dmap); + free(reqp, M_DEVBUF); + } + return (error); +} + +static void +storvsc_sysctl(device_t dev) +{ + struct sysctl_oid_list *child; + struct sysctl_ctx_list *ctx; + struct storvsc_softc *sc; + + sc = device_get_softc(dev); + ctx = device_get_sysctl_ctx(dev); + child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev)); + + SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "data_bio_cnt", CTLFLAG_RW, + &sc->sysctl_data.data_bio_cnt, "# of bio data block"); + SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "data_vaddr_cnt", CTLFLAG_RW, + &sc->sysctl_data.data_vaddr_cnt, "# of vaddr data block"); + SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "data_sg_cnt", CTLFLAG_RW, + &sc->sysctl_data.data_sg_cnt, "# of sg data block"); +} + /** * @brief StorVSC attach function * @@ -925,16 +1033,11 @@ storvsc_attach(device_t dev) sc->hs_unit = device_get_unit(dev); sc->hs_dev = dev; - LIST_INIT(&sc->hs_free_list); mtx_init(&sc->hs_lock, "hvslck", NULL, MTX_DEF); - for (i = 0; i < sc->hs_drv_props->drv_max_ios_per_target; ++i) { - reqp = malloc(sizeof(struct hv_storvsc_request), - M_DEVBUF, M_WAITOK|M_ZERO); - reqp->softc = sc; - - LIST_INSERT_HEAD(&sc->hs_free_list, reqp, link); - } + ret = storvsc_init_requests(dev); + if (ret != 0) + goto cleanup; /* create sg-list page pool */ if (FALSE == g_hv_sgl_page_pool.is_init) { @@ -944,7 +1047,7 @@ storvsc_attach(device_t dev) /* * Pre-create SG list, each SG list with - * VMBUS_CHAN_PRPLIST_MAX segments, each + * STORVSC_DATA_SEGCNT_MAX segments, each * segment has one page buffer */ for (i = 0; i < STORVSC_MAX_IO_REQUESTS; i++) { @@ -952,10 +1055,10 @@ storvsc_attach(device_t dev) M_DEVBUF, M_WAITOK|M_ZERO); sgl_node->sgl_data = - sglist_alloc(VMBUS_CHAN_PRPLIST_MAX, + sglist_alloc(STORVSC_DATA_SEGCNT_MAX, M_WAITOK|M_ZERO); - for (j = 0; j < VMBUS_CHAN_PRPLIST_MAX; j++) { + for (j = 0; j < STORVSC_DATA_SEGCNT_MAX; j++) { tmp_buff = malloc(PAGE_SIZE, M_DEVBUF, M_WAITOK|M_ZERO); @@ -1030,6 +1133,8 @@ storvsc_attach(device_t dev) mtx_unlock(&sc->hs_lock); + storvsc_sysctl(dev); + root_mount_rel(root_mount_token); return (0); @@ -1039,13 +1144,14 @@ cleanup: while (!LIST_EMPTY(&sc->hs_free_list)) { reqp = LIST_FIRST(&sc->hs_free_list); LIST_REMOVE(reqp, link); + bus_dmamap_destroy(sc->storvsc_req_dtag, reqp->data_dmap); free(reqp, M_DEVBUF); } while (!LIST_EMPTY(&g_hv_sgl_page_pool.free_sgl_list)) { sgl_node = LIST_FIRST(&g_hv_sgl_page_pool.free_sgl_list); LIST_REMOVE(sgl_node, link); - for (j = 0; j < VMBUS_CHAN_PRPLIST_MAX; j++) { + for (j = 0; j < STORVSC_DATA_SEGCNT_MAX; j++) { if (NULL != (void*)sgl_node->sgl_data->sg_segs[j].ss_paddr) { free((void*)sgl_node->sgl_data->sg_segs[j].ss_paddr, M_DEVBUF); @@ -1100,7 +1206,7 @@ storvsc_detach(device_t dev) while (!LIST_EMPTY(&sc->hs_free_list)) { reqp = LIST_FIRST(&sc->hs_free_list); LIST_REMOVE(reqp, link); - + bus_dmamap_destroy(sc->storvsc_req_dtag, reqp->data_dmap); free(reqp, M_DEVBUF); } mtx_unlock(&sc->hs_lock); @@ -1108,7 +1214,7 @@ storvsc_detach(device_t dev) while (!LIST_EMPTY(&g_hv_sgl_page_pool.free_sgl_list)) { sgl_node = LIST_FIRST(&g_hv_sgl_page_pool.free_sgl_list); LIST_REMOVE(sgl_node, link); - for (j = 0; j < VMBUS_CHAN_PRPLIST_MAX; j++){ + for (j = 0; j < STORVSC_DATA_SEGCNT_MAX; j++){ if (NULL != (void*)sgl_node->sgl_data->sg_segs[j].ss_paddr) { free((void*)sgl_node->sgl_data->sg_segs[j].ss_paddr, M_DEVBUF); @@ -1293,6 +1399,9 @@ storvsc_action(struct cam_sim *sim, union ccb *ccb) cpi->hba_inquiry = PI_TAG_ABLE|PI_SDTR_ABLE; cpi->target_sprt = 0; cpi->hba_misc = PIM_NOBUSRESET; + if (hv_storvsc_use_pim_unmapped) + cpi->hba_misc |= PIM_UNMAPPED; + cpi->maxio = STORVSC_DATA_SIZE_MAX; cpi->hba_eng_cnt = 0; cpi->max_target = STORVSC_MAX_TARGETS; cpi->max_lun = sc->hs_drv_props->drv_max_luns_per_target; @@ -1367,6 +1476,7 @@ storvsc_action(struct cam_sim *sim, union ccb *ccb) case XPT_SCSI_IO: case XPT_IMMED_NOTIFY: { struct hv_storvsc_request *reqp = NULL; + bus_dmamap_t dmap_saved; if (ccb->csio.cdb_len == 0) { panic("cdl_len is 0\n"); @@ -1385,7 +1495,14 @@ storvsc_action(struct cam_sim *sim, union ccb *ccb) reqp = LIST_FIRST(&sc->hs_free_list); LIST_REMOVE(reqp, link); + /* Save the data_dmap before reset request */ + dmap_saved = reqp->data_dmap; + + /* XXX this is ugly */ bzero(reqp, sizeof(struct hv_storvsc_request)); + + /* Restore necessary bits */ + reqp->data_dmap = dmap_saved; reqp->softc = sc; ccb->ccb_h.status |= CAM_SIM_QUEUED; @@ -1641,6 +1758,50 @@ storvsc_check_bounce_buffer_sgl(bus_dma_segment_t *sgl, } /** + * Copy bus_dma segments to multiple page buffer, which requires + * the pages are compact composed except for the 1st and last pages. + */ +static void +storvsc_xferbuf_prepare(void *arg, bus_dma_segment_t *segs, int nsegs, int error) +{ + struct hv_storvsc_request *reqp = arg; + union ccb *ccb = reqp->ccb; + struct ccb_scsiio *csio = &ccb->csio; + struct storvsc_gpa_range *prplist; + int i; + + prplist = &reqp->prp_list; + prplist->gpa_range.gpa_len = csio->dxfer_len; + prplist->gpa_range.gpa_ofs = segs[0].ds_addr & PAGE_MASK; + + for (i = 0; i < nsegs; i++) { +#ifdef INVARIANTS + if (nsegs > 1) { + if (i == 0) { + KASSERT((segs[i].ds_addr & PAGE_MASK) + + segs[i].ds_len == PAGE_SIZE, + ("invalid 1st page, ofs 0x%jx, len %zu", + (uintmax_t)segs[i].ds_addr, + segs[i].ds_len)); + } else if (i == nsegs - 1) { + KASSERT((segs[i].ds_addr & PAGE_MASK) == 0, + ("invalid last page, ofs 0x%jx", + (uintmax_t)segs[i].ds_addr)); + } else { + KASSERT((segs[i].ds_addr & PAGE_MASK) == 0 && + segs[i].ds_len == PAGE_SIZE, + ("not a full page, ofs 0x%jx, len %zu", + (uintmax_t)segs[i].ds_addr, + segs[i].ds_len)); + } + } +#endif + prplist->gpa_page[i] = atop(segs[i].ds_addr); + } + reqp->prp_cnt = nsegs; +} + +/** * @brief Fill in a request structure based on a CAM control block * * Fills in a request structure based on the contents of a CAM control @@ -1655,11 +1816,9 @@ create_storvsc_request(union ccb *ccb, struct hv_storvsc_request *reqp) { struct ccb_scsiio *csio = &ccb->csio; uint64_t phys_addr; - uint32_t bytes_to_copy = 0; - uint32_t pfn_num = 0; uint32_t pfn; uint64_t not_aligned_seg_bits = 0; - struct hvs_gpa_range *prplist; + int error; /* refer to struct vmscsi_req for meanings of these two fields */ reqp->vstor_packet.u.vm_srb.port = @@ -1703,36 +1862,26 @@ create_storvsc_request(union ccb *ccb, struct hv_storvsc_request *reqp) return (0); } - prplist = &reqp->prp_list; - prplist->gpa_range.gpa_len = csio->dxfer_len; - switch (ccb->ccb_h.flags & CAM_DATA_MASK) { + case CAM_DATA_BIO: case CAM_DATA_VADDR: - { - bytes_to_copy = csio->dxfer_len; - phys_addr = vtophys(csio->data_ptr); - prplist->gpa_range.gpa_ofs = phys_addr & PAGE_MASK; - - while (bytes_to_copy != 0) { - int bytes, page_offset; - phys_addr = - vtophys(&csio->data_ptr[prplist->gpa_range.gpa_len - - bytes_to_copy]); - pfn = phys_addr >> PAGE_SHIFT; - prplist->gpa_page[pfn_num] = pfn; - page_offset = phys_addr & PAGE_MASK; - - bytes = min(PAGE_SIZE - page_offset, bytes_to_copy); - - bytes_to_copy -= bytes; - pfn_num++; + error = bus_dmamap_load_ccb(reqp->softc->storvsc_req_dtag, + reqp->data_dmap, ccb, storvsc_xferbuf_prepare, reqp, + BUS_DMA_NOWAIT); + if (error) { + xpt_print(ccb->ccb_h.path, + "bus_dmamap_load_ccb failed: %d\n", error); + return (error); } - reqp->prp_cnt = pfn_num; + if ((ccb->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_BIO) + reqp->softc->sysctl_data.data_bio_cnt++; + else + reqp->softc->sysctl_data.data_vaddr_cnt++; break; - } case CAM_DATA_SG: { + struct storvsc_gpa_range *prplist; int i = 0; int offset = 0; int ret; @@ -1741,13 +1890,16 @@ create_storvsc_request(union ccb *ccb, struct hv_storvsc_request *reqp) (bus_dma_segment_t *)ccb->csio.data_ptr; u_int16_t storvsc_sg_count = ccb->csio.sglist_cnt; + prplist = &reqp->prp_list; + prplist->gpa_range.gpa_len = csio->dxfer_len; + printf("Storvsc: get SG I/O operation, %d\n", reqp->vstor_packet.u.vm_srb.data_in); - if (storvsc_sg_count > VMBUS_CHAN_PRPLIST_MAX){ + if (storvsc_sg_count > STORVSC_DATA_SEGCNT_MAX){ printf("Storvsc: %d segments is too much, " "only support %d segments\n", - storvsc_sg_count, VMBUS_CHAN_PRPLIST_MAX); + storvsc_sg_count, STORVSC_DATA_SEGCNT_MAX); return (EINVAL); } @@ -1844,6 +1996,7 @@ create_storvsc_request(union ccb *ccb, struct hv_storvsc_request *reqp) reqp->bounce_sgl_count = 0; } + reqp->softc->sysctl_data.data_sg_cnt++; break; } default: @@ -1971,8 +2124,8 @@ storvsc_io_done(struct hv_storvsc_request *reqp) * For more information about INQUIRY, please refer to: * ftp://ftp.avc-pioneer.com/Mtfuji_7/Proposal/Jun09/INQUIRY.pdf */ - const struct scsi_inquiry_data *inq_data = - (const struct scsi_inquiry_data *)csio->data_ptr; + struct scsi_inquiry_data *inq_data = + (struct scsi_inquiry_data *)csio->data_ptr; uint8_t* resp_buf = (uint8_t*)csio->data_ptr; /* Get the buffer length reported by host */ int resp_xfer_len = vm_srb->transfer_len; @@ -2001,6 +2154,25 @@ storvsc_io_done(struct hv_storvsc_request *reqp) mtx_unlock(&sc->hs_lock); } } else { + char vendor[16]; + cam_strvis(vendor, inq_data->vendor, sizeof(inq_data->vendor), + sizeof(vendor)); + /** + * XXX: upgrade SPC2 to SPC3 if host is WIN8 or WIN2012 R2 + * in order to support UNMAP feature + */ + if (!strncmp(vendor,"Msft",4) && + SID_ANSI_REV(inq_data) == SCSI_REV_SPC2 && + (vmstor_proto_version == VMSTOR_PROTOCOL_VERSION_WIN8_1 || + vmstor_proto_version== VMSTOR_PROTOCOL_VERSION_WIN8)) { + inq_data->version = SCSI_REV_SPC3; + if (bootverbose) { + mtx_lock(&sc->hs_lock); + xpt_print(ccb->ccb_h.path, + "storvsc upgrades SPC2 to SPC3\n"); + mtx_unlock(&sc->hs_lock); + } + } ccb->ccb_h.status |= CAM_REQ_CMP; if (bootverbose) { mtx_lock(&sc->hs_lock); diff --git a/sys/dev/hyperv/utilities/hv_heartbeat.c b/sys/dev/hyperv/utilities/hv_heartbeat.c index 5c23f725b230..703380b6c3ce 100644 --- a/sys/dev/hyperv/utilities/hv_heartbeat.c +++ b/sys/dev/hyperv/utilities/hv_heartbeat.c @@ -33,6 +33,7 @@ #include <sys/module.h> #include <sys/timetc.h> #include <sys/syscallsubr.h> +#include <sys/systm.h> #include <dev/hyperv/include/hyperv.h> #include <dev/hyperv/include/vmbus.h> @@ -63,7 +64,7 @@ hv_heartbeat_cb(struct vmbus_channel *channel, void *context) softc = (hv_util_sc*)context; buf = softc->receive_buffer; - recvlen = PAGE_SIZE; + recvlen = softc->ic_buflen; ret = vmbus_chan_recv(channel, buf, &recvlen, &requestid); KASSERT(ret != ENOBUFS, ("hvheartbeat recvbuf is not large enough")); /* XXX check recvlen to make sure that it contains enough data */ @@ -74,8 +75,7 @@ hv_heartbeat_cb(struct vmbus_channel *channel, void *context) &buf[sizeof(struct hv_vmbus_pipe_hdr)]; if (icmsghdrp->icmsgtype == HV_ICMSGTYPE_NEGOTIATE) { - hv_negotiate_version(icmsghdrp, NULL, buf); - + hv_negotiate_version(icmsghdrp, buf); } else { heartbeat_msg = (struct hv_vmbus_heartbeat_msg_data *) @@ -109,11 +109,7 @@ hv_heartbeat_probe(device_t dev) static int hv_heartbeat_attach(device_t dev) { - hv_util_sc *softc = (hv_util_sc*)device_get_softc(dev); - - softc->callback = hv_heartbeat_cb; - - return hv_util_attach(dev); + return hv_util_attach(dev, hv_heartbeat_cb); } static device_method_t heartbeat_methods[] = { diff --git a/sys/dev/hyperv/utilities/hv_kvp.c b/sys/dev/hyperv/utilities/hv_kvp.c index 6c5d0b179b71..d83601251575 100644 --- a/sys/dev/hyperv/utilities/hv_kvp.c +++ b/sys/dev/hyperv/utilities/hv_kvp.c @@ -629,7 +629,7 @@ hv_kvp_process_request(void *context, int pending) kvp_buf = sc->util_sc.receive_buffer; channel = vmbus_get_channel(sc->dev); - recvlen = 2 * PAGE_SIZE; + recvlen = sc->util_sc.ic_buflen; ret = vmbus_chan_recv(channel, kvp_buf, &recvlen, &requestid); KASSERT(ret != ENOBUFS, ("hvkvp recvbuf is not large enough")); /* XXX check recvlen to make sure that it contains enough data */ @@ -696,7 +696,7 @@ hv_kvp_process_request(void *context, int pending) /* * Try reading next buffer */ - recvlen = 2 * PAGE_SIZE; + recvlen = sc->util_sc.ic_buflen; ret = vmbus_chan_recv(channel, kvp_buf, &recvlen, &requestid); KASSERT(ret != ENOBUFS, ("hvkvp recvbuf is not large enough")); /* XXX check recvlen to make sure that it contains enough data */ @@ -892,7 +892,6 @@ hv_kvp_attach(device_t dev) hv_kvp_sc *sc = (hv_kvp_sc*)device_get_softc(dev); - sc->util_sc.callback = hv_kvp_callback; sc->dev = dev; sema_init(&sc->dev_sema, 0, "hv_kvp device semaphore"); mtx_init(&sc->pending_mutex, "hv-kvp pending mutex", @@ -920,7 +919,7 @@ hv_kvp_attach(device_t dev) return (error); sc->hv_kvp_dev->si_drv1 = sc; - return hv_util_attach(dev); + return hv_util_attach(dev, hv_kvp_callback); } static int diff --git a/sys/dev/hyperv/utilities/hv_shutdown.c b/sys/dev/hyperv/utilities/hv_shutdown.c index 521b2dae174e..fab1a227e5d6 100644 --- a/sys/dev/hyperv/utilities/hv_shutdown.c +++ b/sys/dev/hyperv/utilities/hv_shutdown.c @@ -38,6 +38,7 @@ #include <sys/reboot.h> #include <sys/timetc.h> #include <sys/syscallsubr.h> +#include <sys/systm.h> #include <dev/hyperv/include/hyperv.h> #include <dev/hyperv/include/vmbus.h> @@ -67,7 +68,7 @@ hv_shutdown_cb(struct vmbus_channel *channel, void *context) softc = (hv_util_sc*)context; buf = softc->receive_buffer; - recv_len = PAGE_SIZE; + recv_len = softc->ic_buflen; ret = vmbus_chan_recv(channel, buf, &recv_len, &request_id); KASSERT(ret != ENOBUFS, ("hvshutdown recvbuf is not large enough")); /* XXX check recv_len to make sure that it contains enough data */ @@ -78,8 +79,7 @@ hv_shutdown_cb(struct vmbus_channel *channel, void *context) &buf[sizeof(struct hv_vmbus_pipe_hdr)]; if (icmsghdrp->icmsgtype == HV_ICMSGTYPE_NEGOTIATE) { - hv_negotiate_version(icmsghdrp, NULL, buf); - + hv_negotiate_version(icmsghdrp, buf); } else { shutdown_msg = (struct hv_vmbus_shutdown_msg_data *) @@ -131,11 +131,7 @@ hv_shutdown_probe(device_t dev) static int hv_shutdown_attach(device_t dev) { - hv_util_sc *softc = (hv_util_sc*)device_get_softc(dev); - - softc->callback = hv_shutdown_cb; - - return hv_util_attach(dev); + return hv_util_attach(dev, hv_shutdown_cb); } static device_method_t shutdown_methods[] = { diff --git a/sys/dev/hyperv/utilities/hv_timesync.c b/sys/dev/hyperv/utilities/hv_timesync.c index b46e4c799e4c..1cd7bc2274f9 100644 --- a/sys/dev/hyperv/utilities/hv_timesync.c +++ b/sys/dev/hyperv/utilities/hv_timesync.c @@ -38,6 +38,8 @@ #include <sys/reboot.h> #include <sys/timetc.h> #include <sys/syscallsubr.h> +#include <sys/systm.h> +#include <sys/taskqueue.h> #include <dev/hyperv/include/hyperv.h> #include <dev/hyperv/include/vmbus.h> @@ -143,7 +145,7 @@ hv_timesync_cb(struct vmbus_channel *channel, void *context) softc = (hv_timesync_sc*)context; time_buf = softc->util_sc.receive_buffer; - recvlen = PAGE_SIZE; + recvlen = softc->util_sc.ic_buflen; ret = vmbus_chan_recv(channel, time_buf, &recvlen, &requestId); KASSERT(ret != ENOBUFS, ("hvtimesync recvbuf is not large enough")); /* XXX check recvlen to make sure that it contains enough data */ @@ -153,7 +155,7 @@ hv_timesync_cb(struct vmbus_channel *channel, void *context) sizeof(struct hv_vmbus_pipe_hdr)]; if (icmsghdrp->icmsgtype == HV_ICMSGTYPE_NEGOTIATE) { - hv_negotiate_version(icmsghdrp, NULL, time_buf); + hv_negotiate_version(icmsghdrp, time_buf); } else { timedatap = (struct hv_ictimesync_data *) &time_buf[ sizeof(struct hv_vmbus_pipe_hdr) + @@ -187,18 +189,16 @@ hv_timesync_attach(device_t dev) { hv_timesync_sc *softc = device_get_softc(dev); - softc->util_sc.callback = hv_timesync_cb; TASK_INIT(&softc->task, 1, hv_set_host_time, softc); - - return hv_util_attach(dev); + return hv_util_attach(dev, hv_timesync_cb); } static int hv_timesync_detach(device_t dev) { hv_timesync_sc *softc = device_get_softc(dev); - taskqueue_drain(taskqueue_thread, &softc->task); + taskqueue_drain(taskqueue_thread, &softc->task); return hv_util_detach(dev); } diff --git a/sys/dev/hyperv/utilities/hv_util.c b/sys/dev/hyperv/utilities/hv_util.c index e398faa75ae6..e7ecf32ca9fb 100644 --- a/sys/dev/hyperv/utilities/hv_util.c +++ b/sys/dev/hyperv/utilities/hv_util.c @@ -44,12 +44,13 @@ #include <dev/hyperv/utilities/hv_utilreg.h> #include "hv_util.h" +#define VMBUS_IC_BRSIZE (4 * PAGE_SIZE) + void -hv_negotiate_version( - struct hv_vmbus_icmsg_hdr* icmsghdrp, - struct hv_vmbus_icmsg_negotiate* negop, - uint8_t* buf) +hv_negotiate_version(struct hv_vmbus_icmsg_hdr *icmsghdrp, uint8_t *buf) { + struct hv_vmbus_icmsg_negotiate *negop; + icmsghdrp->icmsgsize = 0x10; negop = (struct hv_vmbus_icmsg_negotiate *)&buf[ @@ -74,16 +75,15 @@ hv_negotiate_version( } int -hv_util_attach(device_t dev) +hv_util_attach(device_t dev, vmbus_chan_callback_t cb) { - struct hv_util_sc* softc; - struct vmbus_channel *chan; - int ret; + struct hv_util_sc *sc = device_get_softc(dev); + struct vmbus_channel *chan = vmbus_get_channel(dev); + int error; - softc = device_get_softc(dev); - softc->receive_buffer = - malloc(4 * PAGE_SIZE, M_DEVBUF, M_WAITOK | M_ZERO); - chan = vmbus_get_channel(dev); + sc->ic_buflen = VMBUS_IC_BRSIZE; + sc->receive_buffer = malloc(VMBUS_IC_BRSIZE, M_DEVBUF, + M_WAITOK | M_ZERO); /* * These services are not performance critical and do not need @@ -94,17 +94,13 @@ hv_util_attach(device_t dev) */ vmbus_chan_set_readbatch(chan, false); - ret = vmbus_chan_open(chan, 4 * PAGE_SIZE, 4 * PAGE_SIZE, NULL, 0, - softc->callback, softc); - - if (ret) - goto error0; - + error = vmbus_chan_open(chan, VMBUS_IC_BRSIZE, VMBUS_IC_BRSIZE, NULL, 0, + cb, sc); + if (error) { + free(sc->receive_buffer, M_DEVBUF); + return (error); + } return (0); - -error0: - free(softc->receive_buffer, M_DEVBUF); - return (ret); } int diff --git a/sys/dev/hyperv/utilities/hv_util.h b/sys/dev/hyperv/utilities/hv_util.h index 09202e79ca5e..8cc9fdaa7e62 100644 --- a/sys/dev/hyperv/utilities/hv_util.h +++ b/sys/dev/hyperv/utilities/hv_util.h @@ -31,23 +31,20 @@ #ifndef _HVUTIL_H_ #define _HVUTIL_H_ +#include <dev/hyperv/include/vmbus.h> + /** * hv_util related structures * */ typedef struct hv_util_sc { - /* - * function to process Hyper-V messages - */ - void (*callback)(struct vmbus_channel *, void *); uint8_t *receive_buffer; + int ic_buflen; } hv_util_sc; -void hv_negotiate_version( - struct hv_vmbus_icmsg_hdr* icmsghdrp, - struct hv_vmbus_icmsg_negotiate* negop, - uint8_t* buf); +void hv_negotiate_version(struct hv_vmbus_icmsg_hdr *icmsghdrp, uint8_t *buf); -int hv_util_attach(device_t dev); +int hv_util_attach(device_t dev, vmbus_chan_callback_t cb); int hv_util_detach(device_t dev); + #endif diff --git a/sys/dev/hyperv/vmbus/hv_ring_buffer.c b/sys/dev/hyperv/vmbus/hv_ring_buffer.c deleted file mode 100644 index e8dd836e9e3f..000000000000 --- a/sys/dev/hyperv/vmbus/hv_ring_buffer.c +++ /dev/null @@ -1,524 +0,0 @@ -/*- - * Copyright (c) 2009-2012,2016 Microsoft Corp. - * Copyright (c) 2012 NetApp Inc. - * Copyright (c) 2012 Citrix Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice unmodified, this list of conditions, and the following - * disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include <sys/param.h> -#include <sys/lock.h> -#include <sys/mutex.h> -#include <sys/sysctl.h> - -#include "hv_vmbus_priv.h" -#include <dev/hyperv/vmbus/vmbus_reg.h> - -/* Amount of space to write to */ -#define HV_BYTES_AVAIL_TO_WRITE(r, w, z) \ - ((w) >= (r)) ? ((z) - ((w) - (r))) : ((r) - (w)) - -static uint32_t copy_to_ring_buffer(hv_vmbus_ring_buffer_info *ring_info, - uint32_t start_write_offset, const uint8_t *src, - uint32_t src_len); -static uint32_t copy_from_ring_buffer(hv_vmbus_ring_buffer_info *ring_info, - char *dest, uint32_t dest_len, uint32_t start_read_offset); - -static int -vmbus_br_sysctl_state(SYSCTL_HANDLER_ARGS) -{ - const hv_vmbus_ring_buffer_info *br = arg1; - uint32_t rindex, windex, intr_mask, ravail, wavail; - char state[256]; - - rindex = br->ring_buffer->br_rindex; - windex = br->ring_buffer->br_windex; - intr_mask = br->ring_buffer->br_imask; - wavail = HV_BYTES_AVAIL_TO_WRITE(rindex, windex, br->ring_data_size); - ravail = br->ring_data_size - wavail; - - snprintf(state, sizeof(state), - "rindex:%u windex:%u intr_mask:%u ravail:%u wavail:%u", - rindex, windex, intr_mask, ravail, wavail); - return sysctl_handle_string(oidp, state, sizeof(state), req); -} - -/* - * Binary bufring states. - */ -static int -vmbus_br_sysctl_state_bin(SYSCTL_HANDLER_ARGS) -{ -#define BR_STATE_RIDX 0 -#define BR_STATE_WIDX 1 -#define BR_STATE_IMSK 2 -#define BR_STATE_RSPC 3 -#define BR_STATE_WSPC 4 -#define BR_STATE_MAX 5 - - const hv_vmbus_ring_buffer_info *br = arg1; - uint32_t rindex, windex, wavail, state[BR_STATE_MAX]; - - rindex = br->ring_buffer->br_rindex; - windex = br->ring_buffer->br_windex; - wavail = HV_BYTES_AVAIL_TO_WRITE(rindex, windex, br->ring_data_size); - - state[BR_STATE_RIDX] = rindex; - state[BR_STATE_WIDX] = windex; - state[BR_STATE_IMSK] = br->ring_buffer->br_imask; - state[BR_STATE_WSPC] = wavail; - state[BR_STATE_RSPC] = br->ring_data_size - wavail; - - return sysctl_handle_opaque(oidp, state, sizeof(state), req); -} - -void -vmbus_br_sysctl_create(struct sysctl_ctx_list *ctx, struct sysctl_oid *br_tree, - hv_vmbus_ring_buffer_info *br, const char *name) -{ - struct sysctl_oid *tree; - char desc[64]; - - tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(br_tree), OID_AUTO, - name, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); - if (tree == NULL) - return; - - snprintf(desc, sizeof(desc), "%s state", name); - SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "state", - CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, - br, 0, vmbus_br_sysctl_state, "A", desc); - - snprintf(desc, sizeof(desc), "%s binary state", name); - SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "state_bin", - CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, - br, 0, vmbus_br_sysctl_state_bin, "IU", desc); -} - -/** - * @brief Get number of bytes available to read and to write to - * for the specified ring buffer - */ -static __inline void -get_ring_buffer_avail_bytes(hv_vmbus_ring_buffer_info *rbi, uint32_t *read, - uint32_t *write) -{ - uint32_t read_loc, write_loc; - - /* - * Capture the read/write indices before they changed - */ - read_loc = rbi->ring_buffer->br_rindex; - write_loc = rbi->ring_buffer->br_windex; - - *write = HV_BYTES_AVAIL_TO_WRITE(read_loc, write_loc, - rbi->ring_data_size); - *read = rbi->ring_data_size - *write; -} - -/** - * @brief Get the next write location for the specified ring buffer - */ -static __inline uint32_t -get_next_write_location(hv_vmbus_ring_buffer_info *ring_info) -{ - return ring_info->ring_buffer->br_windex; -} - -/** - * @brief Set the next write location for the specified ring buffer - */ -static __inline void -set_next_write_location(hv_vmbus_ring_buffer_info *ring_info, - uint32_t next_write_location) -{ - ring_info->ring_buffer->br_windex = next_write_location; -} - -/** - * @brief Get the next read location for the specified ring buffer - */ -static __inline uint32_t -get_next_read_location(hv_vmbus_ring_buffer_info *ring_info) -{ - return ring_info->ring_buffer->br_rindex; -} - -/** - * @brief Get the next read location + offset for the specified ring buffer. - * This allows the caller to skip. - */ -static __inline uint32_t -get_next_read_location_with_offset(hv_vmbus_ring_buffer_info *ring_info, - uint32_t offset) -{ - uint32_t next = ring_info->ring_buffer->br_rindex; - - next += offset; - next %= ring_info->ring_data_size; - return (next); -} - -/** - * @brief Set the next read location for the specified ring buffer - */ -static __inline void -set_next_read_location(hv_vmbus_ring_buffer_info *ring_info, - uint32_t next_read_location) -{ - ring_info->ring_buffer->br_rindex = next_read_location; -} - -/** - * @brief Get the start of the ring buffer - */ -static __inline void * -get_ring_buffer(hv_vmbus_ring_buffer_info *ring_info) -{ - return ring_info->ring_buffer->br_data; -} - -/** - * @brief Get the size of the ring buffer. - */ -static __inline uint32_t -get_ring_buffer_size(hv_vmbus_ring_buffer_info *ring_info) -{ - return ring_info->ring_data_size; -} - -/** - * Get the read and write indices as uint64_t of the specified ring buffer. - */ -static __inline uint64_t -get_ring_buffer_indices(hv_vmbus_ring_buffer_info *ring_info) -{ - return ((uint64_t)ring_info->ring_buffer->br_windex) << 32; -} - -void -hv_ring_buffer_read_begin(hv_vmbus_ring_buffer_info *ring_info) -{ - ring_info->ring_buffer->br_imask = 1; - mb(); -} - -uint32_t -hv_ring_buffer_read_end(hv_vmbus_ring_buffer_info *ring_info) -{ - uint32_t read, write; - - ring_info->ring_buffer->br_imask = 0; - mb(); - - /* - * Now check to see if the ring buffer is still empty. - * If it is not, we raced and we need to process new - * incoming messages. - */ - get_ring_buffer_avail_bytes(ring_info, &read, &write); - return (read); -} - -/* - * When we write to the ring buffer, check if the host needs to - * be signaled. Here is the details of this protocol: - * - * 1. The host guarantees that while it is draining the - * ring buffer, it will set the interrupt_mask to - * indicate it does not need to be interrupted when - * new data is placed. - * - * 2. The host guarantees that it will completely drain - * the ring buffer before exiting the read loop. Further, - * once the ring buffer is empty, it will clear the - * interrupt_mask and re-check to see if new data has - * arrived. - */ -static boolean_t -hv_ring_buffer_needsig_on_write(uint32_t old_write_location, - hv_vmbus_ring_buffer_info *rbi) -{ - mb(); - if (rbi->ring_buffer->br_imask) - return (FALSE); - - /* Read memory barrier */ - rmb(); - /* - * This is the only case we need to signal when the - * ring transitions from being empty to non-empty. - */ - if (old_write_location == rbi->ring_buffer->br_rindex) - return (TRUE); - - return (FALSE); -} - -/** - * @brief Initialize the ring buffer. - */ -int -hv_vmbus_ring_buffer_init(hv_vmbus_ring_buffer_info *ring_info, void *buffer, - uint32_t buffer_len) -{ - memset(ring_info, 0, sizeof(hv_vmbus_ring_buffer_info)); - - ring_info->ring_buffer = buffer; - ring_info->ring_buffer->br_rindex = 0; - ring_info->ring_buffer->br_windex = 0; - - ring_info->ring_data_size = buffer_len - sizeof(struct vmbus_bufring); - mtx_init(&ring_info->ring_lock, "vmbus ring buffer", NULL, MTX_SPIN); - - return (0); -} - -/** - * @brief Cleanup the ring buffer. - */ -void -hv_ring_buffer_cleanup(hv_vmbus_ring_buffer_info *ring_info) -{ - mtx_destroy(&ring_info->ring_lock); -} - -/** - * @brief Write to the ring buffer. - */ -int -hv_ring_buffer_write(hv_vmbus_ring_buffer_info *out_ring_info, - const struct iovec iov[], uint32_t iovlen, boolean_t *need_sig) -{ - int i = 0; - uint32_t byte_avail_to_write; - uint32_t byte_avail_to_read; - uint32_t old_write_location; - uint32_t total_bytes_to_write = 0; - volatile uint32_t next_write_location; - uint64_t prev_indices = 0; - - for (i = 0; i < iovlen; i++) - total_bytes_to_write += iov[i].iov_len; - - total_bytes_to_write += sizeof(uint64_t); - - mtx_lock_spin(&out_ring_info->ring_lock); - - get_ring_buffer_avail_bytes(out_ring_info, &byte_avail_to_read, - &byte_avail_to_write); - - /* - * If there is only room for the packet, assume it is full. - * Otherwise, the next time around, we think the ring buffer - * is empty since the read index == write index - */ - if (byte_avail_to_write <= total_bytes_to_write) { - mtx_unlock_spin(&out_ring_info->ring_lock); - return (EAGAIN); - } - - /* - * Write to the ring buffer - */ - next_write_location = get_next_write_location(out_ring_info); - - old_write_location = next_write_location; - - for (i = 0; i < iovlen; i++) { - next_write_location = copy_to_ring_buffer(out_ring_info, - next_write_location, iov[i].iov_base, iov[i].iov_len); - } - - /* - * Set previous packet start - */ - prev_indices = get_ring_buffer_indices(out_ring_info); - - next_write_location = copy_to_ring_buffer(out_ring_info, - next_write_location, (char *)&prev_indices, sizeof(uint64_t)); - - /* - * Full memory barrier before upding the write index. - */ - mb(); - - /* - * Now, update the write location - */ - set_next_write_location(out_ring_info, next_write_location); - - mtx_unlock_spin(&out_ring_info->ring_lock); - - *need_sig = hv_ring_buffer_needsig_on_write(old_write_location, - out_ring_info); - - return (0); -} - -/** - * @brief Read without advancing the read index. - */ -int -hv_ring_buffer_peek(hv_vmbus_ring_buffer_info *in_ring_info, void *buffer, - uint32_t buffer_len) -{ - uint32_t bytesAvailToWrite; - uint32_t bytesAvailToRead; - uint32_t nextReadLocation = 0; - - mtx_lock_spin(&in_ring_info->ring_lock); - - get_ring_buffer_avail_bytes(in_ring_info, &bytesAvailToRead, - &bytesAvailToWrite); - - /* - * Make sure there is something to read - */ - if (bytesAvailToRead < buffer_len) { - mtx_unlock_spin(&in_ring_info->ring_lock); - return (EAGAIN); - } - - /* - * Convert to byte offset - */ - nextReadLocation = get_next_read_location(in_ring_info); - - nextReadLocation = copy_from_ring_buffer(in_ring_info, - (char *)buffer, buffer_len, nextReadLocation); - - mtx_unlock_spin(&in_ring_info->ring_lock); - - return (0); -} - -/** - * @brief Read and advance the read index. - */ -int -hv_ring_buffer_read(hv_vmbus_ring_buffer_info *in_ring_info, void *buffer, - uint32_t buffer_len, uint32_t offset) -{ - uint32_t bytes_avail_to_write; - uint32_t bytes_avail_to_read; - uint32_t next_read_location = 0; - uint64_t prev_indices = 0; - - if (buffer_len <= 0) - return (EINVAL); - - mtx_lock_spin(&in_ring_info->ring_lock); - - get_ring_buffer_avail_bytes(in_ring_info, &bytes_avail_to_read, - &bytes_avail_to_write); - - /* - * Make sure there is something to read - */ - if (bytes_avail_to_read < buffer_len) { - mtx_unlock_spin(&in_ring_info->ring_lock); - return (EAGAIN); - } - - next_read_location = get_next_read_location_with_offset(in_ring_info, - offset); - - next_read_location = copy_from_ring_buffer(in_ring_info, (char *)buffer, - buffer_len, next_read_location); - - next_read_location = copy_from_ring_buffer(in_ring_info, - (char *)&prev_indices, sizeof(uint64_t), next_read_location); - - /* - * Make sure all reads are done before we update the read index since - * the writer may start writing to the read area once the read index - * is updated. - */ - wmb(); - - /* - * Update the read index - */ - set_next_read_location(in_ring_info, next_read_location); - - mtx_unlock_spin(&in_ring_info->ring_lock); - - return (0); -} - -/** - * @brief Helper routine to copy from source to ring buffer. - * - * Assume there is enough room. Handles wrap-around in dest case only! - */ -static uint32_t -copy_to_ring_buffer(hv_vmbus_ring_buffer_info *ring_info, - uint32_t start_write_offset, const uint8_t *src, uint32_t src_len) -{ - char *ring_buffer = get_ring_buffer(ring_info); - uint32_t ring_buffer_size = get_ring_buffer_size(ring_info); - uint32_t fragLen; - - if (src_len > ring_buffer_size - start_write_offset) { - /* wrap-around detected! */ - fragLen = ring_buffer_size - start_write_offset; - memcpy(ring_buffer + start_write_offset, src, fragLen); - memcpy(ring_buffer, src + fragLen, src_len - fragLen); - } else { - memcpy(ring_buffer + start_write_offset, src, src_len); - } - - start_write_offset += src_len; - start_write_offset %= ring_buffer_size; - - return (start_write_offset); -} - -/** - * @brief Helper routine to copy to source from ring buffer. - * - * Assume there is enough room. Handles wrap-around in src case only! - */ -static uint32_t -copy_from_ring_buffer(hv_vmbus_ring_buffer_info *ring_info, char *dest, - uint32_t dest_len, uint32_t start_read_offset) -{ - uint32_t fragLen; - char *ring_buffer = get_ring_buffer(ring_info); - uint32_t ring_buffer_size = get_ring_buffer_size(ring_info); - - if (dest_len > ring_buffer_size - start_read_offset) { - /* wrap-around detected at the src */ - fragLen = ring_buffer_size - start_read_offset; - memcpy(dest, ring_buffer + start_read_offset, fragLen); - memcpy(dest + fragLen, ring_buffer, dest_len - fragLen); - } else { - memcpy(dest, ring_buffer + start_read_offset, dest_len); - } - - start_read_offset += dest_len; - start_read_offset %= ring_buffer_size; - - return (start_read_offset); -} diff --git a/sys/dev/hyperv/vmbus/hv_vmbus_priv.h b/sys/dev/hyperv/vmbus/hv_vmbus_priv.h deleted file mode 100644 index b57a96378dc7..000000000000 --- a/sys/dev/hyperv/vmbus/hv_vmbus_priv.h +++ /dev/null @@ -1,85 +0,0 @@ -/*- - * Copyright (c) 2009-2012,2016 Microsoft Corp. - * Copyright (c) 2012 NetApp Inc. - * Copyright (c) 2012 Citrix Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice unmodified, this list of conditions, and the following - * disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * $FreeBSD$ - */ - -#ifndef __HYPERV_PRIV_H__ -#define __HYPERV_PRIV_H__ - -#include <sys/param.h> -#include <sys/lock.h> -#include <sys/mutex.h> -#include <sys/sema.h> -#include <sys/_iovec.h> - -#include <dev/hyperv/vmbus/vmbus_chanvar.h> - -struct vmbus_softc; - -/* - * Private, VM Bus functions - */ -struct sysctl_ctx_list; -struct sysctl_oid; - -void vmbus_br_sysctl_create(struct sysctl_ctx_list *ctx, - struct sysctl_oid *br_tree, hv_vmbus_ring_buffer_info *br, - const char *name); - -int hv_vmbus_ring_buffer_init( - hv_vmbus_ring_buffer_info *ring_info, - void *buffer, - uint32_t buffer_len); - -void hv_ring_buffer_cleanup( - hv_vmbus_ring_buffer_info *ring_info); - -int hv_ring_buffer_write( - hv_vmbus_ring_buffer_info *ring_info, - const struct iovec iov[], - uint32_t iovlen, - boolean_t *need_sig); - -int hv_ring_buffer_peek( - hv_vmbus_ring_buffer_info *ring_info, - void *buffer, - uint32_t buffer_len); - -int hv_ring_buffer_read( - hv_vmbus_ring_buffer_info *ring_info, - void *buffer, - uint32_t buffer_len, - uint32_t offset); - -void hv_ring_buffer_read_begin( - hv_vmbus_ring_buffer_info *ring_info); - -uint32_t hv_ring_buffer_read_end( - hv_vmbus_ring_buffer_info *ring_info); - -#endif /* __HYPERV_PRIV_H__ */ diff --git a/sys/dev/hyperv/vmbus/hyperv.c b/sys/dev/hyperv/vmbus/hyperv.c index 3040e533b23a..2df313b73507 100644 --- a/sys/dev/hyperv/vmbus/hyperv.c +++ b/sys/dev/hyperv/vmbus/hyperv.c @@ -34,21 +34,14 @@ __FBSDID("$FreeBSD$"); #include <sys/param.h> #include <sys/kernel.h> -#include <sys/malloc.h> -#include <sys/pcpu.h> +#include <sys/systm.h> #include <sys/timetc.h> -#include <machine/bus.h> -#include <machine/md_var.h> -#include <vm/vm.h> -#include <vm/vm_param.h> -#include <vm/pmap.h> +#include <dev/hyperv/include/hyperv.h> #include <dev/hyperv/include/hyperv_busdma.h> -#include <dev/hyperv/vmbus/hv_vmbus_priv.h> #include <dev/hyperv/vmbus/hyperv_machdep.h> #include <dev/hyperv/vmbus/hyperv_reg.h> #include <dev/hyperv/vmbus/hyperv_var.h> -#include <dev/hyperv/vmbus/vmbus_var.h> #define HYPERV_FREEBSD_BUILD 0ULL #define HYPERV_FREEBSD_VERSION ((uint64_t)__FreeBSD_version) @@ -74,13 +67,15 @@ struct hypercall_ctx { struct hyperv_dma hc_dma; }; -static u_int hyperv_get_timecount(struct timecounter *tc); +static u_int hyperv_get_timecount(struct timecounter *); +static bool hyperv_identify(void); +static void hypercall_memfree(void); -u_int hyperv_features; -u_int hyperv_recommends; +u_int hyperv_features; +u_int hyperv_recommends; -static u_int hyperv_pm_features; -static u_int hyperv_features3; +static u_int hyperv_pm_features; +static u_int hyperv_features3; static struct timecounter hyperv_timecounter = { .tc_get_timecount = hyperv_get_timecount, diff --git a/sys/dev/hyperv/vmbus/hyperv_reg.h b/sys/dev/hyperv/vmbus/hyperv_reg.h index 060137491c60..97969585e6aa 100644 --- a/sys/dev/hyperv/vmbus/hyperv_reg.h +++ b/sys/dev/hyperv/vmbus/hyperv_reg.h @@ -30,6 +30,7 @@ #define _HYPERV_REG_H_ #include <sys/param.h> +#include <sys/systm.h> /* * Hyper-V Synthetic MSRs diff --git a/sys/dev/hyperv/vmbus/vmbus.c b/sys/dev/hyperv/vmbus/vmbus.c index 2d7f3a0da535..6753c9e9b524 100644 --- a/sys/dev/hyperv/vmbus/vmbus.c +++ b/sys/dev/hyperv/vmbus/vmbus.c @@ -38,59 +38,45 @@ __FBSDID("$FreeBSD$"); #include <sys/lock.h> #include <sys/malloc.h> #include <sys/module.h> -#include <sys/proc.h> +#include <sys/mutex.h> +#include <sys/smp.h> #include <sys/sysctl.h> -#include <sys/syslog.h> #include <sys/systm.h> -#include <sys/rtprio.h> -#include <sys/interrupt.h> -#include <sys/sx.h> #include <sys/taskqueue.h> -#include <sys/mutex.h> -#include <sys/smp.h> - -#include <machine/resource.h> -#include <sys/rman.h> -#include <machine/stdarg.h> #include <machine/intr_machdep.h> -#include <machine/md_var.h> -#include <machine/segments.h> -#include <sys/pcpu.h> -#include <x86/apicvar.h> +#include <x86/include/apicvar.h> + +#include <contrib/dev/acpica/include/acpi.h> #include <dev/hyperv/include/hyperv.h> -#include <dev/hyperv/vmbus/hv_vmbus_priv.h> +#include <dev/hyperv/include/vmbus_xact.h> #include <dev/hyperv/vmbus/hyperv_reg.h> #include <dev/hyperv/vmbus/hyperv_var.h> #include <dev/hyperv/vmbus/vmbus_reg.h> #include <dev/hyperv/vmbus/vmbus_var.h> +#include <dev/hyperv/vmbus/vmbus_chanvar.h> -#include <contrib/dev/acpica/include/acpi.h> #include "acpi_if.h" #include "vmbus_if.h" #define VMBUS_GPADL_START 0xe1e10 struct vmbus_msghc { - struct hypercall_postmsg_in *mh_inprm; + struct vmbus_xact *mh_xact; struct hypercall_postmsg_in mh_inprm_save; - struct hyperv_dma mh_inprm_dma; - - struct vmbus_message *mh_resp; - struct vmbus_message mh_resp0; -}; - -struct vmbus_msghc_ctx { - struct vmbus_msghc *mhc_free; - struct mtx mhc_free_lock; - uint32_t mhc_flags; - - struct vmbus_msghc *mhc_active; - struct mtx mhc_active_lock; }; -#define VMBUS_MSGHC_CTXF_DESTROY 0x0001 +static int vmbus_probe(device_t); +static int vmbus_attach(device_t); +static int vmbus_detach(device_t); +static int vmbus_read_ivar(device_t, device_t, int, + uintptr_t *); +static int vmbus_child_pnpinfo_str(device_t, device_t, + char *, size_t); +static uint32_t vmbus_get_version_method(device_t, device_t); +static int vmbus_probe_guid_method(device_t, device_t, + const struct hyperv_guid *); static int vmbus_init(struct vmbus_softc *); static int vmbus_connect(struct vmbus_softc *, uint32_t); @@ -104,19 +90,19 @@ static void vmbus_scan_done(struct vmbus_softc *, const struct vmbus_message *); static void vmbus_chanmsg_handle(struct vmbus_softc *, const struct vmbus_message *); - +static void vmbus_msg_task(void *, int); +static void vmbus_synic_setup(void *); +static void vmbus_synic_teardown(void *); static int vmbus_sysctl_version(SYSCTL_HANDLER_ARGS); +static int vmbus_dma_alloc(struct vmbus_softc *); +static void vmbus_dma_free(struct vmbus_softc *); +static int vmbus_intr_setup(struct vmbus_softc *); +static void vmbus_intr_teardown(struct vmbus_softc *); +static int vmbus_doattach(struct vmbus_softc *); +static void vmbus_event_proc_dummy(struct vmbus_softc *, + int); -static struct vmbus_msghc_ctx *vmbus_msghc_ctx_create(bus_dma_tag_t); -static void vmbus_msghc_ctx_destroy( - struct vmbus_msghc_ctx *); -static void vmbus_msghc_ctx_free(struct vmbus_msghc_ctx *); -static struct vmbus_msghc *vmbus_msghc_alloc(bus_dma_tag_t); -static void vmbus_msghc_free(struct vmbus_msghc *); -static struct vmbus_msghc *vmbus_msghc_get1(struct vmbus_msghc_ctx *, - uint32_t); - -struct vmbus_softc *vmbus_sc; +static struct vmbus_softc *vmbus_sc; extern inthand_t IDTVEC(vmbus_isr); @@ -133,83 +119,44 @@ vmbus_chanmsg_handlers[VMBUS_CHANMSG_TYPE_MAX] = { VMBUS_CHANMSG_PROC_WAKEUP(CONNECT_RESP) }; -static struct vmbus_msghc * -vmbus_msghc_alloc(bus_dma_tag_t parent_dtag) -{ - struct vmbus_msghc *mh; - - mh = malloc(sizeof(*mh), M_DEVBUF, M_WAITOK | M_ZERO); - - mh->mh_inprm = hyperv_dmamem_alloc(parent_dtag, - HYPERCALL_PARAM_ALIGN, 0, HYPERCALL_POSTMSGIN_SIZE, - &mh->mh_inprm_dma, BUS_DMA_WAITOK); - if (mh->mh_inprm == NULL) { - free(mh, M_DEVBUF); - return NULL; - } - return mh; -} +static device_method_t vmbus_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, vmbus_probe), + DEVMETHOD(device_attach, vmbus_attach), + DEVMETHOD(device_detach, vmbus_detach), + DEVMETHOD(device_shutdown, bus_generic_shutdown), + DEVMETHOD(device_suspend, bus_generic_suspend), + DEVMETHOD(device_resume, bus_generic_resume), -static void -vmbus_msghc_free(struct vmbus_msghc *mh) -{ - hyperv_dmamem_free(&mh->mh_inprm_dma, mh->mh_inprm); - free(mh, M_DEVBUF); -} + /* Bus interface */ + DEVMETHOD(bus_add_child, bus_generic_add_child), + DEVMETHOD(bus_print_child, bus_generic_print_child), + DEVMETHOD(bus_read_ivar, vmbus_read_ivar), + DEVMETHOD(bus_child_pnpinfo_str, vmbus_child_pnpinfo_str), -static void -vmbus_msghc_ctx_free(struct vmbus_msghc_ctx *mhc) -{ - KASSERT(mhc->mhc_active == NULL, ("still have active msg hypercall")); - KASSERT(mhc->mhc_free == NULL, ("still have hypercall msg")); + /* Vmbus interface */ + DEVMETHOD(vmbus_get_version, vmbus_get_version_method), + DEVMETHOD(vmbus_probe_guid, vmbus_probe_guid_method), - mtx_destroy(&mhc->mhc_free_lock); - mtx_destroy(&mhc->mhc_active_lock); - free(mhc, M_DEVBUF); -} + DEVMETHOD_END +}; -static struct vmbus_msghc_ctx * -vmbus_msghc_ctx_create(bus_dma_tag_t parent_dtag) -{ - struct vmbus_msghc_ctx *mhc; +static driver_t vmbus_driver = { + "vmbus", + vmbus_methods, + sizeof(struct vmbus_softc) +}; - mhc = malloc(sizeof(*mhc), M_DEVBUF, M_WAITOK | M_ZERO); - mtx_init(&mhc->mhc_free_lock, "vmbus msghc free", NULL, MTX_DEF); - mtx_init(&mhc->mhc_active_lock, "vmbus msghc act", NULL, MTX_DEF); +static devclass_t vmbus_devclass; - mhc->mhc_free = vmbus_msghc_alloc(parent_dtag); - if (mhc->mhc_free == NULL) { - vmbus_msghc_ctx_free(mhc); - return NULL; - } - return mhc; -} +DRIVER_MODULE(vmbus, acpi, vmbus_driver, vmbus_devclass, NULL, NULL); +MODULE_DEPEND(vmbus, acpi, 1, 1, 1); +MODULE_VERSION(vmbus, 1); -static struct vmbus_msghc * -vmbus_msghc_get1(struct vmbus_msghc_ctx *mhc, uint32_t dtor_flag) +static __inline struct vmbus_softc * +vmbus_get_softc(void) { - struct vmbus_msghc *mh; - - mtx_lock(&mhc->mhc_free_lock); - - while ((mhc->mhc_flags & dtor_flag) == 0 && mhc->mhc_free == NULL) { - mtx_sleep(&mhc->mhc_free, &mhc->mhc_free_lock, 0, - "gmsghc", 0); - } - if (mhc->mhc_flags & dtor_flag) { - /* Being destroyed */ - mh = NULL; - } else { - mh = mhc->mhc_free; - KASSERT(mh != NULL, ("no free hypercall msg")); - KASSERT(mh->mh_resp == NULL, - ("hypercall msg has pending response")); - mhc->mhc_free = NULL; - } - - mtx_unlock(&mhc->mhc_free_lock); - - return mh; + return vmbus_sc; } void @@ -220,7 +167,7 @@ vmbus_msghc_reset(struct vmbus_msghc *mh, size_t dsize) if (dsize > HYPERCALL_POSTMSGIN_DSIZE_MAX) panic("invalid data size %zu", dsize); - inprm = mh->mh_inprm; + inprm = vmbus_xact_req_data(mh->mh_xact); memset(inprm, 0, HYPERCALL_POSTMSGIN_SIZE); inprm->hc_connid = VMBUS_CONNID_MESSAGE; inprm->hc_msgtype = HYPERV_MSGTYPE_CHANNEL; @@ -231,63 +178,50 @@ struct vmbus_msghc * vmbus_msghc_get(struct vmbus_softc *sc, size_t dsize) { struct vmbus_msghc *mh; + struct vmbus_xact *xact; if (dsize > HYPERCALL_POSTMSGIN_DSIZE_MAX) panic("invalid data size %zu", dsize); - mh = vmbus_msghc_get1(sc->vmbus_msg_hc, VMBUS_MSGHC_CTXF_DESTROY); - if (mh == NULL) - return NULL; + xact = vmbus_xact_get(sc->vmbus_xc, + dsize + __offsetof(struct hypercall_postmsg_in, hc_data[0])); + if (xact == NULL) + return (NULL); + + mh = vmbus_xact_priv(xact, sizeof(*mh)); + mh->mh_xact = xact; vmbus_msghc_reset(mh, dsize); - return mh; + return (mh); } void -vmbus_msghc_put(struct vmbus_softc *sc, struct vmbus_msghc *mh) +vmbus_msghc_put(struct vmbus_softc *sc __unused, struct vmbus_msghc *mh) { - struct vmbus_msghc_ctx *mhc = sc->vmbus_msg_hc; - - KASSERT(mhc->mhc_active == NULL, ("msg hypercall is active")); - mh->mh_resp = NULL; - mtx_lock(&mhc->mhc_free_lock); - KASSERT(mhc->mhc_free == NULL, ("has free hypercall msg")); - mhc->mhc_free = mh; - mtx_unlock(&mhc->mhc_free_lock); - wakeup(&mhc->mhc_free); + vmbus_xact_put(mh->mh_xact); } void * vmbus_msghc_dataptr(struct vmbus_msghc *mh) { - return mh->mh_inprm->hc_data; -} - -static void -vmbus_msghc_ctx_destroy(struct vmbus_msghc_ctx *mhc) -{ - struct vmbus_msghc *mh; - - mtx_lock(&mhc->mhc_free_lock); - mhc->mhc_flags |= VMBUS_MSGHC_CTXF_DESTROY; - mtx_unlock(&mhc->mhc_free_lock); - wakeup(&mhc->mhc_free); - - mh = vmbus_msghc_get1(mhc, 0); - if (mh == NULL) - panic("can't get msghc"); + struct hypercall_postmsg_in *inprm; - vmbus_msghc_free(mh); - vmbus_msghc_ctx_free(mhc); + inprm = vmbus_xact_req_data(mh->mh_xact); + return (inprm->hc_data); } int vmbus_msghc_exec_noresult(struct vmbus_msghc *mh) { sbintime_t time = SBT_1MS; + struct hypercall_postmsg_in *inprm; + bus_addr_t inprm_paddr; int i; + inprm = vmbus_xact_req_data(mh->mh_xact); + inprm_paddr = vmbus_xact_req_paddr(mh->mh_xact); + /* * Save the input parameter so that we could restore the input * parameter if the Hypercall failed. @@ -296,7 +230,7 @@ vmbus_msghc_exec_noresult(struct vmbus_msghc *mh) * Is this really necessary?! i.e. Will the Hypercall ever * overwrite the input parameter? */ - memcpy(&mh->mh_inprm_save, mh->mh_inprm, HYPERCALL_POSTMSGIN_SIZE); + memcpy(&mh->mh_inprm_save, inprm, HYPERCALL_POSTMSGIN_SIZE); /* * In order to cope with transient failures, e.g. insufficient @@ -308,7 +242,7 @@ vmbus_msghc_exec_noresult(struct vmbus_msghc *mh) for (i = 0; i < HC_RETRY_MAX; ++i) { uint64_t status; - status = hypercall_post_message(mh->mh_inprm_dma.hv_paddr); + status = hypercall_post_message(inprm_paddr); if (status == HYPERCALL_STATUS_SUCCESS) return 0; @@ -317,8 +251,7 @@ vmbus_msghc_exec_noresult(struct vmbus_msghc *mh) time *= 2; /* Restore input parameter and try again */ - memcpy(mh->mh_inprm, &mh->mh_inprm_save, - HYPERCALL_POSTMSGIN_SIZE); + memcpy(inprm, &mh->mh_inprm_save, HYPERCALL_POSTMSGIN_SIZE); } #undef HC_RETRY_MAX @@ -327,62 +260,30 @@ vmbus_msghc_exec_noresult(struct vmbus_msghc *mh) } int -vmbus_msghc_exec(struct vmbus_softc *sc, struct vmbus_msghc *mh) +vmbus_msghc_exec(struct vmbus_softc *sc __unused, struct vmbus_msghc *mh) { - struct vmbus_msghc_ctx *mhc = sc->vmbus_msg_hc; int error; - KASSERT(mh->mh_resp == NULL, ("hypercall msg has pending response")); - - mtx_lock(&mhc->mhc_active_lock); - KASSERT(mhc->mhc_active == NULL, ("pending active msg hypercall")); - mhc->mhc_active = mh; - mtx_unlock(&mhc->mhc_active_lock); - + vmbus_xact_activate(mh->mh_xact); error = vmbus_msghc_exec_noresult(mh); - if (error) { - mtx_lock(&mhc->mhc_active_lock); - KASSERT(mhc->mhc_active == mh, ("msghc mismatch")); - mhc->mhc_active = NULL; - mtx_unlock(&mhc->mhc_active_lock); - } + if (error) + vmbus_xact_deactivate(mh->mh_xact); return error; } const struct vmbus_message * -vmbus_msghc_wait_result(struct vmbus_softc *sc, struct vmbus_msghc *mh) +vmbus_msghc_wait_result(struct vmbus_softc *sc __unused, struct vmbus_msghc *mh) { - struct vmbus_msghc_ctx *mhc = sc->vmbus_msg_hc; - - mtx_lock(&mhc->mhc_active_lock); - - KASSERT(mhc->mhc_active == mh, ("msghc mismatch")); - while (mh->mh_resp == NULL) { - mtx_sleep(&mhc->mhc_active, &mhc->mhc_active_lock, 0, - "wmsghc", 0); - } - mhc->mhc_active = NULL; + size_t resp_len; - mtx_unlock(&mhc->mhc_active_lock); - - return mh->mh_resp; + return (vmbus_xact_wait(mh->mh_xact, &resp_len)); } void vmbus_msghc_wakeup(struct vmbus_softc *sc, const struct vmbus_message *msg) { - struct vmbus_msghc_ctx *mhc = sc->vmbus_msg_hc; - struct vmbus_msghc *mh; - - mtx_lock(&mhc->mhc_active_lock); - mh = mhc->mhc_active; - KASSERT(mh != NULL, ("no pending msg hypercall")); - memcpy(&mh->mh_resp0, msg, sizeof(mh->mh_resp0)); - mh->mh_resp = &mh->mh_resp0; - - mtx_unlock(&mhc->mhc_active_lock); - wakeup(&mhc->mhc_active); + vmbus_xact_ctx_wakeup(sc->vmbus_xc, msg, sizeof(*msg)); } uint32_t @@ -1138,9 +1039,10 @@ vmbus_doattach(struct vmbus_softc *sc) /* * Create context for "post message" Hypercalls */ - sc->vmbus_msg_hc = vmbus_msghc_ctx_create( - bus_get_dma_tag(sc->vmbus_dev)); - if (sc->vmbus_msg_hc == NULL) { + sc->vmbus_xc = vmbus_xact_ctx_create(bus_get_dma_tag(sc->vmbus_dev), + HYPERCALL_POSTMSGIN_SIZE, VMBUS_MSG_SIZE, + sizeof(struct vmbus_msghc)); + if (sc->vmbus_xc == NULL) { ret = ENXIO; goto cleanup; } @@ -1195,9 +1097,9 @@ vmbus_doattach(struct vmbus_softc *sc) cleanup: vmbus_intr_teardown(sc); vmbus_dma_free(sc); - if (sc->vmbus_msg_hc != NULL) { - vmbus_msghc_ctx_destroy(sc->vmbus_msg_hc); - sc->vmbus_msg_hc = NULL; + if (sc->vmbus_xc != NULL) { + vmbus_xact_ctx_destroy(sc->vmbus_xc); + sc->vmbus_xc = NULL; } free(sc->vmbus_chmap, M_DEVBUF); mtx_destroy(&sc->vmbus_scan_lock); @@ -1239,26 +1141,6 @@ vmbus_attach(device_t dev) return (0); } -static void -vmbus_sysinit(void *arg __unused) -{ - struct vmbus_softc *sc = vmbus_get_softc(); - - if (vm_guest != VM_GUEST_HV || sc == NULL) - return; - -#ifndef EARLY_AP_STARTUP - /* - * If the system has already booted and thread - * scheduling is possible, as indicated by the - * global cold set to zero, we just call the driver - * initialization directly. - */ - if (!cold) -#endif - vmbus_doattach(sc); -} - static int vmbus_detach(device_t dev) { @@ -1276,9 +1158,9 @@ vmbus_detach(device_t dev) vmbus_intr_teardown(sc); vmbus_dma_free(sc); - if (sc->vmbus_msg_hc != NULL) { - vmbus_msghc_ctx_destroy(sc->vmbus_msg_hc); - sc->vmbus_msg_hc = NULL; + if (sc->vmbus_xc != NULL) { + vmbus_xact_ctx_destroy(sc->vmbus_xc); + sc->vmbus_xc = NULL; } free(sc->vmbus_chmap, M_DEVBUF); @@ -1288,45 +1170,30 @@ vmbus_detach(device_t dev) return (0); } -static device_method_t vmbus_methods[] = { - /* Device interface */ - DEVMETHOD(device_probe, vmbus_probe), - DEVMETHOD(device_attach, vmbus_attach), - DEVMETHOD(device_detach, vmbus_detach), - DEVMETHOD(device_shutdown, bus_generic_shutdown), - DEVMETHOD(device_suspend, bus_generic_suspend), - DEVMETHOD(device_resume, bus_generic_resume), - - /* Bus interface */ - DEVMETHOD(bus_add_child, bus_generic_add_child), - DEVMETHOD(bus_print_child, bus_generic_print_child), - DEVMETHOD(bus_read_ivar, vmbus_read_ivar), - DEVMETHOD(bus_child_pnpinfo_str, vmbus_child_pnpinfo_str), - - /* Vmbus interface */ - DEVMETHOD(vmbus_get_version, vmbus_get_version_method), - DEVMETHOD(vmbus_probe_guid, vmbus_probe_guid_method), - - DEVMETHOD_END -}; - -static driver_t vmbus_driver = { - "vmbus", - vmbus_methods, - sizeof(struct vmbus_softc) -}; +#ifndef EARLY_AP_STARTUP -static devclass_t vmbus_devclass; +static void +vmbus_sysinit(void *arg __unused) +{ + struct vmbus_softc *sc = vmbus_get_softc(); -DRIVER_MODULE(vmbus, acpi, vmbus_driver, vmbus_devclass, NULL, NULL); -MODULE_DEPEND(vmbus, acpi, 1, 1, 1); -MODULE_VERSION(vmbus, 1); + if (vm_guest != VM_GUEST_HV || sc == NULL) + return; -#ifndef EARLY_AP_STARTUP + /* + * If the system has already booted and thread + * scheduling is possible, as indicated by the + * global cold set to zero, we just call the driver + * initialization directly. + */ + if (!cold) + vmbus_doattach(sc); +} /* * NOTE: * We have to start as the last step of SI_SUB_SMP, i.e. after SMP is * initialized. */ SYSINIT(vmbus_initialize, SI_SUB_SMP, SI_ORDER_ANY, vmbus_sysinit, NULL); -#endif + +#endif /* !EARLY_AP_STARTUP */ diff --git a/sys/dev/hyperv/vmbus/vmbus_br.c b/sys/dev/hyperv/vmbus/vmbus_br.c new file mode 100644 index 000000000000..99f4f8826b17 --- /dev/null +++ b/sys/dev/hyperv/vmbus/vmbus_br.c @@ -0,0 +1,404 @@ +/*- + * Copyright (c) 2009-2012,2016 Microsoft Corp. + * Copyright (c) 2012 NetApp Inc. + * Copyright (c) 2012 Citrix Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <sys/param.h> +#include <sys/lock.h> +#include <sys/mutex.h> +#include <sys/sysctl.h> + +#include <dev/hyperv/vmbus/vmbus_reg.h> +#include <dev/hyperv/vmbus/vmbus_brvar.h> + +/* Amount of space available for write */ +#define VMBUS_BR_WAVAIL(r, w, z) \ + (((w) >= (r)) ? ((z) - ((w) - (r))) : ((r) - (w))) + +/* Increase bufing index */ +#define VMBUS_BR_IDXINC(idx, inc, sz) (((idx) + (inc)) % (sz)) + +static int vmbus_br_sysctl_state(SYSCTL_HANDLER_ARGS); +static int vmbus_br_sysctl_state_bin(SYSCTL_HANDLER_ARGS); +static void vmbus_br_setup(struct vmbus_br *, void *, int); + +static int +vmbus_br_sysctl_state(SYSCTL_HANDLER_ARGS) +{ + const struct vmbus_br *br = arg1; + uint32_t rindex, windex, imask, ravail, wavail; + char state[256]; + + rindex = br->vbr_rindex; + windex = br->vbr_windex; + imask = br->vbr_imask; + wavail = VMBUS_BR_WAVAIL(rindex, windex, br->vbr_dsize); + ravail = br->vbr_dsize - wavail; + + snprintf(state, sizeof(state), + "rindex:%u windex:%u imask:%u ravail:%u wavail:%u", + rindex, windex, imask, ravail, wavail); + return sysctl_handle_string(oidp, state, sizeof(state), req); +} + +/* + * Binary bufring states. + */ +static int +vmbus_br_sysctl_state_bin(SYSCTL_HANDLER_ARGS) +{ +#define BR_STATE_RIDX 0 +#define BR_STATE_WIDX 1 +#define BR_STATE_IMSK 2 +#define BR_STATE_RSPC 3 +#define BR_STATE_WSPC 4 +#define BR_STATE_MAX 5 + + const struct vmbus_br *br = arg1; + uint32_t rindex, windex, wavail, state[BR_STATE_MAX]; + + rindex = br->vbr_rindex; + windex = br->vbr_windex; + wavail = VMBUS_BR_WAVAIL(rindex, windex, br->vbr_dsize); + + state[BR_STATE_RIDX] = rindex; + state[BR_STATE_WIDX] = windex; + state[BR_STATE_IMSK] = br->vbr_imask; + state[BR_STATE_WSPC] = wavail; + state[BR_STATE_RSPC] = br->vbr_dsize - wavail; + + return sysctl_handle_opaque(oidp, state, sizeof(state), req); +} + +void +vmbus_br_sysctl_create(struct sysctl_ctx_list *ctx, struct sysctl_oid *br_tree, + struct vmbus_br *br, const char *name) +{ + struct sysctl_oid *tree; + char desc[64]; + + tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(br_tree), OID_AUTO, + name, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); + if (tree == NULL) + return; + + snprintf(desc, sizeof(desc), "%s state", name); + SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "state", + CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, + br, 0, vmbus_br_sysctl_state, "A", desc); + + snprintf(desc, sizeof(desc), "%s binary state", name); + SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "state_bin", + CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, + br, 0, vmbus_br_sysctl_state_bin, "IU", desc); +} + +void +vmbus_rxbr_intr_mask(struct vmbus_rxbr *rbr) +{ + rbr->rxbr_imask = 1; + mb(); +} + +static __inline uint32_t +vmbus_rxbr_avail(const struct vmbus_rxbr *rbr) +{ + uint32_t rindex, windex; + + /* Get snapshot */ + rindex = rbr->rxbr_rindex; + windex = rbr->rxbr_windex; + + return (rbr->rxbr_dsize - + VMBUS_BR_WAVAIL(rindex, windex, rbr->rxbr_dsize)); +} + +uint32_t +vmbus_rxbr_intr_unmask(struct vmbus_rxbr *rbr) +{ + rbr->rxbr_imask = 0; + mb(); + + /* + * Now check to see if the ring buffer is still empty. + * If it is not, we raced and we need to process new + * incoming channel packets. + */ + return vmbus_rxbr_avail(rbr); +} + +static void +vmbus_br_setup(struct vmbus_br *br, void *buf, int blen) +{ + br->vbr = buf; + br->vbr_dsize = blen - sizeof(struct vmbus_bufring); +} + +void +vmbus_rxbr_init(struct vmbus_rxbr *rbr) +{ + mtx_init(&rbr->rxbr_lock, "vmbus_rxbr", NULL, MTX_SPIN); +} + +void +vmbus_rxbr_deinit(struct vmbus_rxbr *rbr) +{ + mtx_destroy(&rbr->rxbr_lock); +} + +void +vmbus_rxbr_setup(struct vmbus_rxbr *rbr, void *buf, int blen) +{ + vmbus_br_setup(&rbr->rxbr, buf, blen); +} + +void +vmbus_txbr_init(struct vmbus_txbr *tbr) +{ + mtx_init(&tbr->txbr_lock, "vmbus_txbr", NULL, MTX_SPIN); +} + +void +vmbus_txbr_deinit(struct vmbus_txbr *tbr) +{ + mtx_destroy(&tbr->txbr_lock); +} + +void +vmbus_txbr_setup(struct vmbus_txbr *tbr, void *buf, int blen) +{ + vmbus_br_setup(&tbr->txbr, buf, blen); +} + +/* + * When we write to the ring buffer, check if the host needs to be + * signaled. + * + * The contract: + * - The host guarantees that while it is draining the TX bufring, + * it will set the br_imask to indicate it does not need to be + * interrupted when new data are added. + * - The host guarantees that it will completely drain the TX bufring + * before exiting the read loop. Further, once the TX bufring is + * empty, it will clear the br_imask and re-check to see if new + * data have arrived. + */ +static __inline boolean_t +vmbus_txbr_need_signal(const struct vmbus_txbr *tbr, uint32_t old_windex) +{ + mb(); + if (tbr->txbr_imask) + return (FALSE); + + __compiler_membar(); + + /* + * This is the only case we need to signal when the + * ring transitions from being empty to non-empty. + */ + if (old_windex == tbr->txbr_rindex) + return (TRUE); + + return (FALSE); +} + +static __inline uint32_t +vmbus_txbr_avail(const struct vmbus_txbr *tbr) +{ + uint32_t rindex, windex; + + /* Get snapshot */ + rindex = tbr->txbr_rindex; + windex = tbr->txbr_windex; + + return VMBUS_BR_WAVAIL(rindex, windex, tbr->txbr_dsize); +} + +static __inline uint32_t +vmbus_txbr_copyto(const struct vmbus_txbr *tbr, uint32_t windex, + const void *src0, uint32_t cplen) +{ + const uint8_t *src = src0; + uint8_t *br_data = tbr->txbr_data; + uint32_t br_dsize = tbr->txbr_dsize; + + if (cplen > br_dsize - windex) { + uint32_t fraglen = br_dsize - windex; + + /* Wrap-around detected */ + memcpy(br_data + windex, src, fraglen); + memcpy(br_data, src + fraglen, cplen - fraglen); + } else { + memcpy(br_data + windex, src, cplen); + } + return VMBUS_BR_IDXINC(windex, cplen, br_dsize); +} + +/* + * Write scattered channel packet to TX bufring. + * + * The offset of this channel packet is written as a 64bits value + * immediately after this channel packet. + */ +int +vmbus_txbr_write(struct vmbus_txbr *tbr, const struct iovec iov[], int iovlen, + boolean_t *need_sig) +{ + uint32_t old_windex, windex, total; + uint64_t save_windex; + int i; + + total = 0; + for (i = 0; i < iovlen; i++) + total += iov[i].iov_len; + total += sizeof(save_windex); + + mtx_lock_spin(&tbr->txbr_lock); + + /* + * NOTE: + * If this write is going to make br_windex same as br_rindex, + * i.e. the available space for write is same as the write size, + * we can't do it then, since br_windex == br_rindex means that + * the bufring is empty. + */ + if (vmbus_txbr_avail(tbr) <= total) { + mtx_unlock_spin(&tbr->txbr_lock); + return (EAGAIN); + } + + /* Save br_windex for later use */ + old_windex = tbr->txbr_windex; + + /* + * Copy the scattered channel packet to the TX bufring. + */ + windex = old_windex; + for (i = 0; i < iovlen; i++) { + windex = vmbus_txbr_copyto(tbr, windex, + iov[i].iov_base, iov[i].iov_len); + } + + /* + * Set the offset of the current channel packet. + */ + save_windex = ((uint64_t)old_windex) << 32; + windex = vmbus_txbr_copyto(tbr, windex, &save_windex, + sizeof(save_windex)); + + /* + * Update the write index _after_ the channel packet + * is copied. + */ + __compiler_membar(); + tbr->txbr_windex = windex; + + mtx_unlock_spin(&tbr->txbr_lock); + + *need_sig = vmbus_txbr_need_signal(tbr, old_windex); + + return (0); +} + +static __inline uint32_t +vmbus_rxbr_copyfrom(const struct vmbus_rxbr *rbr, uint32_t rindex, + void *dst0, int cplen) +{ + uint8_t *dst = dst0; + const uint8_t *br_data = rbr->rxbr_data; + uint32_t br_dsize = rbr->rxbr_dsize; + + if (cplen > br_dsize - rindex) { + uint32_t fraglen = br_dsize - rindex; + + /* Wrap-around detected. */ + memcpy(dst, br_data + rindex, fraglen); + memcpy(dst + fraglen, br_data, cplen - fraglen); + } else { + memcpy(dst, br_data + rindex, cplen); + } + return VMBUS_BR_IDXINC(rindex, cplen, br_dsize); +} + +int +vmbus_rxbr_peek(struct vmbus_rxbr *rbr, void *data, int dlen) +{ + mtx_lock_spin(&rbr->rxbr_lock); + + /* + * The requested data and the 64bits channel packet + * offset should be there at least. + */ + if (vmbus_rxbr_avail(rbr) < dlen + sizeof(uint64_t)) { + mtx_unlock_spin(&rbr->rxbr_lock); + return (EAGAIN); + } + vmbus_rxbr_copyfrom(rbr, rbr->rxbr_rindex, data, dlen); + + mtx_unlock_spin(&rbr->rxbr_lock); + + return (0); +} + +/* + * NOTE: + * We assume (dlen + skip) == sizeof(channel packet). + */ +int +vmbus_rxbr_read(struct vmbus_rxbr *rbr, void *data, int dlen, uint32_t skip) +{ + uint32_t rindex, br_dsize = rbr->rxbr_dsize; + + KASSERT(dlen + skip > 0, ("invalid dlen %d, offset %u", dlen, skip)); + + mtx_lock_spin(&rbr->rxbr_lock); + + if (vmbus_rxbr_avail(rbr) < dlen + skip + sizeof(uint64_t)) { + mtx_unlock_spin(&rbr->rxbr_lock); + return (EAGAIN); + } + + /* + * Copy channel packet from RX bufring. + */ + rindex = VMBUS_BR_IDXINC(rbr->rxbr_rindex, skip, br_dsize); + rindex = vmbus_rxbr_copyfrom(rbr, rindex, data, dlen); + + /* + * Discard this channel packet's 64bits offset, which is useless to us. + */ + rindex = VMBUS_BR_IDXINC(rindex, sizeof(uint64_t), br_dsize); + + /* + * Update the read index _after_ the channel packet is fetched. + */ + __compiler_membar(); + rbr->rxbr_rindex = rindex; + + mtx_unlock_spin(&rbr->rxbr_lock); + + return (0); +} diff --git a/sys/dev/hyperv/vmbus/vmbus_brvar.h b/sys/dev/hyperv/vmbus/vmbus_brvar.h new file mode 100644 index 000000000000..da0ca9d8f535 --- /dev/null +++ b/sys/dev/hyperv/vmbus/vmbus_brvar.h @@ -0,0 +1,104 @@ +/*- + * Copyright (c) 2009-2012,2016 Microsoft Corp. + * Copyright (c) 2012 NetApp Inc. + * Copyright (c) 2012 Citrix Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _VMBUS_BRVAR_H_ +#define _VMBUS_BRVAR_H_ + +#include <sys/param.h> +#include <sys/lock.h> +#include <sys/mutex.h> +#include <sys/_iovec.h> + +struct vmbus_br { + struct vmbus_bufring *vbr; + uint32_t vbr_dsize; /* total data size */ +}; + +#define vbr_windex vbr->br_windex +#define vbr_rindex vbr->br_rindex +#define vbr_imask vbr->br_imask +#define vbr_data vbr->br_data + +struct vmbus_rxbr { + struct mtx rxbr_lock; + struct vmbus_br rxbr; +}; + +#define rxbr_windex rxbr.vbr_windex +#define rxbr_rindex rxbr.vbr_rindex +#define rxbr_imask rxbr.vbr_imask +#define rxbr_data rxbr.vbr_data +#define rxbr_dsize rxbr.vbr_dsize + +struct vmbus_txbr { + struct mtx txbr_lock; + struct vmbus_br txbr; +}; + +#define txbr_windex txbr.vbr_windex +#define txbr_rindex txbr.vbr_rindex +#define txbr_imask txbr.vbr_imask +#define txbr_data txbr.vbr_data +#define txbr_dsize txbr.vbr_dsize + +struct sysctl_ctx_list; +struct sysctl_oid; + +static __inline int +vmbus_txbr_maxpktsz(const struct vmbus_txbr *tbr) +{ + /* + * - 64 bits for the trailing start index (- sizeof(uint64_t)). + * - The rindex and windex can't be same (- 1). See + * the comment near vmbus_bufring.br_{r,w}index. + */ + return (tbr->txbr_dsize - sizeof(uint64_t) - 1); +} + +void vmbus_br_sysctl_create(struct sysctl_ctx_list *ctx, + struct sysctl_oid *br_tree, struct vmbus_br *br, + const char *name); + +void vmbus_rxbr_init(struct vmbus_rxbr *rbr); +void vmbus_rxbr_deinit(struct vmbus_rxbr *rbr); +void vmbus_rxbr_setup(struct vmbus_rxbr *rbr, void *buf, int blen); +int vmbus_rxbr_peek(struct vmbus_rxbr *rbr, void *data, int dlen); +int vmbus_rxbr_read(struct vmbus_rxbr *rbr, void *data, int dlen, + uint32_t skip); +void vmbus_rxbr_intr_mask(struct vmbus_rxbr *rbr); +uint32_t vmbus_rxbr_intr_unmask(struct vmbus_rxbr *rbr); + +void vmbus_txbr_init(struct vmbus_txbr *tbr); +void vmbus_txbr_deinit(struct vmbus_txbr *tbr); +void vmbus_txbr_setup(struct vmbus_txbr *tbr, void *buf, int blen); +int vmbus_txbr_write(struct vmbus_txbr *tbr, + const struct iovec iov[], int iovlen, boolean_t *need_sig); + +#endif /* _VMBUS_BRVAR_H_ */ diff --git a/sys/dev/hyperv/vmbus/vmbus_chan.c b/sys/dev/hyperv/vmbus/vmbus_chan.c index b7a4b2ccd71f..bb88c0c7be4f 100644 --- a/sys/dev/hyperv/vmbus/vmbus_chan.c +++ b/sys/dev/hyperv/vmbus/vmbus_chan.c @@ -31,37 +31,44 @@ __FBSDID("$FreeBSD$"); #include <sys/param.h> #include <sys/kernel.h> -#include <sys/malloc.h> -#include <sys/systm.h> -#include <sys/mbuf.h> #include <sys/lock.h> +#include <sys/malloc.h> #include <sys/mutex.h> +#include <sys/smp.h> #include <sys/sysctl.h> +#include <sys/systm.h> #include <machine/atomic.h> -#include <machine/bus.h> - -#include <vm/vm.h> -#include <vm/vm_param.h> -#include <vm/pmap.h> #include <dev/hyperv/include/hyperv_busdma.h> -#include <dev/hyperv/vmbus/hv_vmbus_priv.h> #include <dev/hyperv/vmbus/hyperv_var.h> #include <dev/hyperv/vmbus/vmbus_reg.h> #include <dev/hyperv/vmbus/vmbus_var.h> - -static void vmbus_chan_update_evtflagcnt(struct vmbus_softc *, - const struct vmbus_channel *); - -static void vmbus_chan_task(void *, int); -static void vmbus_chan_task_nobatch(void *, int); -static void vmbus_chan_detach_task(void *, int); - -static void vmbus_chan_msgproc_choffer(struct vmbus_softc *, - const struct vmbus_message *); -static void vmbus_chan_msgproc_chrescind(struct vmbus_softc *, - const struct vmbus_message *); +#include <dev/hyperv/vmbus/vmbus_brvar.h> +#include <dev/hyperv/vmbus/vmbus_chanvar.h> + +static void vmbus_chan_update_evtflagcnt( + struct vmbus_softc *, + const struct vmbus_channel *); +static void vmbus_chan_close_internal( + struct vmbus_channel *); +static int vmbus_chan_sysctl_mnf(SYSCTL_HANDLER_ARGS); +static void vmbus_chan_sysctl_create( + struct vmbus_channel *); +static struct vmbus_channel *vmbus_chan_alloc(struct vmbus_softc *); +static void vmbus_chan_free(struct vmbus_channel *); +static int vmbus_chan_add(struct vmbus_channel *); +static void vmbus_chan_cpu_default(struct vmbus_channel *); + +static void vmbus_chan_task(void *, int); +static void vmbus_chan_task_nobatch(void *, int); +static void vmbus_chan_detach_task(void *, int); + +static void vmbus_chan_msgproc_choffer(struct vmbus_softc *, + const struct vmbus_message *); +static void vmbus_chan_msgproc_chrescind( + struct vmbus_softc *, + const struct vmbus_message *); /* * Vmbus channel message processing. @@ -177,11 +184,11 @@ vmbus_chan_sysctl_create(struct vmbus_channel *chan) /* * Create sysctl tree for RX bufring. */ - vmbus_br_sysctl_create(ctx, br_tree, &chan->ch_rxbr, "rx"); + vmbus_br_sysctl_create(ctx, br_tree, &chan->ch_rxbr.rxbr, "rx"); /* * Create sysctl tree for TX bufring. */ - vmbus_br_sysctl_create(ctx, br_tree, &chan->ch_txbr, "tx"); + vmbus_br_sysctl_create(ctx, br_tree, &chan->ch_txbr.txbr, "tx"); } } @@ -238,9 +245,9 @@ vmbus_chan_open(struct vmbus_channel *chan, int txbr_size, int rxbr_size, chan->ch_bufring = br; /* TX bufring comes first */ - hv_vmbus_ring_buffer_init(&chan->ch_txbr, br, txbr_size); + vmbus_txbr_setup(&chan->ch_txbr, br, txbr_size); /* RX bufring immediately follows TX bufring */ - hv_vmbus_ring_buffer_init(&chan->ch_rxbr, br + txbr_size, rxbr_size); + vmbus_rxbr_setup(&chan->ch_rxbr, br + txbr_size, rxbr_size); /* Create sysctl tree for this channel */ vmbus_chan_sysctl_create(chan); @@ -548,8 +555,6 @@ vmbus_chan_close_internal(struct vmbus_channel *chan) /* * Destroy the TX+RX bufrings. */ - hv_ring_buffer_cleanup(&chan->ch_txbr); - hv_ring_buffer_cleanup(&chan->ch_rxbr); if (chan->ch_bufring != NULL) { hyperv_dmamem_free(&chan->ch_bufring_dma, chan->ch_bufring); chan->ch_bufring = NULL; @@ -605,6 +610,8 @@ vmbus_chan_send(struct vmbus_channel *chan, uint16_t type, uint16_t flags, hlen = sizeof(pkt); pktlen = hlen + dlen; pad_pktlen = VMBUS_CHANPKT_TOTLEN(pktlen); + KASSERT(pad_pktlen <= vmbus_txbr_maxpktsz(&chan->ch_txbr), + ("invalid packet size %d", pad_pktlen)); pkt.cp_hdr.cph_type = type; pkt.cp_hdr.cph_flags = flags; @@ -619,7 +626,7 @@ vmbus_chan_send(struct vmbus_channel *chan, uint16_t type, uint16_t flags, iov[2].iov_base = &pad; iov[2].iov_len = pad_pktlen - pktlen; - error = hv_ring_buffer_write(&chan->ch_txbr, iov, 3, &send_evt); + error = vmbus_txbr_write(&chan->ch_txbr, iov, 3, &send_evt); if (!error && send_evt) vmbus_chan_signal_tx(chan); return error; @@ -635,12 +642,11 @@ vmbus_chan_send_sglist(struct vmbus_channel *chan, boolean_t send_evt; uint64_t pad = 0; - KASSERT(sglen < VMBUS_CHAN_SGLIST_MAX, - ("invalid sglist len %d", sglen)); - hlen = __offsetof(struct vmbus_chanpkt_sglist, cp_gpa[sglen]); pktlen = hlen + dlen; pad_pktlen = VMBUS_CHANPKT_TOTLEN(pktlen); + KASSERT(pad_pktlen <= vmbus_txbr_maxpktsz(&chan->ch_txbr), + ("invalid packet size %d", pad_pktlen)); pkt.cp_hdr.cph_type = VMBUS_CHANPKT_TYPE_GPA; pkt.cp_hdr.cph_flags = VMBUS_CHANPKT_FLAG_RC; @@ -659,7 +665,7 @@ vmbus_chan_send_sglist(struct vmbus_channel *chan, iov[3].iov_base = &pad; iov[3].iov_len = pad_pktlen - pktlen; - error = hv_ring_buffer_write(&chan->ch_txbr, iov, 4, &send_evt); + error = vmbus_txbr_write(&chan->ch_txbr, iov, 4, &send_evt); if (!error && send_evt) vmbus_chan_signal_tx(chan); return error; @@ -676,13 +682,12 @@ vmbus_chan_send_prplist(struct vmbus_channel *chan, boolean_t send_evt; uint64_t pad = 0; - KASSERT(prp_cnt < VMBUS_CHAN_PRPLIST_MAX, - ("invalid prplist entry count %d", prp_cnt)); - hlen = __offsetof(struct vmbus_chanpkt_prplist, cp_range[0].gpa_page[prp_cnt]); pktlen = hlen + dlen; pad_pktlen = VMBUS_CHANPKT_TOTLEN(pktlen); + KASSERT(pad_pktlen <= vmbus_txbr_maxpktsz(&chan->ch_txbr), + ("invalid packet size %d", pad_pktlen)); pkt.cp_hdr.cph_type = VMBUS_CHANPKT_TYPE_GPA; pkt.cp_hdr.cph_flags = VMBUS_CHANPKT_FLAG_RC; @@ -701,7 +706,7 @@ vmbus_chan_send_prplist(struct vmbus_channel *chan, iov[3].iov_base = &pad; iov[3].iov_len = pad_pktlen - pktlen; - error = hv_ring_buffer_write(&chan->ch_txbr, iov, 4, &send_evt); + error = vmbus_txbr_write(&chan->ch_txbr, iov, 4, &send_evt); if (!error && send_evt) vmbus_chan_signal_tx(chan); return error; @@ -714,7 +719,7 @@ vmbus_chan_recv(struct vmbus_channel *chan, void *data, int *dlen0, struct vmbus_chanpkt_hdr pkt; int error, dlen, hlen; - error = hv_ring_buffer_peek(&chan->ch_rxbr, &pkt, sizeof(pkt)); + error = vmbus_rxbr_peek(&chan->ch_rxbr, &pkt, sizeof(pkt)); if (error) return error; @@ -731,8 +736,8 @@ vmbus_chan_recv(struct vmbus_channel *chan, void *data, int *dlen0, *dlen0 = dlen; /* Skip packet header */ - error = hv_ring_buffer_read(&chan->ch_rxbr, data, dlen, hlen); - KASSERT(!error, ("hv_ring_buffer_read failed")); + error = vmbus_rxbr_read(&chan->ch_rxbr, data, dlen, hlen); + KASSERT(!error, ("vmbus_rxbr_read failed")); return 0; } @@ -744,7 +749,7 @@ vmbus_chan_recv_pkt(struct vmbus_channel *chan, struct vmbus_chanpkt_hdr pkt; int error, pktlen; - error = hv_ring_buffer_peek(&chan->ch_rxbr, &pkt, sizeof(pkt)); + error = vmbus_rxbr_peek(&chan->ch_rxbr, &pkt, sizeof(pkt)); if (error) return error; @@ -757,8 +762,8 @@ vmbus_chan_recv_pkt(struct vmbus_channel *chan, *pktlen0 = pktlen; /* Include packet header */ - error = hv_ring_buffer_read(&chan->ch_rxbr, pkt0, pktlen, 0); - KASSERT(!error, ("hv_ring_buffer_read failed")); + error = vmbus_rxbr_read(&chan->ch_rxbr, pkt0, pktlen, 0); + KASSERT(!error, ("vmbus_rxbr_read failed")); return 0; } @@ -787,12 +792,12 @@ vmbus_chan_task(void *xchan, int pending __unused) cb(chan, cbarg); - left = hv_ring_buffer_read_end(&chan->ch_rxbr); + left = vmbus_rxbr_intr_unmask(&chan->ch_rxbr); if (left == 0) { /* No more data in RX bufring; done */ break; } - hv_ring_buffer_read_begin(&chan->ch_rxbr); + vmbus_rxbr_intr_mask(&chan->ch_rxbr); } } @@ -834,7 +839,7 @@ vmbus_event_flags_proc(struct vmbus_softc *sc, volatile u_long *event_flags, continue; if (chan->ch_flags & VMBUS_CHAN_FLAG_BATCHREAD) - hv_ring_buffer_read_begin(&chan->ch_rxbr); + vmbus_rxbr_intr_mask(&chan->ch_rxbr); taskqueue_enqueue(chan->ch_tq, &chan->ch_task); } } @@ -913,6 +918,8 @@ vmbus_chan_alloc(struct vmbus_softc *sc) mtx_init(&chan->ch_subchan_lock, "vmbus subchan", NULL, MTX_DEF); TAILQ_INIT(&chan->ch_subchans); TASK_INIT(&chan->ch_detach_task, 0, vmbus_chan_detach_task, chan); + vmbus_rxbr_init(&chan->ch_rxbr); + vmbus_txbr_init(&chan->ch_txbr); return chan; } @@ -925,6 +932,8 @@ vmbus_chan_free(struct vmbus_channel *chan) /* TODO: asset no longer on the vmbus channel list */ hyperv_dmamem_free(&chan->ch_monprm_dma, chan->ch_monprm); mtx_destroy(&chan->ch_subchan_lock); + vmbus_rxbr_deinit(&chan->ch_rxbr); + vmbus_txbr_deinit(&chan->ch_txbr); free(chan, M_DEVBUF); } diff --git a/sys/dev/hyperv/vmbus/vmbus_chanvar.h b/sys/dev/hyperv/vmbus/vmbus_chanvar.h index 593d09d2974b..68a134d3f2fb 100644 --- a/sys/dev/hyperv/vmbus/vmbus_chanvar.h +++ b/sys/dev/hyperv/vmbus/vmbus_chanvar.h @@ -39,12 +39,7 @@ #include <dev/hyperv/include/hyperv.h> #include <dev/hyperv/include/hyperv_busdma.h> #include <dev/hyperv/include/vmbus.h> - -typedef struct { - struct vmbus_bufring *ring_buffer; - struct mtx ring_lock; - uint32_t ring_data_size; /* ring_size */ -} hv_vmbus_ring_buffer_info; +#include <dev/hyperv/vmbus/vmbus_brvar.h> struct vmbus_channel { /* @@ -57,7 +52,7 @@ struct vmbus_channel { /* * RX bufring; immediately following ch_txbr. */ - hv_vmbus_ring_buffer_info ch_rxbr; + struct vmbus_rxbr ch_rxbr; struct taskqueue *ch_tq; struct task ch_task; @@ -76,7 +71,7 @@ struct vmbus_channel { * TX bufring and following MNF/evtflags do _not_ fit in * one 64B cacheline. */ - hv_vmbus_ring_buffer_info ch_txbr __aligned(CACHE_LINE_SIZE); + struct vmbus_txbr ch_txbr __aligned(CACHE_LINE_SIZE); uint32_t ch_txflags; /* VMBUS_CHAN_TXF_ */ /* @@ -160,9 +155,10 @@ struct vmbus_channel { struct vmbus_softc; struct vmbus_message; -void vmbus_event_proc(struct vmbus_softc *, int); -void vmbus_event_proc_compat(struct vmbus_softc *, int); -void vmbus_chan_msgproc(struct vmbus_softc *, const struct vmbus_message *); -void vmbus_chan_destroy_all(struct vmbus_softc *); +void vmbus_event_proc(struct vmbus_softc *, int); +void vmbus_event_proc_compat(struct vmbus_softc *, int); +void vmbus_chan_msgproc(struct vmbus_softc *, + const struct vmbus_message *); +void vmbus_chan_destroy_all(struct vmbus_softc *); #endif /* !_VMBUS_CHANVAR_H_ */ diff --git a/sys/dev/hyperv/vmbus/vmbus_et.c b/sys/dev/hyperv/vmbus/vmbus_et.c index 6e3fbfda402d..9e217ece8a22 100644 --- a/sys/dev/hyperv/vmbus/vmbus_et.c +++ b/sys/dev/hyperv/vmbus/vmbus_et.c @@ -32,9 +32,8 @@ __FBSDID("$FreeBSD$"); #include <sys/kernel.h> #include <sys/module.h> #include <sys/proc.h> -#include <sys/systm.h> #include <sys/smp.h> -#include <sys/time.h> +#include <sys/systm.h> #include <sys/timeet.h> #include <dev/hyperv/vmbus/hyperv_reg.h> @@ -57,8 +56,35 @@ __FBSDID("$FreeBSD$"); CPUID_HV_MSR_SYNIC | \ CPUID_HV_MSR_SYNTIMER) +static void vmbus_et_identify(driver_t *, device_t); +static int vmbus_et_probe(device_t); +static int vmbus_et_attach(device_t); +static int vmbus_et_detach(device_t); +static int vmbus_et_start(struct eventtimer *, sbintime_t, + sbintime_t); + static struct eventtimer vmbus_et; +static device_method_t vmbus_et_methods[] = { + DEVMETHOD(device_identify, vmbus_et_identify), + DEVMETHOD(device_probe, vmbus_et_probe), + DEVMETHOD(device_attach, vmbus_et_attach), + DEVMETHOD(device_detach, vmbus_et_detach), + + DEVMETHOD_END +}; + +static driver_t vmbus_et_driver = { + VMBUS_ET_NAME, + vmbus_et_methods, + 0 +}; + +static devclass_t vmbus_et_devclass; + +DRIVER_MODULE(hv_et, vmbus, vmbus_et_driver, vmbus_et_devclass, NULL, NULL); +MODULE_VERSION(hv_et, 1); + static __inline uint64_t hyperv_sbintime2count(sbintime_t time) { @@ -175,22 +201,3 @@ vmbus_et_detach(device_t dev) { return (et_deregister(&vmbus_et)); } - -static device_method_t vmbus_et_methods[] = { - DEVMETHOD(device_identify, vmbus_et_identify), - DEVMETHOD(device_probe, vmbus_et_probe), - DEVMETHOD(device_attach, vmbus_et_attach), - DEVMETHOD(device_detach, vmbus_et_detach), - - DEVMETHOD_END -}; - -static driver_t vmbus_et_driver = { - VMBUS_ET_NAME, - vmbus_et_methods, - 0 -}; - -static devclass_t vmbus_et_devclass; -DRIVER_MODULE(hv_et, vmbus, vmbus_et_driver, vmbus_et_devclass, NULL, NULL); -MODULE_VERSION(hv_et, 1); diff --git a/sys/dev/hyperv/vmbus/vmbus_var.h b/sys/dev/hyperv/vmbus/vmbus_var.h index 1316d7443665..47d9004e5719 100644 --- a/sys/dev/hyperv/vmbus/vmbus_var.h +++ b/sys/dev/hyperv/vmbus/vmbus_var.h @@ -30,7 +30,6 @@ #define _VMBUS_VAR_H_ #include <sys/param.h> -#include <sys/bus_dma.h> #include <sys/taskqueue.h> #include <dev/hyperv/include/hyperv_busdma.h> @@ -87,7 +86,7 @@ struct vmbus_softc { u_long *vmbus_rx_evtflags; /* compat evtflgs from host */ struct vmbus_channel **vmbus_chmap; - struct vmbus_msghc_ctx *vmbus_msg_hc; + struct vmbus_xact_ctx *vmbus_xc; struct vmbus_pcpu_data vmbus_pcpu[MAXCPU]; /* @@ -121,20 +120,6 @@ struct vmbus_softc { #define VMBUS_FLAG_ATTACHED 0x0001 /* vmbus was attached */ #define VMBUS_FLAG_SYNIC 0x0002 /* SynIC was setup */ -extern struct vmbus_softc *vmbus_sc; - -static __inline struct vmbus_softc * -vmbus_get_softc(void) -{ - return vmbus_sc; -} - -static __inline device_t -vmbus_get_device(void) -{ - return vmbus_sc->vmbus_dev; -} - #define VMBUS_PCPU_GET(sc, field, cpu) (sc)->vmbus_pcpu[(cpu)].field #define VMBUS_PCPU_PTR(sc, field, cpu) &(sc)->vmbus_pcpu[(cpu)].field @@ -143,20 +128,23 @@ struct trapframe; struct vmbus_message; struct vmbus_msghc; -void vmbus_handle_intr(struct trapframe *); -int vmbus_add_child(struct vmbus_channel *); -int vmbus_delete_child(struct vmbus_channel *); -void vmbus_et_intr(struct trapframe *); -uint32_t vmbus_gpadl_alloc(struct vmbus_softc *); - -struct vmbus_msghc *vmbus_msghc_get(struct vmbus_softc *, size_t); -void vmbus_msghc_put(struct vmbus_softc *, struct vmbus_msghc *); -void *vmbus_msghc_dataptr(struct vmbus_msghc *); -int vmbus_msghc_exec_noresult(struct vmbus_msghc *); -int vmbus_msghc_exec(struct vmbus_softc *, struct vmbus_msghc *); -const struct vmbus_message *vmbus_msghc_wait_result(struct vmbus_softc *, - struct vmbus_msghc *); -void vmbus_msghc_wakeup(struct vmbus_softc *, const struct vmbus_message *); -void vmbus_msghc_reset(struct vmbus_msghc *, size_t); +void vmbus_handle_intr(struct trapframe *); +int vmbus_add_child(struct vmbus_channel *); +int vmbus_delete_child(struct vmbus_channel *); +void vmbus_et_intr(struct trapframe *); +uint32_t vmbus_gpadl_alloc(struct vmbus_softc *); + +struct vmbus_msghc * + vmbus_msghc_get(struct vmbus_softc *, size_t); +void vmbus_msghc_put(struct vmbus_softc *, struct vmbus_msghc *); +void *vmbus_msghc_dataptr(struct vmbus_msghc *); +int vmbus_msghc_exec_noresult(struct vmbus_msghc *); +int vmbus_msghc_exec(struct vmbus_softc *, struct vmbus_msghc *); +const struct vmbus_message * + vmbus_msghc_wait_result(struct vmbus_softc *, + struct vmbus_msghc *); +void vmbus_msghc_wakeup(struct vmbus_softc *, + const struct vmbus_message *); +void vmbus_msghc_reset(struct vmbus_msghc *, size_t); #endif /* !_VMBUS_VAR_H_ */ diff --git a/sys/dev/hyperv/vmbus/vmbus_xact.c b/sys/dev/hyperv/vmbus/vmbus_xact.c new file mode 100644 index 000000000000..642c165bc293 --- /dev/null +++ b/sys/dev/hyperv/vmbus/vmbus_xact.c @@ -0,0 +1,313 @@ +/*- + * Copyright (c) 2016 Microsoft Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/lock.h> +#include <sys/malloc.h> +#include <sys/mutex.h> +#include <sys/proc.h> +#include <sys/systm.h> + +#include <dev/hyperv/include/hyperv_busdma.h> +#include <dev/hyperv/include/vmbus_xact.h> + +struct vmbus_xact { + struct vmbus_xact_ctx *x_ctx; + void *x_priv; + + void *x_req; + struct hyperv_dma x_req_dma; + + const void *x_resp; + size_t x_resp_len; + void *x_resp0; +}; + +struct vmbus_xact_ctx { + uint32_t xc_flags; + size_t xc_req_size; + size_t xc_resp_size; + size_t xc_priv_size; + + struct vmbus_xact *xc_free; + struct mtx xc_free_lock; + + struct vmbus_xact *xc_active; + struct mtx xc_active_lock; +}; + +#define VMBUS_XACT_CTXF_DESTROY 0x0001 + +static struct vmbus_xact *vmbus_xact_alloc(struct vmbus_xact_ctx *, + bus_dma_tag_t); +static void vmbus_xact_free(struct vmbus_xact *); +static struct vmbus_xact *vmbus_xact_get1(struct vmbus_xact_ctx *, + uint32_t); + +static struct vmbus_xact * +vmbus_xact_alloc(struct vmbus_xact_ctx *ctx, bus_dma_tag_t parent_dtag) +{ + struct vmbus_xact *xact; + + xact = malloc(sizeof(*xact), M_DEVBUF, M_WAITOK | M_ZERO); + xact->x_ctx = ctx; + + /* XXX assume that page aligned is enough */ + xact->x_req = hyperv_dmamem_alloc(parent_dtag, PAGE_SIZE, 0, + ctx->xc_req_size, &xact->x_req_dma, BUS_DMA_WAITOK); + if (xact->x_req == NULL) { + free(xact, M_DEVBUF); + return (NULL); + } + if (ctx->xc_priv_size != 0) + xact->x_priv = malloc(ctx->xc_priv_size, M_DEVBUF, M_WAITOK); + xact->x_resp0 = malloc(ctx->xc_resp_size, M_DEVBUF, M_WAITOK); + + return (xact); +} + +static void +vmbus_xact_free(struct vmbus_xact *xact) +{ + + hyperv_dmamem_free(&xact->x_req_dma, xact->x_req); + free(xact->x_resp0, M_DEVBUF); + if (xact->x_priv != NULL) + free(xact->x_priv, M_DEVBUF); + free(xact, M_DEVBUF); +} + +static struct vmbus_xact * +vmbus_xact_get1(struct vmbus_xact_ctx *ctx, uint32_t dtor_flag) +{ + struct vmbus_xact *xact; + + mtx_lock(&ctx->xc_free_lock); + + while ((ctx->xc_flags & dtor_flag) == 0 && ctx->xc_free == NULL) + mtx_sleep(&ctx->xc_free, &ctx->xc_free_lock, 0, "gxact", 0); + if (ctx->xc_flags & dtor_flag) { + /* Being destroyed */ + xact = NULL; + } else { + xact = ctx->xc_free; + KASSERT(xact != NULL, ("no free xact")); + KASSERT(xact->x_resp == NULL, ("xact has pending response")); + ctx->xc_free = NULL; + } + + mtx_unlock(&ctx->xc_free_lock); + + return (xact); +} + +struct vmbus_xact_ctx * +vmbus_xact_ctx_create(bus_dma_tag_t dtag, size_t req_size, size_t resp_size, + size_t priv_size) +{ + struct vmbus_xact_ctx *ctx; + + ctx = malloc(sizeof(*ctx), M_DEVBUF, M_WAITOK | M_ZERO); + ctx->xc_req_size = req_size; + ctx->xc_resp_size = resp_size; + ctx->xc_priv_size = priv_size; + + ctx->xc_free = vmbus_xact_alloc(ctx, dtag); + if (ctx->xc_free == NULL) { + free(ctx, M_DEVBUF); + return (NULL); + } + + mtx_init(&ctx->xc_free_lock, "vmbus xact free", NULL, MTX_DEF); + mtx_init(&ctx->xc_active_lock, "vmbus xact active", NULL, MTX_DEF); + + return (ctx); +} + +void +vmbus_xact_ctx_destroy(struct vmbus_xact_ctx *ctx) +{ + struct vmbus_xact *xact; + + mtx_lock(&ctx->xc_free_lock); + ctx->xc_flags |= VMBUS_XACT_CTXF_DESTROY; + mtx_unlock(&ctx->xc_free_lock); + wakeup(&ctx->xc_free); + + xact = vmbus_xact_get1(ctx, 0); + if (xact == NULL) + panic("can't get xact"); + + vmbus_xact_free(xact); + mtx_destroy(&ctx->xc_free_lock); + mtx_destroy(&ctx->xc_active_lock); + free(ctx, M_DEVBUF); +} + +struct vmbus_xact * +vmbus_xact_get(struct vmbus_xact_ctx *ctx, size_t req_len) +{ + struct vmbus_xact *xact; + + if (req_len > ctx->xc_req_size) + panic("invalid request size %zu", req_len); + + xact = vmbus_xact_get1(ctx, VMBUS_XACT_CTXF_DESTROY); + if (xact == NULL) + return (NULL); + + memset(xact->x_req, 0, req_len); + return (xact); +} + +void +vmbus_xact_put(struct vmbus_xact *xact) +{ + struct vmbus_xact_ctx *ctx = xact->x_ctx; + + KASSERT(ctx->xc_active == NULL, ("pending active xact")); + xact->x_resp = NULL; + + mtx_lock(&ctx->xc_free_lock); + KASSERT(ctx->xc_free == NULL, ("has free xact")); + ctx->xc_free = xact; + mtx_unlock(&ctx->xc_free_lock); + wakeup(&ctx->xc_free); +} + +void * +vmbus_xact_req_data(const struct vmbus_xact *xact) +{ + + return (xact->x_req); +} + +bus_addr_t +vmbus_xact_req_paddr(const struct vmbus_xact *xact) +{ + + return (xact->x_req_dma.hv_paddr); +} + +void * +vmbus_xact_priv(const struct vmbus_xact *xact, size_t priv_len) +{ + + if (priv_len > xact->x_ctx->xc_priv_size) + panic("invalid priv size %zu", priv_len); + return (xact->x_priv); +} + +void +vmbus_xact_activate(struct vmbus_xact *xact) +{ + struct vmbus_xact_ctx *ctx = xact->x_ctx; + + KASSERT(xact->x_resp == NULL, ("xact has pending response")); + + mtx_lock(&ctx->xc_active_lock); + KASSERT(ctx->xc_active == NULL, ("pending active xact")); + ctx->xc_active = xact; + mtx_unlock(&ctx->xc_active_lock); +} + +void +vmbus_xact_deactivate(struct vmbus_xact *xact) +{ + struct vmbus_xact_ctx *ctx = xact->x_ctx; + + mtx_lock(&ctx->xc_active_lock); + KASSERT(ctx->xc_active == xact, ("xact mismatch")); + ctx->xc_active = NULL; + mtx_unlock(&ctx->xc_active_lock); +} + +const void * +vmbus_xact_wait(struct vmbus_xact *xact, size_t *resp_len) +{ + struct vmbus_xact_ctx *ctx = xact->x_ctx; + const void *resp; + + mtx_lock(&ctx->xc_active_lock); + + KASSERT(ctx->xc_active == xact, ("xact mismatch")); + while (xact->x_resp == NULL) { + mtx_sleep(&ctx->xc_active, &ctx->xc_active_lock, 0, + "wxact", 0); + } + ctx->xc_active = NULL; + + resp = xact->x_resp; + *resp_len = xact->x_resp_len; + + mtx_unlock(&ctx->xc_active_lock); + + return (resp); +} + +static void +vmbus_xact_save_resp(struct vmbus_xact *xact, const void *data, size_t dlen) +{ + struct vmbus_xact_ctx *ctx = xact->x_ctx; + size_t cplen = dlen; + + mtx_assert(&ctx->xc_active_lock, MA_OWNED); + + if (cplen > ctx->xc_resp_size) { + printf("vmbus: xact response truncated %zu -> %zu\n", + cplen, ctx->xc_resp_size); + cplen = ctx->xc_resp_size; + } + + KASSERT(ctx->xc_active == xact, ("xact mismatch")); + memcpy(xact->x_resp0, data, cplen); + xact->x_resp_len = cplen; + xact->x_resp = xact->x_resp0; +} + +void +vmbus_xact_wakeup(struct vmbus_xact *xact, const void *data, size_t dlen) +{ + struct vmbus_xact_ctx *ctx = xact->x_ctx; + + mtx_lock(&ctx->xc_active_lock); + vmbus_xact_save_resp(xact, data, dlen); + mtx_unlock(&ctx->xc_active_lock); + wakeup(&ctx->xc_active); +} + +void +vmbus_xact_ctx_wakeup(struct vmbus_xact_ctx *ctx, const void *data, size_t dlen) +{ + mtx_lock(&ctx->xc_active_lock); + KASSERT(ctx->xc_active != NULL, ("no pending xact")); + vmbus_xact_save_resp(ctx->xc_active, data, dlen); + mtx_unlock(&ctx->xc_active_lock); + wakeup(&ctx->xc_active); +} diff --git a/sys/dev/ioat/ioat.c b/sys/dev/ioat/ioat.c index 0b04e13266a2..45613498adfc 100644 --- a/sys/dev/ioat/ioat.c +++ b/sys/dev/ioat/ioat.c @@ -663,7 +663,7 @@ ioat_process_events(struct ioat_softc *ioat) boolean_t pending; int error; - CTR0(KTR_IOAT, __func__); + CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx); mtx_lock(&ioat->cleanup_lock); @@ -693,8 +693,9 @@ ioat_process_events(struct ioat_softc *ioat) while (1) { desc = ioat_get_ring_entry(ioat, ioat->tail); dmadesc = &desc->bus_dmadesc; - CTR3(KTR_IOAT, "completing desc %u ok cb %p(%p)", ioat->tail, - dmadesc->callback_fn, dmadesc->callback_arg); + CTR4(KTR_IOAT, "channel=%u completing desc %u ok cb %p(%p)", + ioat->chan_idx, ioat->tail, dmadesc->callback_fn, + dmadesc->callback_arg); if (dmadesc->callback_fn != NULL) dmadesc->callback_fn(dmadesc->callback_arg, 0); @@ -764,8 +765,9 @@ out: while (ioat_get_active(ioat) > 0) { desc = ioat_get_ring_entry(ioat, ioat->tail); dmadesc = &desc->bus_dmadesc; - CTR3(KTR_IOAT, "completing desc %u err cb %p(%p)", ioat->tail, - dmadesc->callback_fn, dmadesc->callback_arg); + CTR4(KTR_IOAT, "channel=%u completing desc %u err cb %p(%p)", + ioat->chan_idx, ioat->tail, dmadesc->callback_fn, + dmadesc->callback_arg); if (dmadesc->callback_fn != NULL) dmadesc->callback_fn(dmadesc->callback_arg, @@ -919,7 +921,7 @@ ioat_acquire(bus_dmaengine_t dmaengine) ioat = to_ioat_softc(dmaengine); mtx_lock(&ioat->submit_lock); - CTR0(KTR_IOAT, __func__); + CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx); } int @@ -943,7 +945,7 @@ ioat_release(bus_dmaengine_t dmaengine) struct ioat_softc *ioat; ioat = to_ioat_softc(dmaengine); - CTR0(KTR_IOAT, __func__); + CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx); ioat_write_2(ioat, IOAT_DMACOUNT_OFFSET, (uint16_t)ioat->hw_head); mtx_unlock(&ioat->submit_lock); } @@ -1005,8 +1007,8 @@ ioat_null(bus_dmaengine_t dmaengine, bus_dmaengine_callback_t callback_fn, struct ioat_descriptor *desc; struct ioat_softc *ioat; - CTR0(KTR_IOAT, __func__); ioat = to_ioat_softc(dmaengine); + CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx); desc = ioat_op_generic(ioat, IOAT_OP_COPY, 8, 0, 0, callback_fn, callback_arg, flags); @@ -1028,8 +1030,8 @@ ioat_copy(bus_dmaengine_t dmaengine, bus_addr_t dst, struct ioat_descriptor *desc; struct ioat_softc *ioat; - CTR0(KTR_IOAT, __func__); ioat = to_ioat_softc(dmaengine); + CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx); if (((src | dst) & (0xffffull << 48)) != 0) { ioat_log_message(0, "%s: High 16 bits of src/dst invalid\n", @@ -1059,8 +1061,8 @@ ioat_copy_8k_aligned(bus_dmaengine_t dmaengine, bus_addr_t dst1, struct ioat_descriptor *desc; struct ioat_softc *ioat; - CTR0(KTR_IOAT, __func__); ioat = to_ioat_softc(dmaengine); + CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx); if (((src1 | src2 | dst1 | dst2) & (0xffffull << 48)) != 0) { ioat_log_message(0, "%s: High 16 bits of src/dst invalid\n", @@ -1106,8 +1108,8 @@ ioat_copy_crc(bus_dmaengine_t dmaengine, bus_addr_t dst, bus_addr_t src, uint32_t teststore; uint8_t op; - CTR0(KTR_IOAT, __func__); ioat = to_ioat_softc(dmaengine); + CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx); if ((ioat->capabilities & IOAT_DMACAP_MOVECRC) == 0) { ioat_log_message(0, "%s: Device lacks MOVECRC capability\n", @@ -1185,8 +1187,8 @@ ioat_crc(bus_dmaengine_t dmaengine, bus_addr_t src, bus_size_t len, uint32_t teststore; uint8_t op; - CTR0(KTR_IOAT, __func__); ioat = to_ioat_softc(dmaengine); + CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx); if ((ioat->capabilities & IOAT_DMACAP_CRC) == 0) { ioat_log_message(0, "%s: Device lacks CRC capability\n", @@ -1262,8 +1264,8 @@ ioat_blockfill(bus_dmaengine_t dmaengine, bus_addr_t dst, uint64_t fillpattern, struct ioat_descriptor *desc; struct ioat_softc *ioat; - CTR0(KTR_IOAT, __func__); ioat = to_ioat_softc(dmaengine); + CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx); if ((ioat->capabilities & IOAT_DMACAP_BFILL) == 0) { ioat_log_message(0, "%s: Device lacks BFILL capability\n", @@ -1528,7 +1530,7 @@ ring_grow(struct ioat_softc *ioat, uint32_t oldorder, uint32_t oldsize, newsize, head, tail, i, end; int error; - CTR0(KTR_IOAT, __func__); + CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx); mtx_assert(&ioat->submit_lock, MA_OWNED); @@ -1626,7 +1628,7 @@ ring_shrink(struct ioat_softc *ioat, uint32_t oldorder, uint32_t oldsize, newsize, current_idx, new_idx, i; int error; - CTR0(KTR_IOAT, __func__); + CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx); mtx_assert(&ioat->submit_lock, MA_OWNED); @@ -1802,7 +1804,7 @@ ioat_reset_hw(struct ioat_softc *ioat) unsigned timeout; int error; - CTR0(KTR_IOAT, __func__); + CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx); mtx_lock(IOAT_REFLK); while (ioat->resetting && !ioat->destroying) diff --git a/sys/dev/iwm/if_iwm.c b/sys/dev/iwm/if_iwm.c index 87db721ddd95..8d39c4004490 100644 --- a/sys/dev/iwm/if_iwm.c +++ b/sys/dev/iwm/if_iwm.c @@ -1,4 +1,4 @@ -/* $OpenBSD: if_iwm.c,v 1.39 2015/03/23 00:35:19 jsg Exp $ */ +/* $OpenBSD: if_iwm.c,v 1.42 2015/05/30 02:49:23 deraadt Exp $ */ /* * Copyright (c) 2014 genua mbh <info@genua.de> @@ -173,11 +173,23 @@ const uint8_t iwm_nvm_channels[] = { 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165 }; -#define IWM_NUM_2GHZ_CHANNELS 14 - _Static_assert(nitems(iwm_nvm_channels) <= IWM_NUM_CHANNELS, "IWM_NUM_CHANNELS is too small"); +const uint8_t iwm_nvm_channels_8000[] = { + /* 2.4 GHz */ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + /* 5 GHz */ + 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92, + 96, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, + 149, 153, 157, 161, 165, 169, 173, 177, 181 +}; +_Static_assert(nitems(iwm_nvm_channels_8000) <= IWM_NUM_CHANNELS_8000, + "IWM_NUM_CHANNELS_8000 is too small"); + +#define IWM_NUM_2GHZ_CHANNELS 14 +#define IWM_N_HW_ADDR_MASK 0xF + /* * XXX For now, there's simply a fixed set of rate table entries * that are populated. @@ -205,6 +217,11 @@ const struct iwm_rate { #define IWM_RIDX_IS_CCK(_i_) ((_i_) < IWM_RIDX_OFDM) #define IWM_RIDX_IS_OFDM(_i_) ((_i_) >= IWM_RIDX_OFDM) +struct iwm_nvm_section { + uint16_t length; + uint8_t *data; +}; + static int iwm_store_cscheme(struct iwm_softc *, const uint8_t *, size_t); static int iwm_firmware_store_section(struct iwm_softc *, enum iwm_ucode_type, @@ -242,27 +259,45 @@ static void iwm_mvm_nic_config(struct iwm_softc *); static int iwm_nic_rx_init(struct iwm_softc *); static int iwm_nic_tx_init(struct iwm_softc *); static int iwm_nic_init(struct iwm_softc *); -static void iwm_enable_txq(struct iwm_softc *, int, int); +static int iwm_enable_txq(struct iwm_softc *, int, int, int); static int iwm_post_alive(struct iwm_softc *); static int iwm_nvm_read_chunk(struct iwm_softc *, uint16_t, uint16_t, uint16_t, uint8_t *, uint16_t *); static int iwm_nvm_read_section(struct iwm_softc *, uint16_t, uint8_t *, - uint16_t *); + uint16_t *, size_t); static uint32_t iwm_eeprom_channel_flags(uint16_t); static void iwm_add_channel_band(struct iwm_softc *, - struct ieee80211_channel[], int, int *, int, int, + struct ieee80211_channel[], int, int *, int, size_t, const uint8_t[]); static void iwm_init_channel_map(struct ieee80211com *, int, int *, struct ieee80211_channel[]); static int iwm_parse_nvm_data(struct iwm_softc *, const uint16_t *, - const uint16_t *, const uint16_t *, uint8_t, - uint8_t); -struct iwm_nvm_section; + const uint16_t *, const uint16_t *, + const uint16_t *, const uint16_t *, + const uint16_t *); +static void iwm_set_hw_address_8000(struct iwm_softc *, + struct iwm_nvm_data *, + const uint16_t *, const uint16_t *); +static int iwm_get_sku(const struct iwm_softc *, const uint16_t *, + const uint16_t *); +static int iwm_get_nvm_version(const struct iwm_softc *, const uint16_t *); +static int iwm_get_radio_cfg(const struct iwm_softc *, const uint16_t *, + const uint16_t *); +static int iwm_get_n_hw_addrs(const struct iwm_softc *, + const uint16_t *); +static void iwm_set_radio_cfg(const struct iwm_softc *, + struct iwm_nvm_data *, uint32_t); static int iwm_parse_nvm_sections(struct iwm_softc *, struct iwm_nvm_section *); static int iwm_nvm_init(struct iwm_softc *); +static int iwm_firmware_load_sect(struct iwm_softc *, uint32_t, + const uint8_t *, uint32_t); static int iwm_firmware_load_chunk(struct iwm_softc *, uint32_t, const uint8_t *, uint32_t); +static int iwm_load_firmware_7000(struct iwm_softc *, enum iwm_ucode_type); +static int iwm_load_cpu_sections_8000(struct iwm_softc *, + struct iwm_fw_sects *, int , int *); +static int iwm_load_firmware_8000(struct iwm_softc *, enum iwm_ucode_type); static int iwm_load_firmware(struct iwm_softc *, enum iwm_ucode_type); static int iwm_start_fw(struct iwm_softc *, enum iwm_ucode_type); static int iwm_send_tx_ant_cfg(struct iwm_softc *, uint8_t); @@ -297,10 +332,8 @@ static int iwm_tx(struct iwm_softc *, struct mbuf *, struct ieee80211_node *, int); static int iwm_raw_xmit(struct ieee80211_node *, struct mbuf *, const struct ieee80211_bpf_params *); -static void iwm_mvm_add_sta_cmd_v6_to_v5(struct iwm_mvm_add_sta_cmd_v6 *, - struct iwm_mvm_add_sta_cmd_v5 *); static int iwm_mvm_send_add_sta_cmd_status(struct iwm_softc *, - struct iwm_mvm_add_sta_cmd_v6 *, + struct iwm_mvm_add_sta_cmd_v7 *, int *); static int iwm_mvm_sta_send_to_fw(struct iwm_softc *, struct iwm_node *, int); @@ -321,6 +354,13 @@ static void iwm_setrates(struct iwm_softc *, struct iwm_node *); static int iwm_media_change(struct ifnet *); static int iwm_newstate(struct ieee80211vap *, enum ieee80211_state, int); static void iwm_endscan_cb(void *, int); +static void iwm_mvm_fill_sf_command(struct iwm_softc *, + struct iwm_sf_cfg_cmd *, + struct ieee80211_node *); +static int iwm_mvm_sf_config(struct iwm_softc *, enum iwm_sf_state); +static int iwm_send_bt_init_conf(struct iwm_softc *); +static int iwm_send_update_mcc_cmd(struct iwm_softc *, const char *); +static void iwm_mvm_tt_tx_backoff(struct iwm_softc *, uint32_t); static int iwm_init_hw(struct iwm_softc *); static void iwm_init(struct iwm_softc *); static void iwm_start(struct iwm_softc *); @@ -331,10 +371,12 @@ static void iwm_parent(struct ieee80211com *); static const char * iwm_desc_lookup(uint32_t); static void iwm_nic_error(struct iwm_softc *); +static void iwm_nic_umac_error(struct iwm_softc *); #endif static void iwm_notif_intr(struct iwm_softc *); static void iwm_intr(void *); static int iwm_attach(device_t); +static int iwm_is_valid_ether_addr(uint8_t *); static void iwm_preinit(void *); static int iwm_detach_local(struct iwm_softc *sc, int); static void iwm_init_task(void *); @@ -477,6 +519,12 @@ iwm_read_firmware(struct iwm_softc *sc, enum iwm_ucode_type ucode_type) } fw->fw_fp = fwp; + /* (Re-)Initialize default values. */ + sc->sc_capaflags = 0; + sc->sc_capa_n_scan_channels = IWM_MAX_NUM_SCAN_CHANNELS; + memset(sc->sc_enabled_capa, 0, sizeof(sc->sc_enabled_capa)); + memset(sc->sc_fw_mcc, 0, sizeof(sc->sc_fw_mcc)); + /* * Parse firmware contents */ @@ -490,7 +538,10 @@ iwm_read_firmware(struct iwm_softc *sc, enum iwm_ucode_type ucode_type) goto out; } - sc->sc_fwver = le32toh(uhdr->ver); + snprintf(sc->sc_fwver, sizeof(sc->sc_fwver), "%d.%d (API ver %d)", + IWM_UCODE_MAJOR(le32toh(uhdr->ver)), + IWM_UCODE_MINOR(le32toh(uhdr->ver)), + IWM_UCODE_API(le32toh(uhdr->ver))); data = uhdr->data; len = fw->fw_fp->datasize - sizeof(*uhdr); @@ -527,7 +578,8 @@ iwm_read_firmware(struct iwm_softc *sc, enum iwm_ucode_type ucode_type) sc->sc_capa_max_probe_len = le32toh(*(const uint32_t *)tlv_data); /* limit it to something sensible */ - if (sc->sc_capa_max_probe_len > (1<<16)) { + if (sc->sc_capa_max_probe_len > + IWM_SCAN_OFFLOAD_PROBE_REQ_SIZE) { IWM_DPRINTF(sc, IWM_DEBUG_FIRMWARE_TLV, "%s: IWM_UCODE_TLV_PROBE_MAX_LEN " "ridiculous\n", __func__); @@ -578,7 +630,8 @@ iwm_read_firmware(struct iwm_softc *sc, enum iwm_ucode_type ucode_type) goto parse_out; } break; - case IWM_UCODE_TLV_NUM_OF_CPU: + case IWM_UCODE_TLV_NUM_OF_CPU: { + uint32_t num_cpu; if (tlv_len != sizeof(uint32_t)) { device_printf(sc->sc_dev, "%s: IWM_UCODE_TLV_NUM_OF_CPU: tlv_len (%d) < sizeof(uint32_t)\n", @@ -587,15 +640,16 @@ iwm_read_firmware(struct iwm_softc *sc, enum iwm_ucode_type ucode_type) error = EINVAL; goto parse_out; } - if (le32toh(*(const uint32_t*)tlv_data) != 1) { + num_cpu = le32toh(*(const uint32_t *)tlv_data); + if (num_cpu < 1 || num_cpu > 2) { device_printf(sc->sc_dev, - "%s: driver supports " - "only TLV_NUM_OF_CPU == 1", + "%s: Driver supports only 1 or 2 CPUs\n", __func__); error = EINVAL; goto parse_out; } break; + } case IWM_UCODE_TLV_SEC_RT: if ((error = iwm_firmware_store_section(sc, IWM_UCODE_TYPE_REGULAR, tlv_data, tlv_len)) != 0) { @@ -657,11 +711,80 @@ iwm_read_firmware(struct iwm_softc *sc, enum iwm_ucode_type ucode_type) le32toh(*(const uint32_t *)tlv_data); break; - case IWM_UCODE_TLV_API_CHANGES_SET: - case IWM_UCODE_TLV_ENABLED_CAPABILITIES: + case IWM_UCODE_TLV_API_CHANGES_SET: { + const struct iwm_ucode_api *api; + if (tlv_len != sizeof(*api)) { + error = EINVAL; + goto parse_out; + } + api = (const struct iwm_ucode_api *)tlv_data; + /* Flags may exceed 32 bits in future firmware. */ + if (le32toh(api->api_index) > 0) { + device_printf(sc->sc_dev, + "unsupported API index %d\n", + le32toh(api->api_index)); + goto parse_out; + } + sc->sc_ucode_api = le32toh(api->api_flags); + break; + } + + case IWM_UCODE_TLV_ENABLED_CAPABILITIES: { + const struct iwm_ucode_capa *capa; + int idx, i; + if (tlv_len != sizeof(*capa)) { + error = EINVAL; + goto parse_out; + } + capa = (const struct iwm_ucode_capa *)tlv_data; + idx = le32toh(capa->api_index); + if (idx > howmany(IWM_NUM_UCODE_TLV_CAPA, 32)) { + device_printf(sc->sc_dev, + "unsupported API index %d\n", idx); + goto parse_out; + } + for (i = 0; i < 32; i++) { + if ((le32toh(capa->api_capa) & (1U << i)) == 0) + continue; + setbit(sc->sc_enabled_capa, i + (32 * idx)); + } + break; + } + + case 48: /* undocumented TLV */ + case IWM_UCODE_TLV_SDIO_ADMA_ADDR: + case IWM_UCODE_TLV_FW_GSCAN_CAPA: /* ignore, not used by current driver */ break; + case IWM_UCODE_TLV_SEC_RT_USNIFFER: + if ((error = iwm_firmware_store_section(sc, + IWM_UCODE_TYPE_REGULAR_USNIFFER, tlv_data, + tlv_len)) != 0) + goto parse_out; + break; + + case IWM_UCODE_TLV_N_SCAN_CHANNELS: + if (tlv_len != sizeof(uint32_t)) { + error = EINVAL; + goto parse_out; + } + sc->sc_capa_n_scan_channels = + le32toh(*(const uint32_t *)tlv_data); + break; + + case IWM_UCODE_TLV_FW_VERSION: + if (tlv_len != sizeof(uint32_t) * 3) { + error = EINVAL; + goto parse_out; + } + snprintf(sc->sc_fwver, sizeof(sc->sc_fwver), + "%d.%d.%d", + le32toh(((const uint32_t *)tlv_data)[0]), + le32toh(((const uint32_t *)tlv_data)[1]), + le32toh(((const uint32_t *)tlv_data)[2])); + break; + default: device_printf(sc->sc_dev, "%s: unknown firmware section %d, abort\n", @@ -710,7 +833,7 @@ iwm_dma_map_addr(void *arg, bus_dma_segment_t *segs, int nsegs, int error) if (error != 0) return; KASSERT(nsegs == 1, ("too many DMA segments, %d should be 1", nsegs)); - *(bus_addr_t *)arg = segs[0].ds_addr; + *(bus_addr_t *)arg = segs[0].ds_addr; } static int @@ -720,6 +843,7 @@ iwm_dma_contig_alloc(bus_dma_tag_t tag, struct iwm_dma_info *dma, int error; dma->tag = NULL; + dma->map = NULL; dma->size = size; dma->vaddr = NULL; @@ -739,14 +863,16 @@ iwm_dma_contig_alloc(bus_dma_tag_t tag, struct iwm_dma_info *dma, if (error != 0) { bus_dmamem_free(dma->tag, dma->vaddr, dma->map); dma->vaddr = NULL; - goto fail; + goto fail; } bus_dmamap_sync(dma->tag, dma->map, BUS_DMASYNC_PREWRITE); return 0; -fail: iwm_dma_contig_free(dma); +fail: + iwm_dma_contig_free(dma); + return error; } @@ -764,7 +890,6 @@ iwm_dma_contig_free(struct iwm_dma_info *dma) bus_dma_tag_destroy(dma->tag); dma->tag = NULL; } - } /* fwmem is used to load firmware onto the card */ @@ -786,12 +911,9 @@ iwm_free_fwmem(struct iwm_softc *sc) static int iwm_alloc_sched(struct iwm_softc *sc) { - int rv; - /* TX scheduler rings must be aligned on a 1KB boundary. */ - rv = iwm_dma_contig_alloc(sc->sc_dmat, &sc->sched_dma, + return iwm_dma_contig_alloc(sc->sc_dmat, &sc->sched_dma, nitems(sc->txq) * sizeof(struct iwm_agn_scd_bc_tbl), 1024); - return rv; } static void @@ -901,7 +1023,7 @@ fail: iwm_free_rx_ring(sc, ring); static void iwm_disable_rx_dma(struct iwm_softc *sc) { - + /* XXX conditional nic locks are stupid */ /* XXX print out if we can't lock the NIC? */ if (iwm_nic_lock(sc)) { /* XXX handle if RX stop doesn't finish? */ @@ -915,6 +1037,11 @@ iwm_reset_rx_ring(struct iwm_softc *sc, struct iwm_rx_ring *ring) { /* Reset the ring state */ ring->cur = 0; + + /* + * The hw rx ring index in shared memory must also be cleared, + * otherwise the discrepancy can cause reprocessing chaos. + */ memset(sc->rxq.stat, 0, sizeof(*sc->rxq.stat)); } @@ -1125,6 +1252,7 @@ iwm_ict_reset(struct iwm_softc *sc) /* Set physical address of ICT table (4KB aligned). */ IWM_WRITE(sc, IWM_CSR_DRAM_INT_TBL_REG, IWM_CSR_DRAM_INT_TBL_ENABLE + | IWM_CSR_DRAM_INIT_TBL_WRITE_POINTER | IWM_CSR_DRAM_INIT_TBL_WRAP_CHECK | sc->ict_dma.paddr >> IWM_ICT_PADDR_SHIFT); @@ -1150,8 +1278,8 @@ iwm_stop_device(struct iwm_softc *sc) { struct ieee80211com *ic = &sc->sc_ic; struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); - int chnl, ntries; - int qid; + int chnl, qid; + uint32_t mask = 0; /* tell the device to stop sending interrupts */ iwm_disable_interrupts(sc); @@ -1173,20 +1301,20 @@ iwm_stop_device(struct iwm_softc *sc) iwm_write_prph(sc, IWM_SCD_TXFACT, 0); - /* Stop all DMA channels. */ if (iwm_nic_lock(sc)) { + /* Stop each Tx DMA channel */ for (chnl = 0; chnl < IWM_FH_TCSR_CHNL_NUM; chnl++) { IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_CONFIG_REG(chnl), 0); - for (ntries = 0; ntries < 200; ntries++) { - uint32_t r; - - r = IWM_READ(sc, IWM_FH_TSSR_TX_STATUS_REG); - if (r & IWM_FH_TSSR_TX_STATUS_REG_MSK_CHNL_IDLE( - chnl)) - break; - DELAY(20); - } + mask |= IWM_FH_TSSR_TX_STATUS_REG_MSK_CHNL_IDLE(chnl); + } + + /* Wait for DMA channels to be idle */ + if (!iwm_poll_bit(sc, IWM_FH_TSSR_TX_STATUS_REG, mask, mask, + 5000)) { + device_printf(sc->sc_dev, + "Failing on timeout while stopping DMA channel: [0x%08x]\n", + IWM_READ(sc, IWM_FH_TSSR_TX_STATUS_REG)); } iwm_nic_unlock(sc); } @@ -1217,7 +1345,7 @@ iwm_stop_device(struct iwm_softc *sc) */ iwm_disable_interrupts(sc); /* stop and reset the on-board processor */ - IWM_WRITE(sc, IWM_CSR_RESET, IWM_CSR_RESET_REG_FLAG_NEVO_RESET); + IWM_WRITE(sc, IWM_CSR_RESET, IWM_CSR_RESET_REG_FLAG_SW_RESET); /* * Even if we stop the HW, we still want the RF kill @@ -1263,9 +1391,11 @@ iwm_mvm_nic_config(struct iwm_softc *sc) * (PCIe power is lost before PERST# is asserted), causing ME FW * to lose ownership and not being able to obtain it back. */ - iwm_set_bits_mask_prph(sc, IWM_APMG_PS_CTRL_REG, - IWM_APMG_PS_CTRL_EARLY_PWR_OFF_RESET_DIS, - ~IWM_APMG_PS_CTRL_EARLY_PWR_OFF_RESET_DIS); + if (sc->sc_device_family == IWM_DEVICE_FAMILY_7000) { + iwm_set_bits_mask_prph(sc, IWM_APMG_PS_CTRL_REG, + IWM_APMG_PS_CTRL_EARLY_PWR_OFF_RESET_DIS, + ~IWM_APMG_PS_CTRL_EARLY_PWR_OFF_RESET_DIS); + } } static int @@ -1295,16 +1425,12 @@ iwm_nic_rx_init(struct iwm_softc *sc) IWM_FH_RSCSR_CHNL0_STTS_WPTR_REG, sc->rxq.stat_dma.paddr >> 4); /* Enable RX. */ - /* - * Note: Linux driver also sets this: - * (IWM_RX_RB_TIMEOUT << IWM_FH_RCSR_RX_CONFIG_REG_IRQ_RBTH_POS) | - * - * It causes weird behavior. YMMV. - */ IWM_WRITE(sc, IWM_FH_MEM_RCSR_CHNL0_CONFIG_REG, IWM_FH_RCSR_RX_CONFIG_CHNL_EN_ENABLE_VAL | IWM_FH_RCSR_CHNL0_RX_IGNORE_RXF_EMPTY | /* HW bug */ IWM_FH_RCSR_CHNL0_RX_CONFIG_IRQ_DEST_INT_HOST_VAL | + IWM_FH_RCSR_CHNL0_RX_CONFIG_SINGLE_FRAME_MSK | + (IWM_RX_RB_TIMEOUT << IWM_FH_RCSR_RX_CONFIG_REG_IRQ_RBTH_POS) | IWM_FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_4K | IWM_RX_QUEUE_SIZE_LOG << IWM_FH_RCSR_RX_CONFIG_RBDCB_SIZE_POS); @@ -1318,7 +1444,7 @@ iwm_nic_rx_init(struct iwm_softc *sc) * Thus sayeth el jefe (iwlwifi) via a comment: * * This value should initially be 0 (before preparing any - * RBs), should be 8 after preparing the first 8 RBs (for example) + * RBs), should be 8 after preparing the first 8 RBs (for example) */ IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_WPTR, 8); @@ -1354,6 +1480,9 @@ iwm_nic_tx_init(struct iwm_softc *sc) qid, txq->desc, (unsigned long) (txq->desc_dma.paddr >> 8)); } + + iwm_write_prph(sc, IWM_SCD_GP_CTRL, IWM_SCD_GP_CTRL_AUTO_ACTIVE_MODE); + iwm_nic_unlock(sc); return 0; @@ -1365,7 +1494,8 @@ iwm_nic_init(struct iwm_softc *sc) int error; iwm_apm_init(sc); - iwm_set_pwr(sc); + if (sc->sc_device_family == IWM_DEVICE_FAMILY_7000) + iwm_set_pwr(sc); iwm_mvm_nic_config(sc); @@ -1392,52 +1522,79 @@ const uint8_t iwm_mvm_ac_to_tx_fifo[] = { IWM_MVM_TX_FIFO_BK, }; -static void -iwm_enable_txq(struct iwm_softc *sc, int qid, int fifo) +static int +iwm_enable_txq(struct iwm_softc *sc, int sta_id, int qid, int fifo) { if (!iwm_nic_lock(sc)) { device_printf(sc->sc_dev, "%s: cannot enable txq %d\n", __func__, qid); - return; /* XXX return EBUSY */ + return EBUSY; } - /* unactivate before configuration */ - iwm_write_prph(sc, IWM_SCD_QUEUE_STATUS_BITS(qid), - (0 << IWM_SCD_QUEUE_STTS_REG_POS_ACTIVE) - | (1 << IWM_SCD_QUEUE_STTS_REG_POS_SCD_ACT_EN)); + IWM_WRITE(sc, IWM_HBUS_TARG_WRPTR, qid << 8 | 0); + + if (qid == IWM_MVM_CMD_QUEUE) { + /* unactivate before configuration */ + iwm_write_prph(sc, IWM_SCD_QUEUE_STATUS_BITS(qid), + (0 << IWM_SCD_QUEUE_STTS_REG_POS_ACTIVE) + | (1 << IWM_SCD_QUEUE_STTS_REG_POS_SCD_ACT_EN)); + + iwm_clear_bits_prph(sc, IWM_SCD_AGGR_SEL, (1 << qid)); + + iwm_write_prph(sc, IWM_SCD_QUEUE_RDPTR(qid), 0); + + iwm_write_mem32(sc, sc->sched_base + IWM_SCD_CONTEXT_QUEUE_OFFSET(qid), 0); + /* Set scheduler window size and frame limit. */ + iwm_write_mem32(sc, + sc->sched_base + IWM_SCD_CONTEXT_QUEUE_OFFSET(qid) + + sizeof(uint32_t), + ((IWM_FRAME_LIMIT << IWM_SCD_QUEUE_CTX_REG2_WIN_SIZE_POS) & + IWM_SCD_QUEUE_CTX_REG2_WIN_SIZE_MSK) | + ((IWM_FRAME_LIMIT << IWM_SCD_QUEUE_CTX_REG2_FRAME_LIMIT_POS) & + IWM_SCD_QUEUE_CTX_REG2_FRAME_LIMIT_MSK)); + + iwm_write_prph(sc, IWM_SCD_QUEUE_STATUS_BITS(qid), + (1 << IWM_SCD_QUEUE_STTS_REG_POS_ACTIVE) | + (fifo << IWM_SCD_QUEUE_STTS_REG_POS_TXF) | + (1 << IWM_SCD_QUEUE_STTS_REG_POS_WSL) | + IWM_SCD_QUEUE_STTS_REG_MSK); + } else { + struct iwm_scd_txq_cfg_cmd cmd; + int error; - if (qid != IWM_MVM_CMD_QUEUE) { - iwm_set_bits_prph(sc, IWM_SCD_QUEUECHAIN_SEL, (1 << qid)); - } + iwm_nic_unlock(sc); - iwm_clear_bits_prph(sc, IWM_SCD_AGGR_SEL, (1 << qid)); + memset(&cmd, 0, sizeof(cmd)); + cmd.scd_queue = qid; + cmd.enable = 1; + cmd.sta_id = sta_id; + cmd.tx_fifo = fifo; + cmd.aggregate = 0; + cmd.window = IWM_FRAME_LIMIT; - IWM_WRITE(sc, IWM_HBUS_TARG_WRPTR, qid << 8 | 0); - iwm_write_prph(sc, IWM_SCD_QUEUE_RDPTR(qid), 0); - - iwm_write_mem32(sc, sc->sched_base + IWM_SCD_CONTEXT_QUEUE_OFFSET(qid), 0); - /* Set scheduler window size and frame limit. */ - iwm_write_mem32(sc, - sc->sched_base + IWM_SCD_CONTEXT_QUEUE_OFFSET(qid) + - sizeof(uint32_t), - ((IWM_FRAME_LIMIT << IWM_SCD_QUEUE_CTX_REG2_WIN_SIZE_POS) & - IWM_SCD_QUEUE_CTX_REG2_WIN_SIZE_MSK) | - ((IWM_FRAME_LIMIT << IWM_SCD_QUEUE_CTX_REG2_FRAME_LIMIT_POS) & - IWM_SCD_QUEUE_CTX_REG2_FRAME_LIMIT_MSK)); - - iwm_write_prph(sc, IWM_SCD_QUEUE_STATUS_BITS(qid), - (1 << IWM_SCD_QUEUE_STTS_REG_POS_ACTIVE) | - (fifo << IWM_SCD_QUEUE_STTS_REG_POS_TXF) | - (1 << IWM_SCD_QUEUE_STTS_REG_POS_WSL) | - IWM_SCD_QUEUE_STTS_REG_MSK); + error = iwm_mvm_send_cmd_pdu(sc, IWM_SCD_QUEUE_CFG, IWM_CMD_SYNC, + sizeof(cmd), &cmd); + if (error) { + device_printf(sc->sc_dev, + "cannot enable txq %d\n", qid); + return error; + } + + if (!iwm_nic_lock(sc)) + return EBUSY; + } + + iwm_write_prph(sc, IWM_SCD_EN_CTRL, + iwm_read_prph(sc, IWM_SCD_EN_CTRL) | qid); iwm_nic_unlock(sc); - IWM_DPRINTF(sc, IWM_DEBUG_XMIT, - "%s: enabled txq %d FIFO %d\n", + IWM_DPRINTF(sc, IWM_DEBUG_XMIT, "%s: enabled txq %d FIFO %d\n", __func__, qid, fifo); + + return 0; } static int @@ -1445,16 +1602,16 @@ iwm_post_alive(struct iwm_softc *sc) { int nwords; int error, chnl; + uint32_t base; if (!iwm_nic_lock(sc)) return EBUSY; - if (sc->sched_base != iwm_read_prph(sc, IWM_SCD_SRAM_BASE_ADDR)) { + base = iwm_read_prph(sc, IWM_SCD_SRAM_BASE_ADDR); + if (sc->sched_base != base) { device_printf(sc->sc_dev, - "%s: sched addr mismatch", - __func__); - error = EINVAL; - goto out; + "%s: sched addr mismatch: alive: 0x%x prph: 0x%x\n", + __func__, sc->sched_base, base); } iwm_ict_reset(sc); @@ -1474,8 +1631,15 @@ iwm_post_alive(struct iwm_softc *sc) iwm_write_prph(sc, IWM_SCD_CHAINEXT_EN, 0); + iwm_nic_unlock(sc); + /* enable command channel */ - iwm_enable_txq(sc, IWM_MVM_CMD_QUEUE, 7); + error = iwm_enable_txq(sc, 0 /* unused */, IWM_MVM_CMD_QUEUE, 7); + if (error) + return error; + + if (!iwm_nic_lock(sc)) + return EBUSY; iwm_write_prph(sc, IWM_SCD_TXFACT, 0xff); @@ -1490,11 +1654,13 @@ iwm_post_alive(struct iwm_softc *sc) IWM_FH_TX_CHICKEN_BITS_SCD_AUTO_RETRY_EN); /* Enable L1-Active */ - iwm_clear_bits_prph(sc, IWM_APMG_PCIDEV_STT_REG, - IWM_APMG_PCIDEV_STT_VAL_L1_ACT_DIS); + if (sc->sc_device_family != IWM_DEVICE_FAMILY_8000) { + iwm_clear_bits_prph(sc, IWM_APMG_PCIDEV_STT_REG, + IWM_APMG_PCIDEV_STT_VAL_L1_ACT_DIS); + } out: - iwm_nic_unlock(sc); + iwm_nic_unlock(sc); return error; } @@ -1508,17 +1674,25 @@ iwm_post_alive(struct iwm_softc *sc) const int nvm_to_read[] = { IWM_NVM_SECTION_TYPE_HW, IWM_NVM_SECTION_TYPE_SW, + IWM_NVM_SECTION_TYPE_REGULATORY, IWM_NVM_SECTION_TYPE_CALIBRATION, IWM_NVM_SECTION_TYPE_PRODUCTION, + IWM_NVM_SECTION_TYPE_HW_8000, + IWM_NVM_SECTION_TYPE_MAC_OVERRIDE, + IWM_NVM_SECTION_TYPE_PHY_SKU, }; /* Default NVM size to read */ -#define IWM_NVM_DEFAULT_CHUNK_SIZE (2*1024) -#define IWM_MAX_NVM_SECTION_SIZE 7000 +#define IWM_NVM_DEFAULT_CHUNK_SIZE (2*1024) +#define IWM_MAX_NVM_SECTION_SIZE 8192 #define IWM_NVM_WRITE_OPCODE 1 #define IWM_NVM_READ_OPCODE 0 +/* load nvm chunk response */ +#define IWM_READ_NVM_CHUNK_SUCCEED 0 +#define IWM_READ_NVM_CHUNK_INVALID_ADDRESS 1 + static int iwm_nvm_read_chunk(struct iwm_softc *sc, uint16_t section, uint16_t offset, uint16_t length, uint8_t *data, uint16_t *len) @@ -1538,20 +1712,24 @@ iwm_nvm_read_chunk(struct iwm_softc *sc, uint16_t section, IWM_CMD_SEND_IN_RFKILL, .data = { &nvm_access_cmd, }, }; - int ret, bytes_read, offset_read; + int ret, offset_read; + size_t bytes_read; uint8_t *resp_data; cmd.len[0] = sizeof(struct iwm_nvm_access_cmd); ret = iwm_send_cmd(sc, &cmd); - if (ret) + if (ret) { + device_printf(sc->sc_dev, + "Could not send NVM_ACCESS command (error=%d)\n", ret); return ret; + } pkt = cmd.resp_pkt; if (pkt->hdr.flags & IWM_CMD_FAILED_MSK) { device_printf(sc->sc_dev, - "%s: Bad return from IWM_NVM_ACCES_COMMAND (0x%08X)\n", - __func__, pkt->hdr.flags); + "Bad return from IWM_NVM_ACCES_COMMAND (0x%08X)\n", + pkt->hdr.flags); ret = EIO; goto exit; } @@ -1564,17 +1742,25 @@ iwm_nvm_read_chunk(struct iwm_softc *sc, uint16_t section, offset_read = le16toh(nvm_resp->offset); resp_data = nvm_resp->data; if (ret) { - device_printf(sc->sc_dev, - "%s: NVM access command failed with status %d\n", - __func__, ret); + IWM_DPRINTF(sc, IWM_DEBUG_RESET, + "NVM access command failed with status %d\n", ret); ret = EINVAL; goto exit; } if (offset_read != offset) { device_printf(sc->sc_dev, - "%s: NVM ACCESS response with invalid offset %d\n", - __func__, offset_read); + "NVM ACCESS response with invalid offset %d\n", + offset_read); + ret = EINVAL; + goto exit; + } + + if (bytes_read > length) { + device_printf(sc->sc_dev, + "NVM ACCESS response with too much data " + "(%d bytes requested, %zd bytes received)\n", + length, bytes_read); ret = EINVAL; goto exit; } @@ -1589,7 +1775,7 @@ iwm_nvm_read_chunk(struct iwm_softc *sc, uint16_t section, /* * Reads an NVM section completely. - * NICs prior to 7000 family doesn't have a real NVM, but just read + * NICs prior to 7000 family don't have a real NVM, but just read * section 0 which is the EEPROM. Because the EEPROM reading is unlimited * by uCode, we need to manually check in this case that we don't * overflow and try to read more than the EEPROM size. @@ -1599,32 +1785,34 @@ iwm_nvm_read_chunk(struct iwm_softc *sc, uint16_t section, */ static int iwm_nvm_read_section(struct iwm_softc *sc, - uint16_t section, uint8_t *data, uint16_t *len) + uint16_t section, uint8_t *data, uint16_t *len, size_t max_len) { - uint16_t length, seglen; - int error; + uint16_t chunklen, seglen; + int error = 0; - /* Set nvm section read length */ - length = seglen = IWM_NVM_DEFAULT_CHUNK_SIZE; + IWM_DPRINTF(sc, IWM_DEBUG_RESET, + "reading NVM section %d\n", section); + + chunklen = seglen = IWM_NVM_DEFAULT_CHUNK_SIZE; *len = 0; - /* Read the NVM until exhausted (reading less than requested) */ - while (seglen == length) { + /* Read NVM chunks until exhausted (reading less than requested) */ + while (seglen == chunklen && *len < max_len) { error = iwm_nvm_read_chunk(sc, - section, *len, length, data, &seglen); + section, *len, chunklen, data, &seglen); if (error) { - device_printf(sc->sc_dev, - "Cannot read NVM from section " - "%d offset %d, length %d\n", - section, *len, length); + IWM_DPRINTF(sc, IWM_DEBUG_RESET, + "Cannot read from NVM section " + "%d at offset %d\n", section, *len); return error; } *len += seglen; } IWM_DPRINTF(sc, IWM_DEBUG_RESET, - "NVM section %d read completed\n", section); - return 0; + "NVM section %d read completed (%d bytes, error=%d)\n", + section, *len, error); + return error; } /* @@ -1634,7 +1822,7 @@ iwm_nvm_read_section(struct iwm_softc *sc, /* iwlwifi/iwl-nvm-parse.c */ /* NVM offsets (in words) definitions */ -enum wkp_nvm_offsets { +enum iwm_nvm_offsets { /* NVM HW-Section offset (in words) definitions */ IWM_HW_ADDR = 0x15, @@ -1651,6 +1839,32 @@ enum wkp_nvm_offsets { IWM_XTAL_CALIB = 0x316 - IWM_NVM_CALIB_SECTION }; +enum iwm_8000_nvm_offsets { + /* NVM HW-Section offset (in words) definitions */ + IWM_HW_ADDR0_WFPM_8000 = 0x12, + IWM_HW_ADDR1_WFPM_8000 = 0x16, + IWM_HW_ADDR0_PCIE_8000 = 0x8A, + IWM_HW_ADDR1_PCIE_8000 = 0x8E, + IWM_MAC_ADDRESS_OVERRIDE_8000 = 1, + + /* NVM SW-Section offset (in words) definitions */ + IWM_NVM_SW_SECTION_8000 = 0x1C0, + IWM_NVM_VERSION_8000 = 0, + IWM_RADIO_CFG_8000 = 0, + IWM_SKU_8000 = 2, + IWM_N_HW_ADDRS_8000 = 3, + + /* NVM REGULATORY -Section offset (in words) definitions */ + IWM_NVM_CHANNELS_8000 = 0, + IWM_NVM_LAR_OFFSET_8000_OLD = 0x4C7, + IWM_NVM_LAR_OFFSET_8000 = 0x507, + IWM_NVM_LAR_ENABLED_8000 = 0x7, + + /* NVM calibration section offset (in words) definitions */ + IWM_NVM_CALIB_SECTION_8000 = 0x2B8, + IWM_XTAL_CALIB_8000 = 0x316 - IWM_NVM_CALIB_SECTION_8000 +}; + /* SKU Capabilities (actual values from NVM definition) */ enum nvm_sku_bits { IWM_NVM_SKU_CAP_BAND_24GHZ = (1 << 0), @@ -1667,6 +1881,13 @@ enum nvm_sku_bits { #define IWM_NVM_RF_CFG_TX_ANT_MSK(x) ((x >> 8) & 0xF) /* bits 8-11 */ #define IWM_NVM_RF_CFG_RX_ANT_MSK(x) ((x >> 12) & 0xF) /* bits 12-15 */ +#define IWM_NVM_RF_CFG_FLAVOR_MSK_8000(x) (x & 0xF) +#define IWM_NVM_RF_CFG_DASH_MSK_8000(x) ((x >> 4) & 0xF) +#define IWM_NVM_RF_CFG_STEP_MSK_8000(x) ((x >> 8) & 0xF) +#define IWM_NVM_RF_CFG_TYPE_MSK_8000(x) ((x >> 12) & 0xFFF) +#define IWM_NVM_RF_CFG_TX_ANT_MSK_8000(x) ((x >> 24) & 0xF) +#define IWM_NVM_RF_CFG_RX_ANT_MSK_8000(x) ((x >> 28) & 0xF) + #define DEFAULT_MAX_TX_POWER 16 /** @@ -1718,7 +1939,8 @@ iwm_eeprom_channel_flags(uint16_t ch_flags) static void iwm_add_channel_band(struct iwm_softc *sc, struct ieee80211_channel chans[], - int maxchans, int *nchans, int ch_idx, int ch_num, const uint8_t bands[]) + int maxchans, int *nchans, int ch_idx, size_t ch_num, + const uint8_t bands[]) { const uint16_t * const nvm_ch_flags = sc->sc_nvm.nvm_ch_flags; uint32_t nflags; @@ -1728,7 +1950,10 @@ iwm_add_channel_band(struct iwm_softc *sc, struct ieee80211_channel chans[], for (; ch_idx < ch_num; ch_idx++) { ch_flags = le16_to_cpup(nvm_ch_flags + ch_idx); - ieee = iwm_nvm_channels[ch_idx]; + if (sc->sc_device_family == IWM_DEVICE_FAMILY_7000) + ieee = iwm_nvm_channels[ch_idx]; + else + ieee = iwm_nvm_channels_8000[ch_idx]; if (!(ch_flags & IWM_NVM_CHANNEL_VALID)) { IWM_DPRINTF(sc, IWM_DEBUG_EEPROM, @@ -1760,6 +1985,7 @@ iwm_init_channel_map(struct ieee80211com *ic, int maxchans, int *nchans, struct iwm_softc *sc = ic->ic_softc; struct iwm_nvm_data *data = &sc->sc_nvm; uint8_t bands[IEEE80211_MODE_BYTES]; + size_t ch_num; memset(bands, 0, sizeof(bands)); /* 1-13: 11b/g channels. */ @@ -1774,51 +2000,182 @@ iwm_init_channel_map(struct ieee80211com *ic, int maxchans, int *nchans, IWM_NUM_2GHZ_CHANNELS - 1, IWM_NUM_2GHZ_CHANNELS, bands); if (data->sku_cap_band_52GHz_enable) { + if (sc->sc_device_family == IWM_DEVICE_FAMILY_7000) + ch_num = nitems(iwm_nvm_channels); + else + ch_num = nitems(iwm_nvm_channels_8000); memset(bands, 0, sizeof(bands)); setbit(bands, IEEE80211_MODE_11A); iwm_add_channel_band(sc, chans, maxchans, nchans, - IWM_NUM_2GHZ_CHANNELS, nitems(iwm_nvm_channels), bands); + IWM_NUM_2GHZ_CHANNELS, ch_num, bands); } } +static void +iwm_set_hw_address_8000(struct iwm_softc *sc, struct iwm_nvm_data *data, + const uint16_t *mac_override, const uint16_t *nvm_hw) +{ + const uint8_t *hw_addr; + + if (mac_override) { + static const uint8_t reserved_mac[] = { + 0x02, 0xcc, 0xaa, 0xff, 0xee, 0x00 + }; + + hw_addr = (const uint8_t *)(mac_override + + IWM_MAC_ADDRESS_OVERRIDE_8000); + + /* + * Store the MAC address from MAO section. + * No byte swapping is required in MAO section + */ + IEEE80211_ADDR_COPY(data->hw_addr, hw_addr); + + /* + * Force the use of the OTP MAC address in case of reserved MAC + * address in the NVM, or if address is given but invalid. + */ + if (!IEEE80211_ADDR_EQ(reserved_mac, hw_addr) && + !IEEE80211_ADDR_EQ(ieee80211broadcastaddr, data->hw_addr) && + iwm_is_valid_ether_addr(data->hw_addr) && + !IEEE80211_IS_MULTICAST(data->hw_addr)) + return; + + IWM_DPRINTF(sc, IWM_DEBUG_RESET, + "%s: mac address from nvm override section invalid\n", + __func__); + } + + if (nvm_hw) { + /* read the mac address from WFMP registers */ + uint32_t mac_addr0 = + htole32(iwm_read_prph(sc, IWM_WFMP_MAC_ADDR_0)); + uint32_t mac_addr1 = + htole32(iwm_read_prph(sc, IWM_WFMP_MAC_ADDR_1)); + + hw_addr = (const uint8_t *)&mac_addr0; + data->hw_addr[0] = hw_addr[3]; + data->hw_addr[1] = hw_addr[2]; + data->hw_addr[2] = hw_addr[1]; + data->hw_addr[3] = hw_addr[0]; + + hw_addr = (const uint8_t *)&mac_addr1; + data->hw_addr[4] = hw_addr[1]; + data->hw_addr[5] = hw_addr[0]; + + return; + } + + device_printf(sc->sc_dev, "%s: mac address not found\n", __func__); + memset(data->hw_addr, 0, sizeof(data->hw_addr)); +} + +static int +iwm_get_sku(const struct iwm_softc *sc, const uint16_t *nvm_sw, + const uint16_t *phy_sku) +{ + if (sc->sc_device_family != IWM_DEVICE_FAMILY_8000) + return le16_to_cpup(nvm_sw + IWM_SKU); + + return le32_to_cpup((const uint32_t *)(phy_sku + IWM_SKU_8000)); +} + +static int +iwm_get_nvm_version(const struct iwm_softc *sc, const uint16_t *nvm_sw) +{ + if (sc->sc_device_family != IWM_DEVICE_FAMILY_8000) + return le16_to_cpup(nvm_sw + IWM_NVM_VERSION); + else + return le32_to_cpup((const uint32_t *)(nvm_sw + + IWM_NVM_VERSION_8000)); +} + +static int +iwm_get_radio_cfg(const struct iwm_softc *sc, const uint16_t *nvm_sw, + const uint16_t *phy_sku) +{ + if (sc->sc_device_family != IWM_DEVICE_FAMILY_8000) + return le16_to_cpup(nvm_sw + IWM_RADIO_CFG); + + return le32_to_cpup((const uint32_t *)(phy_sku + IWM_RADIO_CFG_8000)); +} + +static int +iwm_get_n_hw_addrs(const struct iwm_softc *sc, const uint16_t *nvm_sw) +{ + int n_hw_addr; + + if (sc->sc_device_family != IWM_DEVICE_FAMILY_8000) + return le16_to_cpup(nvm_sw + IWM_N_HW_ADDRS); + + n_hw_addr = le32_to_cpup((const uint32_t *)(nvm_sw + IWM_N_HW_ADDRS_8000)); + + return n_hw_addr & IWM_N_HW_ADDR_MASK; +} + +static void +iwm_set_radio_cfg(const struct iwm_softc *sc, struct iwm_nvm_data *data, + uint32_t radio_cfg) +{ + if (sc->sc_device_family != IWM_DEVICE_FAMILY_8000) { + data->radio_cfg_type = IWM_NVM_RF_CFG_TYPE_MSK(radio_cfg); + data->radio_cfg_step = IWM_NVM_RF_CFG_STEP_MSK(radio_cfg); + data->radio_cfg_dash = IWM_NVM_RF_CFG_DASH_MSK(radio_cfg); + data->radio_cfg_pnum = IWM_NVM_RF_CFG_PNUM_MSK(radio_cfg); + return; + } + + /* set the radio configuration for family 8000 */ + data->radio_cfg_type = IWM_NVM_RF_CFG_TYPE_MSK_8000(radio_cfg); + data->radio_cfg_step = IWM_NVM_RF_CFG_STEP_MSK_8000(radio_cfg); + data->radio_cfg_dash = IWM_NVM_RF_CFG_DASH_MSK_8000(radio_cfg); + data->radio_cfg_pnum = IWM_NVM_RF_CFG_FLAVOR_MSK_8000(radio_cfg); + data->valid_tx_ant = IWM_NVM_RF_CFG_TX_ANT_MSK_8000(radio_cfg); + data->valid_rx_ant = IWM_NVM_RF_CFG_RX_ANT_MSK_8000(radio_cfg); +} + static int iwm_parse_nvm_data(struct iwm_softc *sc, - const uint16_t *nvm_hw, const uint16_t *nvm_sw, - const uint16_t *nvm_calib, uint8_t tx_chains, uint8_t rx_chains) + const uint16_t *nvm_hw, const uint16_t *nvm_sw, + const uint16_t *nvm_calib, const uint16_t *mac_override, + const uint16_t *phy_sku, const uint16_t *regulatory) { struct iwm_nvm_data *data = &sc->sc_nvm; uint8_t hw_addr[IEEE80211_ADDR_LEN]; - uint16_t radio_cfg, sku; + uint32_t sku, radio_cfg; - data->nvm_version = le16_to_cpup(nvm_sw + IWM_NVM_VERSION); + data->nvm_version = iwm_get_nvm_version(sc, nvm_sw); - radio_cfg = le16_to_cpup(nvm_sw + IWM_RADIO_CFG); - data->radio_cfg_type = IWM_NVM_RF_CFG_TYPE_MSK(radio_cfg); - data->radio_cfg_step = IWM_NVM_RF_CFG_STEP_MSK(radio_cfg); - data->radio_cfg_dash = IWM_NVM_RF_CFG_DASH_MSK(radio_cfg); - data->radio_cfg_pnum = IWM_NVM_RF_CFG_PNUM_MSK(radio_cfg); + radio_cfg = iwm_get_radio_cfg(sc, nvm_sw, phy_sku); + iwm_set_radio_cfg(sc, data, radio_cfg); - sku = le16_to_cpup(nvm_sw + IWM_SKU); + sku = iwm_get_sku(sc, nvm_sw, phy_sku); data->sku_cap_band_24GHz_enable = sku & IWM_NVM_SKU_CAP_BAND_24GHZ; data->sku_cap_band_52GHz_enable = sku & IWM_NVM_SKU_CAP_BAND_52GHZ; data->sku_cap_11n_enable = 0; - data->n_hw_addrs = le16_to_cpup(nvm_sw + IWM_N_HW_ADDRS); - - data->xtal_calib[0] = *(nvm_calib + IWM_XTAL_CALIB); - data->xtal_calib[1] = *(nvm_calib + IWM_XTAL_CALIB + 1); + data->n_hw_addrs = iwm_get_n_hw_addrs(sc, nvm_sw); /* The byte order is little endian 16 bit, meaning 214365 */ - IEEE80211_ADDR_COPY(hw_addr, nvm_hw + IWM_HW_ADDR); - data->hw_addr[0] = hw_addr[1]; - data->hw_addr[1] = hw_addr[0]; - data->hw_addr[2] = hw_addr[3]; - data->hw_addr[3] = hw_addr[2]; - data->hw_addr[4] = hw_addr[5]; - data->hw_addr[5] = hw_addr[4]; - - memcpy(data->nvm_ch_flags, &nvm_sw[IWM_NVM_CHANNELS], - sizeof(data->nvm_ch_flags)); + if (sc->sc_device_family == IWM_DEVICE_FAMILY_7000) { + IEEE80211_ADDR_COPY(hw_addr, nvm_hw + IWM_HW_ADDR); + data->hw_addr[0] = hw_addr[1]; + data->hw_addr[1] = hw_addr[0]; + data->hw_addr[2] = hw_addr[3]; + data->hw_addr[3] = hw_addr[2]; + data->hw_addr[4] = hw_addr[5]; + data->hw_addr[5] = hw_addr[4]; + } else { + iwm_set_hw_address_8000(sc, data, mac_override, nvm_hw); + } + + if (sc->sc_device_family == IWM_DEVICE_FAMILY_7000) { + memcpy(data->nvm_ch_flags, &nvm_sw[IWM_NVM_CHANNELS], + IWM_NUM_CHANNELS * sizeof(uint16_t)); + } else { + memcpy(data->nvm_ch_flags, ®ulatory[IWM_NVM_CHANNELS_8000], + IWM_NUM_CHANNELS_8000 * sizeof(uint16_t)); + } data->calib_version = 255; /* TODO: this value will prevent some checks from failing, we need to check if this @@ -1832,30 +2189,61 @@ iwm_parse_nvm_data(struct iwm_softc *sc, * END NVM PARSE */ -struct iwm_nvm_section { - uint16_t length; - uint8_t *data; -}; - static int iwm_parse_nvm_sections(struct iwm_softc *sc, struct iwm_nvm_section *sections) { - const uint16_t *hw, *sw, *calib; + const uint16_t *hw, *sw, *calib, *regulatory, *mac_override, *phy_sku; /* Checking for required sections */ - if (!sections[IWM_NVM_SECTION_TYPE_SW].data || - !sections[IWM_NVM_SECTION_TYPE_HW].data) { - device_printf(sc->sc_dev, - "%s: Can't parse empty NVM sections\n", - __func__); - return ENOENT; + if (sc->sc_device_family == IWM_DEVICE_FAMILY_7000) { + if (!sections[IWM_NVM_SECTION_TYPE_SW].data || + !sections[IWM_NVM_SECTION_TYPE_HW].data) { + device_printf(sc->sc_dev, + "Can't parse empty OTP/NVM sections\n"); + return ENOENT; + } + + hw = (const uint16_t *) sections[IWM_NVM_SECTION_TYPE_HW].data; + } else if (sc->sc_device_family == IWM_DEVICE_FAMILY_8000) { + /* SW and REGULATORY sections are mandatory */ + if (!sections[IWM_NVM_SECTION_TYPE_SW].data || + !sections[IWM_NVM_SECTION_TYPE_REGULATORY].data) { + device_printf(sc->sc_dev, + "Can't parse empty OTP/NVM sections\n"); + return ENOENT; + } + /* MAC_OVERRIDE or at least HW section must exist */ + if (!sections[IWM_NVM_SECTION_TYPE_HW_8000].data && + !sections[IWM_NVM_SECTION_TYPE_MAC_OVERRIDE].data) { + device_printf(sc->sc_dev, + "Can't parse mac_address, empty sections\n"); + return ENOENT; + } + + /* PHY_SKU section is mandatory in B0 */ + if (!sections[IWM_NVM_SECTION_TYPE_PHY_SKU].data) { + device_printf(sc->sc_dev, + "Can't parse phy_sku in B0, empty sections\n"); + return ENOENT; + } + + hw = (const uint16_t *) + sections[IWM_NVM_SECTION_TYPE_HW_8000].data; + } else { + panic("unknown device family %d\n", sc->sc_device_family); } - hw = (const uint16_t *)sections[IWM_NVM_SECTION_TYPE_HW].data; sw = (const uint16_t *)sections[IWM_NVM_SECTION_TYPE_SW].data; - calib = (const uint16_t *)sections[IWM_NVM_SECTION_TYPE_CALIBRATION].data; - return iwm_parse_nvm_data(sc, hw, sw, calib, - IWM_FW_VALID_TX_ANT(sc), IWM_FW_VALID_RX_ANT(sc)); + calib = (const uint16_t *) + sections[IWM_NVM_SECTION_TYPE_CALIBRATION].data; + regulatory = (const uint16_t *) + sections[IWM_NVM_SECTION_TYPE_REGULATORY].data; + mac_override = (const uint16_t *) + sections[IWM_NVM_SECTION_TYPE_MAC_OVERRIDE].data; + phy_sku = (const uint16_t *)sections[IWM_NVM_SECTION_TYPE_PHY_SKU].data; + + return iwm_parse_nvm_data(sc, hw, sw, calib, mac_override, + phy_sku, regulatory); } static int @@ -1864,38 +2252,34 @@ iwm_nvm_init(struct iwm_softc *sc) struct iwm_nvm_section nvm_sections[IWM_NVM_NUM_OF_SECTIONS]; int i, section, error; uint16_t len; - uint8_t *nvm_buffer, *temp; + uint8_t *buf; + const size_t bufsz = IWM_MAX_NVM_SECTION_SIZE; - /* Read From FW NVM */ - IWM_DPRINTF(sc, IWM_DEBUG_EEPROM, - "%s: Read NVM\n", - __func__); + memset(nvm_sections, 0 , sizeof(nvm_sections)); - memset(nvm_sections, 0, sizeof(nvm_sections)); + buf = malloc(bufsz, M_DEVBUF, M_NOWAIT); + if (buf == NULL) + return ENOMEM; - /* TODO: find correct NVM max size for a section */ - nvm_buffer = malloc(IWM_OTP_LOW_IMAGE_SIZE, M_DEVBUF, M_NOWAIT); - if (nvm_buffer == NULL) - return (ENOMEM); for (i = 0; i < nitems(nvm_to_read); i++) { section = nvm_to_read[i]; KASSERT(section <= nitems(nvm_sections), ("too many sections")); - error = iwm_nvm_read_section(sc, section, nvm_buffer, &len); - if (error) - break; - - temp = malloc(len, M_DEVBUF, M_NOWAIT); - if (temp == NULL) { + error = iwm_nvm_read_section(sc, section, buf, &len, bufsz); + if (error) { + error = 0; + continue; + } + nvm_sections[section].data = malloc(len, M_DEVBUF, M_NOWAIT); + if (nvm_sections[section].data == NULL) { error = ENOMEM; break; } - memcpy(temp, nvm_buffer, len); - nvm_sections[section].data = temp; + memcpy(nvm_sections[section].data, buf, len); nvm_sections[section].length = len; } - free(nvm_buffer, M_DEVBUF); + free(buf, M_DEVBUF); if (error == 0) error = iwm_parse_nvm_sections(sc, nvm_sections); @@ -1913,21 +2297,52 @@ iwm_nvm_init(struct iwm_softc *sc) */ static int -iwm_firmware_load_chunk(struct iwm_softc *sc, uint32_t dst_addr, +iwm_firmware_load_sect(struct iwm_softc *sc, uint32_t dst_addr, const uint8_t *section, uint32_t byte_cnt) { + int error = EINVAL; + uint32_t chunk_sz, offset; + + chunk_sz = MIN(IWM_FH_MEM_TB_MAX_LENGTH, byte_cnt); + + for (offset = 0; offset < byte_cnt; offset += chunk_sz) { + uint32_t addr, len; + const uint8_t *data; + + addr = dst_addr + offset; + len = MIN(chunk_sz, byte_cnt - offset); + data = section + offset; + + error = iwm_firmware_load_chunk(sc, addr, data, len); + if (error) + break; + } + + return error; +} + +static int +iwm_firmware_load_chunk(struct iwm_softc *sc, uint32_t dst_addr, + const uint8_t *chunk, uint32_t byte_cnt) +{ struct iwm_dma_info *dma = &sc->fw_dma; int error; - /* Copy firmware section into pre-allocated DMA-safe memory. */ - memcpy(dma->vaddr, section, byte_cnt); + /* Copy firmware chunk into pre-allocated DMA-safe memory. */ + memcpy(dma->vaddr, chunk, byte_cnt); bus_dmamap_sync(dma->tag, dma->map, BUS_DMASYNC_PREWRITE); - if (!iwm_nic_lock(sc)) - return EBUSY; + if (dst_addr >= IWM_FW_MEM_EXTENDED_START && + dst_addr <= IWM_FW_MEM_EXTENDED_END) { + iwm_set_bits_prph(sc, IWM_LMPM_CHICK, + IWM_LMPM_CHICK_EXTENDED_ADDR_SPACE); + } sc->sc_fw_chunk_done = 0; + if (!iwm_nic_lock(sc)) + return EBUSY; + IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_CONFIG_REG(IWM_FH_SRVC_CHNL), IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_PAUSE); IWM_WRITE(sc, IWM_FH_SRVC_CHNL_SRAM_ADDR_REG(IWM_FH_SRVC_CHNL), @@ -1953,14 +2368,133 @@ iwm_firmware_load_chunk(struct iwm_softc *sc, uint32_t dst_addr, if ((error = msleep(&sc->sc_fw, &sc->sc_mtx, 0, "iwmfw", hz)) != 0) break; + if (!sc->sc_fw_chunk_done) { + device_printf(sc->sc_dev, + "fw chunk addr 0x%x len %d failed to load\n", + dst_addr, byte_cnt); + } + + if (dst_addr >= IWM_FW_MEM_EXTENDED_START && + dst_addr <= IWM_FW_MEM_EXTENDED_END && iwm_nic_lock(sc)) { + iwm_clear_bits_prph(sc, IWM_LMPM_CHICK, + IWM_LMPM_CHICK_EXTENDED_ADDR_SPACE); + iwm_nic_unlock(sc); + } + return error; } +int +iwm_load_cpu_sections_8000(struct iwm_softc *sc, struct iwm_fw_sects *fws, + int cpu, int *first_ucode_section) +{ + int shift_param; + int i, error = 0, sec_num = 0x1; + uint32_t val, last_read_idx = 0; + const void *data; + uint32_t dlen; + uint32_t offset; + + if (cpu == 1) { + shift_param = 0; + *first_ucode_section = 0; + } else { + shift_param = 16; + (*first_ucode_section)++; + } + + for (i = *first_ucode_section; i < IWM_UCODE_SECT_MAX; i++) { + last_read_idx = i; + data = fws->fw_sect[i].fws_data; + dlen = fws->fw_sect[i].fws_len; + offset = fws->fw_sect[i].fws_devoff; + + /* + * CPU1_CPU2_SEPARATOR_SECTION delimiter - separate between + * CPU1 to CPU2. + * PAGING_SEPARATOR_SECTION delimiter - separate between + * CPU2 non paged to CPU2 paging sec. + */ + if (!data || offset == IWM_CPU1_CPU2_SEPARATOR_SECTION || + offset == IWM_PAGING_SEPARATOR_SECTION) + break; + + IWM_DPRINTF(sc, IWM_DEBUG_RESET, + "LOAD FIRMWARE chunk %d offset 0x%x len %d for cpu %d\n", + i, offset, dlen, cpu); + + if (dlen > sc->sc_fwdmasegsz) { + IWM_DPRINTF(sc, IWM_DEBUG_RESET, + "chunk %d too large (%d bytes)\n", i, dlen); + error = EFBIG; + } else { + error = iwm_firmware_load_sect(sc, offset, data, dlen); + } + if (error) { + device_printf(sc->sc_dev, + "could not load firmware chunk %d (error %d)\n", + i, error); + return error; + } + + /* Notify the ucode of the loaded section number and status */ + if (iwm_nic_lock(sc)) { + val = IWM_READ(sc, IWM_FH_UCODE_LOAD_STATUS); + val = val | (sec_num << shift_param); + IWM_WRITE(sc, IWM_FH_UCODE_LOAD_STATUS, val); + sec_num = (sec_num << 1) | 0x1; + iwm_nic_unlock(sc); + + /* + * The firmware won't load correctly without this delay. + */ + DELAY(8000); + } + } + + *first_ucode_section = last_read_idx; + + if (iwm_nic_lock(sc)) { + if (cpu == 1) + IWM_WRITE(sc, IWM_FH_UCODE_LOAD_STATUS, 0xFFFF); + else + IWM_WRITE(sc, IWM_FH_UCODE_LOAD_STATUS, 0xFFFFFFFF); + iwm_nic_unlock(sc); + } + + return 0; +} + +int +iwm_load_firmware_8000(struct iwm_softc *sc, enum iwm_ucode_type ucode_type) +{ + struct iwm_fw_sects *fws; + int error = 0; + int first_ucode_section; + + IWM_DPRINTF(sc, IWM_DEBUG_RESET, "loading ucode type %d\n", + ucode_type); + + fws = &sc->sc_fw.fw_sects[ucode_type]; + + /* configure the ucode to be ready to get the secured image */ + /* release CPU reset */ + iwm_write_prph(sc, IWM_RELEASE_CPU_RESET, IWM_RELEASE_CPU_RESET_BIT); + + /* load to FW the binary Secured sections of CPU1 */ + error = iwm_load_cpu_sections_8000(sc, fws, 1, &first_ucode_section); + if (error) + return error; + + /* load to FW the binary sections of CPU2 */ + return iwm_load_cpu_sections_8000(sc, fws, 2, &first_ucode_section); +} + static int -iwm_load_firmware(struct iwm_softc *sc, enum iwm_ucode_type ucode_type) +iwm_load_firmware_7000(struct iwm_softc *sc, enum iwm_ucode_type ucode_type) { struct iwm_fw_sects *fws; - int error, i, w; + int error, i; const void *data; uint32_t dlen; uint32_t offset; @@ -1975,21 +2509,57 @@ iwm_load_firmware(struct iwm_softc *sc, enum iwm_ucode_type ucode_type) IWM_DPRINTF(sc, IWM_DEBUG_FIRMWARE_TLV, "LOAD FIRMWARE type %d offset %u len %d\n", ucode_type, offset, dlen); - error = iwm_firmware_load_chunk(sc, offset, data, dlen); + if (dlen > sc->sc_fwdmasegsz) { + IWM_DPRINTF(sc, IWM_DEBUG_FIRMWARE_TLV, + "chunk %d too large (%d bytes)\n", i, dlen); + error = EFBIG; + } else { + error = iwm_firmware_load_sect(sc, offset, data, dlen); + } if (error) { device_printf(sc->sc_dev, - "%s: chunk %u of %u returned error %02d\n", - __func__, i, fws->fw_count, error); + "could not load firmware chunk %u of %u " + "(error=%d)\n", i, fws->fw_count, error); return error; } } - /* wait for the firmware to load */ IWM_WRITE(sc, IWM_CSR_RESET, 0); + return 0; +} + +static int +iwm_load_firmware(struct iwm_softc *sc, enum iwm_ucode_type ucode_type) +{ + int error, w; + + if (sc->sc_device_family == IWM_DEVICE_FAMILY_8000) + error = iwm_load_firmware_8000(sc, ucode_type); + else + error = iwm_load_firmware_7000(sc, ucode_type); + if (error) + return error; + + /* wait for the firmware to load */ for (w = 0; !sc->sc_uc.uc_intr && w < 10; w++) { error = msleep(&sc->sc_uc, &sc->sc_mtx, 0, "iwmuc", hz/10); } + if (error || !sc->sc_uc.uc_ok) { + device_printf(sc->sc_dev, "could not load firmware\n"); + if (sc->sc_device_family == IWM_DEVICE_FAMILY_8000) { + device_printf(sc->sc_dev, "cpu1 status: 0x%x\n", + iwm_read_prph(sc, IWM_SB_CPU_1_STATUS)); + device_printf(sc->sc_dev, "cpu2 status: 0x%x\n", + iwm_read_prph(sc, IWM_SB_CPU_2_STATUS)); + } + } + + /* + * Give the firmware some time to initialize. + * Accessing it too early causes errors. + */ + msleep(&w, &sc->sc_mtx, 0, "iwmfwinit", hz); return error; } @@ -2063,17 +2633,25 @@ iwm_mvm_load_ucode_wait_alive(struct iwm_softc *sc, enum iwm_ucode_type old_type = sc->sc_uc_current; int error; - if ((error = iwm_read_firmware(sc, ucode_type)) != 0) + if ((error = iwm_read_firmware(sc, ucode_type)) != 0) { + device_printf(sc->sc_dev, "iwm_read_firmware: failed %d\n", + error); return error; + } sc->sc_uc_current = ucode_type; error = iwm_start_fw(sc, ucode_type); if (error) { + device_printf(sc->sc_dev, "iwm_start_fw: failed %d\n", error); sc->sc_uc_current = old_type; return error; } - return iwm_post_alive(sc); + error = iwm_post_alive(sc); + if (error) { + device_printf(sc->sc_dev, "iwm_fw_alive: failed %d\n", error); + } + return error; } /* @@ -2109,26 +2687,31 @@ iwm_run_init_mvm_ucode(struct iwm_softc *sc, int justnvm) } IEEE80211_ADDR_COPY(sc->sc_ic.ic_macaddr, sc->sc_nvm.hw_addr); - sc->sc_scan_cmd_len = sizeof(struct iwm_scan_cmd) - + sc->sc_capa_max_probe_len - + IWM_MAX_NUM_SCAN_CHANNELS - * sizeof(struct iwm_scan_channel); - sc->sc_scan_cmd = malloc(sc->sc_scan_cmd_len, M_DEVBUF, - M_NOWAIT); - if (sc->sc_scan_cmd == NULL) - return (ENOMEM); - return 0; } + if ((error = iwm_send_bt_init_conf(sc)) != 0) { + device_printf(sc->sc_dev, + "failed to send bt coex configuration: %d\n", error); + return error; + } + + /* Init Smart FIFO. */ + error = iwm_mvm_sf_config(sc, IWM_SF_INIT_OFF); + if (error != 0) + return error; + /* Send TX valid antennas before triggering calibrations */ - if ((error = iwm_send_tx_ant_cfg(sc, IWM_FW_VALID_TX_ANT(sc))) != 0) + if ((error = iwm_send_tx_ant_cfg(sc, iwm_fw_valid_tx_ant(sc))) != 0) { + device_printf(sc->sc_dev, + "failed to send antennas before calibration: %d\n", error); return error; + } /* - * Send phy configurations command to init uCode - * to start the 16.0 uCode init image internal calibrations. - */ + * Send phy configurations command to init uCode + * to start the 16.0 uCode init image internal calibrations. + */ if ((error = iwm_send_phy_cfg_cmd(sc)) != 0 ) { device_printf(sc->sc_dev, "%s: failed to run internal calibration: %d\n", @@ -2140,10 +2723,18 @@ iwm_run_init_mvm_ucode(struct iwm_softc *sc, int justnvm) * Nothing to do but wait for the init complete notification * from the firmware */ - while (!sc->sc_init_complete) - if ((error = msleep(&sc->sc_init_complete, &sc->sc_mtx, - 0, "iwminit", 2*hz)) != 0) + while (!sc->sc_init_complete) { + error = msleep(&sc->sc_init_complete, &sc->sc_mtx, + 0, "iwminit", 2*hz); + if (error) { + device_printf(sc->sc_dev, "init complete failed: %d\n", + sc->sc_init_complete); break; + } + } + + IWM_DPRINTF(sc, IWM_DEBUG_RESET, "init %scomplete\n", + sc->sc_init_complete ? "" : "not "); return error; } @@ -2160,18 +2751,17 @@ iwm_rx_addbuf(struct iwm_softc *sc, int size, int idx) struct iwm_rx_data *data = &ring->data[idx]; struct mbuf *m; bus_dmamap_t dmamap = NULL; - int error; - bus_addr_t paddr; + bus_dma_segment_t seg; + int nsegs, error; m = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, IWM_RBUF_SIZE); if (m == NULL) return ENOBUFS; m->m_len = m->m_pkthdr.len = m->m_ext.ext_size; - error = bus_dmamap_load(ring->data_dmat, ring->spare_map, - mtod(m, void *), IWM_RBUF_SIZE, iwm_dma_map_addr, - &paddr, BUS_DMA_NOWAIT); - if (error != 0 && error != EFBIG) { + error = bus_dmamap_load_mbuf_sg(ring->data_dmat, ring->spare_map, m, + &seg, &nsegs, BUS_DMA_NOWAIT); + if (error != 0) { device_printf(sc->sc_dev, "%s: can't map mbuf, error %d\n", __func__, error); goto fail; @@ -2189,13 +2779,14 @@ iwm_rx_addbuf(struct iwm_softc *sc, int size, int idx) data->m = m; /* Update RX descriptor. */ - ring->desc[idx] = htole32(paddr >> 8); + KASSERT((seg.ds_addr & 255) == 0, ("seg.ds_addr not aligned")); + ring->desc[idx] = htole32(seg.ds_addr >> 8); bus_dmamap_sync(ring->desc_dma.tag, ring->desc_dma.map, BUS_DMASYNC_PREWRITE); return 0; fail: - m_free(m); + m_freem(m); return error; } @@ -2750,7 +3341,7 @@ iwm_tx(struct iwm_softc *sc, struct mbuf *m, struct ieee80211_node *ni, int ac) flags |= IWM_TX_CMD_FLG_ACK; } - if (type != IEEE80211_FC0_TYPE_DATA + if (type == IEEE80211_FC0_TYPE_DATA && (totlen + IEEE80211_CRC_LEN > vap->iv_rtsthreshold) && !IEEE80211_IS_MULTICAST(wh->i_addr1)) { flags |= IWM_TX_CMD_FLG_PROT_REQUIRE; @@ -2766,12 +3357,15 @@ iwm_tx(struct iwm_softc *sc, struct mbuf *m, struct ieee80211_node *ni, int ac) uint8_t subtype = wh->i_fc[0] & IEEE80211_FC0_SUBTYPE_MASK; if (subtype == IEEE80211_FC0_SUBTYPE_ASSOC_REQ || - subtype == IEEE80211_FC0_SUBTYPE_REASSOC_REQ) - tx->pm_frame_timeout = htole16(3); - else - tx->pm_frame_timeout = htole16(2); + subtype == IEEE80211_FC0_SUBTYPE_REASSOC_REQ) { + tx->pm_frame_timeout = htole16(IWM_PM_FRAME_ASSOC); + } else if (subtype == IEEE80211_FC0_SUBTYPE_ACTION) { + tx->pm_frame_timeout = htole16(IWM_PM_FRAME_NONE); + } else { + tx->pm_frame_timeout = htole16(IWM_PM_FRAME_MGMT); + } } else { - tx->pm_frame_timeout = htole16(0); + tx->pm_frame_timeout = htole16(IWM_PM_FRAME_NONE); } if (hdrlen & 3) { @@ -2839,11 +3433,11 @@ iwm_tx(struct iwm_softc *sc, struct mbuf *m, struct ieee80211_node *ni, int ac) KASSERT(data->in != NULL, ("node is NULL")); IWM_DPRINTF(sc, IWM_DEBUG_XMIT, - "sending data: qid=%d idx=%d len=%d nsegs=%d txflags=0x%08x rate_n_flags=0x%08x rateidx=%d\n", + "sending data: qid=%d idx=%d len=%d nsegs=%d txflags=0x%08x rate_n_flags=0x%08x rateidx=%u\n", ring->qid, ring->cur, totlen, nsegs, le32toh(tx->tx_flags), le32toh(tx->rate_n_flags), - (int) tx->initial_rate_index + tx->initial_rate_index ); /* Fill TX descriptor. */ @@ -2956,52 +3550,19 @@ iwm_mvm_flush_tx_path(struct iwm_softc *sc, int tfd_msk, int sync) * BEGIN mvm/sta.c */ -static void -iwm_mvm_add_sta_cmd_v6_to_v5(struct iwm_mvm_add_sta_cmd_v6 *cmd_v6, - struct iwm_mvm_add_sta_cmd_v5 *cmd_v5) -{ - memset(cmd_v5, 0, sizeof(*cmd_v5)); - - cmd_v5->add_modify = cmd_v6->add_modify; - cmd_v5->tid_disable_tx = cmd_v6->tid_disable_tx; - cmd_v5->mac_id_n_color = cmd_v6->mac_id_n_color; - IEEE80211_ADDR_COPY(cmd_v5->addr, cmd_v6->addr); - cmd_v5->sta_id = cmd_v6->sta_id; - cmd_v5->modify_mask = cmd_v6->modify_mask; - cmd_v5->station_flags = cmd_v6->station_flags; - cmd_v5->station_flags_msk = cmd_v6->station_flags_msk; - cmd_v5->add_immediate_ba_tid = cmd_v6->add_immediate_ba_tid; - cmd_v5->remove_immediate_ba_tid = cmd_v6->remove_immediate_ba_tid; - cmd_v5->add_immediate_ba_ssn = cmd_v6->add_immediate_ba_ssn; - cmd_v5->sleep_tx_count = cmd_v6->sleep_tx_count; - cmd_v5->sleep_state_flags = cmd_v6->sleep_state_flags; - cmd_v5->assoc_id = cmd_v6->assoc_id; - cmd_v5->beamform_flags = cmd_v6->beamform_flags; - cmd_v5->tfd_queue_msk = cmd_v6->tfd_queue_msk; -} - static int iwm_mvm_send_add_sta_cmd_status(struct iwm_softc *sc, - struct iwm_mvm_add_sta_cmd_v6 *cmd, int *status) + struct iwm_mvm_add_sta_cmd_v7 *cmd, int *status) { - struct iwm_mvm_add_sta_cmd_v5 cmd_v5; - - if (sc->sc_capaflags & IWM_UCODE_TLV_FLAGS_STA_KEY_CMD) { - return iwm_mvm_send_cmd_pdu_status(sc, IWM_ADD_STA, - sizeof(*cmd), cmd, status); - } - - iwm_mvm_add_sta_cmd_v6_to_v5(cmd, &cmd_v5); - - return iwm_mvm_send_cmd_pdu_status(sc, IWM_ADD_STA, sizeof(cmd_v5), - &cmd_v5, status); + return iwm_mvm_send_cmd_pdu_status(sc, IWM_ADD_STA, sizeof(*cmd), + cmd, status); } /* send station add/update command to firmware */ static int iwm_mvm_sta_send_to_fw(struct iwm_softc *sc, struct iwm_node *in, int update) { - struct iwm_mvm_add_sta_cmd_v6 add_sta_cmd; + struct iwm_mvm_add_sta_cmd_v7 add_sta_cmd; int ret; uint32_t status; @@ -3012,12 +3573,19 @@ iwm_mvm_sta_send_to_fw(struct iwm_softc *sc, struct iwm_node *in, int update) = htole32(IWM_FW_CMD_ID_AND_COLOR(IWM_DEFAULT_MACID, IWM_DEFAULT_COLOR)); if (!update) { - add_sta_cmd.tfd_queue_msk = htole32(0xf); + int ac; + for (ac = 0; ac < WME_NUM_AC; ac++) { + add_sta_cmd.tfd_queue_msk |= + htole32(1 << iwm_mvm_ac_to_tx_fifo[ac]); + } IEEE80211_ADDR_COPY(&add_sta_cmd.addr, in->in_ni.ni_bssid); } add_sta_cmd.add_modify = update ? 1 : 0; add_sta_cmd.station_flags_msk |= htole32(IWM_STA_FLG_FAT_EN_MSK | IWM_STA_FLG_MIMO_EN_MSK); + add_sta_cmd.tid_disable_tx = htole16(0xffff); + if (update) + add_sta_cmd.modify_mask |= (IWM_STA_MODIFY_TID_DISABLE_TX); status = IWM_ADD_STA_SUCCESS; ret = iwm_mvm_send_add_sta_cmd_status(sc, &add_sta_cmd, &status); @@ -3052,7 +3620,7 @@ static int iwm_mvm_add_int_sta_common(struct iwm_softc *sc, struct iwm_int_sta *sta, const uint8_t *addr, uint16_t mac_id, uint16_t color) { - struct iwm_mvm_add_sta_cmd_v6 cmd; + struct iwm_mvm_add_sta_cmd_v7 cmd; int ret; uint32_t status; @@ -3061,6 +3629,7 @@ iwm_mvm_add_int_sta_common(struct iwm_softc *sc, struct iwm_int_sta *sta, cmd.mac_id_n_color = htole32(IWM_FW_CMD_ID_AND_COLOR(mac_id, color)); cmd.tfd_queue_msk = htole32(sta->tfd_queue_msk); + cmd.tid_disable_tx = htole16(0xffff); if (addr) IEEE80211_ADDR_COPY(cmd.addr, addr); @@ -3089,8 +3658,12 @@ iwm_mvm_add_aux_sta(struct iwm_softc *sc) { int ret; - sc->sc_aux_sta.sta_id = 3; - sc->sc_aux_sta.tfd_queue_msk = 0; + sc->sc_aux_sta.sta_id = IWM_AUX_STA_ID; + sc->sc_aux_sta.tfd_queue_msk = (1 << IWM_MVM_AUX_QUEUE); + + ret = iwm_enable_txq(sc, 0, IWM_MVM_AUX_QUEUE, IWM_MVM_TX_FIFO_MCAST); + if (ret) + return ret; ret = iwm_mvm_add_int_sta_common(sc, &sc->sc_aux_sta, NULL, IWM_MAC_INDEX_AUX, 0); @@ -3210,6 +3783,10 @@ iwm_auth(struct ieee80211vap *vap, struct iwm_softc *sc) in->in_assoc = 0; + error = iwm_mvm_sf_config(sc, IWM_SF_FULL_ON); + if (error != 0) + return error; + error = iwm_allow_mcast(vap, sc); if (error) { device_printf(sc->sc_dev, @@ -3222,7 +3799,7 @@ iwm_auth(struct ieee80211vap *vap, struct iwm_softc *sc) * * Linux iwlwifi doesn't reset the nic each time, nor does it * call ctxt_add() here. Instead, it adds it during vap creation, - * and always does does a mac_ctx_changed(). + * and always does a mac_ctx_changed(). * * The openbsd port doesn't attempt to do that - it reset things * at odd states and does the add here. @@ -3342,7 +3919,7 @@ iwm_release(struct iwm_softc *sc, struct iwm_node *in) * back to nothing anyway, we'll just do a complete device reset. * Up your's, device! */ - //iwm_mvm_flush_tx_path(sc, 0xf, 1); + /* iwm_mvm_flush_tx_path(sc, 0xf, 1); */ iwm_stop_device(sc); iwm_init_hw(sc); if (in) @@ -3448,6 +4025,10 @@ iwm_setrates(struct iwm_softc *sc, struct iwm_node *in) memset(lq, 0, sizeof(*lq)); lq->sta_id = IWM_STATION_ID; + /* For HT, always enable RTS/CTS to avoid excessive retries. */ + if (ni->ni_flags & IEEE80211_NODE_HT) + lq->flags |= IWM_LQ_FLAG_USE_RTS_MSK; + /* * are these used? (we don't do SISO or MIMO) * need to set them to non-zero, though, or we get an error. @@ -3471,7 +4052,7 @@ iwm_setrates(struct iwm_softc *sc, struct iwm_node *in) int nextant; if (txant == 0) - txant = IWM_FW_VALID_TX_ANT(sc); + txant = iwm_fw_valid_tx_ant(sc); nextant = 1<<(ffs(txant)-1); txant &= ~nextant; @@ -3567,7 +4148,12 @@ iwm_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg) "Force transition to INIT; MGT=%d\n", arg); IWM_UNLOCK(sc); IEEE80211_LOCK(ic); - vap->iv_newstate(vap, IEEE80211_S_INIT, arg); + /* Always pass arg as -1 since we can't Tx right now. */ + /* + * XXX arg is just ignored anyway when transitioning + * to IEEE80211_S_INIT. + */ + vap->iv_newstate(vap, IEEE80211_S_INIT, -1); IWM_DPRINTF(sc, IWM_DEBUG_STATE, "Going INIT->SCAN\n"); nstate = IEEE80211_S_SCAN; @@ -3578,7 +4164,6 @@ iwm_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg) switch (nstate) { case IEEE80211_S_INIT: - sc->sc_scanband = 0; break; case IEEE80211_S_AUTH: @@ -3630,6 +4215,7 @@ iwm_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg) "%s: IWM_LQ_CMD failed\n", __func__); } + iwm_mvm_led_enable(sc); break; } @@ -3647,46 +4233,258 @@ iwm_endscan_cb(void *arg, int pending) { struct iwm_softc *sc = arg; struct ieee80211com *ic = &sc->sc_ic; - int done; - int error; IWM_DPRINTF(sc, IWM_DEBUG_SCAN | IWM_DEBUG_TRACE, "%s: scan ended\n", __func__); - IWM_LOCK(sc); - if (sc->sc_scanband == IEEE80211_CHAN_2GHZ && - sc->sc_nvm.sku_cap_band_52GHz_enable) { - done = 0; - if ((error = iwm_mvm_scan_request(sc, - IEEE80211_CHAN_5GHZ, 0, NULL, 0)) != 0) { - device_printf(sc->sc_dev, - "could not initiate 5 GHz scan\n"); - done = 1; + ieee80211_scan_done(TAILQ_FIRST(&ic->ic_vaps)); +} + +/* + * Aging and idle timeouts for the different possible scenarios + * in default configuration + */ +static const uint32_t +iwm_sf_full_timeout_def[IWM_SF_NUM_SCENARIO][IWM_SF_NUM_TIMEOUT_TYPES] = { + { + htole32(IWM_SF_SINGLE_UNICAST_AGING_TIMER_DEF), + htole32(IWM_SF_SINGLE_UNICAST_IDLE_TIMER_DEF) + }, + { + htole32(IWM_SF_AGG_UNICAST_AGING_TIMER_DEF), + htole32(IWM_SF_AGG_UNICAST_IDLE_TIMER_DEF) + }, + { + htole32(IWM_SF_MCAST_AGING_TIMER_DEF), + htole32(IWM_SF_MCAST_IDLE_TIMER_DEF) + }, + { + htole32(IWM_SF_BA_AGING_TIMER_DEF), + htole32(IWM_SF_BA_IDLE_TIMER_DEF) + }, + { + htole32(IWM_SF_TX_RE_AGING_TIMER_DEF), + htole32(IWM_SF_TX_RE_IDLE_TIMER_DEF) + }, +}; + +/* + * Aging and idle timeouts for the different possible scenarios + * in single BSS MAC configuration. + */ +static const uint32_t +iwm_sf_full_timeout[IWM_SF_NUM_SCENARIO][IWM_SF_NUM_TIMEOUT_TYPES] = { + { + htole32(IWM_SF_SINGLE_UNICAST_AGING_TIMER), + htole32(IWM_SF_SINGLE_UNICAST_IDLE_TIMER) + }, + { + htole32(IWM_SF_AGG_UNICAST_AGING_TIMER), + htole32(IWM_SF_AGG_UNICAST_IDLE_TIMER) + }, + { + htole32(IWM_SF_MCAST_AGING_TIMER), + htole32(IWM_SF_MCAST_IDLE_TIMER) + }, + { + htole32(IWM_SF_BA_AGING_TIMER), + htole32(IWM_SF_BA_IDLE_TIMER) + }, + { + htole32(IWM_SF_TX_RE_AGING_TIMER), + htole32(IWM_SF_TX_RE_IDLE_TIMER) + }, +}; + +static void +iwm_mvm_fill_sf_command(struct iwm_softc *sc, struct iwm_sf_cfg_cmd *sf_cmd, + struct ieee80211_node *ni) +{ + int i, j, watermark; + + sf_cmd->watermark[IWM_SF_LONG_DELAY_ON] = htole32(IWM_SF_W_MARK_SCAN); + + /* + * If we are in association flow - check antenna configuration + * capabilities of the AP station, and choose the watermark accordingly. + */ + if (ni) { + if (ni->ni_flags & IEEE80211_NODE_HT) { +#ifdef notyet + if (ni->ni_rxmcs[2] != 0) + watermark = IWM_SF_W_MARK_MIMO3; + else if (ni->ni_rxmcs[1] != 0) + watermark = IWM_SF_W_MARK_MIMO2; + else +#endif + watermark = IWM_SF_W_MARK_SISO; + } else { + watermark = IWM_SF_W_MARK_LEGACY; } + /* default watermark value for unassociated mode. */ } else { - done = 1; + watermark = IWM_SF_W_MARK_MIMO2; } + sf_cmd->watermark[IWM_SF_FULL_ON] = htole32(watermark); - if (done) { - IWM_UNLOCK(sc); - ieee80211_scan_done(TAILQ_FIRST(&ic->ic_vaps)); - IWM_LOCK(sc); - sc->sc_scanband = 0; + for (i = 0; i < IWM_SF_NUM_SCENARIO; i++) { + for (j = 0; j < IWM_SF_NUM_TIMEOUT_TYPES; j++) { + sf_cmd->long_delay_timeouts[i][j] = + htole32(IWM_SF_LONG_DELAY_AGING_TIMER); + } + } + + if (ni) { + memcpy(sf_cmd->full_on_timeouts, iwm_sf_full_timeout, + sizeof(iwm_sf_full_timeout)); + } else { + memcpy(sf_cmd->full_on_timeouts, iwm_sf_full_timeout_def, + sizeof(iwm_sf_full_timeout_def)); + } +} + +static int +iwm_mvm_sf_config(struct iwm_softc *sc, enum iwm_sf_state new_state) +{ + struct ieee80211com *ic = &sc->sc_ic; + struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); + struct iwm_sf_cfg_cmd sf_cmd = { + .state = htole32(IWM_SF_FULL_ON), + }; + int ret = 0; + + if (sc->sc_device_family == IWM_DEVICE_FAMILY_8000) + sf_cmd.state |= htole32(IWM_SF_CFG_DUMMY_NOTIF_OFF); + + switch (new_state) { + case IWM_SF_UNINIT: + case IWM_SF_INIT_OFF: + iwm_mvm_fill_sf_command(sc, &sf_cmd, NULL); + break; + case IWM_SF_FULL_ON: + iwm_mvm_fill_sf_command(sc, &sf_cmd, vap->iv_bss); + break; + default: + IWM_DPRINTF(sc, IWM_DEBUG_PWRSAVE, + "Invalid state: %d. not sending Smart Fifo cmd\n", + new_state); + return EINVAL; + } + + ret = iwm_mvm_send_cmd_pdu(sc, IWM_REPLY_SF_CFG_CMD, IWM_CMD_ASYNC, + sizeof(sf_cmd), &sf_cmd); + return ret; +} + +static int +iwm_send_bt_init_conf(struct iwm_softc *sc) +{ + struct iwm_bt_coex_cmd bt_cmd; + + bt_cmd.mode = htole32(IWM_BT_COEX_WIFI); + bt_cmd.enabled_modules = htole32(IWM_BT_COEX_HIGH_BAND_RET); + + return iwm_mvm_send_cmd_pdu(sc, IWM_BT_CONFIG, 0, sizeof(bt_cmd), + &bt_cmd); +} + +static int +iwm_send_update_mcc_cmd(struct iwm_softc *sc, const char *alpha2) +{ + struct iwm_mcc_update_cmd mcc_cmd; + struct iwm_host_cmd hcmd = { + .id = IWM_MCC_UPDATE_CMD, + .flags = (IWM_CMD_SYNC | IWM_CMD_WANT_SKB), + .data = { &mcc_cmd }, + }; + int ret; +#ifdef IWM_DEBUG + struct iwm_rx_packet *pkt; + struct iwm_mcc_update_resp_v1 *mcc_resp_v1 = NULL; + struct iwm_mcc_update_resp *mcc_resp; + int n_channels; + uint16_t mcc; +#endif + int resp_v2 = isset(sc->sc_enabled_capa, + IWM_UCODE_TLV_CAPA_LAR_SUPPORT_V2); + + memset(&mcc_cmd, 0, sizeof(mcc_cmd)); + mcc_cmd.mcc = htole16(alpha2[0] << 8 | alpha2[1]); + if ((sc->sc_ucode_api & IWM_UCODE_TLV_API_WIFI_MCC_UPDATE) || + isset(sc->sc_enabled_capa, IWM_UCODE_TLV_CAPA_LAR_MULTI_MCC)) + mcc_cmd.source_id = IWM_MCC_SOURCE_GET_CURRENT; + else + mcc_cmd.source_id = IWM_MCC_SOURCE_OLD_FW; + + if (resp_v2) + hcmd.len[0] = sizeof(struct iwm_mcc_update_cmd); + else + hcmd.len[0] = sizeof(struct iwm_mcc_update_cmd_v1); + + IWM_DPRINTF(sc, IWM_DEBUG_NODE, + "send MCC update to FW with '%c%c' src = %d\n", + alpha2[0], alpha2[1], mcc_cmd.source_id); + + ret = iwm_send_cmd(sc, &hcmd); + if (ret) + return ret; + +#ifdef IWM_DEBUG + pkt = hcmd.resp_pkt; + + /* Extract MCC response */ + if (resp_v2) { + mcc_resp = (void *)pkt->data; + mcc = mcc_resp->mcc; + n_channels = le32toh(mcc_resp->n_channels); + } else { + mcc_resp_v1 = (void *)pkt->data; + mcc = mcc_resp_v1->mcc; + n_channels = le32toh(mcc_resp_v1->n_channels); + } + + /* W/A for a FW/NVM issue - returns 0x00 for the world domain */ + if (mcc == 0) + mcc = 0x3030; /* "00" - world */ + + IWM_DPRINTF(sc, IWM_DEBUG_NODE, + "regulatory domain '%c%c' (%d channels available)\n", + mcc >> 8, mcc & 0xff, n_channels); +#endif + iwm_free_resp(sc, &hcmd); + + return 0; +} + +static void +iwm_mvm_tt_tx_backoff(struct iwm_softc *sc, uint32_t backoff) +{ + struct iwm_host_cmd cmd = { + .id = IWM_REPLY_THERMAL_MNG_BACKOFF, + .len = { sizeof(uint32_t), }, + .data = { &backoff, }, + }; + + if (iwm_send_cmd(sc, &cmd) != 0) { + device_printf(sc->sc_dev, + "failed to change thermal tx backoff\n"); } - IWM_UNLOCK(sc); } static int iwm_init_hw(struct iwm_softc *sc) { struct ieee80211com *ic = &sc->sc_ic; - int error, i, qid; + int error, i, ac; - if ((error = iwm_start_hw(sc)) != 0) + if ((error = iwm_start_hw(sc)) != 0) { + printf("iwm_start_hw: failed %d\n", error); return error; + } if ((error = iwm_run_init_mvm_ucode(sc, 0)) != 0) { + printf("iwm_run_init_mvm_ucode: failed %d\n", error); return error; } @@ -3707,19 +4505,32 @@ iwm_init_hw(struct iwm_softc *sc) goto error; } - if ((error = iwm_send_tx_ant_cfg(sc, IWM_FW_VALID_TX_ANT(sc))) != 0) + if ((error = iwm_send_bt_init_conf(sc)) != 0) { + device_printf(sc->sc_dev, "bt init conf failed\n"); goto error; + } + + if ((error = iwm_send_tx_ant_cfg(sc, iwm_fw_valid_tx_ant(sc))) != 0) { + device_printf(sc->sc_dev, "antenna config failed\n"); + goto error; + } /* Send phy db control command and then phy db calibration*/ - if ((error = iwm_send_phy_db_data(sc)) != 0) + if ((error = iwm_send_phy_db_data(sc)) != 0) { + device_printf(sc->sc_dev, "phy_db_data failed\n"); goto error; + } - if ((error = iwm_send_phy_cfg_cmd(sc)) != 0) + if ((error = iwm_send_phy_cfg_cmd(sc)) != 0) { + device_printf(sc->sc_dev, "phy_cfg_cmd failed\n"); goto error; + } /* Add auxiliary station for scanning */ - if ((error = iwm_mvm_add_aux_sta(sc)) != 0) + if ((error = iwm_mvm_add_aux_sta(sc)) != 0) { + device_printf(sc->sc_dev, "add_aux_sta failed\n"); goto error; + } for (i = 0; i < IWM_NUM_PHY_CTX; i++) { /* @@ -3732,13 +4543,35 @@ iwm_init_hw(struct iwm_softc *sc) goto error; } + /* Initialize tx backoffs to the minimum. */ + if (sc->sc_device_family == IWM_DEVICE_FAMILY_7000) + iwm_mvm_tt_tx_backoff(sc, 0); + error = iwm_mvm_power_update_device(sc); if (error) goto error; - /* Mark TX rings as active. */ - for (qid = 0; qid < 4; qid++) { - iwm_enable_txq(sc, qid, qid); + if (isset(sc->sc_enabled_capa, IWM_UCODE_TLV_CAPA_LAR_SUPPORT)) { + if ((error = iwm_send_update_mcc_cmd(sc, "ZZ")) != 0) + goto error; + } + + if (isset(sc->sc_enabled_capa, IWM_UCODE_TLV_CAPA_UMAC_SCAN)) { + if ((error = iwm_mvm_config_umac_scan(sc)) != 0) + goto error; + } + + /* Enable Tx queues. */ + for (ac = 0; ac < WME_NUM_AC; ac++) { + error = iwm_enable_txq(sc, IWM_STATION_ID, ac, + iwm_mvm_ac_to_tx_fifo[ac]); + if (error) + goto error; + } + + if ((error = iwm_mvm_disable_beacon_filter(sc)) != 0) { + device_printf(sc->sc_dev, "failed to disable beacon filter\n"); + goto error; } return 0; @@ -3774,6 +4607,10 @@ iwm_allow_mcast(struct ieee80211vap *vap, struct iwm_softc *sc) return (error); } +/* + * ifnet interfaces + */ + static void iwm_init(struct iwm_softc *sc) { @@ -3786,12 +4623,13 @@ iwm_init(struct iwm_softc *sc) sc->sc_flags &= ~IWM_FLAG_STOPPED; if ((error = iwm_init_hw(sc)) != 0) { + printf("iwm_init_hw failed %d\n", error); iwm_stop(sc); return; } /* - * Ok, firmware loaded and we are jogging + * Ok, firmware loaded and we are jogging */ sc->sc_flags |= IWM_FLAG_HW_INITED; callout_reset(&sc->sc_watchdog_to, hz, iwm_watchdog, sc); @@ -3852,7 +4690,6 @@ iwm_stop(struct iwm_softc *sc) sc->sc_flags &= ~IWM_FLAG_HW_INITED; sc->sc_flags |= IWM_FLAG_STOPPED; sc->sc_generation++; - sc->sc_scanband = 0; iwm_led_blink_stop(sc); sc->sc_tx_timer = 0; iwm_stop_device(sc); @@ -3871,7 +4708,7 @@ iwm_watchdog(void *arg) iwm_nic_error(sc); #endif ieee80211_restart_all(ic); - counter_u64_add(ic->ic_oerrors, 1); + counter_u64_add(sc->sc_ic.ic_oerrors, 1); return; } } @@ -3914,8 +4751,8 @@ iwm_parent(struct ieee80211com *ic) struct iwm_error_event_table { uint32_t valid; /* (nonzero) valid, (0) log is empty */ uint32_t error_id; /* type of error */ - uint32_t pc; /* program counter */ - uint32_t blink1; /* branch link */ + uint32_t trm_hw_status0; /* TRM HW status */ + uint32_t trm_hw_status1; /* TRM HW status */ uint32_t blink2; /* branch link */ uint32_t ilink1; /* interrupt link */ uint32_t ilink2; /* interrupt link */ @@ -3927,8 +4764,9 @@ struct iwm_error_event_table { uint32_t tsf_hi; /* network timestamp function timer */ uint32_t gp1; /* GP1 timer register */ uint32_t gp2; /* GP2 timer register */ - uint32_t gp3; /* GP3 timer register */ - uint32_t ucode_ver; /* uCode version */ + uint32_t fw_rev_type; /* firmware revision type */ + uint32_t major; /* uCode version major */ + uint32_t minor; /* uCode version minor */ uint32_t hw_ver; /* HW Silicon version */ uint32_t brd_ver; /* HW board version */ uint32_t log_pc; /* log program counter */ @@ -3945,7 +4783,7 @@ struct iwm_error_event_table { * time_flag */ uint32_t isr4; /* isr status register LMPM_NIC_ISR4: * wico interrupt */ - uint32_t isr_pref; /* isr status register LMPM_NIC_PREF_STAT */ + uint32_t last_cmd_id; /* last HCMD id handled by the firmware */ uint32_t wait_event; /* wait event() caller address */ uint32_t l2p_control; /* L2pControlField */ uint32_t l2p_duration; /* L2pDurationField */ @@ -3956,6 +4794,31 @@ struct iwm_error_event_table { uint32_t u_timestamp; /* indicate when the date and time of the * compilation */ uint32_t flow_handler; /* FH read/write pointers, RX credit */ +} __packed /* LOG_ERROR_TABLE_API_S_VER_3 */; + +/* + * UMAC error struct - relevant starting from family 8000 chip. + * Note: This structure is read from the device with IO accesses, + * and the reading already does the endian conversion. As it is + * read with u32-sized accesses, any members with a different size + * need to be ordered correctly though! + */ +struct iwm_umac_error_event_table { + uint32_t valid; /* (nonzero) valid, (0) log is empty */ + uint32_t error_id; /* type of error */ + uint32_t blink1; /* branch link */ + uint32_t blink2; /* branch link */ + uint32_t ilink1; /* interrupt link */ + uint32_t ilink2; /* interrupt link */ + uint32_t data1; /* error-specific data */ + uint32_t data2; /* error-specific data */ + uint32_t data3; /* error-specific data */ + uint32_t umac_major; + uint32_t umac_minor; + uint32_t frame_pointer; /* core register 27*/ + uint32_t stack_pointer; /* core register 28 */ + uint32_t cmd_header; /* latest host cmd sent to UMAC */ + uint32_t nic_isr_pref; /* ISR status register */ } __packed; #define ERROR_START_OFFSET (1 * sizeof(uint32_t)) @@ -3997,6 +4860,53 @@ iwm_desc_lookup(uint32_t num) return advanced_lookup[i].name; } +static void +iwm_nic_umac_error(struct iwm_softc *sc) +{ + struct iwm_umac_error_event_table table; + uint32_t base; + + base = sc->sc_uc.uc_umac_error_event_table; + + if (base < 0x800000) { + device_printf(sc->sc_dev, "Invalid error log pointer 0x%08x\n", + base); + return; + } + + if (iwm_read_mem(sc, base, &table, sizeof(table)/sizeof(uint32_t))) { + device_printf(sc->sc_dev, "reading errlog failed\n"); + return; + } + + if (ERROR_START_OFFSET <= table.valid * ERROR_ELEM_SIZE) { + device_printf(sc->sc_dev, "Start UMAC Error Log Dump:\n"); + device_printf(sc->sc_dev, "Status: 0x%x, count: %d\n", + sc->sc_flags, table.valid); + } + + device_printf(sc->sc_dev, "0x%08X | %s\n", table.error_id, + iwm_desc_lookup(table.error_id)); + device_printf(sc->sc_dev, "0x%08X | umac branchlink1\n", table.blink1); + device_printf(sc->sc_dev, "0x%08X | umac branchlink2\n", table.blink2); + device_printf(sc->sc_dev, "0x%08X | umac interruptlink1\n", + table.ilink1); + device_printf(sc->sc_dev, "0x%08X | umac interruptlink2\n", + table.ilink2); + device_printf(sc->sc_dev, "0x%08X | umac data1\n", table.data1); + device_printf(sc->sc_dev, "0x%08X | umac data2\n", table.data2); + device_printf(sc->sc_dev, "0x%08X | umac data3\n", table.data3); + device_printf(sc->sc_dev, "0x%08X | umac major\n", table.umac_major); + device_printf(sc->sc_dev, "0x%08X | umac minor\n", table.umac_minor); + device_printf(sc->sc_dev, "0x%08X | frame pointer\n", + table.frame_pointer); + device_printf(sc->sc_dev, "0x%08X | stack pointer\n", + table.stack_pointer); + device_printf(sc->sc_dev, "0x%08X | last host cmd\n", table.cmd_header); + device_printf(sc->sc_dev, "0x%08X | isr status reg\n", + table.nic_isr_pref); +} + /* * Support for dumping the error log seemed like a good idea ... * but it's mostly hex junk and the only sensible thing is the @@ -4012,13 +4922,13 @@ iwm_nic_error(struct iwm_softc *sc) device_printf(sc->sc_dev, "dumping device error log\n"); base = sc->sc_uc.uc_error_event_table; - if (base < 0x800000 || base >= 0x80C000) { + if (base < 0x800000) { device_printf(sc->sc_dev, - "Not valid error log pointer 0x%08x\n", base); + "Invalid error log pointer 0x%08x\n", base); return; } - if (iwm_read_mem(sc, base, &table, sizeof(table)/sizeof(uint32_t)) != 0) { + if (iwm_read_mem(sc, base, &table, sizeof(table)/sizeof(uint32_t))) { device_printf(sc->sc_dev, "reading errlog failed\n"); return; } @@ -4029,15 +4939,17 @@ iwm_nic_error(struct iwm_softc *sc) } if (ERROR_START_OFFSET <= table.valid * ERROR_ELEM_SIZE) { - device_printf(sc->sc_dev, "Start IWL Error Log Dump:\n"); + device_printf(sc->sc_dev, "Start Error Log Dump:\n"); device_printf(sc->sc_dev, "Status: 0x%x, count: %d\n", sc->sc_flags, table.valid); } device_printf(sc->sc_dev, "0x%08X | %-28s\n", table.error_id, - iwm_desc_lookup(table.error_id)); - device_printf(sc->sc_dev, "%08X | uPc\n", table.pc); - device_printf(sc->sc_dev, "%08X | branchlink1\n", table.blink1); + iwm_desc_lookup(table.error_id)); + device_printf(sc->sc_dev, "%08X | trm_hw_status0\n", + table.trm_hw_status0); + device_printf(sc->sc_dev, "%08X | trm_hw_status1\n", + table.trm_hw_status1); device_printf(sc->sc_dev, "%08X | branchlink2\n", table.blink2); device_printf(sc->sc_dev, "%08X | interruptlink1\n", table.ilink1); device_printf(sc->sc_dev, "%08X | interruptlink2\n", table.ilink2); @@ -4049,8 +4961,10 @@ iwm_nic_error(struct iwm_softc *sc) device_printf(sc->sc_dev, "%08X | tsf hi\n", table.tsf_hi); device_printf(sc->sc_dev, "%08X | time gp1\n", table.gp1); device_printf(sc->sc_dev, "%08X | time gp2\n", table.gp2); - device_printf(sc->sc_dev, "%08X | time gp3\n", table.gp3); - device_printf(sc->sc_dev, "%08X | uCode version\n", table.ucode_ver); + device_printf(sc->sc_dev, "%08X | uCode revision type\n", + table.fw_rev_type); + device_printf(sc->sc_dev, "%08X | uCode version major\n", table.major); + device_printf(sc->sc_dev, "%08X | uCode version minor\n", table.minor); device_printf(sc->sc_dev, "%08X | hw version\n", table.hw_ver); device_printf(sc->sc_dev, "%08X | board version\n", table.brd_ver); device_printf(sc->sc_dev, "%08X | hcmd\n", table.hcmd); @@ -4059,7 +4973,7 @@ iwm_nic_error(struct iwm_softc *sc) device_printf(sc->sc_dev, "%08X | isr2\n", table.isr2); device_printf(sc->sc_dev, "%08X | isr3\n", table.isr3); device_printf(sc->sc_dev, "%08X | isr4\n", table.isr4); - device_printf(sc->sc_dev, "%08X | isr_pref\n", table.isr_pref); + device_printf(sc->sc_dev, "%08X | last cmd Id\n", table.last_cmd_id); device_printf(sc->sc_dev, "%08X | wait_event\n", table.wait_event); device_printf(sc->sc_dev, "%08X | l2p_control\n", table.l2p_control); device_printf(sc->sc_dev, "%08X | l2p_duration\n", table.l2p_duration); @@ -4068,6 +4982,9 @@ iwm_nic_error(struct iwm_softc *sc) device_printf(sc->sc_dev, "%08X | lmpm_pmg_sel\n", table.lmpm_pmg_sel); device_printf(sc->sc_dev, "%08X | timestamp\n", table.u_timestamp); device_printf(sc->sc_dev, "%08X | flow_handler\n", table.flow_handler); + + if (sc->sc_uc.uc_umac_error_event_table) + iwm_nic_umac_error(sc); } #endif @@ -4092,18 +5009,23 @@ do { \ static void iwm_notif_intr(struct iwm_softc *sc) { + struct ieee80211com *ic = &sc->sc_ic; uint16_t hw; bus_dmamap_sync(sc->rxq.stat_dma.tag, sc->rxq.stat_dma.map, BUS_DMASYNC_POSTREAD); hw = le16toh(sc->rxq.stat->closed_rb_num) & 0xfff; + + /* + * Process responses + */ while (sc->rxq.cur != hw) { struct iwm_rx_ring *ring = &sc->rxq; struct iwm_rx_data *data = &sc->rxq.data[sc->rxq.cur]; struct iwm_rx_packet *pkt; struct iwm_cmd_response *cresp; - int qid, idx; + int qid, idx, code; bus_dmamap_sync(sc->rxq.data_dmat, data->map, BUS_DMASYNC_POSTREAD); @@ -4112,10 +5034,10 @@ iwm_notif_intr(struct iwm_softc *sc) qid = pkt->hdr.qid & ~0x80; idx = pkt->hdr.idx; + code = IWM_WIDE_ID(pkt->hdr.flags, pkt->hdr.code); IWM_DPRINTF(sc, IWM_DEBUG_INTR, - "rx packet qid=%d idx=%d flags=%x type=%x %d %d\n", - pkt->hdr.qid & ~0x80, pkt->hdr.idx, pkt->hdr.flags, - pkt->hdr.code, sc->rxq.cur, hw); + "rx packet qid=%d idx=%d type=%x %d %d\n", + pkt->hdr.qid & ~0x80, pkt->hdr.idx, code, sc->rxq.cur, hw); /* * randomly get these from the firmware, no idea why. @@ -4127,7 +5049,7 @@ iwm_notif_intr(struct iwm_softc *sc) continue; } - switch (pkt->hdr.code) { + switch (code) { case IWM_REPLY_RX_PHY_CMD: iwm_mvm_rx_rx_phy_cmd(sc, pkt, data); break; @@ -4145,7 +5067,6 @@ iwm_notif_intr(struct iwm_softc *sc) int missed; /* XXX look at mac_id to determine interface ID */ - struct ieee80211com *ic = &sc->sc_ic; struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); SYNC_RESP_STRUCT(resp, pkt); @@ -4179,16 +5100,56 @@ iwm_notif_intr(struct iwm_softc *sc) break; } + case IWM_MFUART_LOAD_NOTIFICATION: + break; + case IWM_MVM_ALIVE: { - struct iwm_mvm_alive_resp *resp; - SYNC_RESP_STRUCT(resp, pkt); + struct iwm_mvm_alive_resp_v1 *resp1; + struct iwm_mvm_alive_resp_v2 *resp2; + struct iwm_mvm_alive_resp_v3 *resp3; + + if (iwm_rx_packet_payload_len(pkt) == sizeof(*resp1)) { + SYNC_RESP_STRUCT(resp1, pkt); + sc->sc_uc.uc_error_event_table + = le32toh(resp1->error_event_table_ptr); + sc->sc_uc.uc_log_event_table + = le32toh(resp1->log_event_table_ptr); + sc->sched_base = le32toh(resp1->scd_base_ptr); + if (resp1->status == IWM_ALIVE_STATUS_OK) + sc->sc_uc.uc_ok = 1; + else + sc->sc_uc.uc_ok = 0; + } - sc->sc_uc.uc_error_event_table - = le32toh(resp->error_event_table_ptr); - sc->sc_uc.uc_log_event_table - = le32toh(resp->log_event_table_ptr); - sc->sched_base = le32toh(resp->scd_base_ptr); - sc->sc_uc.uc_ok = resp->status == IWM_ALIVE_STATUS_OK; + if (iwm_rx_packet_payload_len(pkt) == sizeof(*resp2)) { + SYNC_RESP_STRUCT(resp2, pkt); + sc->sc_uc.uc_error_event_table + = le32toh(resp2->error_event_table_ptr); + sc->sc_uc.uc_log_event_table + = le32toh(resp2->log_event_table_ptr); + sc->sched_base = le32toh(resp2->scd_base_ptr); + sc->sc_uc.uc_umac_error_event_table + = le32toh(resp2->error_info_addr); + if (resp2->status == IWM_ALIVE_STATUS_OK) + sc->sc_uc.uc_ok = 1; + else + sc->sc_uc.uc_ok = 0; + } + + if (iwm_rx_packet_payload_len(pkt) == sizeof(*resp3)) { + SYNC_RESP_STRUCT(resp3, pkt); + sc->sc_uc.uc_error_event_table + = le32toh(resp3->error_event_table_ptr); + sc->sc_uc.uc_log_event_table + = le32toh(resp3->log_event_table_ptr); + sc->sched_base = le32toh(resp3->scd_base_ptr); + sc->sc_uc.uc_umac_error_event_table + = le32toh(resp3->error_info_addr); + if (resp3->status == IWM_ALIVE_STATUS_OK) + sc->sc_uc.uc_ok = 1; + else + sc->sc_uc.uc_ok = 0; + } sc->sc_uc.uc_intr = 1; wakeup(&sc->sc_uc); @@ -4210,6 +5171,7 @@ iwm_notif_intr(struct iwm_softc *sc) break; } case IWM_NVM_ACCESS_CMD: + case IWM_MCC_UPDATE_CMD: if (sc->sc_wantresp == ((qid << 16) | idx)) { bus_dmamap_sync(sc->rxq.data_dmat, data->map, BUS_DMASYNC_POSTREAD); @@ -4218,6 +5180,21 @@ iwm_notif_intr(struct iwm_softc *sc) } break; + case IWM_MCC_CHUB_UPDATE_CMD: { + struct iwm_mcc_chub_notif *notif; + SYNC_RESP_STRUCT(notif, pkt); + + sc->sc_fw_mcc[0] = (notif->mcc & 0xff00) >> 8; + sc->sc_fw_mcc[1] = notif->mcc & 0xff; + sc->sc_fw_mcc[2] = '\0'; + IWM_DPRINTF(sc, IWM_DEBUG_RESET, + "fw source %d sent CC '%s'\n", + notif->source_id, sc->sc_fw_mcc); + break; } + + case IWM_DTS_MEASUREMENT_NOTIFICATION: + break; + case IWM_PHY_CONFIGURATION_CMD: case IWM_TX_ANT_CONFIGURATION_CMD: case IWM_ADD_STA: @@ -4228,12 +5205,17 @@ iwm_notif_intr(struct iwm_softc *sc) case IWM_BINDING_CONTEXT_CMD: case IWM_TIME_EVENT_CMD: case IWM_SCAN_REQUEST_CMD: + case IWM_WIDE_ID(IWM_ALWAYS_LONG_GROUP, IWM_SCAN_CFG_CMD): + case IWM_WIDE_ID(IWM_ALWAYS_LONG_GROUP, IWM_SCAN_REQ_UMAC): + case IWM_SCAN_OFFLOAD_REQUEST_CMD: case IWM_REPLY_BEACON_FILTERING_CMD: case IWM_MAC_PM_POWER_TABLE: case IWM_TIME_QUOTA_CMD: case IWM_REMOVE_STA: case IWM_TXPATH_FLUSH: case IWM_LQ_CMD: + case IWM_BT_CONFIG: + case IWM_REPLY_THERMAL_MNG_BACKOFF: SYNC_RESP_STRUCT(cresp, pkt); if (sc->sc_wantresp == ((qid << 16) | idx)) { memcpy(sc->sc_cmd_resp, @@ -4250,11 +5232,42 @@ iwm_notif_intr(struct iwm_softc *sc) wakeup(&sc->sc_init_complete); break; - case IWM_SCAN_COMPLETE_NOTIFICATION: { - struct iwm_scan_complete_notif *notif; + case IWM_SCAN_OFFLOAD_COMPLETE: { + struct iwm_periodic_scan_complete *notif; SYNC_RESP_STRUCT(notif, pkt); + break; + } + + case IWM_SCAN_ITERATION_COMPLETE: { + struct iwm_lmac_scan_complete_notif *notif; + SYNC_RESP_STRUCT(notif, pkt); + ieee80211_runtask(&sc->sc_ic, &sc->sc_es_task); + break; + } + + case IWM_SCAN_COMPLETE_UMAC: { + struct iwm_umac_scan_complete *notif; + SYNC_RESP_STRUCT(notif, pkt); + + IWM_DPRINTF(sc, IWM_DEBUG_SCAN, + "UMAC scan complete, status=0x%x\n", + notif->status); +#if 0 /* XXX This would be a duplicate scan end call */ taskqueue_enqueue(sc->sc_tq, &sc->sc_es_task); - break; } +#endif + break; + } + + case IWM_SCAN_ITERATION_COMPLETE_UMAC: { + struct iwm_umac_scan_iter_complete_notif *notif; + SYNC_RESP_STRUCT(notif, pkt); + + IWM_DPRINTF(sc, IWM_DEBUG_SCAN, "UMAC scan iteration " + "complete, status=0x%x, %d channels scanned\n", + notif->status, notif->scanned_channels); + ieee80211_runtask(&sc->sc_ic, &sc->sc_es_task); + break; + } case IWM_REPLY_ERROR: { struct iwm_error_resp *resp; @@ -4264,7 +5277,8 @@ iwm_notif_intr(struct iwm_softc *sc) "firmware error 0x%x, cmd 0x%x\n", le32toh(resp->error_type), resp->cmd_id); - break; } + break; + } case IWM_TIME_EVENT_NOTIFICATION: { struct iwm_time_event_notif *notif; @@ -4272,12 +5286,25 @@ iwm_notif_intr(struct iwm_softc *sc) IWM_DPRINTF(sc, IWM_DEBUG_INTR, "TE notif status = 0x%x action = 0x%x\n", - notif->status, notif->action); - break; } + notif->status, notif->action); + break; + } case IWM_MCAST_FILTER_CMD: break; + case IWM_SCD_QUEUE_CFG: { + struct iwm_scd_txq_cfg_rsp *rsp; + SYNC_RESP_STRUCT(rsp, pkt); + + IWM_DPRINTF(sc, IWM_DEBUG_CMD, + "queue cfg token=0x%x sta_id=%d " + "tid=%d scd_queue=%d\n", + rsp->token, rsp->sta_id, rsp->tid, + rsp->scd_queue); + break; + } + default: device_printf(sc->sc_dev, "frame %d/%d %x UNHANDLED (this should " @@ -4480,10 +5507,14 @@ iwm_intr(void *arg) #define PCI_VENDOR_INTEL 0x8086 #define PCI_PRODUCT_INTEL_WL_3160_1 0x08b3 #define PCI_PRODUCT_INTEL_WL_3160_2 0x08b4 +#define PCI_PRODUCT_INTEL_WL_3165_1 0x3165 +#define PCI_PRODUCT_INTEL_WL_3165_2 0x3166 #define PCI_PRODUCT_INTEL_WL_7260_1 0x08b1 #define PCI_PRODUCT_INTEL_WL_7260_2 0x08b2 #define PCI_PRODUCT_INTEL_WL_7265_1 0x095a #define PCI_PRODUCT_INTEL_WL_7265_2 0x095b +#define PCI_PRODUCT_INTEL_WL_8260_1 0x24f3 +#define PCI_PRODUCT_INTEL_WL_8260_2 0x24f4 static const struct iwm_devices { uint16_t device; @@ -4491,10 +5522,14 @@ static const struct iwm_devices { } iwm_devices[] = { { PCI_PRODUCT_INTEL_WL_3160_1, "Intel Dual Band Wireless AC 3160" }, { PCI_PRODUCT_INTEL_WL_3160_2, "Intel Dual Band Wireless AC 3160" }, + { PCI_PRODUCT_INTEL_WL_3165_1, "Intel Dual Band Wireless AC 3165" }, + { PCI_PRODUCT_INTEL_WL_3165_2, "Intel Dual Band Wireless AC 3165" }, { PCI_PRODUCT_INTEL_WL_7260_1, "Intel Dual Band Wireless AC 7260" }, { PCI_PRODUCT_INTEL_WL_7260_2, "Intel Dual Band Wireless AC 7260" }, { PCI_PRODUCT_INTEL_WL_7265_1, "Intel Dual Band Wireless AC 7265" }, { PCI_PRODUCT_INTEL_WL_7265_2, "Intel Dual Band Wireless AC 7265" }, + { PCI_PRODUCT_INTEL_WL_8260_1, "Intel Dual Band Wireless AC 8260" }, + { PCI_PRODUCT_INTEL_WL_8260_2, "Intel Dual Band Wireless AC 8260" }, }; static int @@ -4502,12 +5537,13 @@ iwm_probe(device_t dev) { int i; - for (i = 0; i < nitems(iwm_devices); i++) + for (i = 0; i < nitems(iwm_devices); i++) { if (pci_get_vendor(dev) == PCI_VENDOR_INTEL && pci_get_device(dev) == iwm_devices[i].device) { device_set_desc(dev, iwm_devices[i].name); return (BUS_PROBE_DEFAULT); } + } return (ENXIO); } @@ -4519,21 +5555,42 @@ iwm_dev_check(device_t dev) sc = device_get_softc(dev); + sc->sc_hw_rev = IWM_READ(sc, IWM_CSR_HW_REV); switch (pci_get_device(dev)) { case PCI_PRODUCT_INTEL_WL_3160_1: case PCI_PRODUCT_INTEL_WL_3160_2: sc->sc_fwname = "iwm3160fw"; sc->host_interrupt_operation_mode = 1; + sc->sc_device_family = IWM_DEVICE_FAMILY_7000; + sc->sc_fwdmasegsz = IWM_FWDMASEGSZ; + return (0); + case PCI_PRODUCT_INTEL_WL_3165_1: + case PCI_PRODUCT_INTEL_WL_3165_2: + sc->sc_fwname = "iwm7265fw"; + sc->host_interrupt_operation_mode = 0; + sc->sc_device_family = IWM_DEVICE_FAMILY_7000; + sc->sc_fwdmasegsz = IWM_FWDMASEGSZ; return (0); case PCI_PRODUCT_INTEL_WL_7260_1: case PCI_PRODUCT_INTEL_WL_7260_2: sc->sc_fwname = "iwm7260fw"; sc->host_interrupt_operation_mode = 1; + sc->sc_device_family = IWM_DEVICE_FAMILY_7000; + sc->sc_fwdmasegsz = IWM_FWDMASEGSZ; return (0); case PCI_PRODUCT_INTEL_WL_7265_1: case PCI_PRODUCT_INTEL_WL_7265_2: sc->sc_fwname = "iwm7265fw"; sc->host_interrupt_operation_mode = 0; + sc->sc_device_family = IWM_DEVICE_FAMILY_7000; + sc->sc_fwdmasegsz = IWM_FWDMASEGSZ; + return (0); + case PCI_PRODUCT_INTEL_WL_8260_1: + case PCI_PRODUCT_INTEL_WL_8260_2: + sc->sc_fwname = "iwm8000Cfw"; + sc->host_interrupt_operation_mode = 0; + sc->sc_device_family = IWM_DEVICE_FAMILY_8000; + sc->sc_fwdmasegsz = IWM_FWDMASEGSZ_8000; return (0); default: device_printf(dev, "unknown adapter type\n"); @@ -4627,14 +5684,6 @@ iwm_attach(device_t dev) callout_init_mtx(&sc->sc_watchdog_to, &sc->sc_mtx, 0); callout_init_mtx(&sc->sc_led_blink_to, &sc->sc_mtx, 0); TASK_INIT(&sc->sc_es_task, 0, iwm_endscan_cb, sc); - sc->sc_tq = taskqueue_create("iwm_taskq", M_WAITOK, - taskqueue_thread_enqueue, &sc->sc_tq); - error = taskqueue_start_threads(&sc->sc_tq, 1, 0, "iwm_taskq"); - if (error != 0) { - device_printf(dev, "can't start threads, error %d\n", - error); - goto fail; - } /* PCI attach */ error = iwm_pci_attach(dev); @@ -4648,17 +5697,62 @@ iwm_attach(device_t dev) if (error != 0) goto fail; - sc->sc_fwdmasegsz = IWM_FWDMASEGSZ; - /* * We now start fiddling with the hardware */ - sc->sc_hw_rev = IWM_READ(sc, IWM_CSR_HW_REV); + /* + * In the 8000 HW family the format of the 4 bytes of CSR_HW_REV have + * changed, and now the revision step also includes bit 0-1 (no more + * "dash" value). To keep hw_rev backwards compatible - we'll store it + * in the old format. + */ + if (sc->sc_device_family == IWM_DEVICE_FAMILY_8000) + sc->sc_hw_rev = (sc->sc_hw_rev & 0xfff0) | + (IWM_CSR_HW_REV_STEP(sc->sc_hw_rev << 2) << 2); + if (iwm_prepare_card_hw(sc) != 0) { device_printf(dev, "could not initialize hardware\n"); goto fail; } + if (sc->sc_device_family == IWM_DEVICE_FAMILY_8000) { + int ret; + uint32_t hw_step; + + /* + * In order to recognize C step the driver should read the + * chip version id located at the AUX bus MISC address. + */ + IWM_SETBITS(sc, IWM_CSR_GP_CNTRL, + IWM_CSR_GP_CNTRL_REG_FLAG_INIT_DONE); + DELAY(2); + + ret = iwm_poll_bit(sc, IWM_CSR_GP_CNTRL, + IWM_CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY, + IWM_CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY, + 25000); + if (ret < 0) { + device_printf(sc->sc_dev, + "Failed to wake up the nic\n"); + goto fail; + } + + if (iwm_nic_lock(sc)) { + hw_step = iwm_read_prph(sc, IWM_WFPM_CTRL_REG); + hw_step |= IWM_ENABLE_WFPM; + iwm_write_prph(sc, IWM_WFPM_CTRL_REG, hw_step); + hw_step = iwm_read_prph(sc, IWM_AUX_MISC_REG); + hw_step = (hw_step >> IWM_HW_STEP_LOCATION_BITS) & 0xF; + if (hw_step == 0x3) + sc->sc_hw_rev = (sc->sc_hw_rev & 0xFFFFFFF3) | + (IWM_SILICON_C_STEP << 2); + iwm_nic_unlock(sc); + } else { + device_printf(sc->sc_dev, "Failed to lock the nic\n"); + goto fail; + } + } + /* Allocate DMA memory for firmware transfers. */ if ((error = iwm_alloc_fwmem(sc)) != 0) { device_printf(dev, "could not allocate memory for firmware\n"); @@ -4752,6 +5846,17 @@ fail: } static int +iwm_is_valid_ether_addr(uint8_t *addr) +{ + char zero_addr[IEEE80211_ADDR_LEN] = { 0, 0, 0, 0, 0, 0 }; + + if ((addr[0] & 1) || IEEE80211_ADDR_EQ(zero_addr, addr)) + return (FALSE); + + return (TRUE); +} + +static int iwm_update_edca(struct ieee80211com *ic) { struct iwm_softc *sc = ic->ic_softc; @@ -4785,11 +5890,9 @@ iwm_preinit(void *arg) goto fail; } device_printf(dev, - "revision: 0x%x, firmware %d.%d (API ver. %d)\n", + "hw rev 0x%x, fw ver %s, address %s\n", sc->sc_hw_rev & IWM_CSR_HW_REV_TYPE_MSK, - IWM_UCODE_MAJOR(sc->sc_fwver), - IWM_UCODE_MINOR(sc->sc_fwver), - IWM_UCODE_API(sc->sc_fwver)); + sc->sc_fwver, ether_sprintf(sc->sc_nvm.hw_addr)); /* not all hardware can do 5GHz band */ if (!sc->sc_nvm.sku_cap_band_52GHz_enable) @@ -4894,18 +5997,18 @@ static void iwm_scan_start(struct ieee80211com *ic) { struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); - struct iwm_softc *sc = ic->ic_softc; + struct iwm_softc *sc = ic->ic_softc; int error; - if (sc->sc_scanband) - return; IWM_LOCK(sc); - error = iwm_mvm_scan_request(sc, IEEE80211_CHAN_2GHZ, 0, NULL, 0); - if (error) { + if (isset(sc->sc_enabled_capa, IWM_UCODE_TLV_CAPA_UMAC_SCAN)) + error = iwm_mvm_umac_scan(sc); + else + error = iwm_mvm_lmac_scan(sc); + if (error != 0) { device_printf(sc->sc_dev, "could not initiate 2 GHz scan\n"); IWM_UNLOCK(sc); ieee80211_cancel_scan(vap); - sc->sc_scanband = 0; } else { iwm_led_blink_start(sc); IWM_UNLOCK(sc); @@ -4976,8 +6079,8 @@ iwm_resume(device_t dev) iwm_init_task(device_get_softc(dev)); IWM_LOCK(sc); - if (sc->sc_flags & IWM_FLAG_DORESUME) { - sc->sc_flags &= ~IWM_FLAG_DORESUME; + if (sc->sc_flags & IWM_FLAG_SCANNING) { + sc->sc_flags &= ~IWM_FLAG_SCANNING; do_reinit = 1; } IWM_UNLOCK(sc); @@ -5001,7 +6104,7 @@ iwm_suspend(device_t dev) if (do_stop) { IWM_LOCK(sc); iwm_stop(sc); - sc->sc_flags |= IWM_FLAG_DORESUME; + sc->sc_flags |= IWM_FLAG_SCANNING; IWM_UNLOCK(sc); } @@ -5015,15 +6118,14 @@ iwm_detach_local(struct iwm_softc *sc, int do_net80211) device_t dev = sc->sc_dev; int i; - if (sc->sc_tq) { - taskqueue_drain_all(sc->sc_tq); - taskqueue_free(sc->sc_tq); - } + ieee80211_draintask(&sc->sc_ic, &sc->sc_es_task); + callout_drain(&sc->sc_led_blink_to); callout_drain(&sc->sc_watchdog_to); iwm_stop_device(sc); - if (do_net80211) + if (do_net80211) { ieee80211_ifdetach(&sc->sc_ic); + } iwm_phy_db_free(sc); diff --git a/sys/dev/iwm/if_iwm_led.c b/sys/dev/iwm/if_iwm_led.c index 53a385720ddf..dce4e8d1aa49 100644 --- a/sys/dev/iwm/if_iwm_led.c +++ b/sys/dev/iwm/if_iwm_led.c @@ -148,13 +148,13 @@ iwm_mvm_led_disable(struct iwm_softc *sc) IWM_WRITE(sc, IWM_CSR_LED_REG, IWM_CSR_LED_REG_TURN_OFF); } -int +static int iwm_mvm_led_is_enabled(struct iwm_softc *sc) { return (IWM_READ(sc, IWM_CSR_LED_REG) == IWM_CSR_LED_REG_TURN_ON); } -void +static void iwm_led_blink_timeout(void *arg) { struct iwm_softc *sc = arg; @@ -177,6 +177,6 @@ iwm_led_blink_start(struct iwm_softc *sc) void iwm_led_blink_stop(struct iwm_softc *sc) { - callout_drain(&sc->sc_led_blink_to); + callout_stop(&sc->sc_led_blink_to); iwm_mvm_led_disable(sc); } diff --git a/sys/dev/iwm/if_iwm_led.h b/sys/dev/iwm/if_iwm_led.h index b5c7d1ce17b0..4fa6bbaff6d6 100644 --- a/sys/dev/iwm/if_iwm_led.h +++ b/sys/dev/iwm/if_iwm_led.h @@ -91,11 +91,9 @@ #ifndef __IF_IWM_LED_H__ #define __IF_IWM_LED_H__ -void iwm_mvm_led_enable(struct iwm_softc *); -void iwm_mvm_led_disable(struct iwm_softc *); -int iwm_mvm_led_is_enabled(struct iwm_softc *); -void iwm_led_blink_timeout(void *); -void iwm_led_blink_start(struct iwm_softc *); -void iwm_led_blink_stop(struct iwm_softc *); +extern void iwm_mvm_led_enable(struct iwm_softc *); +extern void iwm_mvm_led_disable(struct iwm_softc *); +extern void iwm_led_blink_start(struct iwm_softc *); +extern void iwm_led_blink_stop(struct iwm_softc *); #endif /* __IF_IWM_LED_H__ */ diff --git a/sys/dev/iwm/if_iwm_mac_ctxt.c b/sys/dev/iwm/if_iwm_mac_ctxt.c index 139326056c82..0d2da5f6c799 100644 --- a/sys/dev/iwm/if_iwm_mac_ctxt.c +++ b/sys/dev/iwm/if_iwm_mac_ctxt.c @@ -280,12 +280,24 @@ iwm_mvm_mac_ctxt_cmd_common(struct iwm_softc *sc, struct iwm_node *in, /* * XXX should we error out if in_assoc is 1 and ni == NULL? */ +#if 0 if (in->in_assoc) { IEEE80211_ADDR_COPY(cmd->bssid_addr, ni->ni_bssid); } else { /* eth broadcast address */ - memset(cmd->bssid_addr, 0xff, sizeof(cmd->bssid_addr)); + IEEE80211_ADDR_COPY(cmd->bssid_addr, ieee80211broadcastaddr); } +#else + /* + * XXX This workaround makes the firmware behave more correctly once + * we are associated, regularly giving us statistics notifications, + * as well as signaling missed beacons to us. + * Since we only call iwm_mvm_mac_ctxt_add() and + * iwm_mvm_mac_ctxt_changed() when already authenticating or + * associating, ni->ni_bssid should always make sense here. + */ + IEEE80211_ADDR_COPY(cmd->bssid_addr, ni->ni_bssid); +#endif /* * Default to 2ghz if no node information is given. @@ -457,13 +469,7 @@ static int iwm_mvm_mac_ctx_send(struct iwm_softc *sc, struct ieee80211vap *vap, uint32_t action) { - int ret; - - ret = iwm_mvm_mac_ctxt_cmd_station(sc, vap, action); - if (ret) - return (ret); - - return (0); + return iwm_mvm_mac_ctxt_cmd_station(sc, vap, action); } int @@ -489,17 +495,13 @@ int iwm_mvm_mac_ctxt_changed(struct iwm_softc *sc, struct ieee80211vap *vap) { struct iwm_vap *iv = IWM_VAP(vap); - int ret; if (iv->is_uploaded == 0) { device_printf(sc->sc_dev, "%s: called; uploaded = 0\n", __func__); return (EIO); } - ret = iwm_mvm_mac_ctx_send(sc, vap, IWM_FW_CTXT_ACTION_MODIFY); - if (ret) - return (ret); - return (0); + return iwm_mvm_mac_ctx_send(sc, vap, IWM_FW_CTXT_ACTION_MODIFY); } #if 0 diff --git a/sys/dev/iwm/if_iwm_pcie_trans.c b/sys/dev/iwm/if_iwm_pcie_trans.c index 8f2f72827604..0ef3503faee1 100644 --- a/sys/dev/iwm/if_iwm_pcie_trans.c +++ b/sys/dev/iwm/if_iwm_pcie_trans.c @@ -256,13 +256,18 @@ iwm_nic_lock(struct iwm_softc *sc) IWM_SETBITS(sc, IWM_CSR_GP_CNTRL, IWM_CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ); + if (sc->sc_device_family == IWM_DEVICE_FAMILY_8000) + DELAY(2); + if (iwm_poll_bit(sc, IWM_CSR_GP_CNTRL, IWM_CSR_GP_CNTRL_REG_VAL_MAC_ACCESS_EN, IWM_CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY | IWM_CSR_GP_CNTRL_REG_FLAG_GOING_TO_SLEEP, 15000)) { - rv = 1; + rv = 1; } else { /* jolt */ + IWM_DPRINTF(sc, IWM_DEBUG_RESET, + "%s: resetting device via NMI\n", __func__); IWM_WRITE(sc, IWM_CSR_RESET, IWM_CSR_RESET_REG_FLAG_FORCE_NMI); } @@ -343,13 +348,20 @@ iwm_check_rfkill(struct iwm_softc *sc) int iwm_set_hw_ready(struct iwm_softc *sc) { + int ready; + IWM_SETBITS(sc, IWM_CSR_HW_IF_CONFIG_REG, IWM_CSR_HW_IF_CONFIG_REG_BIT_NIC_READY); - return iwm_poll_bit(sc, IWM_CSR_HW_IF_CONFIG_REG, + ready = iwm_poll_bit(sc, IWM_CSR_HW_IF_CONFIG_REG, IWM_CSR_HW_IF_CONFIG_REG_BIT_NIC_READY, IWM_CSR_HW_IF_CONFIG_REG_BIT_NIC_READY, IWM_HW_READY_TIMEOUT); + if (ready) { + IWM_SETBITS(sc, IWM_CSR_MBOX_SET_REG, + IWM_CSR_MBOX_SET_REG_OS_ALIVE); + } + return ready; } #undef IWM_HW_READY_TIMEOUT @@ -413,8 +425,10 @@ iwm_apm_init(struct iwm_softc *sc) IWM_DPRINTF(sc, IWM_DEBUG_RESET, "iwm apm start\n"); /* Disable L0S exit timer (platform NMI Work/Around) */ - IWM_SETBITS(sc, IWM_CSR_GIO_CHICKEN_BITS, - IWM_CSR_GIO_CHICKEN_BITS_REG_BIT_DIS_L0S_EXIT_TIMER); + if (sc->sc_device_family != IWM_DEVICE_FAMILY_8000) { + IWM_SETBITS(sc, IWM_CSR_GIO_CHICKEN_BITS, + IWM_CSR_GIO_CHICKEN_BITS_REG_BIT_DIS_L0S_EXIT_TIMER); + } /* * Disable L0s without affecting L1; @@ -435,7 +449,7 @@ iwm_apm_init(struct iwm_softc *sc) iwm_apm_config(sc); -#if 0 /* not for 7k */ +#if 0 /* not for 7k/8k */ /* Configure analog phase-lock-loop before activating to D0A */ if (trans->cfg->base_params->pll_cfg_val) IWM_SETBITS(trans, IWM_CSR_ANA_PLL_CFG, @@ -491,18 +505,19 @@ iwm_apm_init(struct iwm_softc *sc) * do not disable clocks. This preserves any hardware bits already * set by default in "CLK_CTRL_REG" after reset. */ - iwm_write_prph(sc, IWM_APMG_CLK_EN_REG, IWM_APMG_CLK_VAL_DMA_CLK_RQT); - //kpause("iwmapm", 0, mstohz(20), NULL); - DELAY(20); - - /* Disable L1-Active */ - iwm_set_bits_prph(sc, IWM_APMG_PCIDEV_STT_REG, - IWM_APMG_PCIDEV_STT_VAL_L1_ACT_DIS); - - /* Clear the interrupt in APMG if the NIC is in RFKILL */ - iwm_write_prph(sc, IWM_APMG_RTC_INT_STT_REG, - IWM_APMG_RTC_INT_STT_RFKILL); - + if (sc->sc_device_family == IWM_DEVICE_FAMILY_7000) { + iwm_write_prph(sc, IWM_APMG_CLK_EN_REG, + IWM_APMG_CLK_VAL_DMA_CLK_RQT); + DELAY(20); + + /* Disable L1-Active */ + iwm_set_bits_prph(sc, IWM_APMG_PCIDEV_STT_REG, + IWM_APMG_PCIDEV_STT_VAL_L1_ACT_DIS); + + /* Clear the interrupt in APMG if the NIC is in RFKILL */ + iwm_write_prph(sc, IWM_APMG_RTC_INT_STT_REG, + IWM_APMG_RTC_INT_STT_RFKILL); + } out: if (error) device_printf(sc->sc_dev, "apm init error %d\n", error); @@ -533,9 +548,7 @@ iwm_start_hw(struct iwm_softc *sc) return error; /* Reset the entire device */ - IWM_WRITE(sc, IWM_CSR_RESET, - IWM_CSR_RESET_REG_FLAG_SW_RESET | - IWM_CSR_RESET_REG_FLAG_NEVO_RESET); + IWM_WRITE(sc, IWM_CSR_RESET, IWM_CSR_RESET_REG_FLAG_SW_RESET); DELAY(10); if ((error = iwm_apm_init(sc)) != 0) diff --git a/sys/dev/iwm/if_iwm_phy_ctxt.c b/sys/dev/iwm/if_iwm_phy_ctxt.c index fc8349d028a7..da6bb1a0b4bd 100644 --- a/sys/dev/iwm/if_iwm_phy_ctxt.c +++ b/sys/dev/iwm/if_iwm_phy_ctxt.c @@ -202,8 +202,8 @@ iwm_mvm_phy_ctxt_cmd_data(struct iwm_softc *sc, ieee80211_chan2ieee(ic, chan), chains_static, chains_dynamic, - IWM_FW_VALID_RX_ANT(sc), - IWM_FW_VALID_TX_ANT(sc)); + iwm_fw_valid_rx_ant(sc), + iwm_fw_valid_tx_ant(sc)); cmd->ci.band = IEEE80211_IS_CHAN_2GHZ(chan) ? @@ -217,13 +217,13 @@ iwm_mvm_phy_ctxt_cmd_data(struct iwm_softc *sc, idle_cnt = chains_static; active_cnt = chains_dynamic; - cmd->rxchain_info = htole32(IWM_FW_VALID_RX_ANT(sc) << + cmd->rxchain_info = htole32(iwm_fw_valid_rx_ant(sc) << IWM_PHY_RX_CHAIN_VALID_POS); cmd->rxchain_info |= htole32(idle_cnt << IWM_PHY_RX_CHAIN_CNT_POS); cmd->rxchain_info |= htole32(active_cnt << IWM_PHY_RX_CHAIN_MIMO_CNT_POS); - cmd->txchain_info = htole32(IWM_FW_VALID_TX_ANT(sc)); + cmd->txchain_info = htole32(iwm_fw_valid_tx_ant(sc)); } /* diff --git a/sys/dev/iwm/if_iwm_phy_db.c b/sys/dev/iwm/if_iwm_phy_db.c index 42d891466cdf..b96f87490d74 100644 --- a/sys/dev/iwm/if_iwm_phy_db.c +++ b/sys/dev/iwm/if_iwm_phy_db.c @@ -343,7 +343,6 @@ iwm_send_phy_db_cmd(struct iwm_softc *sc, uint16_t type, cmd.len[0] = sizeof(struct iwm_phy_db_cmd); cmd.data[1] = data; cmd.len[1] = length; - cmd.dataflags[1] = IWM_HCMD_DFL_NOCOPY; return iwm_send_cmd(sc, &cmd); } @@ -374,6 +373,7 @@ iwm_phy_db_send_all_channel_groups(struct iwm_softc *sc, return err; } + DELAY(1000); IWM_DPRINTF(sc, IWM_DEBUG_CMD, "Sent PHY_DB HCMD, type = %d num = %d\n", type, i); } diff --git a/sys/dev/iwm/if_iwm_power.c b/sys/dev/iwm/if_iwm_power.c index 6503756ac232..aca51f89471b 100644 --- a/sys/dev/iwm/if_iwm_power.c +++ b/sys/dev/iwm/if_iwm_power.c @@ -335,14 +335,3 @@ iwm_mvm_disable_beacon_filter(struct iwm_softc *sc) return ret; } - -#if 0 -static int -iwm_mvm_update_beacon_filter(struct iwm_softc *sc, struct iwm_node *in) -{ - if (!sc->sc_bf.bf_enabled) - return 0; - - return iwm_mvm_enable_beacon_filter(sc, in); -} -#endif diff --git a/sys/dev/iwm/if_iwm_scan.c b/sys/dev/iwm/if_iwm_scan.c index 01b0872c105c..94f7125da424 100644 --- a/sys/dev/iwm/if_iwm_scan.c +++ b/sys/dev/iwm/if_iwm_scan.c @@ -172,7 +172,7 @@ iwm_mvm_scan_rx_chain(struct iwm_softc *sc) uint16_t rx_chain; uint8_t rx_ant; - rx_ant = IWM_FW_VALID_RX_ANT(sc); + rx_ant = iwm_fw_valid_rx_ant(sc); rx_chain = rx_ant << IWM_PHY_RX_CHAIN_VALID_POS; rx_chain |= rx_ant << IWM_PHY_RX_CHAIN_FORCE_MIMO_SEL_POS; rx_chain |= rx_ant << IWM_PHY_RX_CHAIN_FORCE_SEL_POS; @@ -180,6 +180,7 @@ iwm_mvm_scan_rx_chain(struct iwm_softc *sc) return htole16(rx_chain); } +#if 0 static uint32_t iwm_mvm_scan_max_out_time(struct iwm_softc *sc, uint32_t flags, int is_assoc) { @@ -197,15 +198,7 @@ iwm_mvm_scan_suspend_time(struct iwm_softc *sc, int is_assoc) return 0; return htole32(SUSPEND_TIME_PERIOD); } - -static uint32_t -iwm_mvm_scan_rxon_flags(struct iwm_softc *sc, int flags) -{ - if (flags & IEEE80211_CHAN_2GHZ) - return htole32(IWM_PHY_BAND_24); - else - return htole32(IWM_PHY_BAND_5); -} +#endif static uint32_t iwm_mvm_scan_rate_n_flags(struct iwm_softc *sc, int flags, int no_cck) @@ -216,7 +209,7 @@ iwm_mvm_scan_rate_n_flags(struct iwm_softc *sc, int flags, int no_cck) for (i = 0, ind = sc->sc_scan_last_antenna; i < IWM_RATE_MCS_ANT_NUM; i++) { ind = (ind + 1) % IWM_RATE_MCS_ANT_NUM; - if (IWM_FW_VALID_TX_ANT(sc) & (1 << ind)) { + if (iwm_fw_valid_tx_ant(sc) & (1 << ind)) { sc->sc_scan_last_antenna = ind; break; } @@ -230,6 +223,7 @@ iwm_mvm_scan_rate_n_flags(struct iwm_softc *sc, int flags, int no_cck) return htole32(IWM_RATE_6M_PLCP | tx_ant); } +#if 0 /* * If req->n_ssids > 0, it means we should do an active scan. * In case of active scan w/o directed scan, we receive a zero-length SSID @@ -253,24 +247,30 @@ iwm_mvm_get_passive_dwell(struct iwm_softc *sc, int flags) { return (flags & IEEE80211_CHAN_2GHZ) ? 100 + 20 : 100 + 10; } +#endif static int -iwm_mvm_scan_fill_channels(struct iwm_softc *sc, struct iwm_scan_cmd *cmd, - int flags, int n_ssids, int basic_ssid) +iwm_mvm_scan_skip_channel(struct ieee80211_channel *c) +{ + if (IEEE80211_IS_CHAN_2GHZ(c) && IEEE80211_IS_CHAN_B(c)) + return 0; + else if (IEEE80211_IS_CHAN_5GHZ(c) && IEEE80211_IS_CHAN_A(c)) + return 0; + else + return 1; +} + +static uint8_t +iwm_mvm_lmac_scan_fill_channels(struct iwm_softc *sc, + struct iwm_scan_channel_cfg_lmac *chan, int n_ssids) { struct ieee80211com *ic = &sc->sc_ic; - uint16_t passive_dwell = iwm_mvm_get_passive_dwell(sc, flags); - uint16_t active_dwell = iwm_mvm_get_active_dwell(sc, flags, n_ssids); - struct iwm_scan_channel *chan = (struct iwm_scan_channel *) - (cmd->data + le16toh(cmd->tx_cmd.len)); - int type = (1 << n_ssids) - 1; struct ieee80211_channel *c; - int nchan, j; + uint8_t nchan; + int j; - if (!basic_ssid) - type |= (1 << n_ssids); - - for (nchan = j = 0; j < ic->ic_nchans; j++) { + for (nchan = j = 0; + j < ic->ic_nchans && nchan < sc->sc_capa_n_scan_channels; j++) { c = &ic->ic_channels[j]; /* For 2GHz, only populate 11b channels */ /* For 5GHz, only populate 11a channels */ @@ -278,175 +278,458 @@ iwm_mvm_scan_fill_channels(struct iwm_softc *sc, struct iwm_scan_cmd *cmd, * Catch other channels, in case we have 900MHz channels or * something in the chanlist. */ - if ((flags & IEEE80211_CHAN_2GHZ) && (! IEEE80211_IS_CHAN_B(c))) { - continue; - } else if ((flags & IEEE80211_CHAN_5GHZ) && (! IEEE80211_IS_CHAN_A(c))) { + if (iwm_mvm_scan_skip_channel(c)) { + IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_EEPROM, + "%s: skipping channel (freq=%d, ieee=%d, flags=0x%08x)\n", + __func__, c->ic_freq, c->ic_ieee, c->ic_flags); continue; - } else { + } + + IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_EEPROM, + "Adding channel %d (%d Mhz) to the list\n", + nchan, c->ic_freq); + chan->channel_num = htole16(ieee80211_mhz2ieee(c->ic_freq, 0)); + chan->iter_count = htole16(1); + chan->iter_interval = htole32(0); + chan->flags = htole32(IWM_UNIFIED_SCAN_CHANNEL_PARTIAL); +#if 0 /* makes scanning while associated less useful */ + if (n_ssids != 0) + chan->flags |= htole32(1 << 1); /* select SSID 0 */ +#endif + chan++; + nchan++; + } + + return nchan; +} + +static uint8_t +iwm_mvm_umac_scan_fill_channels(struct iwm_softc *sc, + struct iwm_scan_channel_cfg_umac *chan, int n_ssids) +{ + struct ieee80211com *ic = &sc->sc_ic; + struct ieee80211_channel *c; + uint8_t nchan; + int j; + + for (nchan = j = 0; + j < ic->ic_nchans && nchan < sc->sc_capa_n_scan_channels; j++) { + c = &ic->ic_channels[j]; + /* For 2GHz, only populate 11b channels */ + /* For 5GHz, only populate 11a channels */ + /* + * Catch other channels, in case we have 900MHz channels or + * something in the chanlist. + */ + if (iwm_mvm_scan_skip_channel(c)) { IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_EEPROM, "%s: skipping channel (freq=%d, ieee=%d, flags=0x%08x)\n", - __func__, - c->ic_freq, - c->ic_ieee, - c->ic_flags); + __func__, c->ic_freq, c->ic_ieee, c->ic_flags); + continue; } + IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_EEPROM, "Adding channel %d (%d Mhz) to the list\n", - nchan, c->ic_freq); - chan->channel = htole16(ieee80211_mhz2ieee(c->ic_freq, flags)); - chan->type = htole32(type); - if (c->ic_flags & IEEE80211_CHAN_PASSIVE) - chan->type &= htole32(~IWM_SCAN_CHANNEL_TYPE_ACTIVE); - chan->active_dwell = htole16(active_dwell); - chan->passive_dwell = htole16(passive_dwell); - chan->iteration_count = htole16(1); + nchan, c->ic_freq); + chan->channel_num = ieee80211_mhz2ieee(c->ic_freq, 0); + chan->iter_count = 1; + chan->iter_interval = htole16(0); + chan->flags = htole32(0); +#if 0 /* makes scanning while associated less useful */ + if (n_ssids != 0) + chan->flags = htole32(1 << 0); /* select SSID 0 */ +#endif chan++; nchan++; } - if (nchan == 0) - device_printf(sc->sc_dev, - "%s: NO CHANNEL!\n", __func__); + return nchan; } -/* - * Fill in probe request with the following parameters: - * TA is our vif HW address, which mac80211 ensures we have. - * Packet is broadcasted, so this is both SA and DA. - * The probe request IE is made out of two: first comes the most prioritized - * SSID if a directed scan is requested. Second comes whatever extra - * information was given to us as the scan request IE. - */ -static uint16_t -iwm_mvm_fill_probe_req(struct iwm_softc *sc, struct ieee80211_frame *frame, - const uint8_t *ta, int n_ssids, const uint8_t *ssid, int ssid_len, - const uint8_t *ie, int ie_len, int left) +static int +iwm_mvm_fill_probe_req(struct iwm_softc *sc, struct iwm_scan_probe_req *preq) { - uint8_t *pos = NULL; + struct ieee80211com *ic = &sc->sc_ic; + struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); + struct ieee80211_frame *wh = (struct ieee80211_frame *)preq->buf; + struct ieee80211_rateset *rs; + size_t remain = sizeof(preq->buf); + uint8_t *frm, *pos; + int ssid_len = 0; + const uint8_t *ssid = NULL; - /* Make sure there is enough space for the probe request, - * two mandatory IEs and the data */ - left -= sizeof(*frame); - if (left < 0) - return 0; + memset(preq, 0, sizeof(*preq)); - frame->i_fc[0] = IEEE80211_FC0_VERSION_0 | IEEE80211_FC0_TYPE_MGT | + /* Ensure enough space for header and SSID IE. */ + if (remain < sizeof(*wh) + 2 + ssid_len) + return ENOBUFS; + + /* + * Build a probe request frame. Most of the following code is a + * copy & paste of what is done in net80211. + */ + wh->i_fc[0] = IEEE80211_FC0_VERSION_0 | IEEE80211_FC0_TYPE_MGT | IEEE80211_FC0_SUBTYPE_PROBE_REQ; - frame->i_fc[1] = IEEE80211_FC1_DIR_NODS; - IEEE80211_ADDR_COPY(frame->i_addr1, ieee80211broadcastaddr); - IEEE80211_ADDR_COPY(frame->i_addr2, ta); - IEEE80211_ADDR_COPY(frame->i_addr3, ieee80211broadcastaddr); + wh->i_fc[1] = IEEE80211_FC1_DIR_NODS; + IEEE80211_ADDR_COPY(wh->i_addr1, ieee80211broadcastaddr); + IEEE80211_ADDR_COPY(wh->i_addr2, vap ? vap->iv_myaddr : ic->ic_macaddr); + IEEE80211_ADDR_COPY(wh->i_addr3, ieee80211broadcastaddr); + *(uint16_t *)&wh->i_dur[0] = 0; /* filled by HW */ + *(uint16_t *)&wh->i_seq[0] = 0; /* filled by HW */ + + frm = (uint8_t *)(wh + 1); + frm = ieee80211_add_ssid(frm, ssid, ssid_len); + + /* Tell the firmware where the MAC header is. */ + preq->mac_header.offset = 0; + preq->mac_header.len = htole16(frm - (uint8_t *)wh); + remain -= frm - (uint8_t *)wh; + + /* Fill in 2GHz IEs and tell firmware where they are. */ + rs = &ic->ic_sup_rates[IEEE80211_MODE_11G]; + if (rs->rs_nrates > IEEE80211_RATE_SIZE) { + if (remain < 4 + rs->rs_nrates) + return ENOBUFS; + } else if (remain < 2 + rs->rs_nrates) { + return ENOBUFS; + } + preq->band_data[0].offset = htole16(frm - (uint8_t *)wh); + pos = frm; + frm = ieee80211_add_rates(frm, rs); + if (rs->rs_nrates > IEEE80211_RATE_SIZE) + frm = ieee80211_add_xrates(frm, rs); + preq->band_data[0].len = htole16(frm - pos); + remain -= frm - pos; + + if (isset(sc->sc_enabled_capa, + IWM_UCODE_TLV_CAPA_DS_PARAM_SET_IE_SUPPORT)) { + if (remain < 3) + return ENOBUFS; + *frm++ = IEEE80211_ELEMID_DSPARMS; + *frm++ = 1; + *frm++ = 0; + remain -= 3; + } - /* for passive scans, no need to fill anything */ - if (n_ssids == 0) - return sizeof(*frame); + if (sc->sc_nvm.sku_cap_band_52GHz_enable) { + /* Fill in 5GHz IEs. */ + rs = &ic->ic_sup_rates[IEEE80211_MODE_11A]; + if (rs->rs_nrates > IEEE80211_RATE_SIZE) { + if (remain < 4 + rs->rs_nrates) + return ENOBUFS; + } else if (remain < 2 + rs->rs_nrates) { + return ENOBUFS; + } + preq->band_data[1].offset = htole16(frm - (uint8_t *)wh); + pos = frm; + frm = ieee80211_add_rates(frm, rs); + if (rs->rs_nrates > IEEE80211_RATE_SIZE) + frm = ieee80211_add_xrates(frm, rs); + preq->band_data[1].len = htole16(frm - pos); + remain -= frm - pos; + } - /* points to the payload of the request */ - pos = (uint8_t *)frame + sizeof(*frame); + /* Send 11n IEs on both 2GHz and 5GHz bands. */ + preq->common_data.offset = htole16(frm - (uint8_t *)wh); + pos = frm; +#if 0 + if (ic->ic_flags & IEEE80211_F_HTON) { + if (remain < 28) + return ENOBUFS; + frm = ieee80211_add_htcaps(frm, ic); + /* XXX add WME info? */ + } +#endif + preq->common_data.len = htole16(frm - pos); - /* fill in our SSID IE */ - left -= ssid_len + 2; - if (left < 0) - return 0; + return 0; +} - pos = ieee80211_add_ssid(pos, ssid, ssid_len); +int +iwm_mvm_config_umac_scan(struct iwm_softc *sc) +{ + struct ieee80211com *ic = &sc->sc_ic; + struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); - if (ie && ie_len && left >= ie_len) { - memcpy(pos, ie, ie_len); - pos += ie_len; + struct iwm_scan_config *scan_config; + int ret, j, nchan; + size_t cmd_size; + struct ieee80211_channel *c; + struct iwm_host_cmd hcmd = { + .id = iwm_cmd_id(IWM_SCAN_CFG_CMD, IWM_ALWAYS_LONG_GROUP, 0), + .flags = IWM_CMD_SYNC, + }; + static const uint32_t rates = (IWM_SCAN_CONFIG_RATE_1M | + IWM_SCAN_CONFIG_RATE_2M | IWM_SCAN_CONFIG_RATE_5M | + IWM_SCAN_CONFIG_RATE_11M | IWM_SCAN_CONFIG_RATE_6M | + IWM_SCAN_CONFIG_RATE_9M | IWM_SCAN_CONFIG_RATE_12M | + IWM_SCAN_CONFIG_RATE_18M | IWM_SCAN_CONFIG_RATE_24M | + IWM_SCAN_CONFIG_RATE_36M | IWM_SCAN_CONFIG_RATE_48M | + IWM_SCAN_CONFIG_RATE_54M); + + cmd_size = sizeof(*scan_config) + sc->sc_capa_n_scan_channels; + + scan_config = malloc(cmd_size, M_DEVBUF, M_NOWAIT | M_ZERO); + if (scan_config == NULL) + return ENOMEM; + + scan_config->tx_chains = htole32(iwm_fw_valid_tx_ant(sc)); + scan_config->rx_chains = htole32(iwm_fw_valid_rx_ant(sc)); + scan_config->legacy_rates = htole32(rates | + IWM_SCAN_CONFIG_SUPPORTED_RATE(rates)); + + /* These timings correspond to iwlwifi's UNASSOC scan. */ + scan_config->dwell_active = 10; + scan_config->dwell_passive = 110; + scan_config->dwell_fragmented = 44; + scan_config->dwell_extended = 90; + scan_config->out_of_channel_time = htole32(0); + scan_config->suspend_time = htole32(0); + + IEEE80211_ADDR_COPY(scan_config->mac_addr, + vap ? vap->iv_myaddr : ic->ic_macaddr); + + scan_config->bcast_sta_id = sc->sc_aux_sta.sta_id; + scan_config->channel_flags = IWM_CHANNEL_FLAG_EBS | + IWM_CHANNEL_FLAG_ACCURATE_EBS | IWM_CHANNEL_FLAG_EBS_ADD | + IWM_CHANNEL_FLAG_PRE_SCAN_PASSIVE2ACTIVE; + + for (nchan = j = 0; + j < ic->ic_nchans && nchan < sc->sc_capa_n_scan_channels; j++) { + c = &ic->ic_channels[j]; + /* For 2GHz, only populate 11b channels */ + /* For 5GHz, only populate 11a channels */ + /* + * Catch other channels, in case we have 900MHz channels or + * something in the chanlist. + */ + if (iwm_mvm_scan_skip_channel(c)) + continue; + scan_config->channel_array[nchan++] = + ieee80211_mhz2ieee(c->ic_freq, 0); } - return pos - (uint8_t *)frame; + scan_config->flags = htole32(IWM_SCAN_CONFIG_FLAG_ACTIVATE | + IWM_SCAN_CONFIG_FLAG_ALLOW_CHUB_REQS | + IWM_SCAN_CONFIG_FLAG_SET_TX_CHAINS | + IWM_SCAN_CONFIG_FLAG_SET_RX_CHAINS | + IWM_SCAN_CONFIG_FLAG_SET_AUX_STA_ID | + IWM_SCAN_CONFIG_FLAG_SET_ALL_TIMES | + IWM_SCAN_CONFIG_FLAG_SET_LEGACY_RATES | + IWM_SCAN_CONFIG_FLAG_SET_MAC_ADDR | + IWM_SCAN_CONFIG_FLAG_SET_CHANNEL_FLAGS| + IWM_SCAN_CONFIG_N_CHANNELS(nchan) | + IWM_SCAN_CONFIG_FLAG_CLEAR_FRAGMENTED); + + hcmd.data[0] = scan_config; + hcmd.len[0] = cmd_size; + + IWM_DPRINTF(sc, IWM_DEBUG_SCAN, "Sending UMAC scan config\n"); + + ret = iwm_send_cmd(sc, &hcmd); + if (!ret) + IWM_DPRINTF(sc, IWM_DEBUG_SCAN, + "UMAC scan config was sent successfully\n"); + + free(scan_config, M_DEVBUF); + return ret; } int -iwm_mvm_scan_request(struct iwm_softc *sc, int flags, - int n_ssids, uint8_t *ssid, int ssid_len) +iwm_mvm_umac_scan(struct iwm_softc *sc) { struct iwm_host_cmd hcmd = { - .id = IWM_SCAN_REQUEST_CMD, + .id = iwm_cmd_id(IWM_SCAN_REQ_UMAC, IWM_ALWAYS_LONG_GROUP, 0), .len = { 0, }, - .data = { sc->sc_scan_cmd, }, + .data = { NULL, }, .flags = IWM_CMD_SYNC, - .dataflags = { IWM_HCMD_DFL_NOCOPY, }, }; - struct iwm_scan_cmd *cmd = sc->sc_scan_cmd; - struct ieee80211com *ic = &sc->sc_ic; - struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); - int is_assoc = 0; + struct iwm_scan_req_umac *req; + struct iwm_scan_req_umac_tail *tail; + size_t req_len; + int ssid_len = 0; + const uint8_t *ssid = NULL; int ret; - uint32_t status; - int basic_ssid = - !(sc->sc_capaflags & IWM_UCODE_TLV_FLAGS_NO_BASIC_SSID); - sc->sc_scanband = flags & (IEEE80211_CHAN_2GHZ | IEEE80211_CHAN_5GHZ); + req_len = sizeof(struct iwm_scan_req_umac) + + (sizeof(struct iwm_scan_channel_cfg_umac) * + sc->sc_capa_n_scan_channels) + + sizeof(struct iwm_scan_req_umac_tail); + if (req_len > IWM_MAX_CMD_PAYLOAD_SIZE) + return ENOMEM; + req = malloc(req_len, M_DEVBUF, M_NOWAIT | M_ZERO); + if (req == NULL) + return ENOMEM; + + hcmd.len[0] = (uint16_t)req_len; + hcmd.data[0] = (void *)req; + + IWM_DPRINTF(sc, IWM_DEBUG_SCAN, "Handling ieee80211 scan request\n"); + + /* These timings correspond to iwlwifi's UNASSOC scan. */ + req->active_dwell = 10; + req->passive_dwell = 110; + req->fragmented_dwell = 44; + req->extended_dwell = 90; + req->max_out_time = 0; + req->suspend_time = 0; + + req->scan_priority = htole32(IWM_SCAN_PRIORITY_HIGH); + req->ooc_priority = htole32(IWM_SCAN_PRIORITY_HIGH); + + req->n_channels = iwm_mvm_umac_scan_fill_channels(sc, + (struct iwm_scan_channel_cfg_umac *)req->data, ssid_len != 0); + + req->general_flags = htole32(IWM_UMAC_SCAN_GEN_FLAGS_PASS_ALL | + IWM_UMAC_SCAN_GEN_FLAGS_ITER_COMPLETE | + IWM_UMAC_SCAN_GEN_FLAGS_EXTENDED_DWELL); + + tail = (void *)((char *)&req->data + + sizeof(struct iwm_scan_channel_cfg_umac) * + sc->sc_capa_n_scan_channels); + + /* Check if we're doing an active directed scan. */ + if (ssid_len != 0) { + tail->direct_scan[0].id = IEEE80211_ELEMID_SSID; + tail->direct_scan[0].len = ssid_len; + memcpy(tail->direct_scan[0].ssid, ssid, ssid_len); + req->general_flags |= + htole32(IWM_UMAC_SCAN_GEN_FLAGS_PRE_CONNECT); + } else { + req->general_flags |= htole32(IWM_UMAC_SCAN_GEN_FLAGS_PASSIVE); + } + + if (isset(sc->sc_enabled_capa, + IWM_UCODE_TLV_CAPA_DS_PARAM_SET_IE_SUPPORT)) + req->general_flags |= + htole32(IWM_UMAC_SCAN_GEN_FLAGS_RRM_ENABLED); + + ret = iwm_mvm_fill_probe_req(sc, &tail->preq); + if (ret) { + free(req, M_DEVBUF); + return ret; + } + + /* Specify the scan plan: We'll do one iteration. */ + tail->schedule[0].interval = 0; + tail->schedule[0].iter_count = 1; + + ret = iwm_send_cmd(sc, &hcmd); + if (!ret) + IWM_DPRINTF(sc, IWM_DEBUG_SCAN, + "Scan request was sent successfully\n"); + free(req, M_DEVBUF); + return ret; +} + +int +iwm_mvm_lmac_scan(struct iwm_softc *sc) +{ + struct iwm_host_cmd hcmd = { + .id = IWM_SCAN_OFFLOAD_REQUEST_CMD, + .len = { 0, }, + .data = { NULL, }, + .flags = IWM_CMD_SYNC, + }; + struct iwm_scan_req_lmac *req; + size_t req_len; + int ret; + int ssid_len = 0; + const uint8_t *ssid = NULL; IWM_DPRINTF(sc, IWM_DEBUG_SCAN, "Handling ieee80211 scan request\n"); - memset(cmd, 0, sc->sc_scan_cmd_len); - cmd->quiet_time = htole16(IWM_ACTIVE_QUIET_TIME); - cmd->quiet_plcp_th = htole16(IWM_PLCP_QUIET_THRESH); - cmd->rxchain_sel_flags = iwm_mvm_scan_rx_chain(sc); - cmd->max_out_time = iwm_mvm_scan_max_out_time(sc, 0, is_assoc); - cmd->suspend_time = iwm_mvm_scan_suspend_time(sc, is_assoc); - cmd->rxon_flags = iwm_mvm_scan_rxon_flags(sc, flags); - cmd->filter_flags = htole32(IWM_MAC_FILTER_ACCEPT_GRP | - IWM_MAC_FILTER_IN_BEACON); + req_len = sizeof(struct iwm_scan_req_lmac) + + (sizeof(struct iwm_scan_channel_cfg_lmac) * + sc->sc_capa_n_scan_channels) + sizeof(struct iwm_scan_probe_req); + if (req_len > IWM_MAX_CMD_PAYLOAD_SIZE) + return ENOMEM; + req = malloc(req_len, M_DEVBUF, M_NOWAIT | M_ZERO); + if (req == NULL) + return ENOMEM; + + hcmd.len[0] = (uint16_t)req_len; + hcmd.data[0] = (void *)req; + + /* These timings correspond to iwlwifi's UNASSOC scan. */ + req->active_dwell = 10; + req->passive_dwell = 110; + req->fragmented_dwell = 44; + req->extended_dwell = 90; + req->max_out_time = 0; + req->suspend_time = 0; + + req->scan_prio = htole32(IWM_SCAN_PRIORITY_HIGH); + req->rx_chain_select = iwm_mvm_scan_rx_chain(sc); + req->iter_num = htole32(1); + req->delay = 0; + + req->scan_flags = htole32(IWM_MVM_LMAC_SCAN_FLAG_PASS_ALL | + IWM_MVM_LMAC_SCAN_FLAG_ITER_COMPLETE | + IWM_MVM_LMAC_SCAN_FLAG_EXTENDED_DWELL); + if (ssid_len == 0) + req->scan_flags |= htole32(IWM_MVM_LMAC_SCAN_FLAG_PASSIVE); + else + req->scan_flags |= + htole32(IWM_MVM_LMAC_SCAN_FLAG_PRE_CONNECTION); + if (isset(sc->sc_enabled_capa, + IWM_UCODE_TLV_CAPA_DS_PARAM_SET_IE_SUPPORT)) + req->scan_flags |= htole32(IWM_MVM_LMAC_SCAN_FLAGS_RRM_ENABLED); + + req->flags = htole32(IWM_PHY_BAND_24); + if (sc->sc_nvm.sku_cap_band_52GHz_enable) + req->flags |= htole32(IWM_PHY_BAND_5); + req->filter_flags = + htole32(IWM_MAC_FILTER_ACCEPT_GRP | IWM_MAC_FILTER_IN_BEACON); + + /* Tx flags 2 GHz. */ + req->tx_cmd[0].tx_flags = htole32(IWM_TX_CMD_FLG_SEQ_CTL | + IWM_TX_CMD_FLG_BT_DIS); + req->tx_cmd[0].rate_n_flags = + iwm_mvm_scan_rate_n_flags(sc, IEEE80211_CHAN_2GHZ, 1/*XXX*/); + req->tx_cmd[0].sta_id = sc->sc_aux_sta.sta_id; - cmd->type = htole32(IWM_SCAN_TYPE_FORCED); - cmd->repeats = htole32(1); + /* Tx flags 5 GHz. */ + req->tx_cmd[1].tx_flags = htole32(IWM_TX_CMD_FLG_SEQ_CTL | + IWM_TX_CMD_FLG_BT_DIS); + req->tx_cmd[1].rate_n_flags = + iwm_mvm_scan_rate_n_flags(sc, IEEE80211_CHAN_5GHZ, 1/*XXX*/); + req->tx_cmd[1].sta_id = sc->sc_aux_sta.sta_id; + + /* Check if we're doing an active directed scan. */ + if (ssid_len != 0) { + req->direct_scan[0].id = IEEE80211_ELEMID_SSID; + req->direct_scan[0].len = ssid_len; + memcpy(req->direct_scan[0].ssid, ssid, ssid_len); + } - /* - * If the user asked for passive scan, don't change to active scan if - * you see any activity on the channel - remain passive. - */ - if (n_ssids > 0) { - cmd->passive2active = htole16(1); - cmd->scan_flags |= IWM_SCAN_FLAGS_PASSIVE2ACTIVE; -#if 0 - if (basic_ssid) { - ssid = req->ssids[0].ssid; - ssid_len = req->ssids[0].ssid_len; - } -#endif - } else { - cmd->passive2active = 0; - cmd->scan_flags &= ~IWM_SCAN_FLAGS_PASSIVE2ACTIVE; + req->n_channels = iwm_mvm_lmac_scan_fill_channels(sc, + (struct iwm_scan_channel_cfg_lmac *)req->data, + ssid_len != 0); + + ret = iwm_mvm_fill_probe_req(sc, + (struct iwm_scan_probe_req *)(req->data + + (sizeof(struct iwm_scan_channel_cfg_lmac) * + sc->sc_capa_n_scan_channels))); + if (ret) { + free(req, M_DEVBUF); + return ret; } - cmd->tx_cmd.tx_flags = htole32(IWM_TX_CMD_FLG_SEQ_CTL | - IWM_TX_CMD_FLG_BT_DIS); - cmd->tx_cmd.sta_id = sc->sc_aux_sta.sta_id; - cmd->tx_cmd.life_time = htole32(IWM_TX_CMD_LIFE_TIME_INFINITE); - cmd->tx_cmd.rate_n_flags = iwm_mvm_scan_rate_n_flags(sc, flags, 1/*XXX*/); - - cmd->tx_cmd.len = htole16(iwm_mvm_fill_probe_req(sc, - (struct ieee80211_frame *)cmd->data, - vap ? vap->iv_myaddr : ic->ic_macaddr, n_ssids, - ssid, ssid_len, NULL, 0, - sc->sc_capa_max_probe_len)); - - cmd->channel_count - = iwm_mvm_scan_fill_channels(sc, cmd, flags, n_ssids, basic_ssid); - - cmd->len = htole16(sizeof(struct iwm_scan_cmd) + - le16toh(cmd->tx_cmd.len) + - (cmd->channel_count * sizeof(struct iwm_scan_channel))); - hcmd.len[0] = le16toh(cmd->len); - - status = IWM_SCAN_RESPONSE_OK; - ret = iwm_mvm_send_cmd_status(sc, &hcmd, &status); - if (!ret && status == IWM_SCAN_RESPONSE_OK) { + /* Specify the scan plan: We'll do one iteration. */ + req->schedule[0].iterations = 1; + req->schedule[0].full_scan_mul = 1; + + /* Disable EBS. */ + req->channel_opt[0].non_ebs_ratio = 1; + req->channel_opt[1].non_ebs_ratio = 1; + + ret = iwm_send_cmd(sc, &hcmd); + if (!ret) { IWM_DPRINTF(sc, IWM_DEBUG_SCAN, "Scan request was sent successfully\n"); - } else { - /* - * If the scan failed, it usually means that the FW was unable - * to allocate the time events. Warn on it, but maybe we - * should try to send the command again with different params. - */ - ret = EIO; } + free(req, M_DEVBUF); return ret; } diff --git a/sys/dev/iwm/if_iwm_scan.h b/sys/dev/iwm/if_iwm_scan.h index d753ff05c82d..a9e74cdbde06 100644 --- a/sys/dev/iwm/if_iwm_scan.h +++ b/sys/dev/iwm/if_iwm_scan.h @@ -106,8 +106,8 @@ #ifndef __IF_IWN_SCAN_H__ #define __IF_IWN_SCAN_H__ -extern int -iwm_mvm_scan_request(struct iwm_softc *sc, int flags, - int n_ssids, uint8_t *ssid, int ssid_len); +extern int iwm_mvm_lmac_scan(struct iwm_softc *sc); +extern int iwm_mvm_config_umac_scan(struct iwm_softc *); +extern int iwm_mvm_umac_scan(struct iwm_softc *); #endif /* __IF_IWN_SCAN_H__ */ diff --git a/sys/dev/iwm/if_iwm_time_event.c b/sys/dev/iwm/if_iwm_time_event.c index 8396218e2228..706b6c62c54d 100644 --- a/sys/dev/iwm/if_iwm_time_event.c +++ b/sys/dev/iwm/if_iwm_time_event.c @@ -257,8 +257,7 @@ iwm_mvm_protect_session(struct iwm_softc *sc, struct iwm_node *in, htole32(IWM_FW_CMD_ID_AND_COLOR(IWM_DEFAULT_MACID, IWM_DEFAULT_COLOR)); time_cmd.id = htole32(IWM_TE_BSS_STA_AGGRESSIVE_ASSOC); - time_cmd.apply_time = htole32(iwm_read_prph(sc, - IWM_DEVICE_SYSTEM_TIME_REG)); + time_cmd.apply_time = htole32(0); time_cmd.max_frags = IWM_TE_V2_FRAG_NONE; time_cmd.max_delay = htole32(max_delay); @@ -268,7 +267,8 @@ iwm_mvm_protect_session(struct iwm_softc *sc, struct iwm_node *in, time_cmd.repeat = 1; time_cmd.policy = htole32(IWM_TE_V2_NOTIF_HOST_EVENT_START | - IWM_TE_V2_NOTIF_HOST_EVENT_END); + IWM_TE_V2_NOTIF_HOST_EVENT_END | + IWM_T2_V2_START_IMMEDIATELY); iwm_mvm_time_event_send_add(sc, in, /*te_data*/NULL, &time_cmd); } diff --git a/sys/dev/iwm/if_iwm_util.c b/sys/dev/iwm/if_iwm_util.c index 0f0aef8d359c..88d5744debe2 100644 --- a/sys/dev/iwm/if_iwm_util.c +++ b/sys/dev/iwm/if_iwm_util.c @@ -169,7 +169,7 @@ iwm_send_cmd(struct iwm_softc *sc, struct iwm_host_cmd *hcmd) { struct iwm_tx_ring *ring = &sc->txq[IWM_MVM_CMD_QUEUE]; struct iwm_tfd *desc; - struct iwm_tx_data *data; + struct iwm_tx_data *txdata = NULL; struct iwm_device_cmd *cmd; struct mbuf *m; bus_dma_segment_t seg; @@ -178,11 +178,15 @@ iwm_send_cmd(struct iwm_softc *sc, struct iwm_host_cmd *hcmd) int error = 0, i, paylen, off; int code; int async, wantresp; + int group_id; int nsegs; + size_t hdrlen, datasz; + uint8_t *data; code = hcmd->id; async = hcmd->flags & IWM_CMD_ASYNC; wantresp = hcmd->flags & IWM_CMD_WANT_SKB; + data = NULL; for (i = 0, paylen = 0; i < nitems(hcmd->len); i++) { paylen += hcmd->len[i]; @@ -207,17 +211,27 @@ iwm_send_cmd(struct iwm_softc *sc, struct iwm_host_cmd *hcmd) } desc = &ring->desc[ring->cur]; - data = &ring->data[ring->cur]; + txdata = &ring->data[ring->cur]; - if (paylen > sizeof(cmd->data)) { + group_id = iwm_cmd_groupid(code); + if (group_id != 0) { + hdrlen = sizeof(cmd->hdr_wide); + datasz = sizeof(cmd->data_wide); + } else { + hdrlen = sizeof(cmd->hdr); + datasz = sizeof(cmd->data); + } + + if (paylen > datasz) { IWM_DPRINTF(sc, IWM_DEBUG_CMD, "large command paylen=%u len0=%u\n", paylen, hcmd->len[0]); /* Command is too large */ + size_t totlen = hdrlen + paylen; if (paylen > IWM_MAX_CMD_PAYLOAD_SIZE) { device_printf(sc->sc_dev, "firmware command too long (%zd bytes)\n", - paylen + sizeof(cmd->hdr)); + totlen); error = EINVAL; goto out; } @@ -229,30 +243,41 @@ iwm_send_cmd(struct iwm_softc *sc, struct iwm_host_cmd *hcmd) m->m_len = m->m_pkthdr.len = m->m_ext.ext_size; error = bus_dmamap_load_mbuf_sg(ring->data_dmat, - data->map, m, &seg, &nsegs, BUS_DMA_NOWAIT); + txdata->map, m, &seg, &nsegs, BUS_DMA_NOWAIT); if (error != 0) { device_printf(sc->sc_dev, "%s: can't map mbuf, error %d\n", __func__, error); m_freem(m); goto out; } - data->m = m; /* mbuf will be freed in iwm_cmd_done() */ + txdata->m = m; /* mbuf will be freed in iwm_cmd_done() */ cmd = mtod(m, struct iwm_device_cmd *); paddr = seg.ds_addr; } else { cmd = &ring->cmd[ring->cur]; - paddr = data->cmd_paddr; + paddr = txdata->cmd_paddr; } - cmd->hdr.code = code; - cmd->hdr.flags = 0; - cmd->hdr.qid = ring->qid; - cmd->hdr.idx = ring->cur; + if (group_id != 0) { + cmd->hdr_wide.opcode = iwm_cmd_opcode(code); + cmd->hdr_wide.group_id = group_id; + cmd->hdr_wide.qid = ring->qid; + cmd->hdr_wide.idx = ring->cur; + cmd->hdr_wide.length = htole16(paylen); + cmd->hdr_wide.version = iwm_cmd_version(code); + data = cmd->data_wide; + } else { + cmd->hdr.code = iwm_cmd_opcode(code); + cmd->hdr.flags = 0; + cmd->hdr.qid = ring->qid; + cmd->hdr.idx = ring->cur; + data = cmd->data; + } for (i = 0, off = 0; i < nitems(hcmd->data); i++) { if (hcmd->len[i] == 0) continue; - memcpy(cmd->data + off, hcmd->data[i], hcmd->len[i]); + memcpy(data + off, hcmd->data[i], hcmd->len[i]); off += hcmd->len[i]; } KASSERT(off == paylen, ("off %d != paylen %d", off, paylen)); @@ -261,18 +286,17 @@ iwm_send_cmd(struct iwm_softc *sc, struct iwm_host_cmd *hcmd) addr_lo = htole32((uint32_t)paddr); memcpy(&desc->tbs[0].lo, &addr_lo, sizeof(uint32_t)); desc->tbs[0].hi_n_len = htole16(iwm_get_dma_hi_addr(paddr) - | ((sizeof(cmd->hdr) + paylen) << 4)); + | ((hdrlen + paylen) << 4)); desc->num_tbs = 1; IWM_DPRINTF(sc, IWM_DEBUG_CMD, - "%s: iwm_send_cmd 0x%x size=%lu %s\n", - __func__, + "iwm_send_cmd 0x%x size=%lu %s\n", code, - (unsigned long) (hcmd->len[0] + hcmd->len[1] + sizeof(cmd->hdr)), + (unsigned long) (hcmd->len[0] + hcmd->len[1] + hdrlen), async ? " (async)" : ""); - if (paylen > sizeof(cmd->data)) { - bus_dmamap_sync(ring->data_dmat, data->map, + if (paylen > datasz) { + bus_dmamap_sync(ring->data_dmat, txdata->map, BUS_DMASYNC_PREWRITE); } else { bus_dmamap_sync(ring->cmd_dma.tag, ring->cmd_dma.map, @@ -404,3 +428,31 @@ iwm_free_resp(struct iwm_softc *sc, struct iwm_host_cmd *hcmd) sc->sc_wantresp = -1; wakeup(&sc->sc_wantresp); } + +uint8_t +iwm_fw_valid_tx_ant(struct iwm_softc *sc) +{ + uint8_t tx_ant; + + tx_ant = ((sc->sc_fw_phy_config & IWM_FW_PHY_CFG_TX_CHAIN) + >> IWM_FW_PHY_CFG_TX_CHAIN_POS); + + if (sc->sc_nvm.valid_tx_ant) + tx_ant &= sc->sc_nvm.valid_tx_ant; + + return tx_ant; +} + +uint8_t +iwm_fw_valid_rx_ant(struct iwm_softc *sc) +{ + uint8_t rx_ant; + + rx_ant = ((sc->sc_fw_phy_config & IWM_FW_PHY_CFG_RX_CHAIN) + >> IWM_FW_PHY_CFG_RX_CHAIN_POS); + + if (sc->sc_nvm.valid_rx_ant) + rx_ant &= sc->sc_nvm.valid_rx_ant; + + return rx_ant; +} diff --git a/sys/dev/iwm/if_iwm_util.h b/sys/dev/iwm/if_iwm_util.h index 2d25a34ddd98..144f87ae7131 100644 --- a/sys/dev/iwm/if_iwm_util.h +++ b/sys/dev/iwm/if_iwm_util.h @@ -116,4 +116,7 @@ extern int iwm_mvm_send_cmd_pdu_status(struct iwm_softc *sc, uint8_t id, uint16_t len, const void *data, uint32_t *status); extern void iwm_free_resp(struct iwm_softc *sc, struct iwm_host_cmd *hcmd); +extern uint8_t iwm_fw_valid_tx_ant(struct iwm_softc *sc); +extern uint8_t iwm_fw_valid_rx_ant(struct iwm_softc *sc); + #endif /* __IF_IWM_UTIL_H__ */ diff --git a/sys/dev/iwm/if_iwmreg.h b/sys/dev/iwm/if_iwmreg.h index 12cfd966d47c..e68f4d48ac3f 100644 --- a/sys/dev/iwm/if_iwmreg.h +++ b/sys/dev/iwm/if_iwmreg.h @@ -1,4 +1,4 @@ -/* $OpenBSD: if_iwmreg.h,v 1.3 2015/02/23 10:25:20 stsp Exp $ */ +/* $OpenBSD: if_iwmreg.h,v 1.4 2015/06/15 08:06:11 stsp Exp $ */ /* $FreeBSD$ */ /****************************************************************************** @@ -136,6 +136,9 @@ #define IWM_CSR_UCODE_DRV_GP1_CLR (0x05c) #define IWM_CSR_UCODE_DRV_GP2 (0x060) +#define IWM_CSR_MBOX_SET_REG (0x088) +#define IWM_CSR_MBOX_SET_REG_OS_ALIVE 0x20 + #define IWM_CSR_LED_REG (0x094) #define IWM_CSR_DRAM_INT_TBL_REG (0x0A0) #define IWM_CSR_MAC_SHADOW_REG_CTRL (0x0A8) /* 6000 and up */ @@ -182,6 +185,8 @@ #define IWM_CSR_HW_IF_CONFIG_REG_BIT_NIC_READY (0x00400000) /* PCI_OWN_SEM */ #define IWM_CSR_HW_IF_CONFIG_REG_BIT_NIC_PREPARE_DONE (0x02000000) /* ME_OWN */ #define IWM_CSR_HW_IF_CONFIG_REG_PREPARE (0x08000000) /* WAKE_ME */ +#define IWM_CSR_HW_IF_CONFIG_REG_ENABLE_PME (0x10000000) +#define IWM_CSR_HW_IF_CONFIG_REG_PERSIST_MODE (0x40000000) /* PERSISTENCE */ #define IWM_CSR_INT_PERIODIC_DIS (0x00) /* disable periodic int*/ #define IWM_CSR_INT_PERIODIC_ENA (0xFF) /* 255*32 usec ~ 8 msec*/ @@ -308,6 +313,7 @@ #define IWM_CSR_HW_REV_TYPE_2x00 (0x0000100) #define IWM_CSR_HW_REV_TYPE_105 (0x0000110) #define IWM_CSR_HW_REV_TYPE_135 (0x0000120) +#define IWM_CSR_HW_REV_TYPE_7265D (0x0000210) #define IWM_CSR_HW_REV_TYPE_NONE (0x00001F0) /* EEPROM REG */ @@ -402,6 +408,7 @@ /* DRAM INT TABLE */ #define IWM_CSR_DRAM_INT_TBL_ENABLE (1 << 31) +#define IWM_CSR_DRAM_INIT_TBL_WRITE_POINTER (1 << 28) #define IWM_CSR_DRAM_INIT_TBL_WRAP_CHECK (1 << 27) /* SECURE boot registers */ @@ -421,19 +428,37 @@ enum iwm_secure_boot_status_reg { IWM_CSR_SECURE_BOOT_CPU_STATUS_SIGN_VERF_FAIL = 0x00000010, }; -#define IWM_CSR_UCODE_LOAD_STATUS_ADDR (0x100) +#define IWM_FH_UCODE_LOAD_STATUS 0x1af0 +#define IWM_CSR_UCODE_LOAD_STATUS_ADDR 0x1e70 enum iwm_secure_load_status_reg { - IWM_CSR_CPU_STATUS_LOADING_STARTED = 0x00000001, - IWM_CSR_CPU_STATUS_LOADING_COMPLETED = 0x00000002, - IWM_CSR_CPU_STATUS_NUM_OF_LAST_COMPLETED = 0x000000F8, - IWM_CSR_CPU_STATUS_NUM_OF_LAST_LOADED_BLOCK = 0x0000FF00, + IWM_LMPM_CPU_UCODE_LOADING_STARTED = 0x00000001, + IWM_LMPM_CPU_HDRS_LOADING_COMPLETED = 0x00000003, + IWM_LMPM_CPU_UCODE_LOADING_COMPLETED = 0x00000007, + IWM_LMPM_CPU_STATUS_NUM_OF_LAST_COMPLETED = 0x000000F8, + IWM_LMPM_CPU_STATUS_NUM_OF_LAST_LOADED_BLOCK = 0x0000FF00, }; +#define IWM_FH_MEM_TB_MAX_LENGTH 0x20000 + +#define IWM_LMPM_SECURE_INSPECTOR_CODE_ADDR 0x1e38 +#define IWM_LMPM_SECURE_INSPECTOR_DATA_ADDR 0x1e3c +#define IWM_LMPM_SECURE_UCODE_LOAD_CPU1_HDR_ADDR 0x1e78 +#define IWM_LMPM_SECURE_UCODE_LOAD_CPU2_HDR_ADDR 0x1e7c -#define IWM_CSR_SECURE_INSPECTOR_CODE_ADDR (0x100) -#define IWM_CSR_SECURE_INSPECTOR_DATA_ADDR (0x100) +#define IWM_LMPM_SECURE_INSPECTOR_CODE_MEM_SPACE 0x400000 +#define IWM_LMPM_SECURE_INSPECTOR_DATA_MEM_SPACE 0x402000 +#define IWM_LMPM_SECURE_CPU1_HDR_MEM_SPACE 0x420000 +#define IWM_LMPM_SECURE_CPU2_HDR_MEM_SPACE 0x420400 #define IWM_CSR_SECURE_TIME_OUT (100) +/* extended range in FW SRAM */ +#define IWM_FW_MEM_EXTENDED_START 0x40000 +#define IWM_FW_MEM_EXTENDED_END 0x57FFF + +/* FW chicken bits */ +#define IWM_LMPM_CHICK 0xa01ff8 +#define IWM_LMPM_CHICK_EXTENDED_ADDR_SPACE 0x01 + #define IWM_FH_TCSR_0_REG0 (0x1D00) /* @@ -484,6 +509,32 @@ enum iwm_secure_load_status_reg { #define IWM_HBUS_TARG_PRPH_WDAT (IWM_HBUS_BASE+0x04c) #define IWM_HBUS_TARG_PRPH_RDAT (IWM_HBUS_BASE+0x050) +/* enable the ID buf for read */ +#define IWM_WFPM_PS_CTL_CLR 0xa0300c +#define IWM_WFMP_MAC_ADDR_0 0xa03080 +#define IWM_WFMP_MAC_ADDR_1 0xa03084 +#define IWM_LMPM_PMG_EN 0xa01cec +#define IWM_RADIO_REG_SYS_MANUAL_DFT_0 0xad4078 +#define IWM_RFIC_REG_RD 0xad0470 +#define IWM_WFPM_CTRL_REG 0xa03030 +#define IWM_WFPM_AUX_CTL_AUX_IF_MAC_OWNER_MSK 0x08000000 +#define IWM_ENABLE_WFPM 0x80000000 + +#define IWM_AUX_MISC_REG 0xa200b0 +#define IWM_HW_STEP_LOCATION_BITS 24 + +#define IWM_AUX_MISC_MASTER1_EN 0xa20818 +#define IWM_AUX_MISC_MASTER1_EN_SBE_MSK 0x1 +#define IWM_AUX_MISC_MASTER1_SMPHR_STATUS 0xa20800 +#define IWM_RSA_ENABLE 0xa24b08 +#define IWM_PREG_AUX_BUS_WPROT_0 0xa04cc0 +#define IWM_SB_CFG_OVERRIDE_ADDR 0xa26c78 +#define IWM_SB_CFG_OVERRIDE_ENABLE 0x8000 +#define IWM_SB_CFG_BASE_OVERRIDE 0xa20000 +#define IWM_SB_MODIFY_CFG_FLAG 0xa03088 +#define IWM_SB_CPU_1_STATUS 0xa01e30 +#define IWM_SB_CPU_2_STATUS 0Xa01e34 + /* Used to enable DBGM */ #define IWM_HBUS_TARG_TEST_REG (IWM_HBUS_BASE+0x05c) @@ -567,7 +618,12 @@ enum iwm_dtd_diode_reg { * containing CAM (Continuous Active Mode) indication. * @IWM_UCODE_TLV_FLAGS_P2P_PS: P2P client power save is supported (only on a * single bound interface). + * @IWM_UCODE_TLV_FLAGS_UAPSD_SUPPORT: General support for uAPSD + * @IWM_UCODE_TLV_FLAGS_EBS_SUPPORT: this uCode image supports EBS. * @IWM_UCODE_TLV_FLAGS_P2P_PS_UAPSD: P2P client supports uAPSD power save + * @IWM_UCODE_TLV_FLAGS_BCAST_FILTERING: uCode supports broadcast filtering. + * @IWM_UCODE_TLV_FLAGS_GO_UAPSD: AP/GO interfaces support uAPSD clients + * */ enum iwm_ucode_tlv_flag { IWM_UCODE_TLV_FLAGS_PAN = (1 << 0), @@ -590,8 +646,150 @@ enum iwm_ucode_tlv_flag { IWM_UCODE_TLV_FLAGS_STA_KEY_CMD = (1 << 19), IWM_UCODE_TLV_FLAGS_DEVICE_PS_CMD = (1 << 20), IWM_UCODE_TLV_FLAGS_P2P_PS = (1 << 21), + IWM_UCODE_TLV_FLAGS_BSS_P2P_PS_DCM = (1 << 22), + IWM_UCODE_TLV_FLAGS_BSS_P2P_PS_SCM = (1 << 23), IWM_UCODE_TLV_FLAGS_UAPSD_SUPPORT = (1 << 24), + IWM_UCODE_TLV_FLAGS_EBS_SUPPORT = (1 << 25), IWM_UCODE_TLV_FLAGS_P2P_PS_UAPSD = (1 << 26), + IWM_UCODE_TLV_FLAGS_BCAST_FILTERING = (1 << 29), + IWM_UCODE_TLV_FLAGS_GO_UAPSD = (1 << 30), + IWM_UCODE_TLV_FLAGS_LTE_COEX = (1 << 31), +}; + +#define IWM_UCODE_TLV_FLAG_BITS \ + "\020\1PAN\2NEWSCAN\3MFP\4P2P\5DW_BC_TABLE\6NEWBT_COEX\7PM_CMD\10SHORT_BL\11RX_ENERG \ +Y\12TIME_EVENT_V2\13D3_6_IPV6\14BF_UPDATED\15NO_BASIC_SSID\17D3_CONTINUITY\20NEW_NSOFF \ +L_S\21NEW_NSOFFL_L\22SCHED_SCAN\24STA_KEY_CMD\25DEVICE_PS_CMD\26P2P_PS\27P2P_PS_DCM\30 \ +P2P_PS_SCM\31UAPSD_SUPPORT\32EBS\33P2P_PS_UAPSD\36BCAST_FILTERING\37GO_UAPSD\40LTE_COEX" + +/** + * enum iwm_ucode_tlv_api - ucode api + * @IWM_UCODE_TLV_API_FRAGMENTED_SCAN: This ucode supports active dwell time + * longer than the passive one, which is essential for fragmented scan. + * @IWM_UCODE_TLV_API_WIFI_MCC_UPDATE: ucode supports MCC updates with source. + * @IWM_UCODE_TLV_API_WIDE_CMD_HDR: ucode supports wide command header + * @IWM_UCODE_TLV_API_LQ_SS_PARAMS: Configure STBC/BFER via LQ CMD ss_params + * @IWM_UCODE_TLV_API_EXT_SCAN_PRIORITY: scan APIs use 8-level priority + * instead of 3. + * @IWM_UCODE_TLV_API_TX_POWER_CHAIN: TX power API has larger command size + * (command version 3) that supports per-chain limits + * + * @IWM_NUM_UCODE_TLV_API: number of bits used + */ +enum iwm_ucode_tlv_api { + IWM_UCODE_TLV_API_FRAGMENTED_SCAN = (1 << 8), + IWM_UCODE_TLV_API_WIFI_MCC_UPDATE = (1 << 9), + IWM_UCODE_TLV_API_WIDE_CMD_HDR = (1 << 14), + IWM_UCODE_TLV_API_LQ_SS_PARAMS = (1 << 18), + IWM_UCODE_TLV_API_EXT_SCAN_PRIORITY = (1 << 24), + IWM_UCODE_TLV_API_TX_POWER_CHAIN = (1 << 27), + + IWM_NUM_UCODE_TLV_API = 32 +}; + +#define IWM_UCODE_TLV_API_BITS \ + "\020\10FRAGMENTED_SCAN\11WIFI_MCC_UPDATE\16WIDE_CMD_HDR\22LQ_SS_PARAMS\30EXT_SCAN_PRIO\33TX_POWER_CHAIN" + +/** + * enum iwm_ucode_tlv_capa - ucode capabilities + * @IWM_UCODE_TLV_CAPA_D0I3_SUPPORT: supports D0i3 + * @IWM_UCODE_TLV_CAPA_LAR_SUPPORT: supports Location Aware Regulatory + * @IWM_UCODE_TLV_CAPA_UMAC_SCAN: supports UMAC scan. + * @IWM_UCODE_TLV_CAPA_BEAMFORMER: supports Beamformer + * @IWM_UCODE_TLV_CAPA_TOF_SUPPORT: supports Time of Flight (802.11mc FTM) + * @IWM_UCODE_TLV_CAPA_TDLS_SUPPORT: support basic TDLS functionality + * @IWM_UCODE_TLV_CAPA_TXPOWER_INSERTION_SUPPORT: supports insertion of current + * tx power value into TPC Report action frame and Link Measurement Report + * action frame + * @IWM_UCODE_TLV_CAPA_DS_PARAM_SET_IE_SUPPORT: supports updating current + * channel in DS parameter set element in probe requests. + * @IWM_UCODE_TLV_CAPA_WFA_TPC_REP_IE_SUPPORT: supports adding TPC Report IE in + * probe requests. + * @IWM_UCODE_TLV_CAPA_QUIET_PERIOD_SUPPORT: supports Quiet Period requests + * @IWM_UCODE_TLV_CAPA_DQA_SUPPORT: supports dynamic queue allocation (DQA), + * which also implies support for the scheduler configuration command + * @IWM_UCODE_TLV_CAPA_TDLS_CHANNEL_SWITCH: supports TDLS channel switching + * @IWM_UCODE_TLV_CAPA_CNSLDTD_D3_D0_IMG: Consolidated D3-D0 image + * @IWM_UCODE_TLV_CAPA_HOTSPOT_SUPPORT: supports Hot Spot Command + * @IWM_UCODE_TLV_CAPA_DC2DC_SUPPORT: supports DC2DC Command + * @IWM_UCODE_TLV_CAPA_2G_COEX_SUPPORT: supports 2G coex Command + * @IWM_UCODE_TLV_CAPA_CSUM_SUPPORT: supports TCP Checksum Offload + * @IWM_UCODE_TLV_CAPA_RADIO_BEACON_STATS: support radio and beacon statistics + * @IWM_UCODE_TLV_CAPA_P2P_STANDALONE_UAPSD: support p2p standalone U-APSD + * @IWM_UCODE_TLV_CAPA_BT_COEX_PLCR: enabled BT Coex packet level co-running + * @IWM_UCODE_TLV_CAPA_LAR_MULTI_MCC: ucode supports LAR updates with different + * sources for the MCC. This TLV bit is a future replacement to + * IWM_UCODE_TLV_API_WIFI_MCC_UPDATE. When either is set, multi-source LAR + * is supported. + * @IWM_UCODE_TLV_CAPA_BT_COEX_RRC: supports BT Coex RRC + * @IWM_UCODE_TLV_CAPA_GSCAN_SUPPORT: supports gscan + * @IWM_UCODE_TLV_CAPA_NAN_SUPPORT: supports NAN + * @IWM_UCODE_TLV_CAPA_UMAC_UPLOAD: supports upload mode in umac (1=supported, + * 0=no support) + * @IWM_UCODE_TLV_CAPA_EXTENDED_DTS_MEASURE: extended DTS measurement + * @IWM_UCODE_TLV_CAPA_SHORT_PM_TIMEOUTS: supports short PM timeouts + * @IWM_UCODE_TLV_CAPA_BT_MPLUT_SUPPORT: supports bt-coex Multi-priority LUT + * @IWM_UCODE_TLV_CAPA_BEACON_ANT_SELECTION: firmware will decide on what + * antenna the beacon should be transmitted + * @IWM_UCODE_TLV_CAPA_BEACON_STORING: firmware will store the latest beacon + * from AP and will send it upon d0i3 exit. + * @IWM_UCODE_TLV_CAPA_LAR_SUPPORT_V2: support LAR API V2 + * @IWM_UCODE_TLV_CAPA_CT_KILL_BY_FW: firmware responsible for CT-kill + * @IWM_UCODE_TLV_CAPA_TEMP_THS_REPORT_SUPPORT: supports temperature + * thresholds reporting + * @IWM_UCODE_TLV_CAPA_CTDP_SUPPORT: supports cTDP command + * @IWM_UCODE_TLV_CAPA_USNIFFER_UNIFIED: supports usniffer enabled in + * regular image. + * @IWM_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG: support getting more shared + * memory addresses from the firmware. + * @IWM_UCODE_TLV_CAPA_LQM_SUPPORT: supports Link Quality Measurement + * @IWM_UCODE_TLV_CAPA_LMAC_UPLOAD: supports upload mode in lmac (1=supported, + * 0=no support) + * + * @IWM_NUM_UCODE_TLV_CAPA: number of bits used + */ +enum iwm_ucode_tlv_capa { + IWM_UCODE_TLV_CAPA_D0I3_SUPPORT = 0, + IWM_UCODE_TLV_CAPA_LAR_SUPPORT = 1, + IWM_UCODE_TLV_CAPA_UMAC_SCAN = 2, + IWM_UCODE_TLV_CAPA_BEAMFORMER = 3, + IWM_UCODE_TLV_CAPA_TOF_SUPPORT = 5, + IWM_UCODE_TLV_CAPA_TDLS_SUPPORT = 6, + IWM_UCODE_TLV_CAPA_TXPOWER_INSERTION_SUPPORT = 8, + IWM_UCODE_TLV_CAPA_DS_PARAM_SET_IE_SUPPORT = 9, + IWM_UCODE_TLV_CAPA_WFA_TPC_REP_IE_SUPPORT = 10, + IWM_UCODE_TLV_CAPA_QUIET_PERIOD_SUPPORT = 11, + IWM_UCODE_TLV_CAPA_DQA_SUPPORT = 12, + IWM_UCODE_TLV_CAPA_TDLS_CHANNEL_SWITCH = 13, + IWM_UCODE_TLV_CAPA_CNSLDTD_D3_D0_IMG = 17, + IWM_UCODE_TLV_CAPA_HOTSPOT_SUPPORT = 18, + IWM_UCODE_TLV_CAPA_DC2DC_CONFIG_SUPPORT = 19, + IWM_UCODE_TLV_CAPA_2G_COEX_SUPPORT = 20, + IWM_UCODE_TLV_CAPA_CSUM_SUPPORT = 21, + IWM_UCODE_TLV_CAPA_RADIO_BEACON_STATS = 22, + IWM_UCODE_TLV_CAPA_P2P_STANDALONE_UAPSD = 26, + IWM_UCODE_TLV_CAPA_BT_COEX_PLCR = 28, + IWM_UCODE_TLV_CAPA_LAR_MULTI_MCC = 29, + IWM_UCODE_TLV_CAPA_BT_COEX_RRC = 30, + IWM_UCODE_TLV_CAPA_GSCAN_SUPPORT = 31, + IWM_UCODE_TLV_CAPA_NAN_SUPPORT = 34, + IWM_UCODE_TLV_CAPA_UMAC_UPLOAD = 35, + IWM_UCODE_TLV_CAPA_EXTENDED_DTS_MEASURE = 64, + IWM_UCODE_TLV_CAPA_SHORT_PM_TIMEOUTS = 65, + IWM_UCODE_TLV_CAPA_BT_MPLUT_SUPPORT = 67, + IWM_UCODE_TLV_CAPA_MULTI_QUEUE_RX_SUPPORT = 68, + IWM_UCODE_TLV_CAPA_BEACON_ANT_SELECTION = 71, + IWM_UCODE_TLV_CAPA_BEACON_STORING = 72, + IWM_UCODE_TLV_CAPA_LAR_SUPPORT_V2 = 73, + IWM_UCODE_TLV_CAPA_CT_KILL_BY_FW = 74, + IWM_UCODE_TLV_CAPA_TEMP_THS_REPORT_SUPPORT = 75, + IWM_UCODE_TLV_CAPA_CTDP_SUPPORT = 76, + IWM_UCODE_TLV_CAPA_USNIFFER_UNIFIED = 77, + IWM_UCODE_TLV_CAPA_LMAC_UPLOAD = 79, + IWM_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG = 80, + IWM_UCODE_TLV_CAPA_LQM_SUPPORT = 81, + + IWM_NUM_UCODE_TLV_CAPA = 128 }; /* The default calibrate table size if not specified by firmware file */ @@ -617,8 +815,8 @@ enum iwm_ucode_sec { * For 16.0 uCode and above, there is no differentiation between sections, * just an offset to the HW address. */ -#define IWM_UCODE_SECTION_MAX 6 -#define IWM_UCODE_FIRST_SECTION_OF_SECOND_CPU (IWM_UCODE_SECTION_MAX/2) +#define IWM_CPU1_CPU2_SEPARATOR_SECTION 0xFFFFCCCC +#define IWM_PAGING_SEPARATOR_SECTION 0xAAAABBBB /* uCode version contains 4 values: Major/Minor/API/Serial */ #define IWM_UCODE_MAJOR(ver) (((ver) & 0xFF000000) >> 24) @@ -765,7 +963,17 @@ enum iwm_ucode_tlv_type { * handling ucode version 9. */ IWM_UCODE_TLV_API_CHANGES_SET = 29, - IWM_UCODE_TLV_ENABLED_CAPABILITIES = 30 + IWM_UCODE_TLV_ENABLED_CAPABILITIES = 30, + + IWM_UCODE_TLV_N_SCAN_CHANNELS = 31, + IWM_UCODE_TLV_PAGING = 32, + IWM_UCODE_TLV_SEC_RT_USNIFFER = 34, + IWM_UCODE_TLV_SDIO_ADMA_ADDR = 35, + IWM_UCODE_TLV_FW_VERSION = 36, + IWM_UCODE_TLV_FW_DBG_DEST = 38, + IWM_UCODE_TLV_FW_DBG_CONF = 39, + IWM_UCODE_TLV_FW_DBG_TRIGGER = 40, + IWM_UCODE_TLV_FW_GSCAN_CAPA = 50, }; struct iwm_ucode_tlv { @@ -774,6 +982,16 @@ struct iwm_ucode_tlv { uint8_t data[0]; }; +struct iwm_ucode_api { + uint32_t api_index; + uint32_t api_flags; +} __packed; + +struct iwm_ucode_capa { + uint32_t api_index; + uint32_t api_capa; +} __packed; + #define IWM_TLV_UCODE_MAGIC 0x0a4c5749 struct iwm_tlv_ucode_header { @@ -846,7 +1064,19 @@ struct iwm_tlv_ucode_header { #define IWM_DEVICE_SYSTEM_TIME_REG 0xA0206C /* Device NMI register */ -#define IWM_DEVICE_SET_NMI_REG 0x00a01c30 +#define IWM_DEVICE_SET_NMI_REG 0x00a01c30 +#define IWM_DEVICE_SET_NMI_VAL_HW 0x01 +#define IWM_DEVICE_SET_NMI_VAL_DRV 0x80 +#define IWM_DEVICE_SET_NMI_8000_REG 0x00a01c24 +#define IWM_DEVICE_SET_NMI_8000_VAL 0x1000000 + +/* + * Device reset for family 8000 + * write to bit 24 in order to reset the CPU + */ +#define IWM_RELEASE_CPU_RESET 0x300c +#define IWM_RELEASE_CPU_RESET_BIT 0x1000000 + /***************************************************************************** * 7000/3000 series SHR DTS addresses * @@ -960,6 +1190,8 @@ struct iwm_tlv_ucode_header { #define IWM_SCD_QUEUE_CTX_REG2_WIN_SIZE_MSK (0x0000007F) #define IWM_SCD_QUEUE_CTX_REG2_FRAME_LIMIT_POS (16) #define IWM_SCD_QUEUE_CTX_REG2_FRAME_LIMIT_MSK (0x007F0000) +#define IWM_SCD_GP_CTRL_ENABLE_31_QUEUES (1 << 0) +#define IWM_SCD_GP_CTRL_AUTO_ACTIVE_MODE (1 << 18) /* Context Data */ #define IWM_SCD_CONTEXT_MEM_LOWER_BOUND (IWM_SCD_MEM_LOWER_BOUND + 0x600) @@ -993,6 +1225,8 @@ struct iwm_tlv_ucode_header { #define IWM_SCD_CHAINEXT_EN (IWM_SCD_BASE + 0x244) #define IWM_SCD_AGGR_SEL (IWM_SCD_BASE + 0x248) #define IWM_SCD_INTERRUPT_MASK (IWM_SCD_BASE + 0x108) +#define IWM_SCD_GP_CTRL (IWM_SCD_BASE + 0x1a8) +#define IWM_SCD_EN_CTRL (IWM_SCD_BASE + 0x254) static inline unsigned int IWM_SCD_QUEUE_WRPTR(unsigned int chnl) { @@ -1510,13 +1744,14 @@ struct iwm_agn_scd_bc_tbl { * BEGIN mvm/fw-api.h */ -/* maximal number of Tx queues in any platform */ -#define IWM_MVM_MAX_QUEUES 20 +/* Maximum number of Tx queues. */ +#define IWM_MVM_MAX_QUEUES 31 /* Tx queue numbers */ enum { IWM_MVM_OFFCHANNEL_QUEUE = 8, IWM_MVM_CMD_QUEUE = 9, + IWM_MVM_AUX_QUEUE = 15, }; enum iwm_mvm_tx_fifo { @@ -1541,6 +1776,13 @@ enum { IWM_PHY_CONTEXT_CMD = 0x8, IWM_DBG_CFG = 0x9, + /* UMAC scan commands */ + IWM_SCAN_ITERATION_COMPLETE_UMAC = 0xb5, + IWM_SCAN_CFG_CMD = 0xc, + IWM_SCAN_REQ_UMAC = 0xd, + IWM_SCAN_ABORT_UMAC = 0xe, + IWM_SCAN_COMPLETE_UMAC = 0xf, + /* station table */ IWM_ADD_STA_KEY = 0x17, IWM_ADD_STA = 0x18, @@ -1551,6 +1793,9 @@ enum { IWM_TXPATH_FLUSH = 0x1e, IWM_MGMT_MCAST_KEY = 0x1f, + /* scheduler config */ + IWM_SCD_QUEUE_CFG = 0x1d, + /* global key */ IWM_WEP_KEY = 0x20, @@ -1574,10 +1819,12 @@ enum { /* Scan offload */ IWM_SCAN_OFFLOAD_REQUEST_CMD = 0x51, IWM_SCAN_OFFLOAD_ABORT_CMD = 0x52, - IWM_SCAN_OFFLOAD_COMPLETE = 0x6D, - IWM_SCAN_OFFLOAD_UPDATE_PROFILES_CMD = 0x6E, + IWM_HOT_SPOT_CMD = 0x53, + IWM_SCAN_OFFLOAD_COMPLETE = 0x6d, + IWM_SCAN_OFFLOAD_UPDATE_PROFILES_CMD = 0x6e, IWM_SCAN_OFFLOAD_CONFIG_CMD = 0x6f, IWM_MATCH_FOUND_NOTIFICATION = 0xd9, + IWM_SCAN_ITERATION_COMPLETE = 0xe7, /* Phy */ IWM_PHY_CONFIGURATION_CMD = 0x6a, @@ -1616,6 +1863,8 @@ enum { IWM_MISSED_BEACONS_NOTIFICATION = 0xa2, + IWM_MFUART_LOAD_NOTIFICATION = 0xb1, + /* Power - new power table command */ IWM_MAC_PM_POWER_TABLE = 0xa9, @@ -1623,6 +1872,10 @@ enum { IWM_REPLY_RX_MPDU_CMD = 0xc1, IWM_BA_NOTIF = 0xc5, + /* Location Aware Regulatory */ + IWM_MCC_UPDATE_CMD = 0xc8, + IWM_MCC_CHUB_UPDATE_CMD = 0xc9, + /* BT Coex */ IWM_BT_COEX_PRIO_TABLE = 0xcc, IWM_BT_COEX_PROT_ENV = 0xcd, @@ -1632,6 +1885,10 @@ enum { IWM_REPLY_SF_CFG_CMD = 0xd1, IWM_REPLY_BEACON_FILTERING_CMD = 0xd2, + /* DTS measurements */ + IWM_CMD_DTS_MEASUREMENT_TRIGGER = 0xdc, + IWM_DTS_MEASUREMENT_NOTIFICATION = 0xdd, + IWM_REPLY_DEBUG_CMD = 0xf0, IWM_DEBUG_LOG_MSG = 0xf7, @@ -1800,10 +2057,14 @@ enum { IWM_NVM_SECTION_TYPE_HW = 0, IWM_NVM_SECTION_TYPE_SW, IWM_NVM_SECTION_TYPE_PAPD, - IWM_NVM_SECTION_TYPE_BT, + IWM_NVM_SECTION_TYPE_REGULATORY, IWM_NVM_SECTION_TYPE_CALIBRATION, IWM_NVM_SECTION_TYPE_PRODUCTION, IWM_NVM_SECTION_TYPE_POST_FCS_CALIB, + /* 7, 8, 9 unknown */ + IWM_NVM_SECTION_TYPE_HW_8000 = 10, + IWM_NVM_SECTION_TYPE_MAC_OVERRIDE, + IWM_NVM_SECTION_TYPE_PHY_SKU, IWM_NVM_NUM_OF_SECTIONS, }; @@ -1874,7 +2135,7 @@ enum { #define IWM_ALIVE_FLG_RFKILL (1 << 0) -struct iwm_mvm_alive_resp { +struct iwm_mvm_alive_resp_v1 { uint16_t status; uint16_t flags; uint8_t ucode_minor; @@ -1896,6 +2157,59 @@ struct iwm_mvm_alive_resp { uint32_t scd_base_ptr; /* SRAM address for SCD */ } __packed; /* IWM_ALIVE_RES_API_S_VER_1 */ +struct iwm_mvm_alive_resp_v2 { + uint16_t status; + uint16_t flags; + uint8_t ucode_minor; + uint8_t ucode_major; + uint16_t id; + uint8_t api_minor; + uint8_t api_major; + uint8_t ver_subtype; + uint8_t ver_type; + uint8_t mac; + uint8_t opt; + uint16_t reserved2; + uint32_t timestamp; + uint32_t error_event_table_ptr; /* SRAM address for error log */ + uint32_t log_event_table_ptr; /* SRAM address for LMAC event log */ + uint32_t cpu_register_ptr; + uint32_t dbgm_config_ptr; + uint32_t alive_counter_ptr; + uint32_t scd_base_ptr; /* SRAM address for SCD */ + uint32_t st_fwrd_addr; /* pointer to Store and forward */ + uint32_t st_fwrd_size; + uint8_t umac_minor; /* UMAC version: minor */ + uint8_t umac_major; /* UMAC version: major */ + uint16_t umac_id; /* UMAC version: id */ + uint32_t error_info_addr; /* SRAM address for UMAC error log */ + uint32_t dbg_print_buff_addr; +} __packed; /* ALIVE_RES_API_S_VER_2 */ + +struct iwm_mvm_alive_resp_v3 { + uint16_t status; + uint16_t flags; + uint32_t ucode_minor; + uint32_t ucode_major; + uint8_t ver_subtype; + uint8_t ver_type; + uint8_t mac; + uint8_t opt; + uint32_t timestamp; + uint32_t error_event_table_ptr; /* SRAM address for error log */ + uint32_t log_event_table_ptr; /* SRAM address for LMAC event log */ + uint32_t cpu_register_ptr; + uint32_t dbgm_config_ptr; + uint32_t alive_counter_ptr; + uint32_t scd_base_ptr; /* SRAM address for SCD */ + uint32_t st_fwrd_addr; /* pointer to Store and forward */ + uint32_t st_fwrd_size; + uint32_t umac_minor; /* UMAC version: minor */ + uint32_t umac_major; /* UMAC version: major */ + uint32_t error_info_addr; /* SRAM address for UMAC error log */ + uint32_t dbg_print_buff_addr; +} __packed; /* ALIVE_RES_API_S_VER_3 */ + /* Error response/notification */ enum { IWM_FW_ERR_UNKNOWN_CMD = 0x0, @@ -2055,6 +2369,7 @@ enum { IWM_TE_V1_NOTIF_HOST_FRAG_END = (1 << 5), IWM_TE_V1_NOTIF_INTERNAL_FRAG_START = (1 << 6), IWM_TE_V1_NOTIF_INTERNAL_FRAG_END = (1 << 7), + IWM_T2_V2_START_IMMEDIATELY = (1 << 11), }; /* IWM_MAC_EVENT_ACTION_API_E_VER_2 */ @@ -2608,6 +2923,21 @@ struct iwm_missed_beacons_notif { } __packed; /* IWM_MISSED_BEACON_NTFY_API_S_VER_3 */ /** + * struct iwm_mfuart_load_notif - mfuart image version & status + * ( IWM_MFUART_LOAD_NOTIFICATION = 0xb1 ) + * @installed_ver: installed image version + * @external_ver: external image version + * @status: MFUART loading status + * @duration: MFUART loading time +*/ +struct iwm_mfuart_load_notif { + uint32_t installed_ver; + uint32_t external_ver; + uint32_t status; + uint32_t duration; +} __packed; /*MFU_LOADER_NTFY_API_S_VER_1*/ + +/** * struct iwm_set_calib_default_cmd - set default value for calibration. * ( IWM_SET_CALIB_DEFAULT_CMD = 0x8e ) * @calib_index: the calibration to set value for @@ -2884,6 +3214,18 @@ enum iwm_sf_scenario { #define IWM_SF_W_MARK_LEGACY 4096 #define IWM_SF_W_MARK_SCAN 4096 +/* SF Scenarios timers for default configuration (aligned to 32 uSec) */ +#define IWM_SF_SINGLE_UNICAST_IDLE_TIMER_DEF 160 /* 150 uSec */ +#define IWM_SF_SINGLE_UNICAST_AGING_TIMER_DEF 400 /* 0.4 mSec */ +#define IWM_SF_AGG_UNICAST_IDLE_TIMER_DEF 160 /* 150 uSec */ +#define IWM_SF_AGG_UNICAST_AGING_TIMER_DEF 400 /* 0.4 mSec */ +#define IWM_SF_MCAST_IDLE_TIMER_DEF 160 /* 150 mSec */ +#define IWM_SF_MCAST_AGING_TIMER_DEF 400 /* 0.4 mSec */ +#define IWM_SF_BA_IDLE_TIMER_DEF 160 /* 150 uSec */ +#define IWM_SF_BA_AGING_TIMER_DEF 400 /* 0.4 mSec */ +#define IWM_SF_TX_RE_IDLE_TIMER_DEF 160 /* 150 uSec */ +#define IWM_SF_TX_RE_AGING_TIMER_DEF 400 /* 0.4 mSec */ + /* SF Scenarios timers for FULL_ON state (aligned to 32 uSec) */ #define IWM_SF_SINGLE_UNICAST_IDLE_TIMER 320 /* 300 uSec */ #define IWM_SF_SINGLE_UNICAST_AGING_TIMER 2016 /* 2 mSec */ @@ -2898,6 +3240,8 @@ enum iwm_sf_scenario { #define IWM_SF_LONG_DELAY_AGING_TIMER 1000000 /* 1 Sec */ +#define IWM_SF_CFG_DUMMY_NOTIF_OFF (1 << 16) + /** * Smart Fifo configuration command. * @state: smart fifo state, types listed in iwm_sf_sate. @@ -3900,6 +4244,18 @@ enum iwm_tx_flags { IWM_TX_CMD_FLG_HCCA_CHUNK = (1 << 31) }; /* IWM_TX_FLAGS_BITS_API_S_VER_1 */ +/** + * enum iwm_tx_pm_timeouts - pm timeout values in TX command + * @IWM_PM_FRAME_NONE: no need to suspend sleep mode + * @IWM_PM_FRAME_MGMT: fw suspend sleep mode for 100TU + * @IWM_PM_FRAME_ASSOC: fw suspend sleep mode for 10sec + */ +enum iwm_tx_pm_timeouts { + IWM_PM_FRAME_NONE = 0, + IWM_PM_FRAME_MGMT = 2, + IWM_PM_FRAME_ASSOC = 3, +}; + /* * TX command security control */ @@ -4366,6 +4722,46 @@ static inline uint32_t iwm_mvm_get_scd_ssn(struct iwm_mvm_tx_resp *tx_resp) * BEGIN mvm/fw-api-scan.h */ +/** + * struct iwm_scd_txq_cfg_cmd - New txq hw scheduler config command + * @token: + * @sta_id: station id + * @tid: + * @scd_queue: scheduler queue to confiug + * @enable: 1 queue enable, 0 queue disable + * @aggregate: 1 aggregated queue, 0 otherwise + * @tx_fifo: %enum iwm_mvm_tx_fifo + * @window: BA window size + * @ssn: SSN for the BA agreement + */ +struct iwm_scd_txq_cfg_cmd { + uint8_t token; + uint8_t sta_id; + uint8_t tid; + uint8_t scd_queue; + uint8_t enable; + uint8_t aggregate; + uint8_t tx_fifo; + uint8_t window; + uint16_t ssn; + uint16_t reserved; +} __packed; /* SCD_QUEUE_CFG_CMD_API_S_VER_1 */ + +/** + * struct iwm_scd_txq_cfg_rsp + * @token: taken from the command + * @sta_id: station id from the command + * @tid: tid from the command + * @scd_queue: scd_queue from the command + */ +struct iwm_scd_txq_cfg_rsp { + uint8_t token; + uint8_t sta_id; + uint8_t tid; + uint8_t scd_queue; +} __packed; /* SCD_QUEUE_CFG_RSP_API_S_VER_1 */ + + /* Scan Commands, Responses, Notifications */ /* Masks for iwm_scan_channel.type flags */ @@ -4430,6 +4826,23 @@ struct iwm_ssid_ie { uint8_t ssid[IEEE80211_NWID_LEN]; } __packed; /* IWM_SCAN_DIRECT_SSID_IE_API_S_VER_1 */ +/* scan offload */ +#define IWM_MAX_SCAN_CHANNELS 40 +#define IWM_SCAN_MAX_BLACKLIST_LEN 64 +#define IWM_SCAN_SHORT_BLACKLIST_LEN 16 +#define IWM_SCAN_MAX_PROFILES 11 +#define IWM_SCAN_OFFLOAD_PROBE_REQ_SIZE 512 + +/* Default watchdog (in MS) for scheduled scan iteration */ +#define IWM_SCHED_SCAN_WATCHDOG cpu_to_le16(15000) + +#define IWM_GOOD_CRC_TH_DEFAULT cpu_to_le16(1) +#define IWM_CAN_ABORT_STATUS 1 + +#define IWM_FULL_SCAN_MULTIPLIER 5 +#define IWM_FAST_SCHED_SCAN_ITERATIONS 3 +#define IWM_MAX_SCHED_SCAN_PLANS 2 + /** * iwm_scan_flags - masks for scan command flags *@IWM_SCAN_FLAGS_PERIODIC_SCAN: @@ -4473,64 +4886,194 @@ enum iwm_scan_type { #define IWM_MAX_NUM_SCAN_CHANNELS 0x24 /** - * struct iwm_scan_cmd - scan request command - * ( IWM_SCAN_REQUEST_CMD = 0x80 ) - * @len: command length in bytes - * @scan_flags: scan flags from IWM_SCAN_FLAGS_* - * @channel_count: num of channels in channel list (1 - IWM_MAX_NUM_SCAN_CHANNELS) - * @quiet_time: in msecs, dwell this time for active scan on quiet channels - * @quiet_plcp_th: quiet PLCP threshold (channel is quiet if less than - * this number of packets were received (typically 1) - * @passive2active: is auto switching from passive to active during scan allowed - * @rxchain_sel_flags: RXON_RX_CHAIN_* - * @max_out_time: in usecs, max out of serving channel time - * @suspend_time: how long to pause scan when returning to service channel: - * bits 0-19: beacon interval in usecs (suspend before executing) - * bits 20-23: reserved - * bits 24-31: number of beacons (suspend between channels) - * @rxon_flags: RXON_FLG_* - * @filter_flags: RXON_FILTER_* - * @tx_cmd: for active scans (zero for passive), w/o payload, - * no RS so specify TX rate - * @direct_scan: direct scan SSIDs - * @type: one of IWM_SCAN_TYPE_* - * @repeats: how many time to repeat the scan - */ -struct iwm_scan_cmd { + * iwm_scan_schedule_lmac - schedule of scan offload + * @delay: delay between iterations, in seconds. + * @iterations: num of scan iterations + * @full_scan_mul: number of partial scans before each full scan + */ +struct iwm_scan_schedule_lmac { + uint16_t delay; + uint8_t iterations; + uint8_t full_scan_mul; +} __packed; /* SCAN_SCHEDULE_API_S */ + +/** + * iwm_scan_req_tx_cmd - SCAN_REQ_TX_CMD_API_S + * @tx_flags: combination of TX_CMD_FLG_* + * @rate_n_flags: rate for *all* Tx attempts, if TX_CMD_FLG_STA_RATE_MSK is + * cleared. Combination of RATE_MCS_* + * @sta_id: index of destination station in FW station table + * @reserved: for alignment and future use + */ +struct iwm_scan_req_tx_cmd { + uint32_t tx_flags; + uint32_t rate_n_flags; + uint8_t sta_id; + uint8_t reserved[3]; +} __packed; + +enum iwm_scan_channel_flags_lmac { + IWM_UNIFIED_SCAN_CHANNEL_FULL = (1 << 27), + IWM_UNIFIED_SCAN_CHANNEL_PARTIAL = (1 << 28), +}; + +/** + * iwm_scan_channel_cfg_lmac - SCAN_CHANNEL_CFG_S_VER2 + * @flags: bits 1-20: directed scan to i'th ssid + * other bits &enum iwm_scan_channel_flags_lmac + * @channel_number: channel number 1-13 etc + * @iter_count: scan iteration on this channel + * @iter_interval: interval in seconds between iterations on one channel + */ +struct iwm_scan_channel_cfg_lmac { + uint32_t flags; + uint16_t channel_num; + uint16_t iter_count; + uint32_t iter_interval; +} __packed; + +/* + * iwm_scan_probe_segment - PROBE_SEGMENT_API_S_VER_1 + * @offset: offset in the data block + * @len: length of the segment + */ +struct iwm_scan_probe_segment { + uint16_t offset; uint16_t len; - uint8_t scan_flags; - uint8_t channel_count; - uint16_t quiet_time; - uint16_t quiet_plcp_th; - uint16_t passive2active; - uint16_t rxchain_sel_flags; +} __packed; + +/* iwm_scan_probe_req - PROBE_REQUEST_FRAME_API_S_VER_2 + * @mac_header: first (and common) part of the probe + * @band_data: band specific data + * @common_data: last (and common) part of the probe + * @buf: raw data block + */ +struct iwm_scan_probe_req { + struct iwm_scan_probe_segment mac_header; + struct iwm_scan_probe_segment band_data[2]; + struct iwm_scan_probe_segment common_data; + uint8_t buf[IWM_SCAN_OFFLOAD_PROBE_REQ_SIZE]; +} __packed; + +enum iwm_scan_channel_flags { + IWM_SCAN_CHANNEL_FLAG_EBS = (1 << 0), + IWM_SCAN_CHANNEL_FLAG_EBS_ACCURATE = (1 << 1), + IWM_SCAN_CHANNEL_FLAG_CACHE_ADD = (1 << 2), +}; + +/* iwm_scan_channel_opt - CHANNEL_OPTIMIZATION_API_S + * @flags: enum iwm_scan_channel_flags + * @non_ebs_ratio: defines the ratio of number of scan iterations where EBS is + * involved. + * 1 - EBS is disabled. + * 2 - every second scan will be full scan(and so on). + */ +struct iwm_scan_channel_opt { + uint16_t flags; + uint16_t non_ebs_ratio; +} __packed; + +/** + * iwm_mvm_lmac_scan_flags + * @IWM_MVM_LMAC_SCAN_FLAG_PASS_ALL: pass all beacons and probe responses + * without filtering. + * @IWM_MVM_LMAC_SCAN_FLAG_PASSIVE: force passive scan on all channels + * @IWM_MVM_LMAC_SCAN_FLAG_PRE_CONNECTION: single channel scan + * @IWM_MVM_LMAC_SCAN_FLAG_ITER_COMPLETE: send iteration complete notification + * @IWM_MVM_LMAC_SCAN_FLAG_MULTIPLE_SSIDS multiple SSID matching + * @IWM_MVM_LMAC_SCAN_FLAG_FRAGMENTED: all passive scans will be fragmented + * @IWM_MVM_LMAC_SCAN_FLAGS_RRM_ENABLED: insert WFA vendor-specific TPC report + * and DS parameter set IEs into probe requests. + * @IWM_MVM_LMAC_SCAN_FLAG_EXTENDED_DWELL: use extended dwell time on channels + * 1, 6 and 11. + * @IWM_MVM_LMAC_SCAN_FLAG_MATCH: Send match found notification on matches + */ +enum iwm_mvm_lmac_scan_flags { + IWM_MVM_LMAC_SCAN_FLAG_PASS_ALL = (1 << 0), + IWM_MVM_LMAC_SCAN_FLAG_PASSIVE = (1 << 1), + IWM_MVM_LMAC_SCAN_FLAG_PRE_CONNECTION = (1 << 2), + IWM_MVM_LMAC_SCAN_FLAG_ITER_COMPLETE = (1 << 3), + IWM_MVM_LMAC_SCAN_FLAG_MULTIPLE_SSIDS = (1 << 4), + IWM_MVM_LMAC_SCAN_FLAG_FRAGMENTED = (1 << 5), + IWM_MVM_LMAC_SCAN_FLAGS_RRM_ENABLED = (1 << 6), + IWM_MVM_LMAC_SCAN_FLAG_EXTENDED_DWELL = (1 << 7), + IWM_MVM_LMAC_SCAN_FLAG_MATCH = (1 << 9), +}; + +enum iwm_scan_priority { + IWM_SCAN_PRIORITY_LOW, + IWM_SCAN_PRIORITY_MEDIUM, + IWM_SCAN_PRIORITY_HIGH, +}; + +/** + * iwm_scan_req_lmac - SCAN_REQUEST_CMD_API_S_VER_1 + * @reserved1: for alignment and future use + * @channel_num: num of channels to scan + * @active-dwell: dwell time for active channels + * @passive-dwell: dwell time for passive channels + * @fragmented-dwell: dwell time for fragmented passive scan + * @extended_dwell: dwell time for channels 1, 6 and 11 (in certain cases) + * @reserved2: for alignment and future use + * @rx_chain_selct: PHY_RX_CHAIN_* flags + * @scan_flags: &enum iwm_mvm_lmac_scan_flags + * @max_out_time: max time (in TU) to be out of associated channel + * @suspend_time: pause scan this long (TUs) when returning to service channel + * @flags: RXON flags + * @filter_flags: RXON filter + * @tx_cmd: tx command for active scan; for 2GHz and for 5GHz + * @direct_scan: list of SSIDs for directed active scan + * @scan_prio: enum iwm_scan_priority + * @iter_num: number of scan iterations + * @delay: delay in seconds before first iteration + * @schedule: two scheduling plans. The first one is finite, the second one can + * be infinite. + * @channel_opt: channel optimization options, for full and partial scan + * @data: channel configuration and probe request packet. + */ +struct iwm_scan_req_lmac { + /* SCAN_REQUEST_FIXED_PART_API_S_VER_7 */ + uint32_t reserved1; + uint8_t n_channels; + uint8_t active_dwell; + uint8_t passive_dwell; + uint8_t fragmented_dwell; + uint8_t extended_dwell; + uint8_t reserved2; + uint16_t rx_chain_select; + uint32_t scan_flags; uint32_t max_out_time; uint32_t suspend_time; - /* IWM_RX_ON_FLAGS_API_S_VER_1 */ - uint32_t rxon_flags; + /* RX_ON_FLAGS_API_S_VER_1 */ + uint32_t flags; uint32_t filter_flags; - struct iwm_tx_cmd tx_cmd; + struct iwm_scan_req_tx_cmd tx_cmd[2]; struct iwm_ssid_ie direct_scan[IWM_PROBE_OPTION_MAX]; - uint32_t type; - uint32_t repeats; + uint32_t scan_prio; + /* SCAN_REQ_PERIODIC_PARAMS_API_S */ + uint32_t iter_num; + uint32_t delay; + struct iwm_scan_schedule_lmac schedule[IWM_MAX_SCHED_SCAN_PLANS]; + struct iwm_scan_channel_opt channel_opt[2]; + uint8_t data[]; +} __packed; - /* - * Probe request frame, followed by channel list. - * - * Size of probe request frame is specified by byte count in tx_cmd. - * Channel list follows immediately after probe request frame. - * Number of channels in list is specified by channel_count. - * Each channel in list is of type: - * - * struct iwm_scan_channel channels[0]; - * - * NOTE: Only one band of channels can be scanned per pass. You - * must not mix 2.4GHz channels and 5.2GHz channels, and you must wait - * for one scan to complete (i.e. receive IWM_SCAN_COMPLETE_NOTIFICATION) - * before requesting another scan. - */ - uint8_t data[0]; -} __packed; /* IWM_SCAN_REQUEST_FIXED_PART_API_S_VER_5 */ +/** + * iwm_scan_offload_complete - PERIODIC_SCAN_COMPLETE_NTF_API_S_VER_2 + * @last_schedule_line: last schedule line executed (fast or regular) + * @last_schedule_iteration: last scan iteration executed before scan abort + * @status: enum iwm_scan_offload_complete_status + * @ebs_status: EBS success status &enum iwm_scan_ebs_status + * @time_after_last_iter; time in seconds elapsed after last iteration + */ +struct iwm_periodic_scan_complete { + uint8_t last_schedule_line; + uint8_t last_schedule_iteration; + uint8_t status; + uint8_t ebs_status; + uint32_t time_after_last_iter; + uint32_t reserved; +} __packed; /* Response to scan request contains only status with one of these values */ #define IWM_SCAN_RESPONSE_OK 0x1 @@ -4653,22 +5196,6 @@ struct iwm_scan_complete_notif { struct iwm_scan_results_notif results[IWM_MAX_NUM_SCAN_CHANNELS]; } __packed; /* IWM_SCAN_COMPLETE_NTF_API_S_VER_2 */ -/* scan offload */ -#define IWM_MAX_SCAN_CHANNELS 40 -#define IWM_SCAN_MAX_BLACKLIST_LEN 64 -#define IWM_SCAN_SHORT_BLACKLIST_LEN 16 -#define IWM_SCAN_MAX_PROFILES 11 -#define IWM_SCAN_OFFLOAD_PROBE_REQ_SIZE 512 - -/* Default watchdog (in MS) for scheduled scan iteration */ -#define IWM_SCHED_SCAN_WATCHDOG cpu_to_le16(15000) - -#define IWM_GOOD_CRC_TH_DEFAULT cpu_to_le16(1) -#define IWM_CAN_ABORT_STATUS 1 - -#define IWM_FULL_SCAN_MULTIPLIER 5 -#define IWM_FAST_SCHED_SCAN_ITERATIONS 3 - enum iwm_scan_framework_client { IWM_SCAN_CLIENT_SCHED_SCAN = (1 << 0), IWM_SCAN_CLIENT_NETDETECT = (1 << 1), @@ -4864,6 +5391,28 @@ enum iwm_scan_offload_compleate_status { }; /** + * struct iwm_lmac_scan_complete_notif - notifies end of scanning (all channels) + * SCAN_COMPLETE_NTF_API_S_VER_3 + * @scanned_channels: number of channels scanned (and number of valid results) + * @status: one of SCAN_COMP_STATUS_* + * @bt_status: BT on/off status + * @last_channel: last channel that was scanned + * @tsf_low: TSF timer (lower half) in usecs + * @tsf_high: TSF timer (higher half) in usecs + * @results: an array of scan results, only "scanned_channels" of them are valid + */ +struct iwm_lmac_scan_complete_notif { + uint8_t scanned_channels; + uint8_t status; + uint8_t bt_status; + uint8_t last_channel; + uint32_t tsf_low; + uint32_t tsf_high; + struct iwm_scan_results_notif results[]; +} __packed; + + +/** * iwm_scan_offload_complete - IWM_SCAN_OFFLOAD_COMPLETE_NTF_API_S_VER_1 * @last_schedule_line: last schedule line executed (fast or regular) * @last_schedule_iteration: last scan iteration executed before scan abort @@ -4895,11 +5444,336 @@ struct iwm_sched_scan_results { * BEGIN mvm/fw-api-sta.h */ +/* UMAC Scan API */ + +/* The maximum of either of these cannot exceed 8, because we use an + * 8-bit mask (see IWM_MVM_SCAN_MASK). + */ +#define IWM_MVM_MAX_UMAC_SCANS 8 +#define IWM_MVM_MAX_LMAC_SCANS 1 + +enum iwm_scan_config_flags { + IWM_SCAN_CONFIG_FLAG_ACTIVATE = (1 << 0), + IWM_SCAN_CONFIG_FLAG_DEACTIVATE = (1 << 1), + IWM_SCAN_CONFIG_FLAG_FORBID_CHUB_REQS = (1 << 2), + IWM_SCAN_CONFIG_FLAG_ALLOW_CHUB_REQS = (1 << 3), + IWM_SCAN_CONFIG_FLAG_SET_TX_CHAINS = (1 << 8), + IWM_SCAN_CONFIG_FLAG_SET_RX_CHAINS = (1 << 9), + IWM_SCAN_CONFIG_FLAG_SET_AUX_STA_ID = (1 << 10), + IWM_SCAN_CONFIG_FLAG_SET_ALL_TIMES = (1 << 11), + IWM_SCAN_CONFIG_FLAG_SET_EFFECTIVE_TIMES = (1 << 12), + IWM_SCAN_CONFIG_FLAG_SET_CHANNEL_FLAGS = (1 << 13), + IWM_SCAN_CONFIG_FLAG_SET_LEGACY_RATES = (1 << 14), + IWM_SCAN_CONFIG_FLAG_SET_MAC_ADDR = (1 << 15), + IWM_SCAN_CONFIG_FLAG_SET_FRAGMENTED = (1 << 16), + IWM_SCAN_CONFIG_FLAG_CLEAR_FRAGMENTED = (1 << 17), + IWM_SCAN_CONFIG_FLAG_SET_CAM_MODE = (1 << 18), + IWM_SCAN_CONFIG_FLAG_CLEAR_CAM_MODE = (1 << 19), + IWM_SCAN_CONFIG_FLAG_SET_PROMISC_MODE = (1 << 20), + IWM_SCAN_CONFIG_FLAG_CLEAR_PROMISC_MODE = (1 << 21), + + /* Bits 26-31 are for num of channels in channel_array */ +#define IWM_SCAN_CONFIG_N_CHANNELS(n) ((n) << 26) +}; + +enum iwm_scan_config_rates { + /* OFDM basic rates */ + IWM_SCAN_CONFIG_RATE_6M = (1 << 0), + IWM_SCAN_CONFIG_RATE_9M = (1 << 1), + IWM_SCAN_CONFIG_RATE_12M = (1 << 2), + IWM_SCAN_CONFIG_RATE_18M = (1 << 3), + IWM_SCAN_CONFIG_RATE_24M = (1 << 4), + IWM_SCAN_CONFIG_RATE_36M = (1 << 5), + IWM_SCAN_CONFIG_RATE_48M = (1 << 6), + IWM_SCAN_CONFIG_RATE_54M = (1 << 7), + /* CCK basic rates */ + IWM_SCAN_CONFIG_RATE_1M = (1 << 8), + IWM_SCAN_CONFIG_RATE_2M = (1 << 9), + IWM_SCAN_CONFIG_RATE_5M = (1 << 10), + IWM_SCAN_CONFIG_RATE_11M = (1 << 11), + + /* Bits 16-27 are for supported rates */ +#define IWM_SCAN_CONFIG_SUPPORTED_RATE(rate) ((rate) << 16) +}; + +enum iwm_channel_flags { + IWM_CHANNEL_FLAG_EBS = (1 << 0), + IWM_CHANNEL_FLAG_ACCURATE_EBS = (1 << 1), + IWM_CHANNEL_FLAG_EBS_ADD = (1 << 2), + IWM_CHANNEL_FLAG_PRE_SCAN_PASSIVE2ACTIVE = (1 << 3), +}; + +/** + * struct iwm_scan_config + * @flags: enum scan_config_flags + * @tx_chains: valid_tx antenna - ANT_* definitions + * @rx_chains: valid_rx antenna - ANT_* definitions + * @legacy_rates: default legacy rates - enum scan_config_rates + * @out_of_channel_time: default max out of serving channel time + * @suspend_time: default max suspend time + * @dwell_active: default dwell time for active scan + * @dwell_passive: default dwell time for passive scan + * @dwell_fragmented: default dwell time for fragmented scan + * @dwell_extended: default dwell time for channels 1, 6 and 11 + * @mac_addr: default mac address to be used in probes + * @bcast_sta_id: the index of the station in the fw + * @channel_flags: default channel flags - enum iwm_channel_flags + * scan_config_channel_flag + * @channel_array: default supported channels + */ +struct iwm_scan_config { + uint32_t flags; + uint32_t tx_chains; + uint32_t rx_chains; + uint32_t legacy_rates; + uint32_t out_of_channel_time; + uint32_t suspend_time; + uint8_t dwell_active; + uint8_t dwell_passive; + uint8_t dwell_fragmented; + uint8_t dwell_extended; + uint8_t mac_addr[IEEE80211_ADDR_LEN]; + uint8_t bcast_sta_id; + uint8_t channel_flags; + uint8_t channel_array[]; +} __packed; /* SCAN_CONFIG_DB_CMD_API_S */ + +/** + * iwm_umac_scan_flags + *@IWM_UMAC_SCAN_FLAG_PREEMPTIVE: scan process triggered by this scan request + * can be preempted by other scan requests with higher priority. + * The low priority scan will be resumed when the higher proirity scan is + * completed. + *@IWM_UMAC_SCAN_FLAG_START_NOTIF: notification will be sent to the driver + * when scan starts. + */ +enum iwm_umac_scan_flags { + IWM_UMAC_SCAN_FLAG_PREEMPTIVE = (1 << 0), + IWM_UMAC_SCAN_FLAG_START_NOTIF = (1 << 1), +}; + +enum iwm_umac_scan_uid_offsets { + IWM_UMAC_SCAN_UID_TYPE_OFFSET = 0, + IWM_UMAC_SCAN_UID_SEQ_OFFSET = 8, +}; + +enum iwm_umac_scan_general_flags { + IWM_UMAC_SCAN_GEN_FLAGS_PERIODIC = (1 << 0), + IWM_UMAC_SCAN_GEN_FLAGS_OVER_BT = (1 << 1), + IWM_UMAC_SCAN_GEN_FLAGS_PASS_ALL = (1 << 2), + IWM_UMAC_SCAN_GEN_FLAGS_PASSIVE = (1 << 3), + IWM_UMAC_SCAN_GEN_FLAGS_PRE_CONNECT = (1 << 4), + IWM_UMAC_SCAN_GEN_FLAGS_ITER_COMPLETE = (1 << 5), + IWM_UMAC_SCAN_GEN_FLAGS_MULTIPLE_SSID = (1 << 6), + IWM_UMAC_SCAN_GEN_FLAGS_FRAGMENTED = (1 << 7), + IWM_UMAC_SCAN_GEN_FLAGS_RRM_ENABLED = (1 << 8), + IWM_UMAC_SCAN_GEN_FLAGS_MATCH = (1 << 9), + IWM_UMAC_SCAN_GEN_FLAGS_EXTENDED_DWELL = (1 << 10), +}; + +/** + * struct iwm_scan_channel_cfg_umac + * @flags: bitmap - 0-19: directed scan to i'th ssid. + * @channel_num: channel number 1-13 etc. + * @iter_count: repetition count for the channel. + * @iter_interval: interval between two scan iterations on one channel. + */ +struct iwm_scan_channel_cfg_umac { + uint32_t flags; + uint8_t channel_num; + uint8_t iter_count; + uint16_t iter_interval; +} __packed; /* SCAN_CHANNEL_CFG_S_VER2 */ + +/** + * struct iwm_scan_umac_schedule + * @interval: interval in seconds between scan iterations + * @iter_count: num of scan iterations for schedule plan, 0xff for infinite loop + * @reserved: for alignment and future use + */ +struct iwm_scan_umac_schedule { + uint16_t interval; + uint8_t iter_count; + uint8_t reserved; +} __packed; /* SCAN_SCHED_PARAM_API_S_VER_1 */ + +/** + * struct iwm_scan_req_umac_tail - the rest of the UMAC scan request command + * parameters following channels configuration array. + * @schedule: two scheduling plans. + * @delay: delay in TUs before starting the first scan iteration + * @reserved: for future use and alignment + * @preq: probe request with IEs blocks + * @direct_scan: list of SSIDs for directed active scan + */ +struct iwm_scan_req_umac_tail { + /* SCAN_PERIODIC_PARAMS_API_S_VER_1 */ + struct iwm_scan_umac_schedule schedule[IWM_MAX_SCHED_SCAN_PLANS]; + uint16_t delay; + uint16_t reserved; + /* SCAN_PROBE_PARAMS_API_S_VER_1 */ + struct iwm_scan_probe_req preq; + struct iwm_ssid_ie direct_scan[IWM_PROBE_OPTION_MAX]; +} __packed; + +/** + * struct iwm_scan_req_umac + * @flags: &enum iwm_umac_scan_flags + * @uid: scan id, &enum iwm_umac_scan_uid_offsets + * @ooc_priority: out of channel priority - &enum iwm_scan_priority + * @general_flags: &enum iwm_umac_scan_general_flags + * @extended_dwell: dwell time for channels 1, 6 and 11 + * @active_dwell: dwell time for active scan + * @passive_dwell: dwell time for passive scan + * @fragmented_dwell: dwell time for fragmented passive scan + * @max_out_time: max out of serving channel time + * @suspend_time: max suspend time + * @scan_priority: scan internal prioritization &enum iwm_scan_priority + * @channel_flags: &enum iwm_scan_channel_flags + * @n_channels: num of channels in scan request + * @reserved: for future use and alignment + * @data: &struct iwm_scan_channel_cfg_umac and + * &struct iwm_scan_req_umac_tail + */ +struct iwm_scan_req_umac { + uint32_t flags; + uint32_t uid; + uint32_t ooc_priority; + /* SCAN_GENERAL_PARAMS_API_S_VER_1 */ + uint32_t general_flags; + uint8_t extended_dwell; + uint8_t active_dwell; + uint8_t passive_dwell; + uint8_t fragmented_dwell; + uint32_t max_out_time; + uint32_t suspend_time; + uint32_t scan_priority; + /* SCAN_CHANNEL_PARAMS_API_S_VER_1 */ + uint8_t channel_flags; + uint8_t n_channels; + uint16_t reserved; + uint8_t data[]; +} __packed; /* SCAN_REQUEST_CMD_UMAC_API_S_VER_1 */ + +/** + * struct iwm_umac_scan_abort + * @uid: scan id, &enum iwm_umac_scan_uid_offsets + * @flags: reserved + */ +struct iwm_umac_scan_abort { + uint32_t uid; + uint32_t flags; +} __packed; /* SCAN_ABORT_CMD_UMAC_API_S_VER_1 */ + +/** + * struct iwm_umac_scan_complete + * @uid: scan id, &enum iwm_umac_scan_uid_offsets + * @last_schedule: last scheduling line + * @last_iter: last scan iteration number + * @scan status: &enum iwm_scan_offload_complete_status + * @ebs_status: &enum iwm_scan_ebs_status + * @time_from_last_iter: time elapsed from last iteration + * @reserved: for future use + */ +struct iwm_umac_scan_complete { + uint32_t uid; + uint8_t last_schedule; + uint8_t last_iter; + uint8_t status; + uint8_t ebs_status; + uint32_t time_from_last_iter; + uint32_t reserved; +} __packed; /* SCAN_COMPLETE_NTF_UMAC_API_S_VER_1 */ + +#define IWM_SCAN_OFFLOAD_MATCHING_CHANNELS_LEN 5 +/** + * struct iwm_scan_offload_profile_match - match information + * @bssid: matched bssid + * @channel: channel where the match occurred + * @energy: + * @matching_feature: + * @matching_channels: bitmap of channels that matched, referencing + * the channels passed in tue scan offload request + */ +struct iwm_scan_offload_profile_match { + uint8_t bssid[IEEE80211_ADDR_LEN]; + uint16_t reserved; + uint8_t channel; + uint8_t energy; + uint8_t matching_feature; + uint8_t matching_channels[IWM_SCAN_OFFLOAD_MATCHING_CHANNELS_LEN]; +} __packed; /* SCAN_OFFLOAD_PROFILE_MATCH_RESULTS_S_VER_1 */ + +/** + * struct iwm_scan_offload_profiles_query - match results query response + * @matched_profiles: bitmap of matched profiles, referencing the + * matches passed in the scan offload request + * @last_scan_age: age of the last offloaded scan + * @n_scans_done: number of offloaded scans done + * @gp2_d0u: GP2 when D0U occurred + * @gp2_invoked: GP2 when scan offload was invoked + * @resume_while_scanning: not used + * @self_recovery: obsolete + * @reserved: reserved + * @matches: array of match information, one for each match + */ +struct iwm_scan_offload_profiles_query { + uint32_t matched_profiles; + uint32_t last_scan_age; + uint32_t n_scans_done; + uint32_t gp2_d0u; + uint32_t gp2_invoked; + uint8_t resume_while_scanning; + uint8_t self_recovery; + uint16_t reserved; + struct iwm_scan_offload_profile_match matches[IWM_SCAN_MAX_PROFILES]; +} __packed; /* SCAN_OFFLOAD_PROFILES_QUERY_RSP_S_VER_2 */ + +/** + * struct iwm_umac_scan_iter_complete_notif - notifies end of scanning iteration + * @uid: scan id, &enum iwm_umac_scan_uid_offsets + * @scanned_channels: number of channels scanned and number of valid elements in + * results array + * @status: one of SCAN_COMP_STATUS_* + * @bt_status: BT on/off status + * @last_channel: last channel that was scanned + * @tsf_low: TSF timer (lower half) in usecs + * @tsf_high: TSF timer (higher half) in usecs + * @results: array of scan results, only "scanned_channels" of them are valid + */ +struct iwm_umac_scan_iter_complete_notif { + uint32_t uid; + uint8_t scanned_channels; + uint8_t status; + uint8_t bt_status; + uint8_t last_channel; + uint32_t tsf_low; + uint32_t tsf_high; + struct iwm_scan_results_notif results[]; +} __packed; /* SCAN_ITER_COMPLETE_NTF_UMAC_API_S_VER_1 */ + +/* Please keep this enum *SORTED* by hex value. + * Needed for binary search, otherwise a warning will be triggered. + */ +enum iwm_scan_subcmd_ids { + IWM_GSCAN_START_CMD = 0x0, + IWM_GSCAN_STOP_CMD = 0x1, + IWM_GSCAN_SET_HOTLIST_CMD = 0x2, + IWM_GSCAN_RESET_HOTLIST_CMD = 0x3, + IWM_GSCAN_SET_SIGNIFICANT_CHANGE_CMD = 0x4, + IWM_GSCAN_RESET_SIGNIFICANT_CHANGE_CMD = 0x5, + IWM_GSCAN_SIGNIFICANT_CHANGE_EVENT = 0xFD, + IWM_GSCAN_HOTLIST_CHANGE_EVENT = 0xFE, + IWM_GSCAN_RESULTS_AVAILABLE_EVENT = 0xFF, +}; + +/* STA API */ + /** * enum iwm_sta_flags - flags for the ADD_STA host command * @IWM_STA_FLG_REDUCED_TX_PWR_CTRL: * @IWM_STA_FLG_REDUCED_TX_PWR_DATA: - * @IWM_STA_FLG_FLG_ANT_MSK: Antenna selection + * @IWM_STA_FLG_DISABLE_TX: set if TX should be disabled * @IWM_STA_FLG_PS: set if STA is in Power Save * @IWM_STA_FLG_INVALID: set if STA is invalid * @IWM_STA_FLG_DLP_EN: Direct Link Protocol is enabled @@ -4923,10 +5797,7 @@ enum iwm_sta_flags { IWM_STA_FLG_REDUCED_TX_PWR_CTRL = (1 << 3), IWM_STA_FLG_REDUCED_TX_PWR_DATA = (1 << 6), - IWM_STA_FLG_FLG_ANT_A = (1 << 4), - IWM_STA_FLG_FLG_ANT_B = (2 << 4), - IWM_STA_FLG_FLG_ANT_MSK = (IWM_STA_FLG_FLG_ANT_A | - IWM_STA_FLG_FLG_ANT_B), + IWM_STA_FLG_DISABLE_TX = (1 << 4), IWM_STA_FLG_PS = (1 << 8), IWM_STA_FLG_DRAIN_FLOW = (1 << 12), @@ -5004,7 +5875,7 @@ enum iwm_sta_key_flag { /** * enum iwm_sta_modify_flag - indicate to the fw what flag are being changed - * @IWM_STA_MODIFY_KEY: this command modifies %key + * @IWM_STA_MODIFY_QUEUE_REMOVAL: this command removes a queue * @IWM_STA_MODIFY_TID_DISABLE_TX: this command modifies %tid_disable_tx * @IWM_STA_MODIFY_TX_RATE: unused * @IWM_STA_MODIFY_ADD_BA_TID: this command modifies %add_immediate_ba_tid @@ -5014,7 +5885,7 @@ enum iwm_sta_key_flag { * @IWM_STA_MODIFY_QUEUES: modify the queues used by this station */ enum iwm_sta_modify_flag { - IWM_STA_MODIFY_KEY = (1 << 0), + IWM_STA_MODIFY_QUEUE_REMOVAL = (1 << 0), IWM_STA_MODIFY_TID_DISABLE_TX = (1 << 1), IWM_STA_MODIFY_TX_RATE = (1 << 2), IWM_STA_MODIFY_ADD_BA_TID = (1 << 3), @@ -5031,11 +5902,14 @@ enum iwm_sta_modify_flag { * @IWM_STA_SLEEP_STATE_AWAKE: * @IWM_STA_SLEEP_STATE_PS_POLL: * @IWM_STA_SLEEP_STATE_UAPSD: + * @IWM_STA_SLEEP_STATE_MOREDATA: set more-data bit on + * (last) released frame */ enum iwm_sta_sleep_flag { IWM_STA_SLEEP_STATE_AWAKE = 0, IWM_STA_SLEEP_STATE_PS_POLL = (1 << 0), IWM_STA_SLEEP_STATE_UAPSD = (1 << 1), + IWM_STA_SLEEP_STATE_MOREDATA = (1 << 2), }; /* STA ID and color bits definitions */ @@ -5083,23 +5957,25 @@ struct iwm_mvm_keyinfo { uint64_t hw_tkip_mic_tx_key; } __packed; +#define IWM_ADD_STA_STATUS_MASK 0xFF +#define IWM_ADD_STA_BAID_VALID_MASK 0x8000 +#define IWM_ADD_STA_BAID_MASK 0x7F00 +#define IWM_ADD_STA_BAID_SHIFT 8 + /** - * struct iwm_mvm_add_sta_cmd_v5 - Add/modify a station in the fw's sta table. - * ( IWM_REPLY_ADD_STA = 0x18 ) + * struct iwm_mvm_add_sta_cmd_v7 - Add/modify a station in the fw's sta table. + * ( REPLY_ADD_STA = 0x18 ) * @add_modify: 1: modify existing, 0: add new station - * @unicast_tx_key_id: unicast tx key id. Relevant only when unicast key sent - * @multicast_tx_key_id: multicast tx key id. Relevant only when multicast key - * sent + * @awake_acs: + * @tid_disable_tx: is tid BIT(tid) enabled for Tx. Clear BIT(x) to enable + * AMPDU for tid x. Set %IWM_STA_MODIFY_TID_DISABLE_TX to change this field. * @mac_id_n_color: the Mac context this station belongs to * @addr[IEEE80211_ADDR_LEN]: station's MAC address * @sta_id: index of station in uCode's station table * @modify_mask: IWM_STA_MODIFY_*, selects which parameters to modify vs. leave * alone. 1 - modify, 0 - don't change. - * @key: look at %iwm_mvm_keyinfo * @station_flags: look at %iwm_sta_flags * @station_flags_msk: what of %station_flags have changed - * @tid_disable_tx: is tid BIT(tid) enabled for Tx. Clear BIT(x) to enable - * AMPDU for tid x. Set %IWM_STA_MODIFY_TID_DISABLE_TX to change this field. * @add_immediate_ba_tid: tid for which to add block-ack support (Rx) * Set %IWM_STA_MODIFY_ADD_BA_TID to use this field, and also set * add_immediate_ba_ssn. @@ -5123,40 +5999,9 @@ struct iwm_mvm_keyinfo { * ADD_STA sets up the table entry for one station, either creating a new * entry, or modifying a pre-existing one. */ -struct iwm_mvm_add_sta_cmd_v5 { +struct iwm_mvm_add_sta_cmd_v7 { uint8_t add_modify; - uint8_t unicast_tx_key_id; - uint8_t multicast_tx_key_id; - uint8_t reserved1; - uint32_t mac_id_n_color; - uint8_t addr[IEEE80211_ADDR_LEN]; - uint16_t reserved2; - uint8_t sta_id; - uint8_t modify_mask; - uint16_t reserved3; - struct iwm_mvm_keyinfo key; - uint32_t station_flags; - uint32_t station_flags_msk; - uint16_t tid_disable_tx; - uint16_t reserved4; - uint8_t add_immediate_ba_tid; - uint8_t remove_immediate_ba_tid; - uint16_t add_immediate_ba_ssn; - uint16_t sleep_tx_count; - uint16_t sleep_state_flags; - uint16_t assoc_id; - uint16_t beamform_flags; - uint32_t tfd_queue_msk; -} __packed; /* IWM_ADD_STA_CMD_API_S_VER_5 */ - -/** - * struct iwm_mvm_add_sta_cmd_v6 - Add / modify a station - * VER_6 of this command is quite similar to VER_5 except - * exclusion of all fields related to the security key installation. - */ -struct iwm_mvm_add_sta_cmd_v6 { - uint8_t add_modify; - uint8_t reserved1; + uint8_t awake_acs; uint16_t tid_disable_tx; uint32_t mac_id_n_color; uint8_t addr[IEEE80211_ADDR_LEN]; /* _STA_ID_MODIFY_INFO_API_S_VER_1 */ @@ -5174,7 +6019,7 @@ struct iwm_mvm_add_sta_cmd_v6 { uint16_t assoc_id; uint16_t beamform_flags; uint32_t tfd_queue_msk; -} __packed; /* IWM_ADD_STA_CMD_API_S_VER_6 */ +} __packed; /* ADD_STA_CMD_API_S_VER_7 */ /** * struct iwm_mvm_add_sta_key_cmd - add/modify sta key @@ -5264,17 +6109,228 @@ struct iwm_mvm_wep_key_cmd { struct iwm_mvm_wep_key wep_key[0]; } __packed; /* SEC_CURR_WEP_KEY_CMD_API_S_VER_2 */ - /* * END mvm/fw-api-sta.h */ /* + * BT coex + */ + +enum iwm_bt_coex_mode { + IWM_BT_COEX_DISABLE = 0x0, + IWM_BT_COEX_NW = 0x1, + IWM_BT_COEX_BT = 0x2, + IWM_BT_COEX_WIFI = 0x3, +}; /* BT_COEX_MODES_E */ + +enum iwm_bt_coex_enabled_modules { + IWM_BT_COEX_MPLUT_ENABLED = (1 << 0), + IWM_BT_COEX_MPLUT_BOOST_ENABLED = (1 << 1), + IWM_BT_COEX_SYNC2SCO_ENABLED = (1 << 2), + IWM_BT_COEX_CORUN_ENABLED = (1 << 3), + IWM_BT_COEX_HIGH_BAND_RET = (1 << 4), +}; /* BT_COEX_MODULES_ENABLE_E_VER_1 */ + +/** + * struct iwm_bt_coex_cmd - bt coex configuration command + * @mode: enum %iwm_bt_coex_mode + * @enabled_modules: enum %iwm_bt_coex_enabled_modules + * + * The structure is used for the BT_COEX command. + */ +struct iwm_bt_coex_cmd { + uint32_t mode; + uint32_t enabled_modules; +} __packed; /* BT_COEX_CMD_API_S_VER_6 */ + + +/* + * Location Aware Regulatory (LAR) API - MCC updates + */ + +/** + * struct iwm_mcc_update_cmd_v1 - Request the device to update geographic + * regulatory profile according to the given MCC (Mobile Country Code). + * The MCC is two letter-code, ascii upper case[A-Z] or '00' for world domain. + * 'ZZ' MCC will be used to switch to NVM default profile; in this case, the + * MCC in the cmd response will be the relevant MCC in the NVM. + * @mcc: given mobile country code + * @source_id: the source from where we got the MCC, see iwm_mcc_source + * @reserved: reserved for alignment + */ +struct iwm_mcc_update_cmd_v1 { + uint16_t mcc; + uint8_t source_id; + uint8_t reserved; +} __packed; /* LAR_UPDATE_MCC_CMD_API_S_VER_1 */ + +/** + * struct iwm_mcc_update_cmd - Request the device to update geographic + * regulatory profile according to the given MCC (Mobile Country Code). + * The MCC is two letter-code, ascii upper case[A-Z] or '00' for world domain. + * 'ZZ' MCC will be used to switch to NVM default profile; in this case, the + * MCC in the cmd response will be the relevant MCC in the NVM. + * @mcc: given mobile country code + * @source_id: the source from where we got the MCC, see iwm_mcc_source + * @reserved: reserved for alignment + * @key: integrity key for MCC API OEM testing + * @reserved2: reserved + */ +struct iwm_mcc_update_cmd { + uint16_t mcc; + uint8_t source_id; + uint8_t reserved; + uint32_t key; + uint32_t reserved2[5]; +} __packed; /* LAR_UPDATE_MCC_CMD_API_S_VER_2 */ + +/** + * iwm_mcc_update_resp_v1 - response to MCC_UPDATE_CMD. + * Contains the new channel control profile map, if changed, and the new MCC + * (mobile country code). + * The new MCC may be different than what was requested in MCC_UPDATE_CMD. + * @status: see &enum iwm_mcc_update_status + * @mcc: the new applied MCC + * @cap: capabilities for all channels which matches the MCC + * @source_id: the MCC source, see iwm_mcc_source + * @n_channels: number of channels in @channels_data (may be 14, 39, 50 or 51 + * channels, depending on platform) + * @channels: channel control data map, DWORD for each channel. Only the first + * 16bits are used. + */ +struct iwm_mcc_update_resp_v1 { + uint32_t status; + uint16_t mcc; + uint8_t cap; + uint8_t source_id; + uint32_t n_channels; + uint32_t channels[0]; +} __packed; /* LAR_UPDATE_MCC_CMD_RESP_S_VER_1 */ + +/** + * iwm_mcc_update_resp - response to MCC_UPDATE_CMD. + * Contains the new channel control profile map, if changed, and the new MCC + * (mobile country code). + * The new MCC may be different than what was requested in MCC_UPDATE_CMD. + * @status: see &enum iwm_mcc_update_status + * @mcc: the new applied MCC + * @cap: capabilities for all channels which matches the MCC + * @source_id: the MCC source, see iwm_mcc_source + * @time: time elapsed from the MCC test start (in 30 seconds TU) + * @reserved: reserved. + * @n_channels: number of channels in @channels_data (may be 14, 39, 50 or 51 + * channels, depending on platform) + * @channels: channel control data map, DWORD for each channel. Only the first + * 16bits are used. + */ +struct iwm_mcc_update_resp { + uint32_t status; + uint16_t mcc; + uint8_t cap; + uint8_t source_id; + uint16_t time; + uint16_t reserved; + uint32_t n_channels; + uint32_t channels[0]; +} __packed; /* LAR_UPDATE_MCC_CMD_RESP_S_VER_2 */ + +/** + * struct iwm_mcc_chub_notif - chub notifies of mcc change + * (MCC_CHUB_UPDATE_CMD = 0xc9) + * The Chub (Communication Hub, CommsHUB) is a HW component that connects to + * the cellular and connectivity cores that gets updates of the mcc, and + * notifies the ucode directly of any mcc change. + * The ucode requests the driver to request the device to update geographic + * regulatory profile according to the given MCC (Mobile Country Code). + * The MCC is two letter-code, ascii upper case[A-Z] or '00' for world domain. + * 'ZZ' MCC will be used to switch to NVM default profile; in this case, the + * MCC in the cmd response will be the relevant MCC in the NVM. + * @mcc: given mobile country code + * @source_id: identity of the change originator, see iwm_mcc_source + * @reserved1: reserved for alignment + */ +struct iwm_mcc_chub_notif { + uint16_t mcc; + uint8_t source_id; + uint8_t reserved1; +} __packed; /* LAR_MCC_NOTIFY_S */ + +enum iwm_mcc_update_status { + IWM_MCC_RESP_NEW_CHAN_PROFILE, + IWM_MCC_RESP_SAME_CHAN_PROFILE, + IWM_MCC_RESP_INVALID, + IWM_MCC_RESP_NVM_DISABLED, + IWM_MCC_RESP_ILLEGAL, + IWM_MCC_RESP_LOW_PRIORITY, + IWM_MCC_RESP_TEST_MODE_ACTIVE, + IWM_MCC_RESP_TEST_MODE_NOT_ACTIVE, + IWM_MCC_RESP_TEST_MODE_DENIAL_OF_SERVICE, +}; + +enum iwm_mcc_source { + IWM_MCC_SOURCE_OLD_FW = 0, + IWM_MCC_SOURCE_ME = 1, + IWM_MCC_SOURCE_BIOS = 2, + IWM_MCC_SOURCE_3G_LTE_HOST = 3, + IWM_MCC_SOURCE_3G_LTE_DEVICE = 4, + IWM_MCC_SOURCE_WIFI = 5, + IWM_MCC_SOURCE_RESERVED = 6, + IWM_MCC_SOURCE_DEFAULT = 7, + IWM_MCC_SOURCE_UNINITIALIZED = 8, + IWM_MCC_SOURCE_MCC_API = 9, + IWM_MCC_SOURCE_GET_CURRENT = 0x10, + IWM_MCC_SOURCE_GETTING_MCC_TEST_MODE = 0x11, +}; + +/* * Some cherry-picked definitions */ #define IWM_FRAME_LIMIT 64 +/* + * From Linux commit ab02165ccec4c78162501acedeef1a768acdb811: + * As the firmware is slowly running out of command IDs and grouping of + * commands is desirable anyway, the firmware is extending the command + * header from 4 bytes to 8 bytes to introduce a group (in place of the + * former flags field, since that's always 0 on commands and thus can + * be easily used to distinguish between the two). + * + * These functions retrieve specific information from the id field in + * the iwm_host_cmd struct which contains the command id, the group id, + * and the version of the command. +*/ +static inline uint8_t +iwm_cmd_opcode(uint32_t cmdid) +{ + return cmdid & 0xff; +} + +static inline uint8_t +iwm_cmd_groupid(uint32_t cmdid) +{ + return ((cmdid & 0Xff00) >> 8); +} + +static inline uint8_t +iwm_cmd_version(uint32_t cmdid) +{ + return ((cmdid & 0xff0000) >> 16); +} + +static inline uint32_t +iwm_cmd_id(uint8_t opcode, uint8_t groupid, uint8_t version) +{ + return opcode + (groupid << 8) + (version << 16); +} + +/* make uint16_t wide id out of uint8_t group and opcode */ +#define IWM_WIDE_ID(grp, opcode) ((grp << 8) | opcode) + +/* due to the conversion, this group is special */ +#define IWM_ALWAYS_LONG_GROUP 1 + struct iwm_cmd_header { uint8_t code; uint8_t flags; @@ -5282,6 +6338,16 @@ struct iwm_cmd_header { uint8_t qid; } __packed; +struct iwm_cmd_header_wide { + uint8_t opcode; + uint8_t group_id; + uint8_t idx; + uint8_t qid; + uint16_t length; + uint8_t reserved; + uint8_t version; +} __packed; + enum iwm_power_scheme { IWM_POWER_SCHEME_CAM = 1, IWM_POWER_SCHEME_BPS, @@ -5292,10 +6358,26 @@ enum iwm_power_scheme { #define IWM_MAX_CMD_PAYLOAD_SIZE ((4096 - 4) - sizeof(struct iwm_cmd_header)) #define IWM_CMD_FAILED_MSK 0x40 +/** + * struct iwm_device_cmd + * + * For allocation of the command and tx queues, this establishes the overall + * size of the largest command we send to uCode, except for commands that + * aren't fully copied and use other TFD space. + */ struct iwm_device_cmd { - struct iwm_cmd_header hdr; - - uint8_t data[IWM_DEF_CMD_PAYLOAD_SIZE]; + union { + struct { + struct iwm_cmd_header hdr; + uint8_t data[IWM_DEF_CMD_PAYLOAD_SIZE]; + }; + struct { + struct iwm_cmd_header_wide hdr_wide; + uint8_t data_wide[IWM_DEF_CMD_PAYLOAD_SIZE - + sizeof(struct iwm_cmd_header_wide) + + sizeof(struct iwm_cmd_header)]; + }; + }; } __packed; struct iwm_rx_packet { @@ -5357,11 +6439,4 @@ iwm_rx_packet_payload_len(const struct iwm_rx_packet *pkt) bus_space_barrier((sc)->sc_st, (sc)->sc_sh, 0, (sc)->sc_sz, \ BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE) -#define IWM_FW_VALID_TX_ANT(sc) \ - ((sc->sc_fw_phy_config & IWM_FW_PHY_CFG_TX_CHAIN) \ - >> IWM_FW_PHY_CFG_TX_CHAIN_POS) -#define IWM_FW_VALID_RX_ANT(sc) \ - ((sc->sc_fw_phy_config & IWM_FW_PHY_CFG_RX_CHAIN) \ - >> IWM_FW_PHY_CFG_RX_CHAIN_POS) - #endif /* __IF_IWM_REG_H__ */ diff --git a/sys/dev/iwm/if_iwmvar.h b/sys/dev/iwm/if_iwmvar.h index e923aa4f70ee..15a4655c03db 100644 --- a/sys/dev/iwm/if_iwmvar.h +++ b/sys/dev/iwm/if_iwmvar.h @@ -137,8 +137,9 @@ struct iwm_tx_radiotap_header { (1 << IEEE80211_RADIOTAP_CHANNEL)) -#define IWM_UCODE_SECT_MAX 6 +#define IWM_UCODE_SECT_MAX 16 #define IWM_FWDMASEGSZ (192*1024) +#define IWM_FWDMASEGSZ_8000 (320*1024) /* sanity check value */ #define IWM_FWMAXSIZE (2*1024*1024) @@ -152,9 +153,10 @@ struct iwm_tx_radiotap_header { #define IWM_FW_STATUS_DONE 2 enum iwm_ucode_type { - IWM_UCODE_TYPE_INIT, IWM_UCODE_TYPE_REGULAR, + IWM_UCODE_TYPE_INIT, IWM_UCODE_TYPE_WOW, + IWM_UCODE_TYPE_REGULAR_USNIFFER, IWM_UCODE_TYPE_MAX }; @@ -197,8 +199,9 @@ struct iwm_nvm_data { uint8_t radio_cfg_pnum; uint8_t valid_tx_ant, valid_rx_ant; #define IWM_NUM_CHANNELS 39 +#define IWM_NUM_CHANNELS_8000 51 - uint16_t nvm_ch_flags[IWM_NUM_CHANNELS]; + uint16_t nvm_ch_flags[IWM_NUM_CHANNELS_8000]; uint16_t nvm_version; uint8_t max_tx_pwr_half_dbm; @@ -216,9 +219,9 @@ struct iwm_host_cmd { int handler_status; uint32_t flags; + uint32_t id; uint16_t len[IWM_MAX_CMD_TBS_PER_TFD]; uint8_t dataflags[IWM_MAX_CMD_TBS_PER_TFD]; - uint8_t id; }; /* @@ -269,13 +272,6 @@ struct iwm_tx_ring { #define IWM_MAX_SCATTER 20 -struct iwm_softc; -struct iwm_rbuf { - struct iwm_softc *sc; - void *vaddr; - bus_addr_t paddr; -}; - struct iwm_rx_data { struct mbuf *m; bus_dmamap_t map; @@ -296,6 +292,7 @@ struct iwm_rx_ring { struct iwm_ucode_status { uint32_t uc_error_event_table; + uint32_t uc_umac_error_event_table; uint32_t uc_log_event_table; int uc_ok; @@ -304,7 +301,9 @@ struct iwm_ucode_status { #define IWM_CMD_RESP_MAX PAGE_SIZE -#define IWM_OTP_LOW_IMAGE_SIZE 2048 +/* lower blocks contain EEPROM image and calibration data */ +#define IWM_OTP_LOW_IMAGE_SIZE_FAMILY_7000 16384 +#define IWM_OTP_LOW_IMAGE_SIZE_FAMILY_8000 32768 #define IWM_MVM_TE_SESSION_PROTECTION_MAX_TIME_MS 500 #define IWM_MVM_TE_SESSION_PROTECTION_MIN_TIME_MS 400 @@ -329,7 +328,7 @@ enum iwm_hcmd_dataflag { * iwlwifi/iwl-phy-db */ -#define IWM_NUM_PAPD_CH_GROUPS 4 +#define IWM_NUM_PAPD_CH_GROUPS 9 #define IWM_NUM_TXP_CH_GROUPS 9 struct iwm_phy_db_entry { @@ -386,6 +385,7 @@ struct iwm_node { #define IWM_NODE(_ni) ((struct iwm_node *)(_ni)) #define IWM_STATION_ID 0 +#define IWM_AUX_STA_ID 1 #define IWM_DEFAULT_MACID 0 #define IWM_DEFAULT_COLOR 0 @@ -409,7 +409,7 @@ struct iwm_softc { #define IWM_FLAG_STOPPED (1 << 2) #define IWM_FLAG_RFKILL (1 << 3) #define IWM_FLAG_BUSY (1 << 4) -#define IWM_FLAG_DORESUME (1 << 5) +#define IWM_FLAG_SCANNING (1 << 5) struct intr_config_hook sc_preinit_hook; struct callout sc_watchdog_to; @@ -441,7 +441,14 @@ struct iwm_softc { int ict_cur; int sc_hw_rev; +#define IWM_SILICON_A_STEP 0 +#define IWM_SILICON_B_STEP 1 +#define IWM_SILICON_C_STEP 2 +#define IWM_SILICON_D_STEP 3 int sc_hw_id; + int sc_device_family; +#define IWM_DEVICE_FAMILY_7000 1 +#define IWM_DEVICE_FAMILY_8000 2 struct iwm_dma_info kw_dma; struct iwm_dma_info fw_dma; @@ -451,10 +458,14 @@ struct iwm_softc { struct iwm_ucode_status sc_uc; enum iwm_ucode_type sc_uc_current; - int sc_fwver; + char sc_fwver[32]; int sc_capaflags; int sc_capa_max_probe_len; + int sc_capa_n_scan_channels; + uint32_t sc_ucode_api; + uint8_t sc_enabled_capa[howmany(IWM_NUM_UCODE_TLV_CAPA, NBBY)]; + char sc_fw_mcc[3]; int sc_intmask; @@ -482,10 +493,7 @@ struct iwm_softc { int sc_tx_timer; - struct iwm_scan_cmd *sc_scan_cmd; - size_t sc_scan_cmd_len; int sc_scan_last_antenna; - int sc_scanband; int sc_fixed_ridx; @@ -495,7 +503,6 @@ struct iwm_softc { uint8_t sc_cmd_resp[IWM_CMD_RESP_MAX]; int sc_wantresp; - struct taskqueue *sc_tq; struct task sc_es_task; struct iwm_rx_phy_info sc_last_phy_info; diff --git a/sys/dev/ixgb/if_ixgb.h b/sys/dev/ixgb/if_ixgb.h index e42dc1deca75..d7044a780f81 100644 --- a/sys/dev/ixgb/if_ixgb.h +++ b/sys/dev/ixgb/if_ixgb.h @@ -277,7 +277,7 @@ struct adapter { /* FreeBSD operating-system-specific structures */ struct ixgb_osdep osdep; - struct device *dev; + device_t dev; struct resource *res_memory; struct resource *res_ioport; struct resource *res_interrupt; diff --git a/sys/dev/ixgb/if_ixgb_osdep.h b/sys/dev/ixgb/if_ixgb_osdep.h index e8ce885525bb..33b550a71f34 100644 --- a/sys/dev/ixgb/if_ixgb_osdep.h +++ b/sys/dev/ixgb/if_ixgb_osdep.h @@ -90,7 +90,7 @@ struct ixgb_osdep { bus_space_tag_t mem_bus_space_tag; bus_space_handle_t mem_bus_space_handle; - struct device *dev; + device_t dev; }; #define IXGB_WRITE_FLUSH(a) IXGB_READ_REG(a, STATUS) diff --git a/sys/dev/ixgbe/ixgbe.h b/sys/dev/ixgbe/ixgbe.h index 6a1522f82564..68f2edb57eef 100644 --- a/sys/dev/ixgbe/ixgbe.h +++ b/sys/dev/ixgbe/ixgbe.h @@ -458,7 +458,7 @@ struct adapter { struct ixgbe_hw hw; struct ixgbe_osdep osdep; - struct device *dev; + device_t dev; struct ifnet *ifp; struct resource *pci_mem; diff --git a/sys/dev/ixl/i40e_adminq.c b/sys/dev/ixl/i40e_adminq.c index 792f832ac70f..cfffc1f01909 100644 --- a/sys/dev/ixl/i40e_adminq.c +++ b/sys/dev/ixl/i40e_adminq.c @@ -39,16 +39,6 @@ #include "i40e_prototype.h" /** - * i40e_is_nvm_update_op - return TRUE if this is an NVM update operation - * @desc: API request descriptor - **/ -static INLINE bool i40e_is_nvm_update_op(struct i40e_aq_desc *desc) -{ - return (desc->opcode == CPU_TO_LE16(i40e_aqc_opc_nvm_erase)) || - (desc->opcode == CPU_TO_LE16(i40e_aqc_opc_nvm_update)); -} - -/** * i40e_adminq_init_regs - Initialize AdminQ registers * @hw: pointer to the hardware structure * @@ -661,13 +651,9 @@ enum i40e_status_code i40e_init_adminq(struct i40e_hw *hw) /* pre-emptive resource lock release */ i40e_aq_release_resource(hw, I40E_NVM_RESOURCE_ID, 0, NULL); - hw->aq.nvm_release_on_done = FALSE; + hw->nvm_release_on_done = FALSE; hw->nvmupd_state = I40E_NVMUPD_STATE_INIT; - ret_code = i40e_aq_set_hmc_resource_profile(hw, - I40E_HMC_PROFILE_DEFAULT, - 0, - NULL); ret_code = I40E_SUCCESS; /* success! */ @@ -1081,26 +1067,7 @@ enum i40e_status_code i40e_clean_arq_element(struct i40e_hw *hw, hw->aq.arq.next_to_clean = ntc; hw->aq.arq.next_to_use = ntu; - if (i40e_is_nvm_update_op(&e->desc)) { - if (hw->aq.nvm_release_on_done) { - i40e_release_nvm(hw); - hw->aq.nvm_release_on_done = FALSE; - } - - switch (hw->nvmupd_state) { - case I40E_NVMUPD_STATE_INIT_WAIT: - hw->nvmupd_state = I40E_NVMUPD_STATE_INIT; - break; - - case I40E_NVMUPD_STATE_WRITE_WAIT: - hw->nvmupd_state = I40E_NVMUPD_STATE_WRITING; - break; - - default: - break; - } - } - + i40e_nvmupd_check_wait_event(hw, LE16_TO_CPU(e->desc.opcode)); clean_arq_element_out: /* Set pending if needed, unlock and return */ if (pending != NULL) diff --git a/sys/dev/ixl/i40e_adminq.h b/sys/dev/ixl/i40e_adminq.h index 6448b8be2cf0..a0279273bd94 100644 --- a/sys/dev/ixl/i40e_adminq.h +++ b/sys/dev/ixl/i40e_adminq.h @@ -105,7 +105,6 @@ struct i40e_adminq_info { u32 fw_build; /* firmware build number */ u16 api_maj_ver; /* api major version */ u16 api_min_ver; /* api minor version */ - bool nvm_release_on_done; struct i40e_spinlock asq_spinlock; /* Send queue spinlock */ struct i40e_spinlock arq_spinlock; /* Receive queue spinlock */ diff --git a/sys/dev/ixl/i40e_adminq_cmd.h b/sys/dev/ixl/i40e_adminq_cmd.h index 7c0ace2e6b75..764ce11fb772 100644 --- a/sys/dev/ixl/i40e_adminq_cmd.h +++ b/sys/dev/ixl/i40e_adminq_cmd.h @@ -140,6 +140,10 @@ enum i40e_admin_queue_opc { i40e_aqc_opc_list_func_capabilities = 0x000A, i40e_aqc_opc_list_dev_capabilities = 0x000B, + /* Proxy commands */ + i40e_aqc_opc_set_proxy_config = 0x0104, + i40e_aqc_opc_set_ns_proxy_table_entry = 0x0105, + /* LAA */ i40e_aqc_opc_mac_address_read = 0x0107, i40e_aqc_opc_mac_address_write = 0x0108, @@ -147,6 +151,10 @@ enum i40e_admin_queue_opc { /* PXE */ i40e_aqc_opc_clear_pxe_mode = 0x0110, + /* WoL commands */ + i40e_aqc_opc_set_wol_filter = 0x0120, + i40e_aqc_opc_get_wake_reason = 0x0121, + /* internal switch commands */ i40e_aqc_opc_get_switch_config = 0x0200, i40e_aqc_opc_add_statistics = 0x0201, @@ -185,6 +193,7 @@ enum i40e_admin_queue_opc { i40e_aqc_opc_remove_control_packet_filter = 0x025B, i40e_aqc_opc_add_cloud_filters = 0x025C, i40e_aqc_opc_remove_cloud_filters = 0x025D, + i40e_aqc_opc_clear_wol_switch_filters = 0x025E, i40e_aqc_opc_add_mirror_rule = 0x0260, i40e_aqc_opc_delete_mirror_rule = 0x0261, @@ -212,7 +221,6 @@ enum i40e_admin_queue_opc { i40e_aqc_opc_suspend_port_tx = 0x041B, i40e_aqc_opc_resume_port_tx = 0x041C, i40e_aqc_opc_configure_partition_bw = 0x041D, - /* hmc */ i40e_aqc_opc_query_hmc_resource_profile = 0x0500, i40e_aqc_opc_set_hmc_resource_profile = 0x0501, @@ -271,6 +279,10 @@ enum i40e_admin_queue_opc { /* Tunnel commands */ i40e_aqc_opc_add_udp_tunnel = 0x0B00, i40e_aqc_opc_del_udp_tunnel = 0x0B01, + i40e_aqc_opc_set_rss_key = 0x0B02, + i40e_aqc_opc_set_rss_lut = 0x0B03, + i40e_aqc_opc_get_rss_key = 0x0B04, + i40e_aqc_opc_get_rss_lut = 0x0B05, /* Async Events */ i40e_aqc_opc_event_lan_overflow = 0x1001, @@ -433,6 +445,7 @@ struct i40e_aqc_list_capabilities_element_resp { #define I40E_AQ_CAP_ID_SDP 0x0062 #define I40E_AQ_CAP_ID_MDIO 0x0063 #define I40E_AQ_CAP_ID_WSR_PROT 0x0064 +#define I40E_AQ_CAP_ID_NVM_MGMT 0x0080 #define I40E_AQ_CAP_ID_FLEX10 0x00F1 #define I40E_AQ_CAP_ID_CEM 0x00F2 @@ -457,13 +470,15 @@ I40E_CHECK_CMD_LENGTH(i40e_aqc_cppm_configuration); /* Set ARP Proxy command / response (indirect 0x0104) */ struct i40e_aqc_arp_proxy_data { __le16 command_flags; -#define I40E_AQ_ARP_INIT_IPV4 0x0008 -#define I40E_AQ_ARP_UNSUP_CTL 0x0010 -#define I40E_AQ_ARP_ENA 0x0020 -#define I40E_AQ_ARP_ADD_IPV4 0x0040 -#define I40E_AQ_ARP_DEL_IPV4 0x0080 +#define I40E_AQ_ARP_INIT_IPV4 0x0800 +#define I40E_AQ_ARP_UNSUP_CTL 0x1000 +#define I40E_AQ_ARP_ENA 0x2000 +#define I40E_AQ_ARP_ADD_IPV4 0x4000 +#define I40E_AQ_ARP_DEL_IPV4 0x8000 __le16 table_id; - __le32 pfpm_proxyfc; + __le32 enabled_offloads; +#define I40E_AQ_ARP_DIRECTED_OFFLOAD_ENABLE 0x00000020 +#define I40E_AQ_ARP_OFFLOAD_ENABLE 0x00000800 __le32 ip_addr; u8 mac_addr[6]; u8 reserved[2]; @@ -478,17 +493,19 @@ struct i40e_aqc_ns_proxy_data { __le16 table_idx_ipv6_0; __le16 table_idx_ipv6_1; __le16 control; -#define I40E_AQ_NS_PROXY_ADD_0 0x0100 -#define I40E_AQ_NS_PROXY_DEL_0 0x0200 -#define I40E_AQ_NS_PROXY_ADD_1 0x0400 -#define I40E_AQ_NS_PROXY_DEL_1 0x0800 -#define I40E_AQ_NS_PROXY_ADD_IPV6_0 0x1000 -#define I40E_AQ_NS_PROXY_DEL_IPV6_0 0x2000 -#define I40E_AQ_NS_PROXY_ADD_IPV6_1 0x4000 -#define I40E_AQ_NS_PROXY_DEL_IPV6_1 0x8000 -#define I40E_AQ_NS_PROXY_COMMAND_SEQ 0x0001 -#define I40E_AQ_NS_PROXY_INIT_IPV6_TBL 0x0002 -#define I40E_AQ_NS_PROXY_INIT_MAC_TBL 0x0004 +#define I40E_AQ_NS_PROXY_ADD_0 0x0001 +#define I40E_AQ_NS_PROXY_DEL_0 0x0002 +#define I40E_AQ_NS_PROXY_ADD_1 0x0004 +#define I40E_AQ_NS_PROXY_DEL_1 0x0008 +#define I40E_AQ_NS_PROXY_ADD_IPV6_0 0x0010 +#define I40E_AQ_NS_PROXY_DEL_IPV6_0 0x0020 +#define I40E_AQ_NS_PROXY_ADD_IPV6_1 0x0040 +#define I40E_AQ_NS_PROXY_DEL_IPV6_1 0x0080 +#define I40E_AQ_NS_PROXY_COMMAND_SEQ 0x0100 +#define I40E_AQ_NS_PROXY_INIT_IPV6_TBL 0x0200 +#define I40E_AQ_NS_PROXY_INIT_MAC_TBL 0x0400 +#define I40E_AQ_NS_PROXY_OFFLOAD_ENABLE 0x0800 +#define I40E_AQ_NS_PROXY_DIRECTED_OFFLOAD_ENABLE 0x1000 u8 mac_addr_0[6]; u8 mac_addr_1[6]; u8 local_mac_addr[6]; @@ -538,6 +555,7 @@ I40E_CHECK_STRUCT_LEN(24, i40e_aqc_mac_address_read_data); /* Manage MAC Address Write Command (0x0108) */ struct i40e_aqc_mac_address_write { __le16 command_flags; +#define I40E_AQC_MC_MAG_EN 0x0100 #define I40E_AQC_WRITE_TYPE_LAA_ONLY 0x0000 #define I40E_AQC_WRITE_TYPE_LAA_WOL 0x4000 #define I40E_AQC_WRITE_TYPE_PORT 0x8000 @@ -561,6 +579,56 @@ struct i40e_aqc_clear_pxe { I40E_CHECK_CMD_LENGTH(i40e_aqc_clear_pxe); +/* Set WoL Filter (0x0120) */ + +struct i40e_aqc_set_wol_filter { + __le16 filter_index; +#define I40E_AQC_MAX_NUM_WOL_FILTERS 8 +#define I40E_AQC_SET_WOL_FILTER_TYPE_MAGIC_SHIFT 15 +#define I40E_AQC_SET_WOL_FILTER_TYPE_MAGIC_MASK (0x1 << \ + I40E_AQC_SET_WOL_FILTER_TYPE_MAGIC_SHIFT) + +#define I40E_AQC_SET_WOL_FILTER_INDEX_SHIFT 0 +#define I40E_AQC_SET_WOL_FILTER_INDEX_MASK (0x7 << \ + I40E_AQC_SET_WOL_FILTER_INDEX_SHIFT) + __le16 cmd_flags; +#define I40E_AQC_SET_WOL_FILTER 0x8000 +#define I40E_AQC_SET_WOL_FILTER_NO_TCO_WOL 0x4000 +#define I40E_AQC_SET_WOL_FILTER_ACTION_CLEAR 0 +#define I40E_AQC_SET_WOL_FILTER_ACTION_SET 1 + __le16 valid_flags; +#define I40E_AQC_SET_WOL_FILTER_ACTION_VALID 0x8000 +#define I40E_AQC_SET_WOL_FILTER_NO_TCO_ACTION_VALID 0x4000 + u8 reserved[2]; + __le32 address_high; + __le32 address_low; +}; + +I40E_CHECK_CMD_LENGTH(i40e_aqc_set_wol_filter); + +struct i40e_aqc_set_wol_filter_data { + u8 filter[128]; + u8 mask[16]; +}; + +I40E_CHECK_STRUCT_LEN(0x90, i40e_aqc_set_wol_filter_data); + +/* Get Wake Reason (0x0121) */ + +struct i40e_aqc_get_wake_reason_completion { + u8 reserved_1[2]; + __le16 wake_reason; +#define I40E_AQC_GET_WAKE_UP_REASON_WOL_REASON_MATCHED_INDEX_SHIFT 0 +#define I40E_AQC_GET_WAKE_UP_REASON_WOL_REASON_MATCHED_INDEX_MASK (0xFF << \ + I40E_AQC_GET_WAKE_UP_REASON_WOL_REASON_MATCHED_INDEX_SHIFT) +#define I40E_AQC_GET_WAKE_UP_REASON_WOL_REASON_RESERVED_SHIFT 8 +#define I40E_AQC_GET_WAKE_UP_REASON_WOL_REASON_RESERVED_MASK (0xFF << \ + I40E_AQC_GET_WAKE_UP_REASON_WOL_REASON_RESERVED_SHIFT) + u8 reserved_2[12]; +}; + +I40E_CHECK_CMD_LENGTH(i40e_aqc_get_wake_reason_completion); + /* Switch configuration commands (0x02xx) */ /* Used by many indirect commands that only pass an seid and a buffer in the @@ -643,6 +711,8 @@ struct i40e_aqc_set_port_parameters { #define I40E_AQ_SET_P_PARAMS_PAD_SHORT_PACKETS 2 /* must set! */ #define I40E_AQ_SET_P_PARAMS_DOUBLE_VLAN_ENA 4 __le16 bad_frame_vsi; +#define I40E_AQ_SET_P_PARAMS_BFRAME_SEID_SHIFT 0x0 +#define I40E_AQ_SET_P_PARAMS_BFRAME_SEID_MASK 0x3FF __le16 default_seid; /* reserved for command */ u8 reserved[10]; }; @@ -694,6 +764,7 @@ I40E_CHECK_STRUCT_LEN(0x10, i40e_aqc_switch_resource_alloc_element_resp); /* Set Switch Configuration (direct 0x0205) */ struct i40e_aqc_set_switch_config { __le16 flags; +/* flags used for both fields below */ #define I40E_AQ_SET_SWITCH_CFG_PROMISC 0x0001 #define I40E_AQ_SET_SWITCH_CFG_L2_FILTER 0x0002 __le16 valid_flags; @@ -862,8 +933,12 @@ struct i40e_aqc_vsi_properties_data { I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT) /* queueing option section */ u8 queueing_opt_flags; +#define I40E_AQ_VSI_QUE_OPT_MULTICAST_UDP_ENA 0x04 +#define I40E_AQ_VSI_QUE_OPT_UNICAST_UDP_ENA 0x08 #define I40E_AQ_VSI_QUE_OPT_TCP_ENA 0x10 #define I40E_AQ_VSI_QUE_OPT_FCOE_ENA 0x20 +#define I40E_AQ_VSI_QUE_OPT_RSS_LUT_PF 0x00 +#define I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI 0x40 u8 queueing_opt_reserved[3]; /* scheduler section */ u8 up_enable_bits; @@ -1597,15 +1672,12 @@ struct i40e_aq_get_set_hmc_resource_profile { I40E_CHECK_CMD_LENGTH(i40e_aq_get_set_hmc_resource_profile); enum i40e_aq_hmc_profile { - /* I40E_HMC_PROFILE_NO_CHANGE = 0, reserved */ + /* I40E_HMC_PROFILE_NO_CHANGE = 0, reserved */ I40E_HMC_PROFILE_DEFAULT = 1, I40E_HMC_PROFILE_FAVOR_VF = 2, I40E_HMC_PROFILE_EQUAL = 3, }; -#define I40E_AQ_GET_HMC_RESOURCE_PROFILE_PM_MASK 0xF -#define I40E_AQ_GET_HMC_RESOURCE_PROFILE_COUNT_MASK 0x3F - /* Get PHY Abilities (indirect 0x0600) uses the generic indirect struct */ /* set in param0 for get phy abilities to report qualified modules */ @@ -1641,6 +1713,10 @@ enum i40e_aq_phy_type { I40E_PHY_TYPE_1000BASE_LX = 0x1C, I40E_PHY_TYPE_1000BASE_T_OPTICAL = 0x1D, I40E_PHY_TYPE_20GBASE_KR2 = 0x1E, + I40E_PHY_TYPE_25GBASE_KR = 0x1F, + I40E_PHY_TYPE_25GBASE_CR = 0x20, + I40E_PHY_TYPE_25GBASE_SR = 0x21, + I40E_PHY_TYPE_25GBASE_LR = 0x22, I40E_PHY_TYPE_MAX }; @@ -1649,6 +1725,7 @@ enum i40e_aq_phy_type { #define I40E_LINK_SPEED_10GB_SHIFT 0x3 #define I40E_LINK_SPEED_40GB_SHIFT 0x4 #define I40E_LINK_SPEED_20GB_SHIFT 0x5 +#define I40E_LINK_SPEED_25GB_SHIFT 0x6 enum i40e_aq_link_speed { I40E_LINK_SPEED_UNKNOWN = 0, @@ -1656,7 +1733,8 @@ enum i40e_aq_link_speed { I40E_LINK_SPEED_1GB = (1 << I40E_LINK_SPEED_1000MB_SHIFT), I40E_LINK_SPEED_10GB = (1 << I40E_LINK_SPEED_10GB_SHIFT), I40E_LINK_SPEED_40GB = (1 << I40E_LINK_SPEED_40GB_SHIFT), - I40E_LINK_SPEED_20GB = (1 << I40E_LINK_SPEED_20GB_SHIFT) + I40E_LINK_SPEED_20GB = (1 << I40E_LINK_SPEED_20GB_SHIFT), + I40E_LINK_SPEED_25GB = (1 << I40E_LINK_SPEED_25GB_SHIFT), }; struct i40e_aqc_module_desc { @@ -1689,7 +1767,13 @@ struct i40e_aq_get_phy_abilities_resp { __le32 eeer_val; u8 d3_lpan; #define I40E_AQ_SET_PHY_D3_LPAN_ENA 0x01 - u8 reserved[3]; + u8 phy_type_ext; +#define I40E_AQ_PHY_TYPE_EXT_25G_KR 0X01 +#define I40E_AQ_PHY_TYPE_EXT_25G_CR 0X02 +#define I40E_AQ_PHY_TYPE_EXT_25G_SR 0x04 +#define I40E_AQ_PHY_TYPE_EXT_25G_LR 0x08 + u8 mod_type_ext; + u8 ext_comp_code; u8 phy_id[4]; u8 module_type[3]; u8 qualified_module_count; @@ -1711,7 +1795,12 @@ struct i40e_aq_set_phy_config { /* same bits as above in all */ __le16 eee_capability; __le32 eeer; u8 low_power_ctrl; - u8 reserved[3]; + u8 phy_type_ext; +#define I40E_AQ_PHY_TYPE_EXT_25G_KR 0X01 +#define I40E_AQ_PHY_TYPE_EXT_25G_CR 0X02 +#define I40E_AQ_PHY_TYPE_EXT_25G_SR 0x04 +#define I40E_AQ_PHY_TYPE_EXT_25G_LR 0x08 + u8 reserved[2]; }; I40E_CHECK_CMD_LENGTH(i40e_aq_set_phy_config); @@ -1791,16 +1880,24 @@ struct i40e_aqc_get_link_status { #define I40E_AQ_LINK_TX_DRAINED 0x01 #define I40E_AQ_LINK_TX_FLUSHED 0x03 #define I40E_AQ_LINK_FORCED_40G 0x10 +/* 25G Error Codes */ +#define I40E_AQ_25G_NO_ERR 0X00 +#define I40E_AQ_25G_NOT_PRESENT 0X01 +#define I40E_AQ_25G_NVM_CRC_ERR 0X02 +#define I40E_AQ_25G_SBUS_UCODE_ERR 0X03 +#define I40E_AQ_25G_SERDES_UCODE_ERR 0X04 +#define I40E_AQ_25G_NIMB_UCODE_ERR 0X05 u8 loopback; /* use defines from i40e_aqc_set_lb_mode */ __le16 max_frame_size; u8 config; #define I40E_AQ_CONFIG_CRC_ENA 0x04 #define I40E_AQ_CONFIG_PACING_MASK 0x78 - u8 external_power_ability; + u8 power_desc; #define I40E_AQ_LINK_POWER_CLASS_1 0x00 #define I40E_AQ_LINK_POWER_CLASS_2 0x01 #define I40E_AQ_LINK_POWER_CLASS_3 0x02 #define I40E_AQ_LINK_POWER_CLASS_4 0x03 +#define I40E_AQ_PWR_CLASS_MASK 0x03 u8 reserved[4]; }; @@ -1857,7 +1954,10 @@ struct i40e_aqc_set_phy_debug { #define I40E_AQ_PHY_DEBUG_RESET_EXTERNAL_NONE 0x00 #define I40E_AQ_PHY_DEBUG_RESET_EXTERNAL_HARD 0x01 #define I40E_AQ_PHY_DEBUG_RESET_EXTERNAL_SOFT 0x02 +/* Disable link manageability on a single port */ #define I40E_AQ_PHY_DEBUG_DISABLE_LINK_FW 0x10 +/* Disable link manageability on all ports needs both bits 4 and 5 */ +#define I40E_AQ_PHY_DEBUG_DISABLE_ALL_LINK_FW 0x20 u8 reserved[15]; }; @@ -2296,6 +2396,46 @@ struct i40e_aqc_del_udp_tunnel_completion { I40E_CHECK_CMD_LENGTH(i40e_aqc_del_udp_tunnel_completion); +struct i40e_aqc_get_set_rss_key { +#define I40E_AQC_SET_RSS_KEY_VSI_VALID (0x1 << 15) +#define I40E_AQC_SET_RSS_KEY_VSI_ID_SHIFT 0 +#define I40E_AQC_SET_RSS_KEY_VSI_ID_MASK (0x3FF << \ + I40E_AQC_SET_RSS_KEY_VSI_ID_SHIFT) + __le16 vsi_id; + u8 reserved[6]; + __le32 addr_high; + __le32 addr_low; +}; + +I40E_CHECK_CMD_LENGTH(i40e_aqc_get_set_rss_key); + +struct i40e_aqc_get_set_rss_key_data { + u8 standard_rss_key[0x28]; + u8 extended_hash_key[0xc]; +}; + +I40E_CHECK_STRUCT_LEN(0x34, i40e_aqc_get_set_rss_key_data); + +struct i40e_aqc_get_set_rss_lut { +#define I40E_AQC_SET_RSS_LUT_VSI_VALID (0x1 << 15) +#define I40E_AQC_SET_RSS_LUT_VSI_ID_SHIFT 0 +#define I40E_AQC_SET_RSS_LUT_VSI_ID_MASK (0x3FF << \ + I40E_AQC_SET_RSS_LUT_VSI_ID_SHIFT) + __le16 vsi_id; +#define I40E_AQC_SET_RSS_LUT_TABLE_TYPE_SHIFT 0 +#define I40E_AQC_SET_RSS_LUT_TABLE_TYPE_MASK (0x1 << \ + I40E_AQC_SET_RSS_LUT_TABLE_TYPE_SHIFT) + +#define I40E_AQC_SET_RSS_LUT_TABLE_TYPE_VSI 0 +#define I40E_AQC_SET_RSS_LUT_TABLE_TYPE_PF 1 + __le16 flags; + u8 reserved[4]; + __le32 addr_high; + __le32 addr_low; +}; + +I40E_CHECK_CMD_LENGTH(i40e_aqc_get_set_rss_lut); + /* tunnel key structure 0x0B10 */ struct i40e_aqc_tunnel_key_structure { diff --git a/sys/dev/ixl/i40e_common.c b/sys/dev/ixl/i40e_common.c index 5a8a8aec73ff..79229752ca29 100644 --- a/sys/dev/ixl/i40e_common.c +++ b/sys/dev/ixl/i40e_common.c @@ -64,8 +64,24 @@ enum i40e_status_code i40e_set_mac_type(struct i40e_hw *hw) case I40E_DEV_ID_10G_BASE_T4: case I40E_DEV_ID_20G_KR2: case I40E_DEV_ID_20G_KR2_A: + case I40E_DEV_ID_25G_B: + case I40E_DEV_ID_25G_SFP28: hw->mac.type = I40E_MAC_XL710; break; + case I40E_DEV_ID_X722_A0: + case I40E_DEV_ID_KX_X722: + case I40E_DEV_ID_QSFP_X722: + case I40E_DEV_ID_SFP_X722: + case I40E_DEV_ID_1G_BASE_T_X722: + case I40E_DEV_ID_10G_BASE_T_X722: + case I40E_DEV_ID_SFP_I_X722: + hw->mac.type = I40E_MAC_X722; + break; + case I40E_DEV_ID_X722_VF: + case I40E_DEV_ID_X722_VF_HV: + case I40E_DEV_ID_X722_A0_VF: + hw->mac.type = I40E_MAC_X722_VF; + break; case I40E_DEV_ID_VF: case I40E_DEV_ID_VF_HV: hw->mac.type = I40E_MAC_VF; @@ -341,14 +357,15 @@ void i40e_debug_aq(struct i40e_hw *hw, enum i40e_debug_mask mask, void *desc, /* the most we could have left is 16 bytes, pad with zeros */ if (i < len) { char d_buf[16]; - int j; + int j, i_sav; + i_sav = i; memset(d_buf, 0, sizeof(d_buf)); for (j = 0; i < len; j++, i++) d_buf[j] = buf[i]; i40e_debug(hw, mask, "\t0x%04X %02X %02X %02X %02X %02X %02X %02X %02X %02X %02X %02X %02X %02X %02X %02X %02X\n", - i, d_buf[0], d_buf[1], d_buf[2], d_buf[3], + i_sav, d_buf[0], d_buf[1], d_buf[2], d_buf[3], d_buf[4], d_buf[5], d_buf[6], d_buf[7], d_buf[8], d_buf[9], d_buf[10], d_buf[11], d_buf[12], d_buf[13], d_buf[14], d_buf[15]); @@ -400,6 +417,164 @@ enum i40e_status_code i40e_aq_queue_shutdown(struct i40e_hw *hw, return status; } +/** + * i40e_aq_get_set_rss_lut + * @hw: pointer to the hardware structure + * @vsi_id: vsi fw index + * @pf_lut: for PF table set TRUE, for VSI table set FALSE + * @lut: pointer to the lut buffer provided by the caller + * @lut_size: size of the lut buffer + * @set: set TRUE to set the table, FALSE to get the table + * + * Internal function to get or set RSS look up table + **/ +static enum i40e_status_code i40e_aq_get_set_rss_lut(struct i40e_hw *hw, + u16 vsi_id, bool pf_lut, + u8 *lut, u16 lut_size, + bool set) +{ + enum i40e_status_code status; + struct i40e_aq_desc desc; + struct i40e_aqc_get_set_rss_lut *cmd_resp = + (struct i40e_aqc_get_set_rss_lut *)&desc.params.raw; + + if (set) + i40e_fill_default_direct_cmd_desc(&desc, + i40e_aqc_opc_set_rss_lut); + else + i40e_fill_default_direct_cmd_desc(&desc, + i40e_aqc_opc_get_rss_lut); + + /* Indirect command */ + desc.flags |= CPU_TO_LE16((u16)I40E_AQ_FLAG_BUF); + desc.flags |= CPU_TO_LE16((u16)I40E_AQ_FLAG_RD); + + cmd_resp->vsi_id = + CPU_TO_LE16((u16)((vsi_id << + I40E_AQC_SET_RSS_LUT_VSI_ID_SHIFT) & + I40E_AQC_SET_RSS_LUT_VSI_ID_MASK)); + cmd_resp->vsi_id |= CPU_TO_LE16((u16)I40E_AQC_SET_RSS_LUT_VSI_VALID); + + if (pf_lut) + cmd_resp->flags |= CPU_TO_LE16((u16) + ((I40E_AQC_SET_RSS_LUT_TABLE_TYPE_PF << + I40E_AQC_SET_RSS_LUT_TABLE_TYPE_SHIFT) & + I40E_AQC_SET_RSS_LUT_TABLE_TYPE_MASK)); + else + cmd_resp->flags |= CPU_TO_LE16((u16) + ((I40E_AQC_SET_RSS_LUT_TABLE_TYPE_VSI << + I40E_AQC_SET_RSS_LUT_TABLE_TYPE_SHIFT) & + I40E_AQC_SET_RSS_LUT_TABLE_TYPE_MASK)); + + status = i40e_asq_send_command(hw, &desc, lut, lut_size, NULL); + + return status; +} + +/** + * i40e_aq_get_rss_lut + * @hw: pointer to the hardware structure + * @vsi_id: vsi fw index + * @pf_lut: for PF table set TRUE, for VSI table set FALSE + * @lut: pointer to the lut buffer provided by the caller + * @lut_size: size of the lut buffer + * + * get the RSS lookup table, PF or VSI type + **/ +enum i40e_status_code i40e_aq_get_rss_lut(struct i40e_hw *hw, u16 vsi_id, + bool pf_lut, u8 *lut, u16 lut_size) +{ + return i40e_aq_get_set_rss_lut(hw, vsi_id, pf_lut, lut, lut_size, + FALSE); +} + +/** + * i40e_aq_set_rss_lut + * @hw: pointer to the hardware structure + * @vsi_id: vsi fw index + * @pf_lut: for PF table set TRUE, for VSI table set FALSE + * @lut: pointer to the lut buffer provided by the caller + * @lut_size: size of the lut buffer + * + * set the RSS lookup table, PF or VSI type + **/ +enum i40e_status_code i40e_aq_set_rss_lut(struct i40e_hw *hw, u16 vsi_id, + bool pf_lut, u8 *lut, u16 lut_size) +{ + return i40e_aq_get_set_rss_lut(hw, vsi_id, pf_lut, lut, lut_size, TRUE); +} + +/** + * i40e_aq_get_set_rss_key + * @hw: pointer to the hw struct + * @vsi_id: vsi fw index + * @key: pointer to key info struct + * @set: set TRUE to set the key, FALSE to get the key + * + * get the RSS key per VSI + **/ +static enum i40e_status_code i40e_aq_get_set_rss_key(struct i40e_hw *hw, + u16 vsi_id, + struct i40e_aqc_get_set_rss_key_data *key, + bool set) +{ + enum i40e_status_code status; + struct i40e_aq_desc desc; + struct i40e_aqc_get_set_rss_key *cmd_resp = + (struct i40e_aqc_get_set_rss_key *)&desc.params.raw; + u16 key_size = sizeof(struct i40e_aqc_get_set_rss_key_data); + + if (set) + i40e_fill_default_direct_cmd_desc(&desc, + i40e_aqc_opc_set_rss_key); + else + i40e_fill_default_direct_cmd_desc(&desc, + i40e_aqc_opc_get_rss_key); + + /* Indirect command */ + desc.flags |= CPU_TO_LE16((u16)I40E_AQ_FLAG_BUF); + desc.flags |= CPU_TO_LE16((u16)I40E_AQ_FLAG_RD); + + cmd_resp->vsi_id = + CPU_TO_LE16((u16)((vsi_id << + I40E_AQC_SET_RSS_KEY_VSI_ID_SHIFT) & + I40E_AQC_SET_RSS_KEY_VSI_ID_MASK)); + cmd_resp->vsi_id |= CPU_TO_LE16((u16)I40E_AQC_SET_RSS_KEY_VSI_VALID); + + status = i40e_asq_send_command(hw, &desc, key, key_size, NULL); + + return status; +} + +/** + * i40e_aq_get_rss_key + * @hw: pointer to the hw struct + * @vsi_id: vsi fw index + * @key: pointer to key info struct + * + **/ +enum i40e_status_code i40e_aq_get_rss_key(struct i40e_hw *hw, + u16 vsi_id, + struct i40e_aqc_get_set_rss_key_data *key) +{ + return i40e_aq_get_set_rss_key(hw, vsi_id, key, FALSE); +} + +/** + * i40e_aq_set_rss_key + * @hw: pointer to the hw struct + * @vsi_id: vsi fw index + * @key: pointer to key info struct + * + * set the RSS key per VSI + **/ +enum i40e_status_code i40e_aq_set_rss_key(struct i40e_hw *hw, + u16 vsi_id, + struct i40e_aqc_get_set_rss_key_data *key) +{ + return i40e_aq_get_set_rss_key(hw, vsi_id, key, TRUE); +} + /* The i40e_ptype_lookup table is used to convert from the 8-bit ptype in the * hardware to a bit-field that can be used by SW to more easily determine the * packet type. @@ -563,7 +738,7 @@ struct i40e_rx_ptype_decoded i40e_ptype_lookup[] = { /* Non Tunneled IPv6 */ I40E_PTT(88, IP, IPV6, FRG, NONE, NONE, NOF, NONE, PAY3), I40E_PTT(89, IP, IPV6, NOF, NONE, NONE, NOF, NONE, PAY3), - I40E_PTT(90, IP, IPV6, NOF, NONE, NONE, NOF, UDP, PAY3), + I40E_PTT(90, IP, IPV6, NOF, NONE, NONE, NOF, UDP, PAY4), I40E_PTT_UNUSED_ENTRY(91), I40E_PTT(92, IP, IPV6, NOF, NONE, NONE, NOF, TCP, PAY4), I40E_PTT(93, IP, IPV6, NOF, NONE, NONE, NOF, SCTP, PAY4), @@ -813,6 +988,7 @@ enum i40e_status_code i40e_init_shared_code(struct i40e_hw *hw) switch (hw->mac.type) { case I40E_MAC_XL710: + case I40E_MAC_X722: break; default: return I40E_ERR_DEVICE_NOT_SUPPORTED; @@ -832,6 +1008,9 @@ enum i40e_status_code i40e_init_shared_code(struct i40e_hw *hw) else hw->pf_id = (u8)(func_rid & 0x7); + if (hw->mac.type == I40E_MAC_X722) + hw->flags |= I40E_HW_FLAG_AQ_SRCTL_ACCESS_ENABLE; + status = i40e_init_nvm(hw); return status; } @@ -1104,8 +1283,7 @@ enum i40e_status_code i40e_pf_reset(struct i40e_hw *hw) I40E_GLGEN_RSTCTL_GRSTDEL_MASK) >> I40E_GLGEN_RSTCTL_GRSTDEL_SHIFT; - /* It can take upto 15 secs for GRST steady state */ - grst_del = grst_del * 20; /* bump it to 16 secs max to be safe */ + grst_del = grst_del * 20; for (cnt = 0; cnt < grst_del; cnt++) { reg = rd32(hw, I40E_GLGEN_RSTAT); @@ -1452,8 +1630,10 @@ enum i40e_status_code i40e_aq_get_phy_capabilities(struct i40e_hw *hw, if (hw->aq.asq_last_status == I40E_AQ_RC_EIO) status = I40E_ERR_UNKNOWN_PHY; - if (report_init) + if (report_init) { hw->phy.phy_types = LE32_TO_CPU(abilities->phy_type); + hw->phy.phy_types |= ((u64)abilities->phy_type_ext << 32); + } return status; } @@ -1997,15 +2177,45 @@ enum i40e_status_code i40e_aq_set_default_vsi(struct i40e_hw *hw, } /** + * i40e_aq_clear_default_vsi + * @hw: pointer to the hw struct + * @seid: vsi number + * @cmd_details: pointer to command details structure or NULL + **/ +enum i40e_status_code i40e_aq_clear_default_vsi(struct i40e_hw *hw, + u16 seid, + struct i40e_asq_cmd_details *cmd_details) +{ + struct i40e_aq_desc desc; + struct i40e_aqc_set_vsi_promiscuous_modes *cmd = + (struct i40e_aqc_set_vsi_promiscuous_modes *) + &desc.params.raw; + enum i40e_status_code status; + + i40e_fill_default_direct_cmd_desc(&desc, + i40e_aqc_opc_set_vsi_promiscuous_modes); + + cmd->promiscuous_flags = CPU_TO_LE16(0); + cmd->valid_flags = CPU_TO_LE16(I40E_AQC_SET_VSI_DEFAULT); + cmd->seid = CPU_TO_LE16(seid); + + status = i40e_asq_send_command(hw, &desc, NULL, 0, cmd_details); + + return status; +} + +/** * i40e_aq_set_vsi_unicast_promiscuous * @hw: pointer to the hw struct * @seid: vsi number * @set: set unicast promiscuous enable/disable * @cmd_details: pointer to command details structure or NULL + * @rx_only_promisc: flag to decide if egress traffic gets mirrored in promisc **/ enum i40e_status_code i40e_aq_set_vsi_unicast_promiscuous(struct i40e_hw *hw, u16 seid, bool set, - struct i40e_asq_cmd_details *cmd_details) + struct i40e_asq_cmd_details *cmd_details, + bool rx_only_promisc) { struct i40e_aq_desc desc; struct i40e_aqc_set_vsi_promiscuous_modes *cmd = @@ -2018,8 +2228,9 @@ enum i40e_status_code i40e_aq_set_vsi_unicast_promiscuous(struct i40e_hw *hw, if (set) { flags |= I40E_AQC_SET_VSI_PROMISC_UNICAST; - if (((hw->aq.api_maj_ver == 1) && (hw->aq.api_min_ver >= 5)) || - (hw->aq.api_maj_ver > 1)) + if (rx_only_promisc && + (((hw->aq.api_maj_ver == 1) && (hw->aq.api_min_ver >= 5)) || + (hw->aq.api_maj_ver > 1))) flags |= I40E_AQC_SET_VSI_PROMISC_TX; } @@ -2192,7 +2403,7 @@ enum i40e_status_code i40e_aq_set_vsi_vlan_promisc(struct i40e_hw *hw, i40e_aqc_opc_set_vsi_promiscuous_modes); if (enable) flags |= I40E_AQC_SET_VSI_PROMISC_VLAN; - + cmd->promiscuous_flags = CPU_TO_LE16(flags); cmd->valid_flags = CPU_TO_LE16(I40E_AQC_SET_VSI_PROMISC_VLAN); cmd->seid = CPU_TO_LE16(seid); @@ -2826,10 +3037,7 @@ enum i40e_status_code i40e_aq_delete_mirrorrule(struct i40e_hw *hw, u16 sw_seid, u16 *rules_used, u16 *rules_free) { /* Rule ID has to be valid except rule_type: INGRESS VLAN mirroring */ - if (rule_type != I40E_AQC_MIRROR_RULE_TYPE_VLAN) { - if (!rule_id) - return I40E_ERR_PARAM; - } else { + if (rule_type == I40E_AQC_MIRROR_RULE_TYPE_VLAN) { /* count and mr_list shall be valid for rule_type INGRESS VLAN * mirroring. For other rule_type, count and rule_type should * not matter. @@ -3026,67 +3234,6 @@ enum i40e_status_code i40e_aq_debug_write_register(struct i40e_hw *hw, } /** - * i40e_aq_get_hmc_resource_profile - * @hw: pointer to the hw struct - * @profile: type of profile the HMC is to be set as - * @pe_vf_enabled_count: the number of PE enabled VFs the system has - * @cmd_details: pointer to command details structure or NULL - * - * query the HMC profile of the device. - **/ -enum i40e_status_code i40e_aq_get_hmc_resource_profile(struct i40e_hw *hw, - enum i40e_aq_hmc_profile *profile, - u8 *pe_vf_enabled_count, - struct i40e_asq_cmd_details *cmd_details) -{ - struct i40e_aq_desc desc; - struct i40e_aq_get_set_hmc_resource_profile *resp = - (struct i40e_aq_get_set_hmc_resource_profile *)&desc.params.raw; - enum i40e_status_code status; - - i40e_fill_default_direct_cmd_desc(&desc, - i40e_aqc_opc_query_hmc_resource_profile); - status = i40e_asq_send_command(hw, &desc, NULL, 0, cmd_details); - - *profile = (enum i40e_aq_hmc_profile)(resp->pm_profile & - I40E_AQ_GET_HMC_RESOURCE_PROFILE_PM_MASK); - *pe_vf_enabled_count = resp->pe_vf_enabled & - I40E_AQ_GET_HMC_RESOURCE_PROFILE_COUNT_MASK; - - return status; -} - -/** - * i40e_aq_set_hmc_resource_profile - * @hw: pointer to the hw struct - * @profile: type of profile the HMC is to be set as - * @pe_vf_enabled_count: the number of PE enabled VFs the system has - * @cmd_details: pointer to command details structure or NULL - * - * set the HMC profile of the device. - **/ -enum i40e_status_code i40e_aq_set_hmc_resource_profile(struct i40e_hw *hw, - enum i40e_aq_hmc_profile profile, - u8 pe_vf_enabled_count, - struct i40e_asq_cmd_details *cmd_details) -{ - struct i40e_aq_desc desc; - struct i40e_aq_get_set_hmc_resource_profile *cmd = - (struct i40e_aq_get_set_hmc_resource_profile *)&desc.params.raw; - enum i40e_status_code status; - - i40e_fill_default_direct_cmd_desc(&desc, - i40e_aqc_opc_set_hmc_resource_profile); - - cmd->pm_profile = (u8)profile; - cmd->pe_vf_enabled = pe_vf_enabled_count; - - status = i40e_asq_send_command(hw, &desc, NULL, 0, cmd_details); - - return status; -} - -/** * i40e_aq_request_resource * @hw: pointer to the hw struct * @resource: resource id @@ -3603,6 +3750,26 @@ static void i40e_parse_discover_capabilities(struct i40e_hw *hw, void *buff, "HW Capability: wr_csr_prot = 0x%llX\n\n", (p->wr_csr_prot & 0xffff)); break; + case I40E_AQ_CAP_ID_NVM_MGMT: + if (number & I40E_NVM_MGMT_SEC_REV_DISABLED) + p->sec_rev_disabled = TRUE; + if (number & I40E_NVM_MGMT_UPDATE_DISABLED) + p->update_disabled = TRUE; + break; + case I40E_AQ_CAP_ID_WOL_AND_PROXY: + hw->num_wol_proxy_filters = (u16)number; + hw->wol_proxy_vsi_seid = (u16)logical_id; + p->apm_wol_support = phys_id & I40E_WOL_SUPPORT_MASK; + if (phys_id & I40E_ACPI_PROGRAMMING_METHOD_MASK) + p->acpi_prog_method = I40E_ACPI_PROGRAMMING_METHOD_AQC_FPK; + else + p->acpi_prog_method = I40E_ACPI_PROGRAMMING_METHOD_HW_FVL; + p->proxy_support = (phys_id & I40E_PROXY_SUPPORT_MASK) ? 1 : 0; + p->proxy_support = p->proxy_support; + i40e_debug(hw, I40E_DEBUG_INIT, + "HW Capability: WOL proxy filters = %d\n", + hw->num_wol_proxy_filters); + break; default: break; } @@ -5211,6 +5378,35 @@ void i40e_add_filter_to_drop_tx_flow_control_frames(struct i40e_hw *hw, } /** + * i40e_fix_up_geneve_vni - adjust Geneve VNI for HW issue + * @filters: list of cloud filters + * @filter_count: length of list + * + * There's an issue in the device where the Geneve VNI layout needs + * to be shifted 1 byte over from the VxLAN VNI + **/ +static void i40e_fix_up_geneve_vni( + struct i40e_aqc_add_remove_cloud_filters_element_data *filters, + u8 filter_count) +{ + struct i40e_aqc_add_remove_cloud_filters_element_data *f = filters; + int i; + + for (i = 0; i < filter_count; i++) { + u16 tnl_type; + u32 ti; + + tnl_type = (LE16_TO_CPU(f[i].flags) & + I40E_AQC_ADD_CLOUD_TNL_TYPE_MASK) >> + I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT; + if (tnl_type == I40E_AQC_ADD_CLOUD_TNL_TYPE_GENEVE) { + ti = LE32_TO_CPU(f[i].tenant_id); + f[i].tenant_id = CPU_TO_LE32(ti << 8); + } + } +} + +/** * i40e_aq_add_cloud_filters * @hw: pointer to the hardware structure * @seid: VSI seid to add cloud filters from @@ -5230,8 +5426,8 @@ enum i40e_status_code i40e_aq_add_cloud_filters(struct i40e_hw *hw, struct i40e_aq_desc desc; struct i40e_aqc_add_remove_cloud_filters *cmd = (struct i40e_aqc_add_remove_cloud_filters *)&desc.params.raw; - u16 buff_len; enum i40e_status_code status; + u16 buff_len; i40e_fill_default_direct_cmd_desc(&desc, i40e_aqc_opc_add_cloud_filters); @@ -5242,6 +5438,8 @@ enum i40e_status_code i40e_aq_add_cloud_filters(struct i40e_hw *hw, cmd->num_filters = filter_count; cmd->seid = CPU_TO_LE16(seid); + i40e_fix_up_geneve_vni(filters, filter_count); + status = i40e_asq_send_command(hw, &desc, filters, buff_len, NULL); return status; @@ -5279,6 +5477,8 @@ enum i40e_status_code i40e_aq_remove_cloud_filters(struct i40e_hw *hw, cmd->num_filters = filter_count; cmd->seid = CPU_TO_LE16(seid); + i40e_fix_up_geneve_vni(filters, filter_count); + status = i40e_asq_send_command(hw, &desc, filters, buff_len, NULL); return status; @@ -6263,3 +6463,158 @@ enum i40e_status_code i40e_vf_reset(struct i40e_hw *hw) return i40e_aq_send_msg_to_pf(hw, I40E_VIRTCHNL_OP_RESET_VF, I40E_SUCCESS, NULL, 0, NULL); } + +/** + * i40e_aq_set_arp_proxy_config + * @hw: pointer to the HW structure + * @proxy_config - pointer to proxy config command table struct + * @cmd_details: pointer to command details + * + * Set ARP offload parameters from pre-populated + * i40e_aqc_arp_proxy_data struct + **/ +enum i40e_status_code i40e_aq_set_arp_proxy_config(struct i40e_hw *hw, + struct i40e_aqc_arp_proxy_data *proxy_config, + struct i40e_asq_cmd_details *cmd_details) +{ + struct i40e_aq_desc desc; + enum i40e_status_code status; + + if (!proxy_config) + return I40E_ERR_PARAM; + + i40e_fill_default_direct_cmd_desc(&desc, i40e_aqc_opc_set_proxy_config); + + desc.params.external.addr_high = + CPU_TO_LE32(I40E_HI_DWORD((u64)proxy_config)); + desc.params.external.addr_low = + CPU_TO_LE32(I40E_LO_DWORD((u64)proxy_config)); + + status = i40e_asq_send_command(hw, &desc, proxy_config, + sizeof(struct i40e_aqc_arp_proxy_data), + cmd_details); + + return status; +} + +/** + * i40e_aq_opc_set_ns_proxy_table_entry + * @hw: pointer to the HW structure + * @ns_proxy_table_entry: pointer to NS table entry command struct + * @cmd_details: pointer to command details + * + * Set IPv6 Neighbor Solicitation (NS) protocol offload parameters + * from pre-populated i40e_aqc_ns_proxy_data struct + **/ +enum i40e_status_code i40e_aq_set_ns_proxy_table_entry(struct i40e_hw *hw, + struct i40e_aqc_ns_proxy_data *ns_proxy_table_entry, + struct i40e_asq_cmd_details *cmd_details) +{ + struct i40e_aq_desc desc; + enum i40e_status_code status; + + if (!ns_proxy_table_entry) + return I40E_ERR_PARAM; + + i40e_fill_default_direct_cmd_desc(&desc, + i40e_aqc_opc_set_ns_proxy_table_entry); + + desc.params.external.addr_high = + CPU_TO_LE32(I40E_HI_DWORD((u64)ns_proxy_table_entry)); + desc.params.external.addr_low = + CPU_TO_LE32(I40E_LO_DWORD((u64)ns_proxy_table_entry)); + + status = i40e_asq_send_command(hw, &desc, ns_proxy_table_entry, + sizeof(struct i40e_aqc_ns_proxy_data), + cmd_details); + + return status; +} + +/** + * i40e_aq_set_clear_wol_filter + * @hw: pointer to the hw struct + * @filter_index: index of filter to modify (0-7) + * @filter: buffer containing filter to be set + * @set_filter: TRUE to set filter, FALSE to clear filter + * @no_wol_tco: if TRUE, pass through packets cannot cause wake-up + * if FALSE, pass through packets may cause wake-up + * @filter_valid: TRUE if filter action is valid + * @no_wol_tco_valid: TRUE if no WoL in TCO traffic action valid + * @cmd_details: pointer to command details structure or NULL + * + * Set or clear WoL filter for port attached to the PF + **/ +enum i40e_status_code i40e_aq_set_clear_wol_filter(struct i40e_hw *hw, + u8 filter_index, + struct i40e_aqc_set_wol_filter_data *filter, + bool set_filter, bool no_wol_tco, + bool filter_valid, bool no_wol_tco_valid, + struct i40e_asq_cmd_details *cmd_details) +{ + struct i40e_aq_desc desc; + struct i40e_aqc_set_wol_filter *cmd = + (struct i40e_aqc_set_wol_filter *)&desc.params.raw; + enum i40e_status_code status; + u16 cmd_flags = 0; + u16 valid_flags = 0; + u16 buff_len = 0; + + i40e_fill_default_direct_cmd_desc(&desc, i40e_aqc_opc_set_wol_filter); + + if (filter_index >= I40E_AQC_MAX_NUM_WOL_FILTERS) + return I40E_ERR_PARAM; + cmd->filter_index = CPU_TO_LE16(filter_index); + + if (set_filter) { + if (!filter) + return I40E_ERR_PARAM; + cmd_flags |= I40E_AQC_SET_WOL_FILTER; + buff_len = sizeof(*filter); + } + if (no_wol_tco) + cmd_flags |= I40E_AQC_SET_WOL_FILTER_NO_TCO_WOL; + cmd->cmd_flags = CPU_TO_LE16(cmd_flags); + + if (filter_valid) + valid_flags |= I40E_AQC_SET_WOL_FILTER_ACTION_VALID; + if (no_wol_tco_valid) + valid_flags |= I40E_AQC_SET_WOL_FILTER_NO_TCO_ACTION_VALID; + cmd->valid_flags = CPU_TO_LE16(valid_flags); + + cmd->address_high = CPU_TO_LE32(I40E_HI_DWORD((u64)filter)); + cmd->address_low = CPU_TO_LE32(I40E_LO_DWORD((u64)filter)); + + status = i40e_asq_send_command(hw, &desc, filter, + buff_len, cmd_details); + + return status; +} + +/** + * i40e_aq_get_wake_event_reason + * @hw: pointer to the hw struct + * @wake_reason: return value, index of matching filter + * @cmd_details: pointer to command details structure or NULL + * + * Get information for the reason of a Wake Up event + **/ +enum i40e_status_code i40e_aq_get_wake_event_reason(struct i40e_hw *hw, + u16 *wake_reason, + struct i40e_asq_cmd_details *cmd_details) +{ + struct i40e_aq_desc desc; + struct i40e_aqc_get_wake_reason_completion *resp = + (struct i40e_aqc_get_wake_reason_completion *)&desc.params.raw; + enum i40e_status_code status; + + i40e_fill_default_direct_cmd_desc(&desc, i40e_aqc_opc_get_wake_reason); + + status = i40e_asq_send_command(hw, &desc, NULL, 0, cmd_details); + + if (status == I40E_SUCCESS) + *wake_reason = LE16_TO_CPU(resp->wake_reason); + + return status; +} + diff --git a/sys/dev/ixl/i40e_devids.h b/sys/dev/ixl/i40e_devids.h index a898f6d53d49..5725cb96754c 100644 --- a/sys/dev/ixl/i40e_devids.h +++ b/sys/dev/ixl/i40e_devids.h @@ -50,8 +50,20 @@ #define I40E_DEV_ID_20G_KR2 0x1587 #define I40E_DEV_ID_20G_KR2_A 0x1588 #define I40E_DEV_ID_10G_BASE_T4 0x1589 +#define I40E_DEV_ID_25G_B 0x158A +#define I40E_DEV_ID_25G_SFP28 0x158B #define I40E_DEV_ID_VF 0x154C #define I40E_DEV_ID_VF_HV 0x1571 +#define I40E_DEV_ID_X722_A0 0x374C +#define I40E_DEV_ID_X722_A0_VF 0x374D +#define I40E_DEV_ID_KX_X722 0x37CE +#define I40E_DEV_ID_QSFP_X722 0x37CF +#define I40E_DEV_ID_SFP_X722 0x37D0 +#define I40E_DEV_ID_1G_BASE_T_X722 0x37D1 +#define I40E_DEV_ID_10G_BASE_T_X722 0x37D2 +#define I40E_DEV_ID_SFP_I_X722 0x37D3 +#define I40E_DEV_ID_X722_VF 0x37CD +#define I40E_DEV_ID_X722_VF_HV 0x37D9 #define i40e_is_40G_device(d) ((d) == I40E_DEV_ID_QSFP_A || \ (d) == I40E_DEV_ID_QSFP_B || \ diff --git a/sys/dev/ixl/i40e_nvm.c b/sys/dev/ixl/i40e_nvm.c index 4556d1734f81..151691ec97e2 100644 --- a/sys/dev/ixl/i40e_nvm.c +++ b/sys/dev/ixl/i40e_nvm.c @@ -220,7 +220,15 @@ enum i40e_status_code i40e_read_nvm_word(struct i40e_hw *hw, u16 offset, { enum i40e_status_code ret_code = I40E_SUCCESS; - ret_code = i40e_read_nvm_word_srctl(hw, offset, data); + if (hw->flags & I40E_HW_FLAG_AQ_SRCTL_ACCESS_ENABLE) { + ret_code = i40e_acquire_nvm(hw, I40E_RESOURCE_READ); + if (!ret_code) { + ret_code = i40e_read_nvm_word_aq(hw, offset, data); + i40e_release_nvm(hw); + } + } else { + ret_code = i40e_read_nvm_word_srctl(hw, offset, data); + } return ret_code; } @@ -238,7 +246,10 @@ enum i40e_status_code __i40e_read_nvm_word(struct i40e_hw *hw, { enum i40e_status_code ret_code = I40E_SUCCESS; - ret_code = i40e_read_nvm_word_srctl(hw, offset, data); + if (hw->flags & I40E_HW_FLAG_AQ_SRCTL_ACCESS_ENABLE) + ret_code = i40e_read_nvm_word_aq(hw, offset, data); + else + ret_code = i40e_read_nvm_word_srctl(hw, offset, data); return ret_code; } @@ -330,7 +341,10 @@ enum i40e_status_code __i40e_read_nvm_buffer(struct i40e_hw *hw, { enum i40e_status_code ret_code = I40E_SUCCESS; - ret_code = i40e_read_nvm_buffer_srctl(hw, offset, words, data); + if (hw->flags & I40E_HW_FLAG_AQ_SRCTL_ACCESS_ENABLE) + ret_code = i40e_read_nvm_buffer_aq(hw, offset, words, data); + else + ret_code = i40e_read_nvm_buffer_srctl(hw, offset, words, data); return ret_code; } @@ -350,7 +364,16 @@ enum i40e_status_code i40e_read_nvm_buffer(struct i40e_hw *hw, u16 offset, { enum i40e_status_code ret_code = I40E_SUCCESS; - ret_code = i40e_read_nvm_buffer_srctl(hw, offset, words, data); + if (hw->flags & I40E_HW_FLAG_AQ_SRCTL_ACCESS_ENABLE) { + ret_code = i40e_acquire_nvm(hw, I40E_RESOURCE_READ); + if (!ret_code) { + ret_code = i40e_read_nvm_buffer_aq(hw, offset, words, + data); + i40e_release_nvm(hw); + } + } else { + ret_code = i40e_read_nvm_buffer_srctl(hw, offset, words, data); + } return ret_code; } @@ -834,10 +857,10 @@ enum i40e_status_code i40e_nvmupd_command(struct i40e_hw *hw, /* early check for status command and debug msgs */ upd_cmd = i40e_nvmupd_validate_command(hw, cmd, perrno); - i40e_debug(hw, I40E_DEBUG_NVM, "%s state %d nvm_release_on_hold %d cmd 0x%08x config 0x%08x offset 0x%08x data_size 0x%08x\n", + i40e_debug(hw, I40E_DEBUG_NVM, "%s state %d nvm_release_on_hold %d opc 0x%04x cmd 0x%08x config 0x%08x offset 0x%08x data_size 0x%08x\n", i40e_nvm_update_state_str[upd_cmd], hw->nvmupd_state, - hw->aq.nvm_release_on_done, + hw->nvm_release_on_done, hw->nvm_wait_opcode, cmd->command, cmd->config, cmd->offset, cmd->data_size); if (upd_cmd == I40E_NVMUPD_INVALID) { @@ -851,7 +874,18 @@ enum i40e_status_code i40e_nvmupd_command(struct i40e_hw *hw, * going into the state machine */ if (upd_cmd == I40E_NVMUPD_STATUS) { + if (!cmd->data_size) { + *perrno = -EFAULT; + return I40E_ERR_BUF_TOO_SHORT; + } + bytes[0] = hw->nvmupd_state; + + if (cmd->data_size >= 4) { + bytes[1] = 0; + *((u16 *)&bytes[2]) = hw->nvm_wait_opcode; + } + return I40E_SUCCESS; } @@ -870,6 +904,14 @@ enum i40e_status_code i40e_nvmupd_command(struct i40e_hw *hw, case I40E_NVMUPD_STATE_INIT_WAIT: case I40E_NVMUPD_STATE_WRITE_WAIT: + /* if we need to stop waiting for an event, clear + * the wait info and return before doing anything else + */ + if (cmd->offset == 0xffff) { + i40e_nvmupd_check_wait_event(hw, hw->nvm_wait_opcode); + return I40E_SUCCESS; + } + status = I40E_ERR_NOT_READY; *perrno = -EBUSY; break; @@ -942,7 +984,8 @@ static enum i40e_status_code i40e_nvmupd_state_init(struct i40e_hw *hw, if (status) { i40e_release_nvm(hw); } else { - hw->aq.nvm_release_on_done = TRUE; + hw->nvm_release_on_done = TRUE; + hw->nvm_wait_opcode = i40e_aqc_opc_nvm_erase; hw->nvmupd_state = I40E_NVMUPD_STATE_INIT_WAIT; } } @@ -958,7 +1001,8 @@ static enum i40e_status_code i40e_nvmupd_state_init(struct i40e_hw *hw, if (status) { i40e_release_nvm(hw); } else { - hw->aq.nvm_release_on_done = TRUE; + hw->nvm_release_on_done = TRUE; + hw->nvm_wait_opcode = i40e_aqc_opc_nvm_update; hw->nvmupd_state = I40E_NVMUPD_STATE_INIT_WAIT; } } @@ -971,10 +1015,12 @@ static enum i40e_status_code i40e_nvmupd_state_init(struct i40e_hw *hw, hw->aq.asq_last_status); } else { status = i40e_nvmupd_nvm_write(hw, cmd, bytes, perrno); - if (status) + if (status) { i40e_release_nvm(hw); - else + } else { + hw->nvm_wait_opcode = i40e_aqc_opc_nvm_update; hw->nvmupd_state = I40E_NVMUPD_STATE_WRITE_WAIT; + } } break; @@ -992,7 +1038,8 @@ static enum i40e_status_code i40e_nvmupd_state_init(struct i40e_hw *hw, -EIO; i40e_release_nvm(hw); } else { - hw->aq.nvm_release_on_done = TRUE; + hw->nvm_release_on_done = TRUE; + hw->nvm_wait_opcode = i40e_aqc_opc_nvm_update; hw->nvmupd_state = I40E_NVMUPD_STATE_INIT_WAIT; } } @@ -1087,8 +1134,10 @@ retry: switch (upd_cmd) { case I40E_NVMUPD_WRITE_CON: status = i40e_nvmupd_nvm_write(hw, cmd, bytes, perrno); - if (!status) + if (!status) { + hw->nvm_wait_opcode = i40e_aqc_opc_nvm_update; hw->nvmupd_state = I40E_NVMUPD_STATE_WRITE_WAIT; + } break; case I40E_NVMUPD_WRITE_LCB: @@ -1100,7 +1149,8 @@ retry: -EIO; hw->nvmupd_state = I40E_NVMUPD_STATE_INIT; } else { - hw->aq.nvm_release_on_done = TRUE; + hw->nvm_release_on_done = TRUE; + hw->nvm_wait_opcode = i40e_aqc_opc_nvm_update; hw->nvmupd_state = I40E_NVMUPD_STATE_INIT_WAIT; } break; @@ -1115,6 +1165,7 @@ retry: -EIO; hw->nvmupd_state = I40E_NVMUPD_STATE_INIT; } else { + hw->nvm_wait_opcode = i40e_aqc_opc_nvm_update; hw->nvmupd_state = I40E_NVMUPD_STATE_WRITE_WAIT; } break; @@ -1129,7 +1180,8 @@ retry: -EIO; hw->nvmupd_state = I40E_NVMUPD_STATE_INIT; } else { - hw->aq.nvm_release_on_done = TRUE; + hw->nvm_release_on_done = TRUE; + hw->nvm_wait_opcode = i40e_aqc_opc_nvm_update; hw->nvmupd_state = I40E_NVMUPD_STATE_INIT_WAIT; } break; @@ -1179,6 +1231,38 @@ retry: } /** + * i40e_nvmupd_check_wait_event - handle NVM update operation events + * @hw: pointer to the hardware structure + * @opcode: the event that just happened + **/ +void i40e_nvmupd_check_wait_event(struct i40e_hw *hw, u16 opcode) +{ + if (opcode == hw->nvm_wait_opcode) { + + i40e_debug(hw, I40E_DEBUG_NVM, + "NVMUPD: clearing wait on opcode 0x%04x\n", opcode); + if (hw->nvm_release_on_done) { + i40e_release_nvm(hw); + hw->nvm_release_on_done = FALSE; + } + hw->nvm_wait_opcode = 0; + + switch (hw->nvmupd_state) { + case I40E_NVMUPD_STATE_INIT_WAIT: + hw->nvmupd_state = I40E_NVMUPD_STATE_INIT; + break; + + case I40E_NVMUPD_STATE_WRITE_WAIT: + hw->nvmupd_state = I40E_NVMUPD_STATE_WRITING; + break; + + default: + break; + } + } +} + +/** * i40e_nvmupd_validate_command - Validate given command * @hw: pointer to hardware structure * @cmd: pointer to nvm update command buffer @@ -1340,6 +1424,12 @@ static enum i40e_status_code i40e_nvmupd_exec_aq(struct i40e_hw *hw, *perrno = i40e_aq_rc_to_posix(status, hw->aq.asq_last_status); } + /* should we wait for a followup event? */ + if (cmd->offset) { + hw->nvm_wait_opcode = cmd->offset; + hw->nvmupd_state = I40E_NVMUPD_STATE_INIT_WAIT; + } + return status; } diff --git a/sys/dev/ixl/i40e_osdep.c b/sys/dev/ixl/i40e_osdep.c index 9b77627942e7..2a771515a9ac 100644 --- a/sys/dev/ixl/i40e_osdep.c +++ b/sys/dev/ixl/i40e_osdep.c @@ -32,7 +32,7 @@ ******************************************************************************/ /*$FreeBSD$*/ -#include <machine/stdarg.h> +#include <sys/limits.h> #include "ixl.h" @@ -137,7 +137,7 @@ void i40e_init_spinlock(struct i40e_spinlock *lock) { mtx_init(&lock->mutex, "mutex", - MTX_NETWORK_LOCK, MTX_DEF | MTX_DUPOK); + "ixl spinlock", MTX_DEF | MTX_DUPOK); } void @@ -159,11 +159,34 @@ i40e_destroy_spinlock(struct i40e_spinlock *lock) mtx_destroy(&lock->mutex); } +void +i40e_msec_pause(int msecs) +{ + int ticks_to_pause = (msecs * hz) / 1000; + int start_ticks = ticks; + + if (cold || SCHEDULER_STOPPED()) { + i40e_msec_delay(msecs); + return; + } + + while (1) { + kern_yield(PRI_USER); + int yielded_ticks = ticks - start_ticks; + if (yielded_ticks > ticks_to_pause) + break; + else if (yielded_ticks < 0 + && (yielded_ticks + INT_MAX + 1 > ticks_to_pause)) { + break; + } + } +} + /* * Helper function for debug statement printing */ void -i40e_debug_d(struct i40e_hw *hw, enum i40e_debug_mask mask, char *fmt, ...) +i40e_debug_shared(struct i40e_hw *hw, enum i40e_debug_mask mask, char *fmt, ...) { va_list args; diff --git a/sys/dev/ixl/i40e_osdep.h b/sys/dev/ixl/i40e_osdep.h index f5ca7f461d30..7f9873d94aba 100644 --- a/sys/dev/ixl/i40e_osdep.h +++ b/sys/dev/ixl/i40e_osdep.h @@ -55,7 +55,7 @@ #include <dev/pci/pcireg.h> #define i40e_usec_delay(x) DELAY(x) -#define i40e_msec_delay(x) DELAY(1000*(x)) +#define i40e_msec_delay(x) DELAY(1000 * (x)) #define DBG 0 #define MSGOUT(S, A, B) printf(S "\n", A, B) @@ -75,12 +75,13 @@ #define DEBUGOUT7(S,A,B,C,D,E,F,G) #endif -#define UNREFERENCED_XPARAMETER +/* Remove unused shared code macros */ #define UNREFERENCED_PARAMETER(_p) #define UNREFERENCED_1PARAMETER(_p) #define UNREFERENCED_2PARAMETER(_p, _q) #define UNREFERENCED_3PARAMETER(_p, _q, _r) #define UNREFERENCED_4PARAMETER(_p, _q, _r, _s) +#define UNREFERENCED_5PARAMETER(_p, _q, _r, _s, _t) #define STATIC static #define INLINE inline @@ -110,9 +111,6 @@ #define FIELD_SIZEOF(x, y) (sizeof(((x*)0)->y)) -#define BIT(a) (1UL << (a)) -#define BIT_ULL(a) (1ULL << (a)) - typedef uint8_t u8; typedef int8_t s8; typedef uint16_t u16; @@ -153,7 +151,7 @@ struct i40e_osdep { bus_space_handle_t mem_bus_space_handle; bus_size_t mem_bus_space_size; uint32_t flush_reg; - struct device *dev; + device_t dev; }; struct i40e_dma_mem { @@ -180,10 +178,13 @@ void i40e_write_pci_cfg(struct i40e_hw *, u32, u16); ** i40e_debug - OS dependent version of shared code debug printing */ enum i40e_debug_mask; -#define i40e_debug(h, m, s, ...) i40e_debug_d(h, m, s, ##__VA_ARGS__) -extern void i40e_debug_d(struct i40e_hw *hw, enum i40e_debug_mask mask, +#define i40e_debug(h, m, s, ...) i40e_debug_shared(h, m, s, ##__VA_ARGS__) +extern void i40e_debug_shared(struct i40e_hw *hw, enum i40e_debug_mask mask, char *fmt_str, ...); +/* Non-busy-wait that uses kern_yield() */ +void i40e_msec_pause(int); + /* ** This hardware supports either 16 or 32 byte rx descriptors; ** the driver only uses the 32 byte kind. diff --git a/sys/dev/ixl/i40e_prototype.h b/sys/dev/ixl/i40e_prototype.h index c53945fa30c2..01d11d6335b2 100644 --- a/sys/dev/ixl/i40e_prototype.h +++ b/sys/dev/ixl/i40e_prototype.h @@ -77,6 +77,17 @@ void i40e_debug_aq(struct i40e_hw *hw, enum i40e_debug_mask mask, void i40e_idle_aq(struct i40e_hw *hw); bool i40e_check_asq_alive(struct i40e_hw *hw); enum i40e_status_code i40e_aq_queue_shutdown(struct i40e_hw *hw, bool unloading); + +enum i40e_status_code i40e_aq_get_rss_lut(struct i40e_hw *hw, u16 seid, + bool pf_lut, u8 *lut, u16 lut_size); +enum i40e_status_code i40e_aq_set_rss_lut(struct i40e_hw *hw, u16 seid, + bool pf_lut, u8 *lut, u16 lut_size); +enum i40e_status_code i40e_aq_get_rss_key(struct i40e_hw *hw, + u16 seid, + struct i40e_aqc_get_set_rss_key_data *key); +enum i40e_status_code i40e_aq_set_rss_key(struct i40e_hw *hw, + u16 seid, + struct i40e_aqc_get_set_rss_key_data *key); const char *i40e_aq_str(struct i40e_hw *hw, enum i40e_admin_queue_err aq_err); const char *i40e_stat_str(struct i40e_hw *hw, enum i40e_status_code stat_err); @@ -107,6 +118,8 @@ enum i40e_status_code i40e_aq_set_phy_debug(struct i40e_hw *hw, u8 cmd_flags, struct i40e_asq_cmd_details *cmd_details); enum i40e_status_code i40e_aq_set_default_vsi(struct i40e_hw *hw, u16 vsi_id, struct i40e_asq_cmd_details *cmd_details); +enum i40e_status_code i40e_aq_clear_default_vsi(struct i40e_hw *hw, u16 vsi_id, + struct i40e_asq_cmd_details *cmd_details); enum i40e_status_code i40e_aq_get_phy_capabilities(struct i40e_hw *hw, bool qualified_modules, bool report_init, struct i40e_aq_get_phy_abilities_resp *abilities, @@ -149,7 +162,8 @@ enum i40e_status_code i40e_aq_set_vsi_broadcast(struct i40e_hw *hw, u16 vsi_id, bool set_filter, struct i40e_asq_cmd_details *cmd_details); enum i40e_status_code i40e_aq_set_vsi_unicast_promiscuous(struct i40e_hw *hw, - u16 vsi_id, bool set, struct i40e_asq_cmd_details *cmd_details); + u16 vsi_id, bool set, struct i40e_asq_cmd_details *cmd_details, + bool rx_only_promisc); enum i40e_status_code i40e_aq_set_vsi_multicast_promiscuous(struct i40e_hw *hw, u16 vsi_id, bool set, struct i40e_asq_cmd_details *cmd_details); enum i40e_status_code i40e_aq_set_vsi_mc_promisc_on_vlan(struct i40e_hw *hw, @@ -329,10 +343,6 @@ enum i40e_status_code i40e_aq_config_vsi_bw_limit(struct i40e_hw *hw, enum i40e_status_code i40e_aq_dcb_ignore_pfc(struct i40e_hw *hw, u8 tcmap, bool request, u8 *tcmap_ret, struct i40e_asq_cmd_details *cmd_details); -enum i40e_status_code i40e_aq_get_hmc_resource_profile(struct i40e_hw *hw, - enum i40e_aq_hmc_profile *profile, - u8 *pe_vf_enabled_count, - struct i40e_asq_cmd_details *cmd_details); enum i40e_status_code i40e_aq_config_switch_comp_ets_bw_limit( struct i40e_hw *hw, u16 seid, struct i40e_aqc_configure_switching_comp_ets_bw_limit_data *bw_data, @@ -343,10 +353,6 @@ enum i40e_status_code i40e_aq_config_vsi_ets_sla_bw_limit(struct i40e_hw *hw, struct i40e_asq_cmd_details *cmd_details); enum i40e_status_code i40e_aq_dcb_updated(struct i40e_hw *hw, struct i40e_asq_cmd_details *cmd_details); -enum i40e_status_code i40e_aq_set_hmc_resource_profile(struct i40e_hw *hw, - enum i40e_aq_hmc_profile profile, - u8 pe_vf_enabled_count, - struct i40e_asq_cmd_details *cmd_details); enum i40e_status_code i40e_aq_config_switch_comp_bw_limit(struct i40e_hw *hw, u16 seid, u16 credit, u8 max_bw, struct i40e_asq_cmd_details *cmd_details); @@ -386,7 +392,6 @@ enum i40e_status_code i40e_aq_remove_cloud_filters(struct i40e_hw *hw, u16 vsi, struct i40e_aqc_add_remove_cloud_filters_element_data *filters, u8 filter_count); - enum i40e_status_code i40e_aq_alternate_read(struct i40e_hw *hw, u32 reg_addr0, u32 *reg_val0, u32 reg_addr1, u32 *reg_val1); @@ -448,6 +453,7 @@ enum i40e_status_code i40e_validate_nvm_checksum(struct i40e_hw *hw, enum i40e_status_code i40e_nvmupd_command(struct i40e_hw *hw, struct i40e_nvm_access *cmd, u8 *bytes, int *); +void i40e_nvmupd_check_wait_event(struct i40e_hw *hw, u16 opcode); void i40e_set_pci_config_data(struct i40e_hw *hw, u16 link_status); enum i40e_status_code i40e_set_mac_type(struct i40e_hw *hw); @@ -496,9 +502,26 @@ enum i40e_status_code i40e_aq_rx_ctl_write_register(struct i40e_hw *hw, u32 reg_addr, u32 reg_val, struct i40e_asq_cmd_details *cmd_details); void i40e_write_rx_ctl(struct i40e_hw *hw, u32 reg_addr, u32 reg_val); +enum i40e_status_code i40e_aq_set_arp_proxy_config(struct i40e_hw *hw, + struct i40e_aqc_arp_proxy_data *proxy_config, + struct i40e_asq_cmd_details *cmd_details); +enum i40e_status_code i40e_aq_set_ns_proxy_table_entry(struct i40e_hw *hw, + struct i40e_aqc_ns_proxy_data *ns_proxy_table_entry, + struct i40e_asq_cmd_details *cmd_details); +enum i40e_status_code i40e_aq_set_clear_wol_filter(struct i40e_hw *hw, + u8 filter_index, + struct i40e_aqc_set_wol_filter_data *filter, + bool set_filter, bool no_wol_tco, + bool filter_valid, bool no_wol_tco_valid, + struct i40e_asq_cmd_details *cmd_details); +enum i40e_status_code i40e_aq_get_wake_event_reason(struct i40e_hw *hw, + u16 *wake_reason, + struct i40e_asq_cmd_details *cmd_details); enum i40e_status_code i40e_read_phy_register(struct i40e_hw *hw, u8 page, u16 reg, u8 phy_addr, u16 *value); enum i40e_status_code i40e_write_phy_register(struct i40e_hw *hw, u8 page, u16 reg, u8 phy_addr, u16 value); u8 i40e_get_phy_address(struct i40e_hw *hw, u8 dev_num); +enum i40e_status_code i40e_blink_phy_link_led(struct i40e_hw *hw, + u32 time, u32 interval); #endif /* _I40E_PROTOTYPE_H_ */ diff --git a/sys/dev/ixl/i40e_register.h b/sys/dev/ixl/i40e_register.h index 9fbfc7b826c2..fb41ea23a3d0 100644 --- a/sys/dev/ixl/i40e_register.h +++ b/sys/dev/ixl/i40e_register.h @@ -3398,4 +3398,1966 @@ #define I40E_VFQF_HREGION_OVERRIDE_ENA_7_MASK I40E_MASK(0x1, I40E_VFQF_HREGION_OVERRIDE_ENA_7_SHIFT) #define I40E_VFQF_HREGION_REGION_7_SHIFT 29 #define I40E_VFQF_HREGION_REGION_7_MASK I40E_MASK(0x7, I40E_VFQF_HREGION_REGION_7_SHIFT) + +#define I40E_MNGSB_FDCRC 0x000B7050 /* Reset: POR */ +#define I40E_MNGSB_FDCRC_CRC_RES_SHIFT 0 +#define I40E_MNGSB_FDCRC_CRC_RES_MASK I40E_MASK(0xFF, I40E_MNGSB_FDCRC_CRC_RES_SHIFT) +#define I40E_MNGSB_FDCS 0x000B7040 /* Reset: POR */ +#define I40E_MNGSB_FDCS_CRC_CONT_SHIFT 2 +#define I40E_MNGSB_FDCS_CRC_CONT_MASK I40E_MASK(0x1, I40E_MNGSB_FDCS_CRC_CONT_SHIFT) +#define I40E_MNGSB_FDCS_CRC_SEED_EN_SHIFT 3 +#define I40E_MNGSB_FDCS_CRC_SEED_EN_MASK I40E_MASK(0x1, I40E_MNGSB_FDCS_CRC_SEED_EN_SHIFT) +#define I40E_MNGSB_FDCS_CRC_WR_INH_SHIFT 4 +#define I40E_MNGSB_FDCS_CRC_WR_INH_MASK I40E_MASK(0x1, I40E_MNGSB_FDCS_CRC_WR_INH_SHIFT) +#define I40E_MNGSB_FDCS_CRC_SEED_SHIFT 8 +#define I40E_MNGSB_FDCS_CRC_SEED_MASK I40E_MASK(0xFF, I40E_MNGSB_FDCS_CRC_SEED_SHIFT) +#define I40E_MNGSB_FDS 0x000B7048 /* Reset: POR */ +#define I40E_MNGSB_FDS_START_BC_SHIFT 0 +#define I40E_MNGSB_FDS_START_BC_MASK I40E_MASK(0xFFF, I40E_MNGSB_FDS_START_BC_SHIFT) +#define I40E_MNGSB_FDS_LAST_BC_SHIFT 16 +#define I40E_MNGSB_FDS_LAST_BC_MASK I40E_MASK(0xFFF, I40E_MNGSB_FDS_LAST_BC_SHIFT) + +#define I40E_GL_VF_CTRL_RX(_VF) (0x00083600 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: EMPR */ +#define I40E_GL_VF_CTRL_RX_MAX_INDEX 127 +#define I40E_GL_VF_CTRL_RX_AQ_RX_EN_SHIFT 0 +#define I40E_GL_VF_CTRL_RX_AQ_RX_EN_MASK I40E_MASK(0x1, I40E_GL_VF_CTRL_RX_AQ_RX_EN_SHIFT) +#define I40E_GL_VF_CTRL_TX(_VF) (0x00083400 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: EMPR */ +#define I40E_GL_VF_CTRL_TX_MAX_INDEX 127 +#define I40E_GL_VF_CTRL_TX_AQ_TX_EN_SHIFT 0 +#define I40E_GL_VF_CTRL_TX_AQ_TX_EN_MASK I40E_MASK(0x1, I40E_GL_VF_CTRL_TX_AQ_TX_EN_SHIFT) + +#define I40E_GLCM_LAN_CACHESIZE 0x0010C4D8 /* Reset: CORER */ +#define I40E_GLCM_LAN_CACHESIZE_WORD_SIZE_SHIFT 0 +#define I40E_GLCM_LAN_CACHESIZE_WORD_SIZE_MASK I40E_MASK(0xFFF, I40E_GLCM_LAN_CACHESIZE_WORD_SIZE_SHIFT) +#define I40E_GLCM_LAN_CACHESIZE_SETS_SHIFT 12 +#define I40E_GLCM_LAN_CACHESIZE_SETS_MASK I40E_MASK(0xF, I40E_GLCM_LAN_CACHESIZE_SETS_SHIFT) +#define I40E_GLCM_LAN_CACHESIZE_WAYS_SHIFT 16 +#define I40E_GLCM_LAN_CACHESIZE_WAYS_MASK I40E_MASK(0x3FF, I40E_GLCM_LAN_CACHESIZE_WAYS_SHIFT) +#define I40E_GLCM_PE_CACHESIZE 0x00138FE4 /* Reset: CORER */ +#define I40E_GLCM_PE_CACHESIZE_WORD_SIZE_SHIFT 0 +#define I40E_GLCM_PE_CACHESIZE_WORD_SIZE_MASK I40E_MASK(0xFFF, I40E_GLCM_PE_CACHESIZE_WORD_SIZE_SHIFT) +#define I40E_GLCM_PE_CACHESIZE_SETS_SHIFT 12 +#define I40E_GLCM_PE_CACHESIZE_SETS_MASK I40E_MASK(0xF, I40E_GLCM_PE_CACHESIZE_SETS_SHIFT) +#define I40E_GLCM_PE_CACHESIZE_WAYS_SHIFT 16 +#define I40E_GLCM_PE_CACHESIZE_WAYS_MASK I40E_MASK(0x1FF, I40E_GLCM_PE_CACHESIZE_WAYS_SHIFT) +#define I40E_PFCM_PE_ERRDATA 0x00138D00 /* Reset: PFR */ +#define I40E_PFCM_PE_ERRDATA_ERROR_CODE_SHIFT 0 +#define I40E_PFCM_PE_ERRDATA_ERROR_CODE_MASK I40E_MASK(0xF, I40E_PFCM_PE_ERRDATA_ERROR_CODE_SHIFT) +#define I40E_PFCM_PE_ERRDATA_Q_TYPE_SHIFT 4 +#define I40E_PFCM_PE_ERRDATA_Q_TYPE_MASK I40E_MASK(0x7, I40E_PFCM_PE_ERRDATA_Q_TYPE_SHIFT) +#define I40E_PFCM_PE_ERRDATA_Q_NUM_SHIFT 8 +#define I40E_PFCM_PE_ERRDATA_Q_NUM_MASK I40E_MASK(0x3FFFF, I40E_PFCM_PE_ERRDATA_Q_NUM_SHIFT) +#define I40E_PFCM_PE_ERRINFO 0x00138C80 /* Reset: PFR */ +#define I40E_PFCM_PE_ERRINFO_ERROR_VALID_SHIFT 0 +#define I40E_PFCM_PE_ERRINFO_ERROR_VALID_MASK I40E_MASK(0x1, I40E_PFCM_PE_ERRINFO_ERROR_VALID_SHIFT) +#define I40E_PFCM_PE_ERRINFO_ERROR_INST_SHIFT 4 +#define I40E_PFCM_PE_ERRINFO_ERROR_INST_MASK I40E_MASK(0x7, I40E_PFCM_PE_ERRINFO_ERROR_INST_SHIFT) +#define I40E_PFCM_PE_ERRINFO_DBL_ERROR_CNT_SHIFT 8 +#define I40E_PFCM_PE_ERRINFO_DBL_ERROR_CNT_MASK I40E_MASK(0xFF, I40E_PFCM_PE_ERRINFO_DBL_ERROR_CNT_SHIFT) +#define I40E_PFCM_PE_ERRINFO_RLU_ERROR_CNT_SHIFT 16 +#define I40E_PFCM_PE_ERRINFO_RLU_ERROR_CNT_MASK I40E_MASK(0xFF, I40E_PFCM_PE_ERRINFO_RLU_ERROR_CNT_SHIFT) +#define I40E_PFCM_PE_ERRINFO_RLS_ERROR_CNT_SHIFT 24 +#define I40E_PFCM_PE_ERRINFO_RLS_ERROR_CNT_MASK I40E_MASK(0xFF, I40E_PFCM_PE_ERRINFO_RLS_ERROR_CNT_SHIFT) + +#define I40E_PRTDCB_TFMSTC(_i) (0x000A0040 + ((_i) * 32)) /* _i=0...7 */ /* Reset: CORER */ +#define I40E_PRTDCB_TFMSTC_MAX_INDEX 7 +#define I40E_PRTDCB_TFMSTC_MSTC_SHIFT 0 +#define I40E_PRTDCB_TFMSTC_MSTC_MASK I40E_MASK(0xFFFFF, I40E_PRTDCB_TFMSTC_MSTC_SHIFT) +#define I40E_GL_FWSTS_FWROWD_SHIFT 8 +#define I40E_GL_FWSTS_FWROWD_MASK I40E_MASK(0x1, I40E_GL_FWSTS_FWROWD_SHIFT) +#define I40E_GLFOC_CACHESIZE 0x000AA0DC /* Reset: CORER */ +#define I40E_GLFOC_CACHESIZE_WORD_SIZE_SHIFT 0 +#define I40E_GLFOC_CACHESIZE_WORD_SIZE_MASK I40E_MASK(0xFF, I40E_GLFOC_CACHESIZE_WORD_SIZE_SHIFT) +#define I40E_GLFOC_CACHESIZE_SETS_SHIFT 8 +#define I40E_GLFOC_CACHESIZE_SETS_MASK I40E_MASK(0xFFF, I40E_GLFOC_CACHESIZE_SETS_SHIFT) +#define I40E_GLFOC_CACHESIZE_WAYS_SHIFT 20 +#define I40E_GLFOC_CACHESIZE_WAYS_MASK I40E_MASK(0xF, I40E_GLFOC_CACHESIZE_WAYS_SHIFT) +#define I40E_GLHMC_APBVTINUSEBASE(_i) (0x000C4a00 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ +#define I40E_GLHMC_APBVTINUSEBASE_MAX_INDEX 15 +#define I40E_GLHMC_APBVTINUSEBASE_FPMAPBINUSEBASE_SHIFT 0 +#define I40E_GLHMC_APBVTINUSEBASE_FPMAPBINUSEBASE_MASK I40E_MASK(0xFFFFFF, I40E_GLHMC_APBVTINUSEBASE_FPMAPBINUSEBASE_SHIFT) +#define I40E_GLHMC_CEQPART(_i) (0x001312C0 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ +#define I40E_GLHMC_CEQPART_MAX_INDEX 15 +#define I40E_GLHMC_CEQPART_PMCEQBASE_SHIFT 0 +#define I40E_GLHMC_CEQPART_PMCEQBASE_MASK I40E_MASK(0xFF, I40E_GLHMC_CEQPART_PMCEQBASE_SHIFT) +#define I40E_GLHMC_CEQPART_PMCEQSIZE_SHIFT 16 +#define I40E_GLHMC_CEQPART_PMCEQSIZE_MASK I40E_MASK(0x1FF, I40E_GLHMC_CEQPART_PMCEQSIZE_SHIFT) +#define I40E_GLHMC_DBCQMAX 0x000C20F0 /* Reset: CORER */ +#define I40E_GLHMC_DBCQMAX_GLHMC_DBCQMAX_SHIFT 0 +#define I40E_GLHMC_DBCQMAX_GLHMC_DBCQMAX_MASK I40E_MASK(0x3FFFF, I40E_GLHMC_DBCQMAX_GLHMC_DBCQMAX_SHIFT) +#define I40E_GLHMC_DBCQPART(_i) (0x00131240 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ +#define I40E_GLHMC_DBCQPART_MAX_INDEX 15 +#define I40E_GLHMC_DBCQPART_PMDBCQBASE_SHIFT 0 +#define I40E_GLHMC_DBCQPART_PMDBCQBASE_MASK I40E_MASK(0x3FFF, I40E_GLHMC_DBCQPART_PMDBCQBASE_SHIFT) +#define I40E_GLHMC_DBCQPART_PMDBCQSIZE_SHIFT 16 +#define I40E_GLHMC_DBCQPART_PMDBCQSIZE_MASK I40E_MASK(0x7FFF, I40E_GLHMC_DBCQPART_PMDBCQSIZE_SHIFT) +#define I40E_GLHMC_DBQPMAX 0x000C20EC /* Reset: CORER */ +#define I40E_GLHMC_DBQPMAX_GLHMC_DBQPMAX_SHIFT 0 +#define I40E_GLHMC_DBQPMAX_GLHMC_DBQPMAX_MASK I40E_MASK(0x7FFFF, I40E_GLHMC_DBQPMAX_GLHMC_DBQPMAX_SHIFT) +#define I40E_GLHMC_DBQPPART(_i) (0x00138D80 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ +#define I40E_GLHMC_DBQPPART_MAX_INDEX 15 +#define I40E_GLHMC_DBQPPART_PMDBQPBASE_SHIFT 0 +#define I40E_GLHMC_DBQPPART_PMDBQPBASE_MASK I40E_MASK(0x3FFF, I40E_GLHMC_DBQPPART_PMDBQPBASE_SHIFT) +#define I40E_GLHMC_DBQPPART_PMDBQPSIZE_SHIFT 16 +#define I40E_GLHMC_DBQPPART_PMDBQPSIZE_MASK I40E_MASK(0x7FFF, I40E_GLHMC_DBQPPART_PMDBQPSIZE_SHIFT) +#define I40E_GLHMC_PEARPBASE(_i) (0x000C4800 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ +#define I40E_GLHMC_PEARPBASE_MAX_INDEX 15 +#define I40E_GLHMC_PEARPBASE_FPMPEARPBASE_SHIFT 0 +#define I40E_GLHMC_PEARPBASE_FPMPEARPBASE_MASK I40E_MASK(0xFFFFFF, I40E_GLHMC_PEARPBASE_FPMPEARPBASE_SHIFT) +#define I40E_GLHMC_PEARPCNT(_i) (0x000C4900 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ +#define I40E_GLHMC_PEARPCNT_MAX_INDEX 15 +#define I40E_GLHMC_PEARPCNT_FPMPEARPCNT_SHIFT 0 +#define I40E_GLHMC_PEARPCNT_FPMPEARPCNT_MASK I40E_MASK(0x1FFFFFFF, I40E_GLHMC_PEARPCNT_FPMPEARPCNT_SHIFT) +#define I40E_GLHMC_PEARPMAX 0x000C2038 /* Reset: CORER */ +#define I40E_GLHMC_PEARPMAX_PMPEARPMAX_SHIFT 0 +#define I40E_GLHMC_PEARPMAX_PMPEARPMAX_MASK I40E_MASK(0x1FFFF, I40E_GLHMC_PEARPMAX_PMPEARPMAX_SHIFT) +#define I40E_GLHMC_PEARPOBJSZ 0x000C2034 /* Reset: CORER */ +#define I40E_GLHMC_PEARPOBJSZ_PMPEARPOBJSZ_SHIFT 0 +#define I40E_GLHMC_PEARPOBJSZ_PMPEARPOBJSZ_MASK I40E_MASK(0x7, I40E_GLHMC_PEARPOBJSZ_PMPEARPOBJSZ_SHIFT) +#define I40E_GLHMC_PECQBASE(_i) (0x000C4200 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ +#define I40E_GLHMC_PECQBASE_MAX_INDEX 15 +#define I40E_GLHMC_PECQBASE_FPMPECQBASE_SHIFT 0 +#define I40E_GLHMC_PECQBASE_FPMPECQBASE_MASK I40E_MASK(0xFFFFFF, I40E_GLHMC_PECQBASE_FPMPECQBASE_SHIFT) +#define I40E_GLHMC_PECQCNT(_i) (0x000C4300 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ +#define I40E_GLHMC_PECQCNT_MAX_INDEX 15 +#define I40E_GLHMC_PECQCNT_FPMPECQCNT_SHIFT 0 +#define I40E_GLHMC_PECQCNT_FPMPECQCNT_MASK I40E_MASK(0x1FFFFFFF, I40E_GLHMC_PECQCNT_FPMPECQCNT_SHIFT) +#define I40E_GLHMC_PECQOBJSZ 0x000C2020 /* Reset: CORER */ +#define I40E_GLHMC_PECQOBJSZ_PMPECQOBJSZ_SHIFT 0 +#define I40E_GLHMC_PECQOBJSZ_PMPECQOBJSZ_MASK I40E_MASK(0xF, I40E_GLHMC_PECQOBJSZ_PMPECQOBJSZ_SHIFT) +#define I40E_GLHMC_PEHTCNT(_i) (0x000C4700 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ +#define I40E_GLHMC_PEHTCNT_MAX_INDEX 15 +#define I40E_GLHMC_PEHTCNT_FPMPEHTCNT_SHIFT 0 +#define I40E_GLHMC_PEHTCNT_FPMPEHTCNT_MASK I40E_MASK(0x1FFFFFFF, I40E_GLHMC_PEHTCNT_FPMPEHTCNT_SHIFT) +#define I40E_GLHMC_PEHTEBASE(_i) (0x000C4600 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ +#define I40E_GLHMC_PEHTEBASE_MAX_INDEX 15 +#define I40E_GLHMC_PEHTEBASE_FPMPEHTEBASE_SHIFT 0 +#define I40E_GLHMC_PEHTEBASE_FPMPEHTEBASE_MASK I40E_MASK(0xFFFFFF, I40E_GLHMC_PEHTEBASE_FPMPEHTEBASE_SHIFT) +#define I40E_GLHMC_PEHTEOBJSZ 0x000C202c /* Reset: CORER */ +#define I40E_GLHMC_PEHTEOBJSZ_PMPEHTEOBJSZ_SHIFT 0 +#define I40E_GLHMC_PEHTEOBJSZ_PMPEHTEOBJSZ_MASK I40E_MASK(0xF, I40E_GLHMC_PEHTEOBJSZ_PMPEHTEOBJSZ_SHIFT) +#define I40E_GLHMC_PEHTMAX 0x000C2030 /* Reset: CORER */ +#define I40E_GLHMC_PEHTMAX_PMPEHTMAX_SHIFT 0 +#define I40E_GLHMC_PEHTMAX_PMPEHTMAX_MASK I40E_MASK(0x1FFFFF, I40E_GLHMC_PEHTMAX_PMPEHTMAX_SHIFT) +#define I40E_GLHMC_PEMRBASE(_i) (0x000C4c00 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ +#define I40E_GLHMC_PEMRBASE_MAX_INDEX 15 +#define I40E_GLHMC_PEMRBASE_FPMPEMRBASE_SHIFT 0 +#define I40E_GLHMC_PEMRBASE_FPMPEMRBASE_MASK I40E_MASK(0xFFFFFF, I40E_GLHMC_PEMRBASE_FPMPEMRBASE_SHIFT) +#define I40E_GLHMC_PEMRCNT(_i) (0x000C4d00 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ +#define I40E_GLHMC_PEMRCNT_MAX_INDEX 15 +#define I40E_GLHMC_PEMRCNT_FPMPEMRSZ_SHIFT 0 +#define I40E_GLHMC_PEMRCNT_FPMPEMRSZ_MASK I40E_MASK(0x1FFFFFFF, I40E_GLHMC_PEMRCNT_FPMPEMRSZ_SHIFT) +#define I40E_GLHMC_PEMRMAX 0x000C2040 /* Reset: CORER */ +#define I40E_GLHMC_PEMRMAX_PMPEMRMAX_SHIFT 0 +#define I40E_GLHMC_PEMRMAX_PMPEMRMAX_MASK I40E_MASK(0x7FFFFF, I40E_GLHMC_PEMRMAX_PMPEMRMAX_SHIFT) +#define I40E_GLHMC_PEMROBJSZ 0x000C203c /* Reset: CORER */ +#define I40E_GLHMC_PEMROBJSZ_PMPEMROBJSZ_SHIFT 0 +#define I40E_GLHMC_PEMROBJSZ_PMPEMROBJSZ_MASK I40E_MASK(0xF, I40E_GLHMC_PEMROBJSZ_PMPEMROBJSZ_SHIFT) +#define I40E_GLHMC_PEPBLBASE(_i) (0x000C5800 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ +#define I40E_GLHMC_PEPBLBASE_MAX_INDEX 15 +#define I40E_GLHMC_PEPBLBASE_FPMPEPBLBASE_SHIFT 0 +#define I40E_GLHMC_PEPBLBASE_FPMPEPBLBASE_MASK I40E_MASK(0xFFFFFF, I40E_GLHMC_PEPBLBASE_FPMPEPBLBASE_SHIFT) +#define I40E_GLHMC_PEPBLCNT(_i) (0x000C5900 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ +#define I40E_GLHMC_PEPBLCNT_MAX_INDEX 15 +#define I40E_GLHMC_PEPBLCNT_FPMPEPBLCNT_SHIFT 0 +#define I40E_GLHMC_PEPBLCNT_FPMPEPBLCNT_MASK I40E_MASK(0x1FFFFFFF, I40E_GLHMC_PEPBLCNT_FPMPEPBLCNT_SHIFT) +#define I40E_GLHMC_PEPBLMAX 0x000C206c /* Reset: CORER */ +#define I40E_GLHMC_PEPBLMAX_PMPEPBLMAX_SHIFT 0 +#define I40E_GLHMC_PEPBLMAX_PMPEPBLMAX_MASK I40E_MASK(0x1FFFFFFF, I40E_GLHMC_PEPBLMAX_PMPEPBLMAX_SHIFT) +#define I40E_GLHMC_PEPFFIRSTSD 0x000C20E4 /* Reset: CORER */ +#define I40E_GLHMC_PEPFFIRSTSD_GLHMC_PEPFFIRSTSD_SHIFT 0 +#define I40E_GLHMC_PEPFFIRSTSD_GLHMC_PEPFFIRSTSD_MASK I40E_MASK(0xFFF, I40E_GLHMC_PEPFFIRSTSD_GLHMC_PEPFFIRSTSD_SHIFT) +#define I40E_GLHMC_PEQ1BASE(_i) (0x000C5200 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ +#define I40E_GLHMC_PEQ1BASE_MAX_INDEX 15 +#define I40E_GLHMC_PEQ1BASE_FPMPEQ1BASE_SHIFT 0 +#define I40E_GLHMC_PEQ1BASE_FPMPEQ1BASE_MASK I40E_MASK(0xFFFFFF, I40E_GLHMC_PEQ1BASE_FPMPEQ1BASE_SHIFT) +#define I40E_GLHMC_PEQ1CNT(_i) (0x000C5300 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ +#define I40E_GLHMC_PEQ1CNT_MAX_INDEX 15 +#define I40E_GLHMC_PEQ1CNT_FPMPEQ1CNT_SHIFT 0 +#define I40E_GLHMC_PEQ1CNT_FPMPEQ1CNT_MASK I40E_MASK(0x1FFFFFFF, I40E_GLHMC_PEQ1CNT_FPMPEQ1CNT_SHIFT) +#define I40E_GLHMC_PEQ1FLBASE(_i) (0x000C5400 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ +#define I40E_GLHMC_PEQ1FLBASE_MAX_INDEX 15 +#define I40E_GLHMC_PEQ1FLBASE_FPMPEQ1FLBASE_SHIFT 0 +#define I40E_GLHMC_PEQ1FLBASE_FPMPEQ1FLBASE_MASK I40E_MASK(0xFFFFFF, I40E_GLHMC_PEQ1FLBASE_FPMPEQ1FLBASE_SHIFT) +#define I40E_GLHMC_PEQ1FLMAX 0x000C2058 /* Reset: CORER */ +#define I40E_GLHMC_PEQ1FLMAX_PMPEQ1FLMAX_SHIFT 0 +#define I40E_GLHMC_PEQ1FLMAX_PMPEQ1FLMAX_MASK I40E_MASK(0x3FFFFFF, I40E_GLHMC_PEQ1FLMAX_PMPEQ1FLMAX_SHIFT) +#define I40E_GLHMC_PEQ1MAX 0x000C2054 /* Reset: CORER */ +#define I40E_GLHMC_PEQ1MAX_PMPEQ1MAX_SHIFT 0 +#define I40E_GLHMC_PEQ1MAX_PMPEQ1MAX_MASK I40E_MASK(0x3FFFFFF, I40E_GLHMC_PEQ1MAX_PMPEQ1MAX_SHIFT) +#define I40E_GLHMC_PEQ1OBJSZ 0x000C2050 /* Reset: CORER */ +#define I40E_GLHMC_PEQ1OBJSZ_PMPEQ1OBJSZ_SHIFT 0 +#define I40E_GLHMC_PEQ1OBJSZ_PMPEQ1OBJSZ_MASK I40E_MASK(0xF, I40E_GLHMC_PEQ1OBJSZ_PMPEQ1OBJSZ_SHIFT) +#define I40E_GLHMC_PEQPBASE(_i) (0x000C4000 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ +#define I40E_GLHMC_PEQPBASE_MAX_INDEX 15 +#define I40E_GLHMC_PEQPBASE_FPMPEQPBASE_SHIFT 0 +#define I40E_GLHMC_PEQPBASE_FPMPEQPBASE_MASK I40E_MASK(0xFFFFFF, I40E_GLHMC_PEQPBASE_FPMPEQPBASE_SHIFT) +#define I40E_GLHMC_PEQPCNT(_i) (0x000C4100 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ +#define I40E_GLHMC_PEQPCNT_MAX_INDEX 15 +#define I40E_GLHMC_PEQPCNT_FPMPEQPCNT_SHIFT 0 +#define I40E_GLHMC_PEQPCNT_FPMPEQPCNT_MASK I40E_MASK(0x1FFFFFFF, I40E_GLHMC_PEQPCNT_FPMPEQPCNT_SHIFT) +#define I40E_GLHMC_PEQPOBJSZ 0x000C201c /* Reset: CORER */ +#define I40E_GLHMC_PEQPOBJSZ_PMPEQPOBJSZ_SHIFT 0 +#define I40E_GLHMC_PEQPOBJSZ_PMPEQPOBJSZ_MASK I40E_MASK(0xF, I40E_GLHMC_PEQPOBJSZ_PMPEQPOBJSZ_SHIFT) +#define I40E_GLHMC_PESRQBASE(_i) (0x000C4400 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ +#define I40E_GLHMC_PESRQBASE_MAX_INDEX 15 +#define I40E_GLHMC_PESRQBASE_FPMPESRQBASE_SHIFT 0 +#define I40E_GLHMC_PESRQBASE_FPMPESRQBASE_MASK I40E_MASK(0xFFFFFF, I40E_GLHMC_PESRQBASE_FPMPESRQBASE_SHIFT) +#define I40E_GLHMC_PESRQCNT(_i) (0x000C4500 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ +#define I40E_GLHMC_PESRQCNT_MAX_INDEX 15 +#define I40E_GLHMC_PESRQCNT_FPMPESRQCNT_SHIFT 0 +#define I40E_GLHMC_PESRQCNT_FPMPESRQCNT_MASK I40E_MASK(0x1FFFFFFF, I40E_GLHMC_PESRQCNT_FPMPESRQCNT_SHIFT) +#define I40E_GLHMC_PESRQMAX 0x000C2028 /* Reset: CORER */ +#define I40E_GLHMC_PESRQMAX_PMPESRQMAX_SHIFT 0 +#define I40E_GLHMC_PESRQMAX_PMPESRQMAX_MASK I40E_MASK(0xFFFF, I40E_GLHMC_PESRQMAX_PMPESRQMAX_SHIFT) +#define I40E_GLHMC_PESRQOBJSZ 0x000C2024 /* Reset: CORER */ +#define I40E_GLHMC_PESRQOBJSZ_PMPESRQOBJSZ_SHIFT 0 +#define I40E_GLHMC_PESRQOBJSZ_PMPESRQOBJSZ_MASK I40E_MASK(0xF, I40E_GLHMC_PESRQOBJSZ_PMPESRQOBJSZ_SHIFT) +#define I40E_GLHMC_PETIMERBASE(_i) (0x000C5A00 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ +#define I40E_GLHMC_PETIMERBASE_MAX_INDEX 15 +#define I40E_GLHMC_PETIMERBASE_FPMPETIMERBASE_SHIFT 0 +#define I40E_GLHMC_PETIMERBASE_FPMPETIMERBASE_MASK I40E_MASK(0xFFFFFF, I40E_GLHMC_PETIMERBASE_FPMPETIMERBASE_SHIFT) +#define I40E_GLHMC_PETIMERCNT(_i) (0x000C5B00 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ +#define I40E_GLHMC_PETIMERCNT_MAX_INDEX 15 +#define I40E_GLHMC_PETIMERCNT_FPMPETIMERCNT_SHIFT 0 +#define I40E_GLHMC_PETIMERCNT_FPMPETIMERCNT_MASK I40E_MASK(0x1FFFFFFF, I40E_GLHMC_PETIMERCNT_FPMPETIMERCNT_SHIFT) +#define I40E_GLHMC_PETIMERMAX 0x000C2084 /* Reset: CORER */ +#define I40E_GLHMC_PETIMERMAX_PMPETIMERMAX_SHIFT 0 +#define I40E_GLHMC_PETIMERMAX_PMPETIMERMAX_MASK I40E_MASK(0x1FFFFFFF, I40E_GLHMC_PETIMERMAX_PMPETIMERMAX_SHIFT) +#define I40E_GLHMC_PETIMEROBJSZ 0x000C2080 /* Reset: CORER */ +#define I40E_GLHMC_PETIMEROBJSZ_PMPETIMEROBJSZ_SHIFT 0 +#define I40E_GLHMC_PETIMEROBJSZ_PMPETIMEROBJSZ_MASK I40E_MASK(0xF, I40E_GLHMC_PETIMEROBJSZ_PMPETIMEROBJSZ_SHIFT) +#define I40E_GLHMC_PEXFBASE(_i) (0x000C4e00 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ +#define I40E_GLHMC_PEXFBASE_MAX_INDEX 15 +#define I40E_GLHMC_PEXFBASE_FPMPEXFBASE_SHIFT 0 +#define I40E_GLHMC_PEXFBASE_FPMPEXFBASE_MASK I40E_MASK(0xFFFFFF, I40E_GLHMC_PEXFBASE_FPMPEXFBASE_SHIFT) +#define I40E_GLHMC_PEXFCNT(_i) (0x000C4f00 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ +#define I40E_GLHMC_PEXFCNT_MAX_INDEX 15 +#define I40E_GLHMC_PEXFCNT_FPMPEXFCNT_SHIFT 0 +#define I40E_GLHMC_PEXFCNT_FPMPEXFCNT_MASK I40E_MASK(0x1FFFFFFF, I40E_GLHMC_PEXFCNT_FPMPEXFCNT_SHIFT) +#define I40E_GLHMC_PEXFFLBASE(_i) (0x000C5000 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ +#define I40E_GLHMC_PEXFFLBASE_MAX_INDEX 15 +#define I40E_GLHMC_PEXFFLBASE_FPMPEXFFLBASE_SHIFT 0 +#define I40E_GLHMC_PEXFFLBASE_FPMPEXFFLBASE_MASK I40E_MASK(0xFFFFFF, I40E_GLHMC_PEXFFLBASE_FPMPEXFFLBASE_SHIFT) +#define I40E_GLHMC_PEXFFLMAX 0x000C204c /* Reset: CORER */ +#define I40E_GLHMC_PEXFFLMAX_PMPEXFFLMAX_SHIFT 0 +#define I40E_GLHMC_PEXFFLMAX_PMPEXFFLMAX_MASK I40E_MASK(0x1FFFFFF, I40E_GLHMC_PEXFFLMAX_PMPEXFFLMAX_SHIFT) +#define I40E_GLHMC_PEXFMAX 0x000C2048 /* Reset: CORER */ +#define I40E_GLHMC_PEXFMAX_PMPEXFMAX_SHIFT 0 +#define I40E_GLHMC_PEXFMAX_PMPEXFMAX_MASK I40E_MASK(0x3FFFFFF, I40E_GLHMC_PEXFMAX_PMPEXFMAX_SHIFT) +#define I40E_GLHMC_PEXFOBJSZ 0x000C2044 /* Reset: CORER */ +#define I40E_GLHMC_PEXFOBJSZ_PMPEXFOBJSZ_SHIFT 0 +#define I40E_GLHMC_PEXFOBJSZ_PMPEXFOBJSZ_MASK I40E_MASK(0xF, I40E_GLHMC_PEXFOBJSZ_PMPEXFOBJSZ_SHIFT) +#define I40E_GLHMC_PFPESDPART(_i) (0x000C0880 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ +#define I40E_GLHMC_PFPESDPART_MAX_INDEX 15 +#define I40E_GLHMC_PFPESDPART_PMSDBASE_SHIFT 0 +#define I40E_GLHMC_PFPESDPART_PMSDBASE_MASK I40E_MASK(0xFFF, I40E_GLHMC_PFPESDPART_PMSDBASE_SHIFT) +#define I40E_GLHMC_PFPESDPART_PMSDSIZE_SHIFT 16 +#define I40E_GLHMC_PFPESDPART_PMSDSIZE_MASK I40E_MASK(0x1FFF, I40E_GLHMC_PFPESDPART_PMSDSIZE_SHIFT) +#define I40E_GLHMC_VFAPBVTINUSEBASE(_i) (0x000Cca00 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLHMC_VFAPBVTINUSEBASE_MAX_INDEX 31 +#define I40E_GLHMC_VFAPBVTINUSEBASE_FPMAPBINUSEBASE_SHIFT 0 +#define I40E_GLHMC_VFAPBVTINUSEBASE_FPMAPBINUSEBASE_MASK I40E_MASK(0xFFFFFF, I40E_GLHMC_VFAPBVTINUSEBASE_FPMAPBINUSEBASE_SHIFT) +#define I40E_GLHMC_VFCEQPART(_i) (0x00132240 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLHMC_VFCEQPART_MAX_INDEX 31 +#define I40E_GLHMC_VFCEQPART_PMCEQBASE_SHIFT 0 +#define I40E_GLHMC_VFCEQPART_PMCEQBASE_MASK I40E_MASK(0xFF, I40E_GLHMC_VFCEQPART_PMCEQBASE_SHIFT) +#define I40E_GLHMC_VFCEQPART_PMCEQSIZE_SHIFT 16 +#define I40E_GLHMC_VFCEQPART_PMCEQSIZE_MASK I40E_MASK(0x1FF, I40E_GLHMC_VFCEQPART_PMCEQSIZE_SHIFT) +#define I40E_GLHMC_VFDBCQPART(_i) (0x00132140 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLHMC_VFDBCQPART_MAX_INDEX 31 +#define I40E_GLHMC_VFDBCQPART_PMDBCQBASE_SHIFT 0 +#define I40E_GLHMC_VFDBCQPART_PMDBCQBASE_MASK I40E_MASK(0x3FFF, I40E_GLHMC_VFDBCQPART_PMDBCQBASE_SHIFT) +#define I40E_GLHMC_VFDBCQPART_PMDBCQSIZE_SHIFT 16 +#define I40E_GLHMC_VFDBCQPART_PMDBCQSIZE_MASK I40E_MASK(0x7FFF, I40E_GLHMC_VFDBCQPART_PMDBCQSIZE_SHIFT) +#define I40E_GLHMC_VFDBQPPART(_i) (0x00138E00 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLHMC_VFDBQPPART_MAX_INDEX 31 +#define I40E_GLHMC_VFDBQPPART_PMDBQPBASE_SHIFT 0 +#define I40E_GLHMC_VFDBQPPART_PMDBQPBASE_MASK I40E_MASK(0x3FFF, I40E_GLHMC_VFDBQPPART_PMDBQPBASE_SHIFT) +#define I40E_GLHMC_VFDBQPPART_PMDBQPSIZE_SHIFT 16 +#define I40E_GLHMC_VFDBQPPART_PMDBQPSIZE_MASK I40E_MASK(0x7FFF, I40E_GLHMC_VFDBQPPART_PMDBQPSIZE_SHIFT) +#define I40E_GLHMC_VFFSIAVBASE(_i) (0x000Cd600 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLHMC_VFFSIAVBASE_MAX_INDEX 31 +#define I40E_GLHMC_VFFSIAVBASE_FPMFSIAVBASE_SHIFT 0 +#define I40E_GLHMC_VFFSIAVBASE_FPMFSIAVBASE_MASK I40E_MASK(0xFFFFFF, I40E_GLHMC_VFFSIAVBASE_FPMFSIAVBASE_SHIFT) +#define I40E_GLHMC_VFFSIAVCNT(_i) (0x000Cd700 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLHMC_VFFSIAVCNT_MAX_INDEX 31 +#define I40E_GLHMC_VFFSIAVCNT_FPMFSIAVCNT_SHIFT 0 +#define I40E_GLHMC_VFFSIAVCNT_FPMFSIAVCNT_MASK I40E_MASK(0x1FFFFFFF, I40E_GLHMC_VFFSIAVCNT_FPMFSIAVCNT_SHIFT) +#define I40E_GLHMC_VFPDINV(_i) (0x000C8300 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLHMC_VFPDINV_MAX_INDEX 31 +#define I40E_GLHMC_VFPDINV_PMSDIDX_SHIFT 0 +#define I40E_GLHMC_VFPDINV_PMSDIDX_MASK I40E_MASK(0xFFF, I40E_GLHMC_VFPDINV_PMSDIDX_SHIFT) +#define I40E_GLHMC_VFPDINV_PMSDPARTSEL_SHIFT 15 +#define I40E_GLHMC_VFPDINV_PMSDPARTSEL_MASK I40E_MASK(0x1, I40E_GLHMC_VFPDINV_PMSDPARTSEL_SHIFT) +#define I40E_GLHMC_VFPDINV_PMPDIDX_SHIFT 16 +#define I40E_GLHMC_VFPDINV_PMPDIDX_MASK I40E_MASK(0x1FF, I40E_GLHMC_VFPDINV_PMPDIDX_SHIFT) +#define I40E_GLHMC_VFPEARPBASE(_i) (0x000Cc800 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLHMC_VFPEARPBASE_MAX_INDEX 31 +#define I40E_GLHMC_VFPEARPBASE_FPMPEARPBASE_SHIFT 0 +#define I40E_GLHMC_VFPEARPBASE_FPMPEARPBASE_MASK I40E_MASK(0xFFFFFF, I40E_GLHMC_VFPEARPBASE_FPMPEARPBASE_SHIFT) +#define I40E_GLHMC_VFPEARPCNT(_i) (0x000Cc900 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLHMC_VFPEARPCNT_MAX_INDEX 31 +#define I40E_GLHMC_VFPEARPCNT_FPMPEARPCNT_SHIFT 0 +#define I40E_GLHMC_VFPEARPCNT_FPMPEARPCNT_MASK I40E_MASK(0x1FFFFFFF, I40E_GLHMC_VFPEARPCNT_FPMPEARPCNT_SHIFT) +#define I40E_GLHMC_VFPECQBASE(_i) (0x000Cc200 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLHMC_VFPECQBASE_MAX_INDEX 31 +#define I40E_GLHMC_VFPECQBASE_FPMPECQBASE_SHIFT 0 +#define I40E_GLHMC_VFPECQBASE_FPMPECQBASE_MASK I40E_MASK(0xFFFFFF, I40E_GLHMC_VFPECQBASE_FPMPECQBASE_SHIFT) +#define I40E_GLHMC_VFPECQCNT(_i) (0x000Cc300 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLHMC_VFPECQCNT_MAX_INDEX 31 +#define I40E_GLHMC_VFPECQCNT_FPMPECQCNT_SHIFT 0 +#define I40E_GLHMC_VFPECQCNT_FPMPECQCNT_MASK I40E_MASK(0x1FFFFFFF, I40E_GLHMC_VFPECQCNT_FPMPECQCNT_SHIFT) +#define I40E_GLHMC_VFPEHTCNT(_i) (0x000Cc700 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLHMC_VFPEHTCNT_MAX_INDEX 31 +#define I40E_GLHMC_VFPEHTCNT_FPMPEHTCNT_SHIFT 0 +#define I40E_GLHMC_VFPEHTCNT_FPMPEHTCNT_MASK I40E_MASK(0x1FFFFFFF, I40E_GLHMC_VFPEHTCNT_FPMPEHTCNT_SHIFT) +#define I40E_GLHMC_VFPEHTEBASE(_i) (0x000Cc600 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLHMC_VFPEHTEBASE_MAX_INDEX 31 +#define I40E_GLHMC_VFPEHTEBASE_FPMPEHTEBASE_SHIFT 0 +#define I40E_GLHMC_VFPEHTEBASE_FPMPEHTEBASE_MASK I40E_MASK(0xFFFFFF, I40E_GLHMC_VFPEHTEBASE_FPMPEHTEBASE_SHIFT) +#define I40E_GLHMC_VFPEMRBASE(_i) (0x000Ccc00 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLHMC_VFPEMRBASE_MAX_INDEX 31 +#define I40E_GLHMC_VFPEMRBASE_FPMPEMRBASE_SHIFT 0 +#define I40E_GLHMC_VFPEMRBASE_FPMPEMRBASE_MASK I40E_MASK(0xFFFFFF, I40E_GLHMC_VFPEMRBASE_FPMPEMRBASE_SHIFT) +#define I40E_GLHMC_VFPEMRCNT(_i) (0x000Ccd00 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLHMC_VFPEMRCNT_MAX_INDEX 31 +#define I40E_GLHMC_VFPEMRCNT_FPMPEMRSZ_SHIFT 0 +#define I40E_GLHMC_VFPEMRCNT_FPMPEMRSZ_MASK I40E_MASK(0x1FFFFFFF, I40E_GLHMC_VFPEMRCNT_FPMPEMRSZ_SHIFT) +#define I40E_GLHMC_VFPEPBLBASE(_i) (0x000Cd800 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLHMC_VFPEPBLBASE_MAX_INDEX 31 +#define I40E_GLHMC_VFPEPBLBASE_FPMPEPBLBASE_SHIFT 0 +#define I40E_GLHMC_VFPEPBLBASE_FPMPEPBLBASE_MASK I40E_MASK(0xFFFFFF, I40E_GLHMC_VFPEPBLBASE_FPMPEPBLBASE_SHIFT) +#define I40E_GLHMC_VFPEPBLCNT(_i) (0x000Cd900 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLHMC_VFPEPBLCNT_MAX_INDEX 31 +#define I40E_GLHMC_VFPEPBLCNT_FPMPEPBLCNT_SHIFT 0 +#define I40E_GLHMC_VFPEPBLCNT_FPMPEPBLCNT_MASK I40E_MASK(0x1FFFFFFF, I40E_GLHMC_VFPEPBLCNT_FPMPEPBLCNT_SHIFT) +#define I40E_GLHMC_VFPEQ1BASE(_i) (0x000Cd200 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLHMC_VFPEQ1BASE_MAX_INDEX 31 +#define I40E_GLHMC_VFPEQ1BASE_FPMPEQ1BASE_SHIFT 0 +#define I40E_GLHMC_VFPEQ1BASE_FPMPEQ1BASE_MASK I40E_MASK(0xFFFFFF, I40E_GLHMC_VFPEQ1BASE_FPMPEQ1BASE_SHIFT) +#define I40E_GLHMC_VFPEQ1CNT(_i) (0x000Cd300 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLHMC_VFPEQ1CNT_MAX_INDEX 31 +#define I40E_GLHMC_VFPEQ1CNT_FPMPEQ1CNT_SHIFT 0 +#define I40E_GLHMC_VFPEQ1CNT_FPMPEQ1CNT_MASK I40E_MASK(0x1FFFFFFF, I40E_GLHMC_VFPEQ1CNT_FPMPEQ1CNT_SHIFT) +#define I40E_GLHMC_VFPEQ1FLBASE(_i) (0x000Cd400 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLHMC_VFPEQ1FLBASE_MAX_INDEX 31 +#define I40E_GLHMC_VFPEQ1FLBASE_FPMPEQ1FLBASE_SHIFT 0 +#define I40E_GLHMC_VFPEQ1FLBASE_FPMPEQ1FLBASE_MASK I40E_MASK(0xFFFFFF, I40E_GLHMC_VFPEQ1FLBASE_FPMPEQ1FLBASE_SHIFT) +#define I40E_GLHMC_VFPEQPBASE(_i) (0x000Cc000 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLHMC_VFPEQPBASE_MAX_INDEX 31 +#define I40E_GLHMC_VFPEQPBASE_FPMPEQPBASE_SHIFT 0 +#define I40E_GLHMC_VFPEQPBASE_FPMPEQPBASE_MASK I40E_MASK(0xFFFFFF, I40E_GLHMC_VFPEQPBASE_FPMPEQPBASE_SHIFT) +#define I40E_GLHMC_VFPEQPCNT(_i) (0x000Cc100 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLHMC_VFPEQPCNT_MAX_INDEX 31 +#define I40E_GLHMC_VFPEQPCNT_FPMPEQPCNT_SHIFT 0 +#define I40E_GLHMC_VFPEQPCNT_FPMPEQPCNT_MASK I40E_MASK(0x1FFFFFFF, I40E_GLHMC_VFPEQPCNT_FPMPEQPCNT_SHIFT) +#define I40E_GLHMC_VFPESRQBASE(_i) (0x000Cc400 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLHMC_VFPESRQBASE_MAX_INDEX 31 +#define I40E_GLHMC_VFPESRQBASE_FPMPESRQBASE_SHIFT 0 +#define I40E_GLHMC_VFPESRQBASE_FPMPESRQBASE_MASK I40E_MASK(0xFFFFFF, I40E_GLHMC_VFPESRQBASE_FPMPESRQBASE_SHIFT) +#define I40E_GLHMC_VFPESRQCNT(_i) (0x000Cc500 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLHMC_VFPESRQCNT_MAX_INDEX 31 +#define I40E_GLHMC_VFPESRQCNT_FPMPESRQCNT_SHIFT 0 +#define I40E_GLHMC_VFPESRQCNT_FPMPESRQCNT_MASK I40E_MASK(0x1FFFFFFF, I40E_GLHMC_VFPESRQCNT_FPMPESRQCNT_SHIFT) +#define I40E_GLHMC_VFPETIMERBASE(_i) (0x000CDA00 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLHMC_VFPETIMERBASE_MAX_INDEX 31 +#define I40E_GLHMC_VFPETIMERBASE_FPMPETIMERBASE_SHIFT 0 +#define I40E_GLHMC_VFPETIMERBASE_FPMPETIMERBASE_MASK I40E_MASK(0xFFFFFF, I40E_GLHMC_VFPETIMERBASE_FPMPETIMERBASE_SHIFT) +#define I40E_GLHMC_VFPETIMERCNT(_i) (0x000CDB00 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLHMC_VFPETIMERCNT_MAX_INDEX 31 +#define I40E_GLHMC_VFPETIMERCNT_FPMPETIMERCNT_SHIFT 0 +#define I40E_GLHMC_VFPETIMERCNT_FPMPETIMERCNT_MASK I40E_MASK(0x1FFFFFFF, I40E_GLHMC_VFPETIMERCNT_FPMPETIMERCNT_SHIFT) +#define I40E_GLHMC_VFPEXFBASE(_i) (0x000Cce00 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLHMC_VFPEXFBASE_MAX_INDEX 31 +#define I40E_GLHMC_VFPEXFBASE_FPMPEXFBASE_SHIFT 0 +#define I40E_GLHMC_VFPEXFBASE_FPMPEXFBASE_MASK I40E_MASK(0xFFFFFF, I40E_GLHMC_VFPEXFBASE_FPMPEXFBASE_SHIFT) +#define I40E_GLHMC_VFPEXFCNT(_i) (0x000Ccf00 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLHMC_VFPEXFCNT_MAX_INDEX 31 +#define I40E_GLHMC_VFPEXFCNT_FPMPEXFCNT_SHIFT 0 +#define I40E_GLHMC_VFPEXFCNT_FPMPEXFCNT_MASK I40E_MASK(0x1FFFFFFF, I40E_GLHMC_VFPEXFCNT_FPMPEXFCNT_SHIFT) +#define I40E_GLHMC_VFPEXFFLBASE(_i) (0x000Cd000 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLHMC_VFPEXFFLBASE_MAX_INDEX 31 +#define I40E_GLHMC_VFPEXFFLBASE_FPMPEXFFLBASE_SHIFT 0 +#define I40E_GLHMC_VFPEXFFLBASE_FPMPEXFFLBASE_MASK I40E_MASK(0xFFFFFF, I40E_GLHMC_VFPEXFFLBASE_FPMPEXFFLBASE_SHIFT) +#define I40E_GLHMC_VFSDPART(_i) (0x000C8800 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLHMC_VFSDPART_MAX_INDEX 31 +#define I40E_GLHMC_VFSDPART_PMSDBASE_SHIFT 0 +#define I40E_GLHMC_VFSDPART_PMSDBASE_MASK I40E_MASK(0xFFF, I40E_GLHMC_VFSDPART_PMSDBASE_SHIFT) +#define I40E_GLHMC_VFSDPART_PMSDSIZE_SHIFT 16 +#define I40E_GLHMC_VFSDPART_PMSDSIZE_MASK I40E_MASK(0x1FFF, I40E_GLHMC_VFSDPART_PMSDSIZE_SHIFT) +#define I40E_GLPBLOC_CACHESIZE 0x000A80BC /* Reset: CORER */ +#define I40E_GLPBLOC_CACHESIZE_WORD_SIZE_SHIFT 0 +#define I40E_GLPBLOC_CACHESIZE_WORD_SIZE_MASK I40E_MASK(0xFF, I40E_GLPBLOC_CACHESIZE_WORD_SIZE_SHIFT) +#define I40E_GLPBLOC_CACHESIZE_SETS_SHIFT 8 +#define I40E_GLPBLOC_CACHESIZE_SETS_MASK I40E_MASK(0xFFF, I40E_GLPBLOC_CACHESIZE_SETS_SHIFT) +#define I40E_GLPBLOC_CACHESIZE_WAYS_SHIFT 20 +#define I40E_GLPBLOC_CACHESIZE_WAYS_MASK I40E_MASK(0xF, I40E_GLPBLOC_CACHESIZE_WAYS_SHIFT) +#define I40E_GLPDOC_CACHESIZE 0x000D0088 /* Reset: CORER */ +#define I40E_GLPDOC_CACHESIZE_WORD_SIZE_SHIFT 0 +#define I40E_GLPDOC_CACHESIZE_WORD_SIZE_MASK I40E_MASK(0xFF, I40E_GLPDOC_CACHESIZE_WORD_SIZE_SHIFT) +#define I40E_GLPDOC_CACHESIZE_SETS_SHIFT 8 +#define I40E_GLPDOC_CACHESIZE_SETS_MASK I40E_MASK(0xFFF, I40E_GLPDOC_CACHESIZE_SETS_SHIFT) +#define I40E_GLPDOC_CACHESIZE_WAYS_SHIFT 20 +#define I40E_GLPDOC_CACHESIZE_WAYS_MASK I40E_MASK(0xF, I40E_GLPDOC_CACHESIZE_WAYS_SHIFT) +#define I40E_GLPEOC_CACHESIZE 0x000A60E8 /* Reset: CORER */ +#define I40E_GLPEOC_CACHESIZE_WORD_SIZE_SHIFT 0 +#define I40E_GLPEOC_CACHESIZE_WORD_SIZE_MASK I40E_MASK(0xFF, I40E_GLPEOC_CACHESIZE_WORD_SIZE_SHIFT) +#define I40E_GLPEOC_CACHESIZE_SETS_SHIFT 8 +#define I40E_GLPEOC_CACHESIZE_SETS_MASK I40E_MASK(0xFFF, I40E_GLPEOC_CACHESIZE_SETS_SHIFT) +#define I40E_GLPEOC_CACHESIZE_WAYS_SHIFT 20 +#define I40E_GLPEOC_CACHESIZE_WAYS_MASK I40E_MASK(0xF, I40E_GLPEOC_CACHESIZE_WAYS_SHIFT) +#define I40E_PFHMC_PDINV_PMSDPARTSEL_SHIFT 15 +#define I40E_PFHMC_PDINV_PMSDPARTSEL_MASK I40E_MASK(0x1, I40E_PFHMC_PDINV_PMSDPARTSEL_SHIFT) +#define I40E_PFHMC_SDCMD_PMSDPARTSEL_SHIFT 15 +#define I40E_PFHMC_SDCMD_PMSDPARTSEL_MASK I40E_MASK(0x1, I40E_PFHMC_SDCMD_PMSDPARTSEL_SHIFT) +#define I40E_GL_PPRS_SPARE 0x000856E0 /* Reset: CORER */ +#define I40E_GL_PPRS_SPARE_GL_PPRS_SPARE_SHIFT 0 +#define I40E_GL_PPRS_SPARE_GL_PPRS_SPARE_MASK I40E_MASK(0xFFFFFFFF, I40E_GL_PPRS_SPARE_GL_PPRS_SPARE_SHIFT) +#define I40E_GL_TLAN_SPARE 0x000E64E0 /* Reset: CORER */ +#define I40E_GL_TLAN_SPARE_GL_TLAN_SPARE_SHIFT 0 +#define I40E_GL_TLAN_SPARE_GL_TLAN_SPARE_MASK I40E_MASK(0xFFFFFFFF, I40E_GL_TLAN_SPARE_GL_TLAN_SPARE_SHIFT) +#define I40E_GL_TUPM_SPARE 0x000a2230 /* Reset: CORER */ +#define I40E_GL_TUPM_SPARE_GL_TUPM_SPARE_SHIFT 0 +#define I40E_GL_TUPM_SPARE_GL_TUPM_SPARE_MASK I40E_MASK(0xFFFFFFFF, I40E_GL_TUPM_SPARE_GL_TUPM_SPARE_SHIFT) +#define I40E_GLGEN_CAR_DEBUG 0x000B81C0 /* Reset: POR */ +#define I40E_GLGEN_CAR_DEBUG_CAR_UPPER_CORE_CLK_EN_SHIFT 0 +#define I40E_GLGEN_CAR_DEBUG_CAR_UPPER_CORE_CLK_EN_MASK I40E_MASK(0x1, I40E_GLGEN_CAR_DEBUG_CAR_UPPER_CORE_CLK_EN_SHIFT) +#define I40E_GLGEN_CAR_DEBUG_CAR_PCIE_HIU_CLK_EN_SHIFT 1 +#define I40E_GLGEN_CAR_DEBUG_CAR_PCIE_HIU_CLK_EN_MASK I40E_MASK(0x1, I40E_GLGEN_CAR_DEBUG_CAR_PCIE_HIU_CLK_EN_SHIFT) +#define I40E_GLGEN_CAR_DEBUG_CAR_PE_CLK_EN_SHIFT 2 +#define I40E_GLGEN_CAR_DEBUG_CAR_PE_CLK_EN_MASK I40E_MASK(0x1, I40E_GLGEN_CAR_DEBUG_CAR_PE_CLK_EN_SHIFT) +#define I40E_GLGEN_CAR_DEBUG_CAR_PCIE_PRIM_CLK_ACTIVE_SHIFT 3 +#define I40E_GLGEN_CAR_DEBUG_CAR_PCIE_PRIM_CLK_ACTIVE_MASK I40E_MASK(0x1, I40E_GLGEN_CAR_DEBUG_CAR_PCIE_PRIM_CLK_ACTIVE_SHIFT) +#define I40E_GLGEN_CAR_DEBUG_CDC_PE_ACTIVE_SHIFT 4 +#define I40E_GLGEN_CAR_DEBUG_CDC_PE_ACTIVE_MASK I40E_MASK(0x1, I40E_GLGEN_CAR_DEBUG_CDC_PE_ACTIVE_SHIFT) +#define I40E_GLGEN_CAR_DEBUG_CAR_PCIE_RAW_PRST_RESET_N_SHIFT 5 +#define I40E_GLGEN_CAR_DEBUG_CAR_PCIE_RAW_PRST_RESET_N_MASK I40E_MASK(0x1, I40E_GLGEN_CAR_DEBUG_CAR_PCIE_RAW_PRST_RESET_N_SHIFT) +#define I40E_GLGEN_CAR_DEBUG_CAR_PCIE_RAW_SCLR_RESET_N_SHIFT 6 +#define I40E_GLGEN_CAR_DEBUG_CAR_PCIE_RAW_SCLR_RESET_N_MASK I40E_MASK(0x1, I40E_GLGEN_CAR_DEBUG_CAR_PCIE_RAW_SCLR_RESET_N_SHIFT) +#define I40E_GLGEN_CAR_DEBUG_CAR_PCIE_RAW_IB_RESET_N_SHIFT 7 +#define I40E_GLGEN_CAR_DEBUG_CAR_PCIE_RAW_IB_RESET_N_MASK I40E_MASK(0x1, I40E_GLGEN_CAR_DEBUG_CAR_PCIE_RAW_IB_RESET_N_SHIFT) +#define I40E_GLGEN_CAR_DEBUG_CAR_PCIE_RAW_IMIB_RESET_N_SHIFT 8 +#define I40E_GLGEN_CAR_DEBUG_CAR_PCIE_RAW_IMIB_RESET_N_MASK I40E_MASK(0x1, I40E_GLGEN_CAR_DEBUG_CAR_PCIE_RAW_IMIB_RESET_N_SHIFT) +#define I40E_GLGEN_CAR_DEBUG_CAR_RAW_EMP_RESET_N_SHIFT 9 +#define I40E_GLGEN_CAR_DEBUG_CAR_RAW_EMP_RESET_N_MASK I40E_MASK(0x1, I40E_GLGEN_CAR_DEBUG_CAR_RAW_EMP_RESET_N_SHIFT) +#define I40E_GLGEN_CAR_DEBUG_CAR_RAW_GLOBAL_RESET_N_SHIFT 10 +#define I40E_GLGEN_CAR_DEBUG_CAR_RAW_GLOBAL_RESET_N_MASK I40E_MASK(0x1, I40E_GLGEN_CAR_DEBUG_CAR_RAW_GLOBAL_RESET_N_SHIFT) +#define I40E_GLGEN_CAR_DEBUG_CAR_RAW_LAN_POWER_GOOD_SHIFT 11 +#define I40E_GLGEN_CAR_DEBUG_CAR_RAW_LAN_POWER_GOOD_MASK I40E_MASK(0x1, I40E_GLGEN_CAR_DEBUG_CAR_RAW_LAN_POWER_GOOD_SHIFT) +#define I40E_GLGEN_CAR_DEBUG_CDC_IOSF_PRIMERY_RST_B_SHIFT 12 +#define I40E_GLGEN_CAR_DEBUG_CDC_IOSF_PRIMERY_RST_B_MASK I40E_MASK(0x1, I40E_GLGEN_CAR_DEBUG_CDC_IOSF_PRIMERY_RST_B_SHIFT) +#define I40E_GLGEN_CAR_DEBUG_GBE_GLOBALRST_B_SHIFT 13 +#define I40E_GLGEN_CAR_DEBUG_GBE_GLOBALRST_B_MASK I40E_MASK(0x1, I40E_GLGEN_CAR_DEBUG_GBE_GLOBALRST_B_SHIFT) +#define I40E_GLGEN_CAR_DEBUG_FLEEP_AL_GLOBR_DONE_SHIFT 14 +#define I40E_GLGEN_CAR_DEBUG_FLEEP_AL_GLOBR_DONE_MASK I40E_MASK(0x1, I40E_GLGEN_CAR_DEBUG_FLEEP_AL_GLOBR_DONE_SHIFT) +#define I40E_GLGEN_MISC_SPARE 0x000880E0 /* Reset: POR */ +#define I40E_GLGEN_MISC_SPARE_GLGEN_MISC_SPARE_SHIFT 0 +#define I40E_GLGEN_MISC_SPARE_GLGEN_MISC_SPARE_MASK I40E_MASK(0xFFFFFFFF, I40E_GLGEN_MISC_SPARE_GLGEN_MISC_SPARE_SHIFT) +#define I40E_GL_UFUSE_SOC 0x000BE550 /* Reset: POR */ +#define I40E_GL_UFUSE_SOC_PORT_MODE_SHIFT 0 +#define I40E_GL_UFUSE_SOC_PORT_MODE_MASK I40E_MASK(0x3, I40E_GL_UFUSE_SOC_PORT_MODE_SHIFT) +#define I40E_GL_UFUSE_SOC_NIC_ID_SHIFT 2 +#define I40E_GL_UFUSE_SOC_NIC_ID_MASK I40E_MASK(0x1, I40E_GL_UFUSE_SOC_NIC_ID_SHIFT) +#define I40E_GL_UFUSE_SOC_SPARE_FUSES_SHIFT 3 +#define I40E_GL_UFUSE_SOC_SPARE_FUSES_MASK I40E_MASK(0x1FFF, I40E_GL_UFUSE_SOC_SPARE_FUSES_SHIFT) +#define I40E_PFINT_DYN_CTL0_WB_ON_ITR_SHIFT 30 +#define I40E_PFINT_DYN_CTL0_WB_ON_ITR_MASK I40E_MASK(0x1, I40E_PFINT_DYN_CTL0_WB_ON_ITR_SHIFT) +#define I40E_PFINT_DYN_CTLN_WB_ON_ITR_SHIFT 30 +#define I40E_PFINT_DYN_CTLN_WB_ON_ITR_MASK I40E_MASK(0x1, I40E_PFINT_DYN_CTLN_WB_ON_ITR_SHIFT) +#define I40E_VFINT_DYN_CTL0_WB_ON_ITR_SHIFT 30 +#define I40E_VFINT_DYN_CTL0_WB_ON_ITR_MASK I40E_MASK(0x1, I40E_VFINT_DYN_CTL0_WB_ON_ITR_SHIFT) +#define I40E_VFINT_DYN_CTLN_WB_ON_ITR_SHIFT 30 +#define I40E_VFINT_DYN_CTLN_WB_ON_ITR_MASK I40E_MASK(0x1, I40E_VFINT_DYN_CTLN_WB_ON_ITR_SHIFT) +#define I40E_VPLAN_QBASE(_VF) (0x00074800 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */ +#define I40E_VPLAN_QBASE_MAX_INDEX 127 +#define I40E_VPLAN_QBASE_VFFIRSTQ_SHIFT 0 +#define I40E_VPLAN_QBASE_VFFIRSTQ_MASK I40E_MASK(0x7FF, I40E_VPLAN_QBASE_VFFIRSTQ_SHIFT) +#define I40E_VPLAN_QBASE_VFNUMQ_SHIFT 11 +#define I40E_VPLAN_QBASE_VFNUMQ_MASK I40E_MASK(0xFF, I40E_VPLAN_QBASE_VFNUMQ_SHIFT) +#define I40E_VPLAN_QBASE_VFQTABLE_ENA_SHIFT 31 +#define I40E_VPLAN_QBASE_VFQTABLE_ENA_MASK I40E_MASK(0x1, I40E_VPLAN_QBASE_VFQTABLE_ENA_SHIFT) +#define I40E_PRTMAC_LINK_DOWN_COUNTER 0x001E2440 /* Reset: GLOBR */ +#define I40E_PRTMAC_LINK_DOWN_COUNTER_LINK_DOWN_COUNTER_SHIFT 0 +#define I40E_PRTMAC_LINK_DOWN_COUNTER_LINK_DOWN_COUNTER_MASK I40E_MASK(0xFFFF, I40E_PRTMAC_LINK_DOWN_COUNTER_LINK_DOWN_COUNTER_SHIFT) +#define I40E_GLNVM_AL_REQ 0x000B6164 /* Reset: POR */ +#define I40E_GLNVM_AL_REQ_POR_SHIFT 0 +#define I40E_GLNVM_AL_REQ_POR_MASK I40E_MASK(0x1, I40E_GLNVM_AL_REQ_POR_SHIFT) +#define I40E_GLNVM_AL_REQ_PCIE_IMIB_SHIFT 1 +#define I40E_GLNVM_AL_REQ_PCIE_IMIB_MASK I40E_MASK(0x1, I40E_GLNVM_AL_REQ_PCIE_IMIB_SHIFT) +#define I40E_GLNVM_AL_REQ_GLOBR_SHIFT 2 +#define I40E_GLNVM_AL_REQ_GLOBR_MASK I40E_MASK(0x1, I40E_GLNVM_AL_REQ_GLOBR_SHIFT) +#define I40E_GLNVM_AL_REQ_CORER_SHIFT 3 +#define I40E_GLNVM_AL_REQ_CORER_MASK I40E_MASK(0x1, I40E_GLNVM_AL_REQ_CORER_SHIFT) +#define I40E_GLNVM_AL_REQ_PE_SHIFT 4 +#define I40E_GLNVM_AL_REQ_PE_MASK I40E_MASK(0x1, I40E_GLNVM_AL_REQ_PE_SHIFT) +#define I40E_GLNVM_AL_REQ_PCIE_IMIB_ASSERT_SHIFT 5 +#define I40E_GLNVM_AL_REQ_PCIE_IMIB_ASSERT_MASK I40E_MASK(0x1, I40E_GLNVM_AL_REQ_PCIE_IMIB_ASSERT_SHIFT) +#define I40E_GLNVM_ALTIMERS 0x000B6140 /* Reset: POR */ +#define I40E_GLNVM_ALTIMERS_PCI_ALTIMER_SHIFT 0 +#define I40E_GLNVM_ALTIMERS_PCI_ALTIMER_MASK I40E_MASK(0xFFF, I40E_GLNVM_ALTIMERS_PCI_ALTIMER_SHIFT) +#define I40E_GLNVM_ALTIMERS_GEN_ALTIMER_SHIFT 12 +#define I40E_GLNVM_ALTIMERS_GEN_ALTIMER_MASK I40E_MASK(0xFFFFF, I40E_GLNVM_ALTIMERS_GEN_ALTIMER_SHIFT) +#define I40E_GLNVM_FLA 0x000B6108 /* Reset: POR */ +#define I40E_GLNVM_FLA_LOCKED_SHIFT 6 +#define I40E_GLNVM_FLA_LOCKED_MASK I40E_MASK(0x1, I40E_GLNVM_FLA_LOCKED_SHIFT) + +#define I40E_GLNVM_ULD 0x000B6008 /* Reset: POR */ +#define I40E_GLNVM_ULD_PCIER_DONE_SHIFT 0 +#define I40E_GLNVM_ULD_PCIER_DONE_MASK I40E_MASK(0x1, I40E_GLNVM_ULD_PCIER_DONE_SHIFT) +#define I40E_GLNVM_ULD_PCIER_DONE_1_SHIFT 1 +#define I40E_GLNVM_ULD_PCIER_DONE_1_MASK I40E_MASK(0x1, I40E_GLNVM_ULD_PCIER_DONE_1_SHIFT) +#define I40E_GLNVM_ULD_CORER_DONE_SHIFT 3 +#define I40E_GLNVM_ULD_CORER_DONE_MASK I40E_MASK(0x1, I40E_GLNVM_ULD_CORER_DONE_SHIFT) +#define I40E_GLNVM_ULD_GLOBR_DONE_SHIFT 4 +#define I40E_GLNVM_ULD_GLOBR_DONE_MASK I40E_MASK(0x1, I40E_GLNVM_ULD_GLOBR_DONE_SHIFT) +#define I40E_GLNVM_ULD_POR_DONE_SHIFT 5 +#define I40E_GLNVM_ULD_POR_DONE_MASK I40E_MASK(0x1, I40E_GLNVM_ULD_POR_DONE_SHIFT) +#define I40E_GLNVM_ULD_POR_DONE_1_SHIFT 8 +#define I40E_GLNVM_ULD_POR_DONE_1_MASK I40E_MASK(0x1, I40E_GLNVM_ULD_POR_DONE_1_SHIFT) +#define I40E_GLNVM_ULD_PCIER_DONE_2_SHIFT 9 +#define I40E_GLNVM_ULD_PCIER_DONE_2_MASK I40E_MASK(0x1, I40E_GLNVM_ULD_PCIER_DONE_2_SHIFT) +#define I40E_GLNVM_ULD_PE_DONE_SHIFT 10 +#define I40E_GLNVM_ULD_PE_DONE_MASK I40E_MASK(0x1, I40E_GLNVM_ULD_PE_DONE_SHIFT) +#define I40E_GLNVM_ULT 0x000B6154 /* Reset: POR */ +#define I40E_GLNVM_ULT_CONF_PCIR_AE_SHIFT 0 +#define I40E_GLNVM_ULT_CONF_PCIR_AE_MASK I40E_MASK(0x1, I40E_GLNVM_ULT_CONF_PCIR_AE_SHIFT) +#define I40E_GLNVM_ULT_CONF_PCIRTL_AE_SHIFT 1 +#define I40E_GLNVM_ULT_CONF_PCIRTL_AE_MASK I40E_MASK(0x1, I40E_GLNVM_ULT_CONF_PCIRTL_AE_SHIFT) +#define I40E_GLNVM_ULT_RESERVED_1_SHIFT 2 +#define I40E_GLNVM_ULT_RESERVED_1_MASK I40E_MASK(0x1, I40E_GLNVM_ULT_RESERVED_1_SHIFT) +#define I40E_GLNVM_ULT_CONF_CORE_AE_SHIFT 3 +#define I40E_GLNVM_ULT_CONF_CORE_AE_MASK I40E_MASK(0x1, I40E_GLNVM_ULT_CONF_CORE_AE_SHIFT) +#define I40E_GLNVM_ULT_CONF_GLOBAL_AE_SHIFT 4 +#define I40E_GLNVM_ULT_CONF_GLOBAL_AE_MASK I40E_MASK(0x1, I40E_GLNVM_ULT_CONF_GLOBAL_AE_SHIFT) +#define I40E_GLNVM_ULT_CONF_POR_AE_SHIFT 5 +#define I40E_GLNVM_ULT_CONF_POR_AE_MASK I40E_MASK(0x1, I40E_GLNVM_ULT_CONF_POR_AE_SHIFT) +#define I40E_GLNVM_ULT_RESERVED_2_SHIFT 6 +#define I40E_GLNVM_ULT_RESERVED_2_MASK I40E_MASK(0x1, I40E_GLNVM_ULT_RESERVED_2_SHIFT) +#define I40E_GLNVM_ULT_RESERVED_3_SHIFT 7 +#define I40E_GLNVM_ULT_RESERVED_3_MASK I40E_MASK(0x1, I40E_GLNVM_ULT_RESERVED_3_SHIFT) +#define I40E_GLNVM_ULT_CONF_EMP_AE_SHIFT 8 +#define I40E_GLNVM_ULT_CONF_EMP_AE_MASK I40E_MASK(0x1, I40E_GLNVM_ULT_CONF_EMP_AE_SHIFT) +#define I40E_GLNVM_ULT_CONF_PCIALT_AE_SHIFT 9 +#define I40E_GLNVM_ULT_CONF_PCIALT_AE_MASK I40E_MASK(0x1, I40E_GLNVM_ULT_CONF_PCIALT_AE_SHIFT) +#define I40E_GLNVM_ULT_RESERVED_4_SHIFT 10 +#define I40E_GLNVM_ULT_RESERVED_4_MASK I40E_MASK(0x3FFFFF, I40E_GLNVM_ULT_RESERVED_4_SHIFT) +#define I40E_MEM_INIT_DONE_STAT 0x000B615C /* Reset: POR */ +#define I40E_MEM_INIT_DONE_STAT_CMLAN_MEM_INIT_DONE_SHIFT 0 +#define I40E_MEM_INIT_DONE_STAT_CMLAN_MEM_INIT_DONE_MASK I40E_MASK(0x1, I40E_MEM_INIT_DONE_STAT_CMLAN_MEM_INIT_DONE_SHIFT) +#define I40E_MEM_INIT_DONE_STAT_PMAT_MEM_INIT_DONE_SHIFT 1 +#define I40E_MEM_INIT_DONE_STAT_PMAT_MEM_INIT_DONE_MASK I40E_MASK(0x1, I40E_MEM_INIT_DONE_STAT_PMAT_MEM_INIT_DONE_SHIFT) +#define I40E_MEM_INIT_DONE_STAT_RCU_MEM_INIT_DONE_SHIFT 2 +#define I40E_MEM_INIT_DONE_STAT_RCU_MEM_INIT_DONE_MASK I40E_MASK(0x1, I40E_MEM_INIT_DONE_STAT_RCU_MEM_INIT_DONE_SHIFT) +#define I40E_MEM_INIT_DONE_STAT_TDPU_MEM_INIT_DONE_SHIFT 3 +#define I40E_MEM_INIT_DONE_STAT_TDPU_MEM_INIT_DONE_MASK I40E_MASK(0x1, I40E_MEM_INIT_DONE_STAT_TDPU_MEM_INIT_DONE_SHIFT) +#define I40E_MEM_INIT_DONE_STAT_TLAN_MEM_INIT_DONE_SHIFT 4 +#define I40E_MEM_INIT_DONE_STAT_TLAN_MEM_INIT_DONE_MASK I40E_MASK(0x1, I40E_MEM_INIT_DONE_STAT_TLAN_MEM_INIT_DONE_SHIFT) +#define I40E_MEM_INIT_DONE_STAT_RLAN_MEM_INIT_DONE_SHIFT 5 +#define I40E_MEM_INIT_DONE_STAT_RLAN_MEM_INIT_DONE_MASK I40E_MASK(0x1, I40E_MEM_INIT_DONE_STAT_RLAN_MEM_INIT_DONE_SHIFT) +#define I40E_MEM_INIT_DONE_STAT_RDPU_MEM_INIT_DONE_SHIFT 6 +#define I40E_MEM_INIT_DONE_STAT_RDPU_MEM_INIT_DONE_MASK I40E_MASK(0x1, I40E_MEM_INIT_DONE_STAT_RDPU_MEM_INIT_DONE_SHIFT) +#define I40E_MEM_INIT_DONE_STAT_PPRS_MEM_INIT_DONE_SHIFT 7 +#define I40E_MEM_INIT_DONE_STAT_PPRS_MEM_INIT_DONE_MASK I40E_MASK(0x1, I40E_MEM_INIT_DONE_STAT_PPRS_MEM_INIT_DONE_SHIFT) +#define I40E_MEM_INIT_DONE_STAT_RPB_MEM_INIT_DONE_SHIFT 8 +#define I40E_MEM_INIT_DONE_STAT_RPB_MEM_INIT_DONE_MASK I40E_MASK(0x1, I40E_MEM_INIT_DONE_STAT_RPB_MEM_INIT_DONE_SHIFT) +#define I40E_MEM_INIT_DONE_STAT_TPB_MEM_INIT_DONE_SHIFT 9 +#define I40E_MEM_INIT_DONE_STAT_TPB_MEM_INIT_DONE_MASK I40E_MASK(0x1, I40E_MEM_INIT_DONE_STAT_TPB_MEM_INIT_DONE_SHIFT) +#define I40E_MEM_INIT_DONE_STAT_FOC_MEM_INIT_DONE_SHIFT 10 +#define I40E_MEM_INIT_DONE_STAT_FOC_MEM_INIT_DONE_MASK I40E_MASK(0x1, I40E_MEM_INIT_DONE_STAT_FOC_MEM_INIT_DONE_SHIFT) +#define I40E_MEM_INIT_DONE_STAT_TSCD_MEM_INIT_DONE_SHIFT 11 +#define I40E_MEM_INIT_DONE_STAT_TSCD_MEM_INIT_DONE_MASK I40E_MASK(0x1, I40E_MEM_INIT_DONE_STAT_TSCD_MEM_INIT_DONE_SHIFT) +#define I40E_MEM_INIT_DONE_STAT_TCB_MEM_INIT_DONE_SHIFT 12 +#define I40E_MEM_INIT_DONE_STAT_TCB_MEM_INIT_DONE_MASK I40E_MASK(0x1, I40E_MEM_INIT_DONE_STAT_TCB_MEM_INIT_DONE_SHIFT) +#define I40E_MEM_INIT_DONE_STAT_RCB_MEM_INIT_DONE_SHIFT 13 +#define I40E_MEM_INIT_DONE_STAT_RCB_MEM_INIT_DONE_MASK I40E_MASK(0x1, I40E_MEM_INIT_DONE_STAT_RCB_MEM_INIT_DONE_SHIFT) +#define I40E_MEM_INIT_DONE_STAT_WUC_MEM_INIT_DONE_SHIFT 14 +#define I40E_MEM_INIT_DONE_STAT_WUC_MEM_INIT_DONE_MASK I40E_MASK(0x1, I40E_MEM_INIT_DONE_STAT_WUC_MEM_INIT_DONE_SHIFT) +#define I40E_MEM_INIT_DONE_STAT_STAT_MEM_INIT_DONE_SHIFT 15 +#define I40E_MEM_INIT_DONE_STAT_STAT_MEM_INIT_DONE_MASK I40E_MASK(0x1, I40E_MEM_INIT_DONE_STAT_STAT_MEM_INIT_DONE_SHIFT) +#define I40E_MEM_INIT_DONE_STAT_ITR_MEM_INIT_DONE_SHIFT 16 +#define I40E_MEM_INIT_DONE_STAT_ITR_MEM_INIT_DONE_MASK I40E_MASK(0x1, I40E_MEM_INIT_DONE_STAT_ITR_MEM_INIT_DONE_SHIFT) +#define I40E_MNGSB_DADD 0x000B7030 /* Reset: POR */ +#define I40E_MNGSB_DADD_ADDR_SHIFT 0 +#define I40E_MNGSB_DADD_ADDR_MASK I40E_MASK(0xFFFFFFFF, I40E_MNGSB_DADD_ADDR_SHIFT) +#define I40E_MNGSB_DCNT 0x000B7034 /* Reset: POR */ +#define I40E_MNGSB_DCNT_BYTE_CNT_SHIFT 0 +#define I40E_MNGSB_DCNT_BYTE_CNT_MASK I40E_MASK(0xFFFFFFFF, I40E_MNGSB_DCNT_BYTE_CNT_SHIFT) +#define I40E_MNGSB_MSGCTL 0x000B7020 /* Reset: POR */ +#define I40E_MNGSB_MSGCTL_HDR_DWS_SHIFT 0 +#define I40E_MNGSB_MSGCTL_HDR_DWS_MASK I40E_MASK(0x3, I40E_MNGSB_MSGCTL_HDR_DWS_SHIFT) +#define I40E_MNGSB_MSGCTL_EXP_RDW_SHIFT 8 +#define I40E_MNGSB_MSGCTL_EXP_RDW_MASK I40E_MASK(0x1FF, I40E_MNGSB_MSGCTL_EXP_RDW_SHIFT) +#define I40E_MNGSB_MSGCTL_MSG_MODE_SHIFT 26 +#define I40E_MNGSB_MSGCTL_MSG_MODE_MASK I40E_MASK(0x3, I40E_MNGSB_MSGCTL_MSG_MODE_SHIFT) +#define I40E_MNGSB_MSGCTL_TOKEN_MODE_SHIFT 28 +#define I40E_MNGSB_MSGCTL_TOKEN_MODE_MASK I40E_MASK(0x3, I40E_MNGSB_MSGCTL_TOKEN_MODE_SHIFT) +#define I40E_MNGSB_MSGCTL_BARCLR_SHIFT 30 +#define I40E_MNGSB_MSGCTL_BARCLR_MASK I40E_MASK(0x1, I40E_MNGSB_MSGCTL_BARCLR_SHIFT) +#define I40E_MNGSB_MSGCTL_CMDV_SHIFT 31 +#define I40E_MNGSB_MSGCTL_CMDV_MASK I40E_MASK(0x1, I40E_MNGSB_MSGCTL_CMDV_SHIFT) +#define I40E_MNGSB_RDATA 0x000B7300 /* Reset: POR */ +#define I40E_MNGSB_RDATA_DATA_SHIFT 0 +#define I40E_MNGSB_RDATA_DATA_MASK I40E_MASK(0xFFFFFFFF, I40E_MNGSB_RDATA_DATA_SHIFT) +#define I40E_MNGSB_RHDR0 0x000B72FC /* Reset: POR */ +#define I40E_MNGSB_RHDR0_DESTINATION_SHIFT 0 +#define I40E_MNGSB_RHDR0_DESTINATION_MASK I40E_MASK(0xFF, I40E_MNGSB_RHDR0_DESTINATION_SHIFT) +#define I40E_MNGSB_RHDR0_SOURCE_SHIFT 8 +#define I40E_MNGSB_RHDR0_SOURCE_MASK I40E_MASK(0xFF, I40E_MNGSB_RHDR0_SOURCE_SHIFT) +#define I40E_MNGSB_RHDR0_OPCODE_SHIFT 16 +#define I40E_MNGSB_RHDR0_OPCODE_MASK I40E_MASK(0xFF, I40E_MNGSB_RHDR0_OPCODE_SHIFT) +#define I40E_MNGSB_RHDR0_TAG_SHIFT 24 +#define I40E_MNGSB_RHDR0_TAG_MASK I40E_MASK(0x7, I40E_MNGSB_RHDR0_TAG_SHIFT) +#define I40E_MNGSB_RHDR0_RESPONSE_SHIFT 27 +#define I40E_MNGSB_RHDR0_RESPONSE_MASK I40E_MASK(0x7, I40E_MNGSB_RHDR0_RESPONSE_SHIFT) +#define I40E_MNGSB_RHDR0_EH_SHIFT 31 +#define I40E_MNGSB_RHDR0_EH_MASK I40E_MASK(0x1, I40E_MNGSB_RHDR0_EH_SHIFT) +#define I40E_MNGSB_RSPCTL 0x000B7024 /* Reset: POR */ +#define I40E_MNGSB_RSPCTL_DMA_MSG_DWORDS_SHIFT 0 +#define I40E_MNGSB_RSPCTL_DMA_MSG_DWORDS_MASK I40E_MASK(0x1FF, I40E_MNGSB_RSPCTL_DMA_MSG_DWORDS_SHIFT) +#define I40E_MNGSB_RSPCTL_RSP_MODE_SHIFT 26 +#define I40E_MNGSB_RSPCTL_RSP_MODE_MASK I40E_MASK(0x3, I40E_MNGSB_RSPCTL_RSP_MODE_SHIFT) +#define I40E_MNGSB_RSPCTL_RSP_BAD_LEN_SHIFT 30 +#define I40E_MNGSB_RSPCTL_RSP_BAD_LEN_MASK I40E_MASK(0x1, I40E_MNGSB_RSPCTL_RSP_BAD_LEN_SHIFT) +#define I40E_MNGSB_RSPCTL_RSP_ERR_SHIFT 31 +#define I40E_MNGSB_RSPCTL_RSP_ERR_MASK I40E_MASK(0x1, I40E_MNGSB_RSPCTL_RSP_ERR_SHIFT) +#define I40E_MNGSB_WDATA 0x000B7100 /* Reset: POR */ +#define I40E_MNGSB_WDATA_DATA_SHIFT 0 +#define I40E_MNGSB_WDATA_DATA_MASK I40E_MASK(0xFFFFFFFF, I40E_MNGSB_WDATA_DATA_SHIFT) +#define I40E_MNGSB_WHDR0 0x000B70F4 /* Reset: POR */ +#define I40E_MNGSB_WHDR0_RAW_DEST_SHIFT 0 +#define I40E_MNGSB_WHDR0_RAW_DEST_MASK I40E_MASK(0xFF, I40E_MNGSB_WHDR0_RAW_DEST_SHIFT) +#define I40E_MNGSB_WHDR0_DEST_SEL_SHIFT 12 +#define I40E_MNGSB_WHDR0_DEST_SEL_MASK I40E_MASK(0xF, I40E_MNGSB_WHDR0_DEST_SEL_SHIFT) +#define I40E_MNGSB_WHDR0_OPCODE_SEL_SHIFT 16 +#define I40E_MNGSB_WHDR0_OPCODE_SEL_MASK I40E_MASK(0xFF, I40E_MNGSB_WHDR0_OPCODE_SEL_SHIFT) +#define I40E_MNGSB_WHDR0_TAG_SHIFT 24 +#define I40E_MNGSB_WHDR0_TAG_MASK I40E_MASK(0x7F, I40E_MNGSB_WHDR0_TAG_SHIFT) +#define I40E_MNGSB_WHDR1 0x000B70F8 /* Reset: POR */ +#define I40E_MNGSB_WHDR1_ADDR_SHIFT 0 +#define I40E_MNGSB_WHDR1_ADDR_MASK I40E_MASK(0xFFFFFFFF, I40E_MNGSB_WHDR1_ADDR_SHIFT) +#define I40E_MNGSB_WHDR2 0x000B70FC /* Reset: POR */ +#define I40E_MNGSB_WHDR2_LENGTH_SHIFT 0 +#define I40E_MNGSB_WHDR2_LENGTH_MASK I40E_MASK(0xFFFFFFFF, I40E_MNGSB_WHDR2_LENGTH_SHIFT) + +#define I40E_GLPCI_CAPSUP_WAKUP_EN_SHIFT 21 +#define I40E_GLPCI_CAPSUP_WAKUP_EN_MASK I40E_MASK(0x1, I40E_GLPCI_CAPSUP_WAKUP_EN_SHIFT) + +#define I40E_GLPCI_CUR_CLNT_COMMON 0x0009CA18 /* Reset: PCIR */ +#define I40E_GLPCI_CUR_CLNT_COMMON_DATA_LINES_SHIFT 0 +#define I40E_GLPCI_CUR_CLNT_COMMON_DATA_LINES_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_CLNT_COMMON_DATA_LINES_SHIFT) +#define I40E_GLPCI_CUR_CLNT_COMMON_OSR_SHIFT 16 +#define I40E_GLPCI_CUR_CLNT_COMMON_OSR_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_CLNT_COMMON_OSR_SHIFT) +#define I40E_GLPCI_CUR_CLNT_PIPEMON 0x0009CA20 /* Reset: PCIR */ +#define I40E_GLPCI_CUR_CLNT_PIPEMON_DATA_LINES_SHIFT 0 +#define I40E_GLPCI_CUR_CLNT_PIPEMON_DATA_LINES_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_CLNT_PIPEMON_DATA_LINES_SHIFT) +#define I40E_GLPCI_CUR_MNG_ALWD 0x0009c514 /* Reset: PCIR */ +#define I40E_GLPCI_CUR_MNG_ALWD_DATA_LINES_SHIFT 0 +#define I40E_GLPCI_CUR_MNG_ALWD_DATA_LINES_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_MNG_ALWD_DATA_LINES_SHIFT) +#define I40E_GLPCI_CUR_MNG_ALWD_OSR_SHIFT 16 +#define I40E_GLPCI_CUR_MNG_ALWD_OSR_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_MNG_ALWD_OSR_SHIFT) +#define I40E_GLPCI_CUR_MNG_RSVD 0x0009c594 /* Reset: PCIR */ +#define I40E_GLPCI_CUR_MNG_RSVD_DATA_LINES_SHIFT 0 +#define I40E_GLPCI_CUR_MNG_RSVD_DATA_LINES_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_MNG_RSVD_DATA_LINES_SHIFT) +#define I40E_GLPCI_CUR_MNG_RSVD_OSR_SHIFT 16 +#define I40E_GLPCI_CUR_MNG_RSVD_OSR_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_MNG_RSVD_OSR_SHIFT) +#define I40E_GLPCI_CUR_PMAT_ALWD 0x0009c510 /* Reset: PCIR */ +#define I40E_GLPCI_CUR_PMAT_ALWD_DATA_LINES_SHIFT 0 +#define I40E_GLPCI_CUR_PMAT_ALWD_DATA_LINES_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_PMAT_ALWD_DATA_LINES_SHIFT) +#define I40E_GLPCI_CUR_PMAT_ALWD_OSR_SHIFT 16 +#define I40E_GLPCI_CUR_PMAT_ALWD_OSR_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_PMAT_ALWD_OSR_SHIFT) +#define I40E_GLPCI_CUR_PMAT_RSVD 0x0009c590 /* Reset: PCIR */ +#define I40E_GLPCI_CUR_PMAT_RSVD_DATA_LINES_SHIFT 0 +#define I40E_GLPCI_CUR_PMAT_RSVD_DATA_LINES_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_PMAT_RSVD_DATA_LINES_SHIFT) +#define I40E_GLPCI_CUR_PMAT_RSVD_OSR_SHIFT 16 +#define I40E_GLPCI_CUR_PMAT_RSVD_OSR_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_PMAT_RSVD_OSR_SHIFT) +#define I40E_GLPCI_CUR_RLAN_ALWD 0x0009c500 /* Reset: PCIR */ +#define I40E_GLPCI_CUR_RLAN_ALWD_DATA_LINES_SHIFT 0 +#define I40E_GLPCI_CUR_RLAN_ALWD_DATA_LINES_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_RLAN_ALWD_DATA_LINES_SHIFT) +#define I40E_GLPCI_CUR_RLAN_ALWD_OSR_SHIFT 16 +#define I40E_GLPCI_CUR_RLAN_ALWD_OSR_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_RLAN_ALWD_OSR_SHIFT) +#define I40E_GLPCI_CUR_RLAN_RSVD 0x0009c580 /* Reset: PCIR */ +#define I40E_GLPCI_CUR_RLAN_RSVD_DATA_LINES_SHIFT 0 +#define I40E_GLPCI_CUR_RLAN_RSVD_DATA_LINES_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_RLAN_RSVD_DATA_LINES_SHIFT) +#define I40E_GLPCI_CUR_RLAN_RSVD_OSR_SHIFT 16 +#define I40E_GLPCI_CUR_RLAN_RSVD_OSR_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_RLAN_RSVD_OSR_SHIFT) +#define I40E_GLPCI_CUR_RXPE_ALWD 0x0009c508 /* Reset: PCIR */ +#define I40E_GLPCI_CUR_RXPE_ALWD_DATA_LINES_SHIFT 0 +#define I40E_GLPCI_CUR_RXPE_ALWD_DATA_LINES_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_RXPE_ALWD_DATA_LINES_SHIFT) +#define I40E_GLPCI_CUR_RXPE_ALWD_OSR_SHIFT 16 +#define I40E_GLPCI_CUR_RXPE_ALWD_OSR_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_RXPE_ALWD_OSR_SHIFT) +#define I40E_GLPCI_CUR_RXPE_RSVD 0x0009c588 /* Reset: PCIR */ +#define I40E_GLPCI_CUR_RXPE_RSVD_DATA_LINES_SHIFT 0 +#define I40E_GLPCI_CUR_RXPE_RSVD_DATA_LINES_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_RXPE_RSVD_DATA_LINES_SHIFT) +#define I40E_GLPCI_CUR_RXPE_RSVD_OSR_SHIFT 16 +#define I40E_GLPCI_CUR_RXPE_RSVD_OSR_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_RXPE_RSVD_OSR_SHIFT) +#define I40E_GLPCI_CUR_TDPU_ALWD 0x0009c518 /* Reset: PCIR */ +#define I40E_GLPCI_CUR_TDPU_ALWD_DATA_LINES_SHIFT 0 +#define I40E_GLPCI_CUR_TDPU_ALWD_DATA_LINES_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_TDPU_ALWD_DATA_LINES_SHIFT) +#define I40E_GLPCI_CUR_TDPU_ALWD_OSR_SHIFT 16 +#define I40E_GLPCI_CUR_TDPU_ALWD_OSR_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_TDPU_ALWD_OSR_SHIFT) +#define I40E_GLPCI_CUR_TDPU_RSVD 0x0009c598 /* Reset: PCIR */ +#define I40E_GLPCI_CUR_TDPU_RSVD_DATA_LINES_SHIFT 0 +#define I40E_GLPCI_CUR_TDPU_RSVD_DATA_LINES_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_TDPU_RSVD_DATA_LINES_SHIFT) +#define I40E_GLPCI_CUR_TDPU_RSVD_OSR_SHIFT 16 +#define I40E_GLPCI_CUR_TDPU_RSVD_OSR_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_TDPU_RSVD_OSR_SHIFT) +#define I40E_GLPCI_CUR_TLAN_ALWD 0x0009c504 /* Reset: PCIR */ +#define I40E_GLPCI_CUR_TLAN_ALWD_DATA_LINES_SHIFT 0 +#define I40E_GLPCI_CUR_TLAN_ALWD_DATA_LINES_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_TLAN_ALWD_DATA_LINES_SHIFT) +#define I40E_GLPCI_CUR_TLAN_ALWD_OSR_SHIFT 16 +#define I40E_GLPCI_CUR_TLAN_ALWD_OSR_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_TLAN_ALWD_OSR_SHIFT) +#define I40E_GLPCI_CUR_TLAN_RSVD 0x0009c584 /* Reset: PCIR */ +#define I40E_GLPCI_CUR_TLAN_RSVD_DATA_LINES_SHIFT 0 +#define I40E_GLPCI_CUR_TLAN_RSVD_DATA_LINES_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_TLAN_RSVD_DATA_LINES_SHIFT) +#define I40E_GLPCI_CUR_TLAN_RSVD_OSR_SHIFT 16 +#define I40E_GLPCI_CUR_TLAN_RSVD_OSR_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_TLAN_RSVD_OSR_SHIFT) +#define I40E_GLPCI_CUR_TXPE_ALWD 0x0009c50C /* Reset: PCIR */ +#define I40E_GLPCI_CUR_TXPE_ALWD_DATA_LINES_SHIFT 0 +#define I40E_GLPCI_CUR_TXPE_ALWD_DATA_LINES_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_TXPE_ALWD_DATA_LINES_SHIFT) +#define I40E_GLPCI_CUR_TXPE_ALWD_OSR_SHIFT 16 +#define I40E_GLPCI_CUR_TXPE_ALWD_OSR_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_TXPE_ALWD_OSR_SHIFT) +#define I40E_GLPCI_CUR_TXPE_RSVD 0x0009c58c /* Reset: PCIR */ +#define I40E_GLPCI_CUR_TXPE_RSVD_DATA_LINES_SHIFT 0 +#define I40E_GLPCI_CUR_TXPE_RSVD_DATA_LINES_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_TXPE_RSVD_DATA_LINES_SHIFT) +#define I40E_GLPCI_CUR_TXPE_RSVD_OSR_SHIFT 16 +#define I40E_GLPCI_CUR_TXPE_RSVD_OSR_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_TXPE_RSVD_OSR_SHIFT) +#define I40E_GLPCI_CUR_WATMK_CLNT_COMMON 0x0009CA28 /* Reset: PCIR */ +#define I40E_GLPCI_CUR_WATMK_CLNT_COMMON_DATA_LINES_SHIFT 0 +#define I40E_GLPCI_CUR_WATMK_CLNT_COMMON_DATA_LINES_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_WATMK_CLNT_COMMON_DATA_LINES_SHIFT) +#define I40E_GLPCI_CUR_WATMK_CLNT_COMMON_OSR_SHIFT 16 +#define I40E_GLPCI_CUR_WATMK_CLNT_COMMON_OSR_MASK I40E_MASK(0xFFFF, I40E_GLPCI_CUR_WATMK_CLNT_COMMON_OSR_SHIFT) + +#define I40E_GLPCI_LBARCTRL_PE_DB_SIZE_SHIFT 4 +#define I40E_GLPCI_LBARCTRL_PE_DB_SIZE_MASK I40E_MASK(0x3, I40E_GLPCI_LBARCTRL_PE_DB_SIZE_SHIFT) +#define I40E_GLPCI_LBARCTRL_VF_PE_DB_SIZE_SHIFT 10 +#define I40E_GLPCI_LBARCTRL_VF_PE_DB_SIZE_MASK I40E_MASK(0x1, I40E_GLPCI_LBARCTRL_VF_PE_DB_SIZE_SHIFT) +#define I40E_GLPCI_NPQ_CFG 0x0009CA00 /* Reset: PCIR */ +#define I40E_GLPCI_NPQ_CFG_EXTEND_TO_SHIFT 0 +#define I40E_GLPCI_NPQ_CFG_EXTEND_TO_MASK I40E_MASK(0x1, I40E_GLPCI_NPQ_CFG_EXTEND_TO_SHIFT) +#define I40E_GLPCI_NPQ_CFG_SMALL_TO_SHIFT 1 +#define I40E_GLPCI_NPQ_CFG_SMALL_TO_MASK I40E_MASK(0x1, I40E_GLPCI_NPQ_CFG_SMALL_TO_SHIFT) +#define I40E_GLPCI_NPQ_CFG_WEIGHT_AVG_SHIFT 2 +#define I40E_GLPCI_NPQ_CFG_WEIGHT_AVG_MASK I40E_MASK(0xF, I40E_GLPCI_NPQ_CFG_WEIGHT_AVG_SHIFT) +#define I40E_GLPCI_NPQ_CFG_NPQ_SPARE_SHIFT 6 +#define I40E_GLPCI_NPQ_CFG_NPQ_SPARE_MASK I40E_MASK(0x3FF, I40E_GLPCI_NPQ_CFG_NPQ_SPARE_SHIFT) +#define I40E_GLPCI_NPQ_CFG_NPQ_ERR_STAT_SHIFT 16 +#define I40E_GLPCI_NPQ_CFG_NPQ_ERR_STAT_MASK I40E_MASK(0xF, I40E_GLPCI_NPQ_CFG_NPQ_ERR_STAT_SHIFT) +#define I40E_GLPCI_WATMK_CLNT_PIPEMON 0x0009CA30 /* Reset: PCIR */ +#define I40E_GLPCI_WATMK_CLNT_PIPEMON_DATA_LINES_SHIFT 0 +#define I40E_GLPCI_WATMK_CLNT_PIPEMON_DATA_LINES_MASK I40E_MASK(0xFFFF, I40E_GLPCI_WATMK_CLNT_PIPEMON_DATA_LINES_SHIFT) +#define I40E_GLPCI_WATMK_MNG_ALWD 0x0009CB14 /* Reset: PCIR */ +#define I40E_GLPCI_WATMK_MNG_ALWD_DATA_LINES_SHIFT 0 +#define I40E_GLPCI_WATMK_MNG_ALWD_DATA_LINES_MASK I40E_MASK(0xFFFF, I40E_GLPCI_WATMK_MNG_ALWD_DATA_LINES_SHIFT) +#define I40E_GLPCI_WATMK_MNG_ALWD_OSR_SHIFT 16 +#define I40E_GLPCI_WATMK_MNG_ALWD_OSR_MASK I40E_MASK(0xFFFF, I40E_GLPCI_WATMK_MNG_ALWD_OSR_SHIFT) +#define I40E_GLPCI_WATMK_PMAT_ALWD 0x0009CB10 /* Reset: PCIR */ +#define I40E_GLPCI_WATMK_PMAT_ALWD_DATA_LINES_SHIFT 0 +#define I40E_GLPCI_WATMK_PMAT_ALWD_DATA_LINES_MASK I40E_MASK(0xFFFF, I40E_GLPCI_WATMK_PMAT_ALWD_DATA_LINES_SHIFT) +#define I40E_GLPCI_WATMK_PMAT_ALWD_OSR_SHIFT 16 +#define I40E_GLPCI_WATMK_PMAT_ALWD_OSR_MASK I40E_MASK(0xFFFF, I40E_GLPCI_WATMK_PMAT_ALWD_OSR_SHIFT) +#define I40E_GLPCI_WATMK_RLAN_ALWD 0x0009CB00 /* Reset: PCIR */ +#define I40E_GLPCI_WATMK_RLAN_ALWD_DATA_LINES_SHIFT 0 +#define I40E_GLPCI_WATMK_RLAN_ALWD_DATA_LINES_MASK I40E_MASK(0xFFFF, I40E_GLPCI_WATMK_RLAN_ALWD_DATA_LINES_SHIFT) +#define I40E_GLPCI_WATMK_RLAN_ALWD_OSR_SHIFT 16 +#define I40E_GLPCI_WATMK_RLAN_ALWD_OSR_MASK I40E_MASK(0xFFFF, I40E_GLPCI_WATMK_RLAN_ALWD_OSR_SHIFT) +#define I40E_GLPCI_WATMK_RXPE_ALWD 0x0009CB08 /* Reset: PCIR */ +#define I40E_GLPCI_WATMK_RXPE_ALWD_DATA_LINES_SHIFT 0 +#define I40E_GLPCI_WATMK_RXPE_ALWD_DATA_LINES_MASK I40E_MASK(0xFFFF, I40E_GLPCI_WATMK_RXPE_ALWD_DATA_LINES_SHIFT) +#define I40E_GLPCI_WATMK_RXPE_ALWD_OSR_SHIFT 16 +#define I40E_GLPCI_WATMK_RXPE_ALWD_OSR_MASK I40E_MASK(0xFFFF, I40E_GLPCI_WATMK_RXPE_ALWD_OSR_SHIFT) +#define I40E_GLPCI_WATMK_TLAN_ALWD 0x0009CB04 /* Reset: PCIR */ +#define I40E_GLPCI_WATMK_TLAN_ALWD_DATA_LINES_SHIFT 0 +#define I40E_GLPCI_WATMK_TLAN_ALWD_DATA_LINES_MASK I40E_MASK(0xFFFF, I40E_GLPCI_WATMK_TLAN_ALWD_DATA_LINES_SHIFT) +#define I40E_GLPCI_WATMK_TLAN_ALWD_OSR_SHIFT 16 +#define I40E_GLPCI_WATMK_TLAN_ALWD_OSR_MASK I40E_MASK(0xFFFF, I40E_GLPCI_WATMK_TLAN_ALWD_OSR_SHIFT) +#define I40E_GLPCI_WATMK_TPDU_ALWD 0x0009CB18 /* Reset: PCIR */ +#define I40E_GLPCI_WATMK_TPDU_ALWD_DATA_LINES_SHIFT 0 +#define I40E_GLPCI_WATMK_TPDU_ALWD_DATA_LINES_MASK I40E_MASK(0xFFFF, I40E_GLPCI_WATMK_TPDU_ALWD_DATA_LINES_SHIFT) +#define I40E_GLPCI_WATMK_TPDU_ALWD_OSR_SHIFT 16 +#define I40E_GLPCI_WATMK_TPDU_ALWD_OSR_MASK I40E_MASK(0xFFFF, I40E_GLPCI_WATMK_TPDU_ALWD_OSR_SHIFT) +#define I40E_GLPCI_WATMK_TXPE_ALWD 0x0009CB0c /* Reset: PCIR */ +#define I40E_GLPCI_WATMK_TXPE_ALWD_DATA_LINES_SHIFT 0 +#define I40E_GLPCI_WATMK_TXPE_ALWD_DATA_LINES_MASK I40E_MASK(0xFFFF, I40E_GLPCI_WATMK_TXPE_ALWD_DATA_LINES_SHIFT) +#define I40E_GLPCI_WATMK_TXPE_ALWD_OSR_SHIFT 16 +#define I40E_GLPCI_WATMK_TXPE_ALWD_OSR_MASK I40E_MASK(0xFFFF, I40E_GLPCI_WATMK_TXPE_ALWD_OSR_SHIFT) +#define I40E_GLPE_CPUSTATUS0 0x0000D040 /* Reset: PE_CORER */ +#define I40E_GLPE_CPUSTATUS0_PECPUSTATUS0_SHIFT 0 +#define I40E_GLPE_CPUSTATUS0_PECPUSTATUS0_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPE_CPUSTATUS0_PECPUSTATUS0_SHIFT) +#define I40E_GLPE_CPUSTATUS1 0x0000D044 /* Reset: PE_CORER */ +#define I40E_GLPE_CPUSTATUS1_PECPUSTATUS1_SHIFT 0 +#define I40E_GLPE_CPUSTATUS1_PECPUSTATUS1_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPE_CPUSTATUS1_PECPUSTATUS1_SHIFT) +#define I40E_GLPE_CPUSTATUS2 0x0000D048 /* Reset: PE_CORER */ +#define I40E_GLPE_CPUSTATUS2_PECPUSTATUS2_SHIFT 0 +#define I40E_GLPE_CPUSTATUS2_PECPUSTATUS2_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPE_CPUSTATUS2_PECPUSTATUS2_SHIFT) +#define I40E_GLPE_CPUTRIG0 0x0000D060 /* Reset: PE_CORER */ +#define I40E_GLPE_CPUTRIG0_PECPUTRIG0_SHIFT 0 +#define I40E_GLPE_CPUTRIG0_PECPUTRIG0_MASK I40E_MASK(0xFFFF, I40E_GLPE_CPUTRIG0_PECPUTRIG0_SHIFT) +#define I40E_GLPE_CPUTRIG0_TEPREQUEST0_SHIFT 17 +#define I40E_GLPE_CPUTRIG0_TEPREQUEST0_MASK I40E_MASK(0x1, I40E_GLPE_CPUTRIG0_TEPREQUEST0_SHIFT) +#define I40E_GLPE_CPUTRIG0_OOPREQUEST0_SHIFT 18 +#define I40E_GLPE_CPUTRIG0_OOPREQUEST0_MASK I40E_MASK(0x1, I40E_GLPE_CPUTRIG0_OOPREQUEST0_SHIFT) +#define I40E_GLPE_DUAL40_RUPM 0x0000DA04 /* Reset: PE_CORER */ +#define I40E_GLPE_DUAL40_RUPM_DUAL_40G_MODE_SHIFT 0 +#define I40E_GLPE_DUAL40_RUPM_DUAL_40G_MODE_MASK I40E_MASK(0x1, I40E_GLPE_DUAL40_RUPM_DUAL_40G_MODE_SHIFT) +#define I40E_GLPE_PFAEQEDROPCNT(_i) (0x00131440 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ +#define I40E_GLPE_PFAEQEDROPCNT_MAX_INDEX 15 +#define I40E_GLPE_PFAEQEDROPCNT_AEQEDROPCNT_SHIFT 0 +#define I40E_GLPE_PFAEQEDROPCNT_AEQEDROPCNT_MASK I40E_MASK(0xFFFF, I40E_GLPE_PFAEQEDROPCNT_AEQEDROPCNT_SHIFT) +#define I40E_GLPE_PFCEQEDROPCNT(_i) (0x001313C0 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ +#define I40E_GLPE_PFCEQEDROPCNT_MAX_INDEX 15 +#define I40E_GLPE_PFCEQEDROPCNT_CEQEDROPCNT_SHIFT 0 +#define I40E_GLPE_PFCEQEDROPCNT_CEQEDROPCNT_MASK I40E_MASK(0xFFFF, I40E_GLPE_PFCEQEDROPCNT_CEQEDROPCNT_SHIFT) +#define I40E_GLPE_PFCQEDROPCNT(_i) (0x00131340 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */ +#define I40E_GLPE_PFCQEDROPCNT_MAX_INDEX 15 +#define I40E_GLPE_PFCQEDROPCNT_CQEDROPCNT_SHIFT 0 +#define I40E_GLPE_PFCQEDROPCNT_CQEDROPCNT_MASK I40E_MASK(0xFFFF, I40E_GLPE_PFCQEDROPCNT_CQEDROPCNT_SHIFT) +#define I40E_GLPE_RUPM_CQPPOOL 0x0000DACC /* Reset: PE_CORER */ +#define I40E_GLPE_RUPM_CQPPOOL_CQPSPADS_SHIFT 0 +#define I40E_GLPE_RUPM_CQPPOOL_CQPSPADS_MASK I40E_MASK(0xFF, I40E_GLPE_RUPM_CQPPOOL_CQPSPADS_SHIFT) +#define I40E_GLPE_RUPM_FLRPOOL 0x0000DAC4 /* Reset: PE_CORER */ +#define I40E_GLPE_RUPM_FLRPOOL_FLRSPADS_SHIFT 0 +#define I40E_GLPE_RUPM_FLRPOOL_FLRSPADS_MASK I40E_MASK(0xFF, I40E_GLPE_RUPM_FLRPOOL_FLRSPADS_SHIFT) +#define I40E_GLPE_RUPM_GCTL 0x0000DA00 /* Reset: PE_CORER */ +#define I40E_GLPE_RUPM_GCTL_ALLOFFTH_SHIFT 0 +#define I40E_GLPE_RUPM_GCTL_ALLOFFTH_MASK I40E_MASK(0xFF, I40E_GLPE_RUPM_GCTL_ALLOFFTH_SHIFT) +#define I40E_GLPE_RUPM_GCTL_RUPM_P0_DIS_SHIFT 26 +#define I40E_GLPE_RUPM_GCTL_RUPM_P0_DIS_MASK I40E_MASK(0x1, I40E_GLPE_RUPM_GCTL_RUPM_P0_DIS_SHIFT) +#define I40E_GLPE_RUPM_GCTL_RUPM_P1_DIS_SHIFT 27 +#define I40E_GLPE_RUPM_GCTL_RUPM_P1_DIS_MASK I40E_MASK(0x1, I40E_GLPE_RUPM_GCTL_RUPM_P1_DIS_SHIFT) +#define I40E_GLPE_RUPM_GCTL_RUPM_P2_DIS_SHIFT 28 +#define I40E_GLPE_RUPM_GCTL_RUPM_P2_DIS_MASK I40E_MASK(0x1, I40E_GLPE_RUPM_GCTL_RUPM_P2_DIS_SHIFT) +#define I40E_GLPE_RUPM_GCTL_RUPM_P3_DIS_SHIFT 29 +#define I40E_GLPE_RUPM_GCTL_RUPM_P3_DIS_MASK I40E_MASK(0x1, I40E_GLPE_RUPM_GCTL_RUPM_P3_DIS_SHIFT) +#define I40E_GLPE_RUPM_GCTL_RUPM_DIS_SHIFT 30 +#define I40E_GLPE_RUPM_GCTL_RUPM_DIS_MASK I40E_MASK(0x1, I40E_GLPE_RUPM_GCTL_RUPM_DIS_SHIFT) +#define I40E_GLPE_RUPM_GCTL_SWLB_MODE_SHIFT 31 +#define I40E_GLPE_RUPM_GCTL_SWLB_MODE_MASK I40E_MASK(0x1, I40E_GLPE_RUPM_GCTL_SWLB_MODE_SHIFT) +#define I40E_GLPE_RUPM_PTXPOOL 0x0000DAC8 /* Reset: PE_CORER */ +#define I40E_GLPE_RUPM_PTXPOOL_PTXSPADS_SHIFT 0 +#define I40E_GLPE_RUPM_PTXPOOL_PTXSPADS_MASK I40E_MASK(0xFF, I40E_GLPE_RUPM_PTXPOOL_PTXSPADS_SHIFT) +#define I40E_GLPE_RUPM_PUSHPOOL 0x0000DAC0 /* Reset: PE_CORER */ +#define I40E_GLPE_RUPM_PUSHPOOL_PUSHSPADS_SHIFT 0 +#define I40E_GLPE_RUPM_PUSHPOOL_PUSHSPADS_MASK I40E_MASK(0xFF, I40E_GLPE_RUPM_PUSHPOOL_PUSHSPADS_SHIFT) +#define I40E_GLPE_RUPM_TXHOST_EN 0x0000DA08 /* Reset: PE_CORER */ +#define I40E_GLPE_RUPM_TXHOST_EN_TXHOST_EN_SHIFT 0 +#define I40E_GLPE_RUPM_TXHOST_EN_TXHOST_EN_MASK I40E_MASK(0x1, I40E_GLPE_RUPM_TXHOST_EN_TXHOST_EN_SHIFT) +#define I40E_GLPE_VFAEQEDROPCNT(_i) (0x00132540 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLPE_VFAEQEDROPCNT_MAX_INDEX 31 +#define I40E_GLPE_VFAEQEDROPCNT_AEQEDROPCNT_SHIFT 0 +#define I40E_GLPE_VFAEQEDROPCNT_AEQEDROPCNT_MASK I40E_MASK(0xFFFF, I40E_GLPE_VFAEQEDROPCNT_AEQEDROPCNT_SHIFT) +#define I40E_GLPE_VFCEQEDROPCNT(_i) (0x00132440 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLPE_VFCEQEDROPCNT_MAX_INDEX 31 +#define I40E_GLPE_VFCEQEDROPCNT_CEQEDROPCNT_SHIFT 0 +#define I40E_GLPE_VFCEQEDROPCNT_CEQEDROPCNT_MASK I40E_MASK(0xFFFF, I40E_GLPE_VFCEQEDROPCNT_CEQEDROPCNT_SHIFT) +#define I40E_GLPE_VFCQEDROPCNT(_i) (0x00132340 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */ +#define I40E_GLPE_VFCQEDROPCNT_MAX_INDEX 31 +#define I40E_GLPE_VFCQEDROPCNT_CQEDROPCNT_SHIFT 0 +#define I40E_GLPE_VFCQEDROPCNT_CQEDROPCNT_MASK I40E_MASK(0xFFFF, I40E_GLPE_VFCQEDROPCNT_CQEDROPCNT_SHIFT) +#define I40E_GLPE_VFFLMOBJCTRL(_i) (0x0000D400 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPE_VFFLMOBJCTRL_MAX_INDEX 31 +#define I40E_GLPE_VFFLMOBJCTRL_XMIT_BLOCKSIZE_SHIFT 0 +#define I40E_GLPE_VFFLMOBJCTRL_XMIT_BLOCKSIZE_MASK I40E_MASK(0x7, I40E_GLPE_VFFLMOBJCTRL_XMIT_BLOCKSIZE_SHIFT) +#define I40E_GLPE_VFFLMOBJCTRL_Q1_BLOCKSIZE_SHIFT 8 +#define I40E_GLPE_VFFLMOBJCTRL_Q1_BLOCKSIZE_MASK I40E_MASK(0x7, I40E_GLPE_VFFLMOBJCTRL_Q1_BLOCKSIZE_SHIFT) +#define I40E_GLPE_VFFLMQ1ALLOCERR(_i) (0x0000C700 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPE_VFFLMQ1ALLOCERR_MAX_INDEX 31 +#define I40E_GLPE_VFFLMQ1ALLOCERR_ERROR_COUNT_SHIFT 0 +#define I40E_GLPE_VFFLMQ1ALLOCERR_ERROR_COUNT_MASK I40E_MASK(0xFFFF, I40E_GLPE_VFFLMQ1ALLOCERR_ERROR_COUNT_SHIFT) +#define I40E_GLPE_VFFLMXMITALLOCERR(_i) (0x0000C600 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPE_VFFLMXMITALLOCERR_MAX_INDEX 31 +#define I40E_GLPE_VFFLMXMITALLOCERR_ERROR_COUNT_SHIFT 0 +#define I40E_GLPE_VFFLMXMITALLOCERR_ERROR_COUNT_MASK I40E_MASK(0xFFFF, I40E_GLPE_VFFLMXMITALLOCERR_ERROR_COUNT_SHIFT) +#define I40E_GLPE_VFUDACTRL(_i) (0x0000C000 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPE_VFUDACTRL_MAX_INDEX 31 +#define I40E_GLPE_VFUDACTRL_IPV4MCFRAGRESBP_SHIFT 0 +#define I40E_GLPE_VFUDACTRL_IPV4MCFRAGRESBP_MASK I40E_MASK(0x1, I40E_GLPE_VFUDACTRL_IPV4MCFRAGRESBP_SHIFT) +#define I40E_GLPE_VFUDACTRL_IPV4UCFRAGRESBP_SHIFT 1 +#define I40E_GLPE_VFUDACTRL_IPV4UCFRAGRESBP_MASK I40E_MASK(0x1, I40E_GLPE_VFUDACTRL_IPV4UCFRAGRESBP_SHIFT) +#define I40E_GLPE_VFUDACTRL_IPV6MCFRAGRESBP_SHIFT 2 +#define I40E_GLPE_VFUDACTRL_IPV6MCFRAGRESBP_MASK I40E_MASK(0x1, I40E_GLPE_VFUDACTRL_IPV6MCFRAGRESBP_SHIFT) +#define I40E_GLPE_VFUDACTRL_IPV6UCFRAGRESBP_SHIFT 3 +#define I40E_GLPE_VFUDACTRL_IPV6UCFRAGRESBP_MASK I40E_MASK(0x1, I40E_GLPE_VFUDACTRL_IPV6UCFRAGRESBP_SHIFT) +#define I40E_GLPE_VFUDACTRL_UDPMCFRAGRESFAIL_SHIFT 4 +#define I40E_GLPE_VFUDACTRL_UDPMCFRAGRESFAIL_MASK I40E_MASK(0x1, I40E_GLPE_VFUDACTRL_UDPMCFRAGRESFAIL_SHIFT) +#define I40E_GLPE_VFUDAUCFBQPN(_i) (0x0000C100 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPE_VFUDAUCFBQPN_MAX_INDEX 31 +#define I40E_GLPE_VFUDAUCFBQPN_QPN_SHIFT 0 +#define I40E_GLPE_VFUDAUCFBQPN_QPN_MASK I40E_MASK(0x3FFFF, I40E_GLPE_VFUDAUCFBQPN_QPN_SHIFT) +#define I40E_GLPE_VFUDAUCFBQPN_VALID_SHIFT 31 +#define I40E_GLPE_VFUDAUCFBQPN_VALID_MASK I40E_MASK(0x1, I40E_GLPE_VFUDAUCFBQPN_VALID_SHIFT) +#define I40E_PFPE_AEQALLOC 0x00131180 /* Reset: PFR */ +#define I40E_PFPE_AEQALLOC_AECOUNT_SHIFT 0 +#define I40E_PFPE_AEQALLOC_AECOUNT_MASK I40E_MASK(0xFFFFFFFF, I40E_PFPE_AEQALLOC_AECOUNT_SHIFT) +#define I40E_PFPE_CCQPHIGH 0x00008200 /* Reset: PFR */ +#define I40E_PFPE_CCQPHIGH_PECCQPHIGH_SHIFT 0 +#define I40E_PFPE_CCQPHIGH_PECCQPHIGH_MASK I40E_MASK(0xFFFFFFFF, I40E_PFPE_CCQPHIGH_PECCQPHIGH_SHIFT) +#define I40E_PFPE_CCQPLOW 0x00008180 /* Reset: PFR */ +#define I40E_PFPE_CCQPLOW_PECCQPLOW_SHIFT 0 +#define I40E_PFPE_CCQPLOW_PECCQPLOW_MASK I40E_MASK(0xFFFFFFFF, I40E_PFPE_CCQPLOW_PECCQPLOW_SHIFT) +#define I40E_PFPE_CCQPSTATUS 0x00008100 /* Reset: PFR */ +#define I40E_PFPE_CCQPSTATUS_CCQP_DONE_SHIFT 0 +#define I40E_PFPE_CCQPSTATUS_CCQP_DONE_MASK I40E_MASK(0x1, I40E_PFPE_CCQPSTATUS_CCQP_DONE_SHIFT) +#define I40E_PFPE_CCQPSTATUS_HMC_PROFILE_SHIFT 4 +#define I40E_PFPE_CCQPSTATUS_HMC_PROFILE_MASK I40E_MASK(0x7, I40E_PFPE_CCQPSTATUS_HMC_PROFILE_SHIFT) +#define I40E_PFPE_CCQPSTATUS_RDMA_EN_VFS_SHIFT 16 +#define I40E_PFPE_CCQPSTATUS_RDMA_EN_VFS_MASK I40E_MASK(0x3F, I40E_PFPE_CCQPSTATUS_RDMA_EN_VFS_SHIFT) +#define I40E_PFPE_CCQPSTATUS_CCQP_ERR_SHIFT 31 +#define I40E_PFPE_CCQPSTATUS_CCQP_ERR_MASK I40E_MASK(0x1, I40E_PFPE_CCQPSTATUS_CCQP_ERR_SHIFT) +#define I40E_PFPE_CQACK 0x00131100 /* Reset: PFR */ +#define I40E_PFPE_CQACK_PECQID_SHIFT 0 +#define I40E_PFPE_CQACK_PECQID_MASK I40E_MASK(0x1FFFF, I40E_PFPE_CQACK_PECQID_SHIFT) +#define I40E_PFPE_CQARM 0x00131080 /* Reset: PFR */ +#define I40E_PFPE_CQARM_PECQID_SHIFT 0 +#define I40E_PFPE_CQARM_PECQID_MASK I40E_MASK(0x1FFFF, I40E_PFPE_CQARM_PECQID_SHIFT) +#define I40E_PFPE_CQPDB 0x00008000 /* Reset: PFR */ +#define I40E_PFPE_CQPDB_WQHEAD_SHIFT 0 +#define I40E_PFPE_CQPDB_WQHEAD_MASK I40E_MASK(0x7FF, I40E_PFPE_CQPDB_WQHEAD_SHIFT) +#define I40E_PFPE_CQPERRCODES 0x00008880 /* Reset: PFR */ +#define I40E_PFPE_CQPERRCODES_CQP_MINOR_CODE_SHIFT 0 +#define I40E_PFPE_CQPERRCODES_CQP_MINOR_CODE_MASK I40E_MASK(0xFFFF, I40E_PFPE_CQPERRCODES_CQP_MINOR_CODE_SHIFT) +#define I40E_PFPE_CQPERRCODES_CQP_MAJOR_CODE_SHIFT 16 +#define I40E_PFPE_CQPERRCODES_CQP_MAJOR_CODE_MASK I40E_MASK(0xFFFF, I40E_PFPE_CQPERRCODES_CQP_MAJOR_CODE_SHIFT) +#define I40E_PFPE_CQPTAIL 0x00008080 /* Reset: PFR */ +#define I40E_PFPE_CQPTAIL_WQTAIL_SHIFT 0 +#define I40E_PFPE_CQPTAIL_WQTAIL_MASK I40E_MASK(0x7FF, I40E_PFPE_CQPTAIL_WQTAIL_SHIFT) +#define I40E_PFPE_CQPTAIL_CQP_OP_ERR_SHIFT 31 +#define I40E_PFPE_CQPTAIL_CQP_OP_ERR_MASK I40E_MASK(0x1, I40E_PFPE_CQPTAIL_CQP_OP_ERR_SHIFT) +#define I40E_PFPE_FLMQ1ALLOCERR 0x00008980 /* Reset: PFR */ +#define I40E_PFPE_FLMQ1ALLOCERR_ERROR_COUNT_SHIFT 0 +#define I40E_PFPE_FLMQ1ALLOCERR_ERROR_COUNT_MASK I40E_MASK(0xFFFF, I40E_PFPE_FLMQ1ALLOCERR_ERROR_COUNT_SHIFT) +#define I40E_PFPE_FLMXMITALLOCERR 0x00008900 /* Reset: PFR */ +#define I40E_PFPE_FLMXMITALLOCERR_ERROR_COUNT_SHIFT 0 +#define I40E_PFPE_FLMXMITALLOCERR_ERROR_COUNT_MASK I40E_MASK(0xFFFF, I40E_PFPE_FLMXMITALLOCERR_ERROR_COUNT_SHIFT) +#define I40E_PFPE_IPCONFIG0 0x00008280 /* Reset: PFR */ +#define I40E_PFPE_IPCONFIG0_PEIPID_SHIFT 0 +#define I40E_PFPE_IPCONFIG0_PEIPID_MASK I40E_MASK(0xFFFF, I40E_PFPE_IPCONFIG0_PEIPID_SHIFT) +#define I40E_PFPE_IPCONFIG0_USEENTIREIDRANGE_SHIFT 16 +#define I40E_PFPE_IPCONFIG0_USEENTIREIDRANGE_MASK I40E_MASK(0x1, I40E_PFPE_IPCONFIG0_USEENTIREIDRANGE_SHIFT) +#define I40E_PFPE_MRTEIDXMASK 0x00008600 /* Reset: PFR */ +#define I40E_PFPE_MRTEIDXMASK_MRTEIDXMASKBITS_SHIFT 0 +#define I40E_PFPE_MRTEIDXMASK_MRTEIDXMASKBITS_MASK I40E_MASK(0x1F, I40E_PFPE_MRTEIDXMASK_MRTEIDXMASKBITS_SHIFT) +#define I40E_PFPE_RCVUNEXPECTEDERROR 0x00008680 /* Reset: PFR */ +#define I40E_PFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_SHIFT 0 +#define I40E_PFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_MASK I40E_MASK(0xFFFFFF, I40E_PFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_SHIFT) +#define I40E_PFPE_TCPNOWTIMER 0x00008580 /* Reset: PFR */ +#define I40E_PFPE_TCPNOWTIMER_TCP_NOW_SHIFT 0 +#define I40E_PFPE_TCPNOWTIMER_TCP_NOW_MASK I40E_MASK(0xFFFFFFFF, I40E_PFPE_TCPNOWTIMER_TCP_NOW_SHIFT) +#define I40E_PFPE_UDACTRL 0x00008700 /* Reset: PFR */ +#define I40E_PFPE_UDACTRL_IPV4MCFRAGRESBP_SHIFT 0 +#define I40E_PFPE_UDACTRL_IPV4MCFRAGRESBP_MASK I40E_MASK(0x1, I40E_PFPE_UDACTRL_IPV4MCFRAGRESBP_SHIFT) +#define I40E_PFPE_UDACTRL_IPV4UCFRAGRESBP_SHIFT 1 +#define I40E_PFPE_UDACTRL_IPV4UCFRAGRESBP_MASK I40E_MASK(0x1, I40E_PFPE_UDACTRL_IPV4UCFRAGRESBP_SHIFT) +#define I40E_PFPE_UDACTRL_IPV6MCFRAGRESBP_SHIFT 2 +#define I40E_PFPE_UDACTRL_IPV6MCFRAGRESBP_MASK I40E_MASK(0x1, I40E_PFPE_UDACTRL_IPV6MCFRAGRESBP_SHIFT) +#define I40E_PFPE_UDACTRL_IPV6UCFRAGRESBP_SHIFT 3 +#define I40E_PFPE_UDACTRL_IPV6UCFRAGRESBP_MASK I40E_MASK(0x1, I40E_PFPE_UDACTRL_IPV6UCFRAGRESBP_SHIFT) +#define I40E_PFPE_UDACTRL_UDPMCFRAGRESFAIL_SHIFT 4 +#define I40E_PFPE_UDACTRL_UDPMCFRAGRESFAIL_MASK I40E_MASK(0x1, I40E_PFPE_UDACTRL_UDPMCFRAGRESFAIL_SHIFT) +#define I40E_PFPE_UDAUCFBQPN 0x00008780 /* Reset: PFR */ +#define I40E_PFPE_UDAUCFBQPN_QPN_SHIFT 0 +#define I40E_PFPE_UDAUCFBQPN_QPN_MASK I40E_MASK(0x3FFFF, I40E_PFPE_UDAUCFBQPN_QPN_SHIFT) +#define I40E_PFPE_UDAUCFBQPN_VALID_SHIFT 31 +#define I40E_PFPE_UDAUCFBQPN_VALID_MASK I40E_MASK(0x1, I40E_PFPE_UDAUCFBQPN_VALID_SHIFT) +#define I40E_PFPE_WQEALLOC 0x00138C00 /* Reset: PFR */ +#define I40E_PFPE_WQEALLOC_PEQPID_SHIFT 0 +#define I40E_PFPE_WQEALLOC_PEQPID_MASK I40E_MASK(0x3FFFF, I40E_PFPE_WQEALLOC_PEQPID_SHIFT) +#define I40E_PFPE_WQEALLOC_WQE_DESC_INDEX_SHIFT 20 +#define I40E_PFPE_WQEALLOC_WQE_DESC_INDEX_MASK I40E_MASK(0xFFF, I40E_PFPE_WQEALLOC_WQE_DESC_INDEX_SHIFT) +#define I40E_PRTDCB_RLPMC 0x0001F140 /* Reset: PE_CORER */ +#define I40E_PRTDCB_RLPMC_TC2PFC_SHIFT 0 +#define I40E_PRTDCB_RLPMC_TC2PFC_MASK I40E_MASK(0xFF, I40E_PRTDCB_RLPMC_TC2PFC_SHIFT) +#define I40E_PRTDCB_TCMSTC_RLPM(_i) (0x0001F040 + ((_i) * 32)) /* _i=0...7 */ /* Reset: PE_CORER */ +#define I40E_PRTDCB_TCMSTC_RLPM_MAX_INDEX 7 +#define I40E_PRTDCB_TCMSTC_RLPM_MSTC_SHIFT 0 +#define I40E_PRTDCB_TCMSTC_RLPM_MSTC_MASK I40E_MASK(0xFFFFF, I40E_PRTDCB_TCMSTC_RLPM_MSTC_SHIFT) +#define I40E_PRTDCB_TCPMC_RLPM 0x0001F1A0 /* Reset: PE_CORER */ +#define I40E_PRTDCB_TCPMC_RLPM_CPM_SHIFT 0 +#define I40E_PRTDCB_TCPMC_RLPM_CPM_MASK I40E_MASK(0x1FFF, I40E_PRTDCB_TCPMC_RLPM_CPM_SHIFT) +#define I40E_PRTDCB_TCPMC_RLPM_LLTC_SHIFT 13 +#define I40E_PRTDCB_TCPMC_RLPM_LLTC_MASK I40E_MASK(0xFF, I40E_PRTDCB_TCPMC_RLPM_LLTC_SHIFT) +#define I40E_PRTDCB_TCPMC_RLPM_TCPM_MODE_SHIFT 30 +#define I40E_PRTDCB_TCPMC_RLPM_TCPM_MODE_MASK I40E_MASK(0x1, I40E_PRTDCB_TCPMC_RLPM_TCPM_MODE_SHIFT) +#define I40E_PRTE_RUPM_TCCNTR03 0x0000DAE0 /* Reset: PE_CORER */ +#define I40E_PRTE_RUPM_TCCNTR03_TC0COUNT_SHIFT 0 +#define I40E_PRTE_RUPM_TCCNTR03_TC0COUNT_MASK I40E_MASK(0xFF, I40E_PRTE_RUPM_TCCNTR03_TC0COUNT_SHIFT) +#define I40E_PRTE_RUPM_TCCNTR03_TC1COUNT_SHIFT 8 +#define I40E_PRTE_RUPM_TCCNTR03_TC1COUNT_MASK I40E_MASK(0xFF, I40E_PRTE_RUPM_TCCNTR03_TC1COUNT_SHIFT) +#define I40E_PRTE_RUPM_TCCNTR03_TC2COUNT_SHIFT 16 +#define I40E_PRTE_RUPM_TCCNTR03_TC2COUNT_MASK I40E_MASK(0xFF, I40E_PRTE_RUPM_TCCNTR03_TC2COUNT_SHIFT) +#define I40E_PRTE_RUPM_TCCNTR03_TC3COUNT_SHIFT 24 +#define I40E_PRTE_RUPM_TCCNTR03_TC3COUNT_MASK I40E_MASK(0xFF, I40E_PRTE_RUPM_TCCNTR03_TC3COUNT_SHIFT) +#define I40E_PRTPE_RUPM_CNTR 0x0000DB20 /* Reset: PE_CORER */ +#define I40E_PRTPE_RUPM_CNTR_COUNT_SHIFT 0 +#define I40E_PRTPE_RUPM_CNTR_COUNT_MASK I40E_MASK(0xFF, I40E_PRTPE_RUPM_CNTR_COUNT_SHIFT) +#define I40E_PRTPE_RUPM_CTL 0x0000DA40 /* Reset: PE_CORER */ +#define I40E_PRTPE_RUPM_CTL_LLTC_SHIFT 13 +#define I40E_PRTPE_RUPM_CTL_LLTC_MASK I40E_MASK(0xFF, I40E_PRTPE_RUPM_CTL_LLTC_SHIFT) +#define I40E_PRTPE_RUPM_CTL_RUPM_MODE_SHIFT 30 +#define I40E_PRTPE_RUPM_CTL_RUPM_MODE_MASK I40E_MASK(0x1, I40E_PRTPE_RUPM_CTL_RUPM_MODE_SHIFT) +#define I40E_PRTPE_RUPM_PFCCTL 0x0000DA60 /* Reset: PE_CORER */ +#define I40E_PRTPE_RUPM_PFCCTL_TC2PFC_SHIFT 0 +#define I40E_PRTPE_RUPM_PFCCTL_TC2PFC_MASK I40E_MASK(0xFF, I40E_PRTPE_RUPM_PFCCTL_TC2PFC_SHIFT) +#define I40E_PRTPE_RUPM_PFCPC 0x0000DA80 /* Reset: PE_CORER */ +#define I40E_PRTPE_RUPM_PFCPC_PORTOFFTH_SHIFT 0 +#define I40E_PRTPE_RUPM_PFCPC_PORTOFFTH_MASK I40E_MASK(0xFF, I40E_PRTPE_RUPM_PFCPC_PORTOFFTH_SHIFT) +#define I40E_PRTPE_RUPM_PFCTCC 0x0000DAA0 /* Reset: PE_CORER */ +#define I40E_PRTPE_RUPM_PFCTCC_TCOFFTH_SHIFT 0 +#define I40E_PRTPE_RUPM_PFCTCC_TCOFFTH_MASK I40E_MASK(0xFF, I40E_PRTPE_RUPM_PFCTCC_TCOFFTH_SHIFT) +#define I40E_PRTPE_RUPM_PFCTCC_LL_PRI_TH_SHIFT 16 +#define I40E_PRTPE_RUPM_PFCTCC_LL_PRI_TH_MASK I40E_MASK(0xFF, I40E_PRTPE_RUPM_PFCTCC_LL_PRI_TH_SHIFT) +#define I40E_PRTPE_RUPM_PFCTCC_LL_PRI_EN_SHIFT 31 +#define I40E_PRTPE_RUPM_PFCTCC_LL_PRI_EN_MASK I40E_MASK(0x1, I40E_PRTPE_RUPM_PFCTCC_LL_PRI_EN_SHIFT) +#define I40E_PRTPE_RUPM_PTCTCCNTR47 0x0000DB60 /* Reset: PE_CORER */ +#define I40E_PRTPE_RUPM_PTCTCCNTR47_TC4COUNT_SHIFT 0 +#define I40E_PRTPE_RUPM_PTCTCCNTR47_TC4COUNT_MASK I40E_MASK(0xFF, I40E_PRTPE_RUPM_PTCTCCNTR47_TC4COUNT_SHIFT) +#define I40E_PRTPE_RUPM_PTCTCCNTR47_TC5COUNT_SHIFT 8 +#define I40E_PRTPE_RUPM_PTCTCCNTR47_TC5COUNT_MASK I40E_MASK(0xFF, I40E_PRTPE_RUPM_PTCTCCNTR47_TC5COUNT_SHIFT) +#define I40E_PRTPE_RUPM_PTCTCCNTR47_TC6COUNT_SHIFT 16 +#define I40E_PRTPE_RUPM_PTCTCCNTR47_TC6COUNT_MASK I40E_MASK(0xFF, I40E_PRTPE_RUPM_PTCTCCNTR47_TC6COUNT_SHIFT) +#define I40E_PRTPE_RUPM_PTCTCCNTR47_TC7COUNT_SHIFT 24 +#define I40E_PRTPE_RUPM_PTCTCCNTR47_TC7COUNT_MASK I40E_MASK(0xFF, I40E_PRTPE_RUPM_PTCTCCNTR47_TC7COUNT_SHIFT) +#define I40E_PRTPE_RUPM_PTXTCCNTR03 0x0000DB40 /* Reset: PE_CORER */ +#define I40E_PRTPE_RUPM_PTXTCCNTR03_TC0COUNT_SHIFT 0 +#define I40E_PRTPE_RUPM_PTXTCCNTR03_TC0COUNT_MASK I40E_MASK(0xFF, I40E_PRTPE_RUPM_PTXTCCNTR03_TC0COUNT_SHIFT) +#define I40E_PRTPE_RUPM_PTXTCCNTR03_TC1COUNT_SHIFT 8 +#define I40E_PRTPE_RUPM_PTXTCCNTR03_TC1COUNT_MASK I40E_MASK(0xFF, I40E_PRTPE_RUPM_PTXTCCNTR03_TC1COUNT_SHIFT) +#define I40E_PRTPE_RUPM_PTXTCCNTR03_TC2COUNT_SHIFT 16 +#define I40E_PRTPE_RUPM_PTXTCCNTR03_TC2COUNT_MASK I40E_MASK(0xFF, I40E_PRTPE_RUPM_PTXTCCNTR03_TC2COUNT_SHIFT) +#define I40E_PRTPE_RUPM_PTXTCCNTR03_TC3COUNT_SHIFT 24 +#define I40E_PRTPE_RUPM_PTXTCCNTR03_TC3COUNT_MASK I40E_MASK(0xFF, I40E_PRTPE_RUPM_PTXTCCNTR03_TC3COUNT_SHIFT) +#define I40E_PRTPE_RUPM_TCCNTR47 0x0000DB00 /* Reset: PE_CORER */ +#define I40E_PRTPE_RUPM_TCCNTR47_TC4COUNT_SHIFT 0 +#define I40E_PRTPE_RUPM_TCCNTR47_TC4COUNT_MASK I40E_MASK(0xFF, I40E_PRTPE_RUPM_TCCNTR47_TC4COUNT_SHIFT) +#define I40E_PRTPE_RUPM_TCCNTR47_TC5COUNT_SHIFT 8 +#define I40E_PRTPE_RUPM_TCCNTR47_TC5COUNT_MASK I40E_MASK(0xFF, I40E_PRTPE_RUPM_TCCNTR47_TC5COUNT_SHIFT) +#define I40E_PRTPE_RUPM_TCCNTR47_TC6COUNT_SHIFT 16 +#define I40E_PRTPE_RUPM_TCCNTR47_TC6COUNT_MASK I40E_MASK(0xFF, I40E_PRTPE_RUPM_TCCNTR47_TC6COUNT_SHIFT) +#define I40E_PRTPE_RUPM_TCCNTR47_TC7COUNT_SHIFT 24 +#define I40E_PRTPE_RUPM_TCCNTR47_TC7COUNT_MASK I40E_MASK(0xFF, I40E_PRTPE_RUPM_TCCNTR47_TC7COUNT_SHIFT) +#define I40E_PRTPE_RUPM_THRES 0x0000DA20 /* Reset: PE_CORER */ +#define I40E_PRTPE_RUPM_THRES_MINSPADSPERTC_SHIFT 0 +#define I40E_PRTPE_RUPM_THRES_MINSPADSPERTC_MASK I40E_MASK(0xFF, I40E_PRTPE_RUPM_THRES_MINSPADSPERTC_SHIFT) +#define I40E_PRTPE_RUPM_THRES_MAXSPADS_SHIFT 8 +#define I40E_PRTPE_RUPM_THRES_MAXSPADS_MASK I40E_MASK(0xFF, I40E_PRTPE_RUPM_THRES_MAXSPADS_SHIFT) +#define I40E_PRTPE_RUPM_THRES_MAXSPADSPERTC_SHIFT 16 +#define I40E_PRTPE_RUPM_THRES_MAXSPADSPERTC_MASK I40E_MASK(0xFF, I40E_PRTPE_RUPM_THRES_MAXSPADSPERTC_SHIFT) +#define I40E_VFPE_AEQALLOC(_VF) (0x00130C00 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */ +#define I40E_VFPE_AEQALLOC_MAX_INDEX 127 +#define I40E_VFPE_AEQALLOC_AECOUNT_SHIFT 0 +#define I40E_VFPE_AEQALLOC_AECOUNT_MASK I40E_MASK(0xFFFFFFFF, I40E_VFPE_AEQALLOC_AECOUNT_SHIFT) +#define I40E_VFPE_CCQPHIGH(_VF) (0x00001000 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */ +#define I40E_VFPE_CCQPHIGH_MAX_INDEX 127 +#define I40E_VFPE_CCQPHIGH_PECCQPHIGH_SHIFT 0 +#define I40E_VFPE_CCQPHIGH_PECCQPHIGH_MASK I40E_MASK(0xFFFFFFFF, I40E_VFPE_CCQPHIGH_PECCQPHIGH_SHIFT) +#define I40E_VFPE_CCQPLOW(_VF) (0x00000C00 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */ +#define I40E_VFPE_CCQPLOW_MAX_INDEX 127 +#define I40E_VFPE_CCQPLOW_PECCQPLOW_SHIFT 0 +#define I40E_VFPE_CCQPLOW_PECCQPLOW_MASK I40E_MASK(0xFFFFFFFF, I40E_VFPE_CCQPLOW_PECCQPLOW_SHIFT) +#define I40E_VFPE_CCQPSTATUS(_VF) (0x00000800 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */ +#define I40E_VFPE_CCQPSTATUS_MAX_INDEX 127 +#define I40E_VFPE_CCQPSTATUS_CCQP_DONE_SHIFT 0 +#define I40E_VFPE_CCQPSTATUS_CCQP_DONE_MASK I40E_MASK(0x1, I40E_VFPE_CCQPSTATUS_CCQP_DONE_SHIFT) +#define I40E_VFPE_CCQPSTATUS_HMC_PROFILE_SHIFT 4 +#define I40E_VFPE_CCQPSTATUS_HMC_PROFILE_MASK I40E_MASK(0x7, I40E_VFPE_CCQPSTATUS_HMC_PROFILE_SHIFT) +#define I40E_VFPE_CCQPSTATUS_RDMA_EN_VFS_SHIFT 16 +#define I40E_VFPE_CCQPSTATUS_RDMA_EN_VFS_MASK I40E_MASK(0x3F, I40E_VFPE_CCQPSTATUS_RDMA_EN_VFS_SHIFT) +#define I40E_VFPE_CCQPSTATUS_CCQP_ERR_SHIFT 31 +#define I40E_VFPE_CCQPSTATUS_CCQP_ERR_MASK I40E_MASK(0x1, I40E_VFPE_CCQPSTATUS_CCQP_ERR_SHIFT) +#define I40E_VFPE_CQACK(_VF) (0x00130800 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */ +#define I40E_VFPE_CQACK_MAX_INDEX 127 +#define I40E_VFPE_CQACK_PECQID_SHIFT 0 +#define I40E_VFPE_CQACK_PECQID_MASK I40E_MASK(0x1FFFF, I40E_VFPE_CQACK_PECQID_SHIFT) +#define I40E_VFPE_CQARM(_VF) (0x00130400 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */ +#define I40E_VFPE_CQARM_MAX_INDEX 127 +#define I40E_VFPE_CQARM_PECQID_SHIFT 0 +#define I40E_VFPE_CQARM_PECQID_MASK I40E_MASK(0x1FFFF, I40E_VFPE_CQARM_PECQID_SHIFT) +#define I40E_VFPE_CQPDB(_VF) (0x00000000 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */ +#define I40E_VFPE_CQPDB_MAX_INDEX 127 +#define I40E_VFPE_CQPDB_WQHEAD_SHIFT 0 +#define I40E_VFPE_CQPDB_WQHEAD_MASK I40E_MASK(0x7FF, I40E_VFPE_CQPDB_WQHEAD_SHIFT) +#define I40E_VFPE_CQPERRCODES(_VF) (0x00001800 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */ +#define I40E_VFPE_CQPERRCODES_MAX_INDEX 127 +#define I40E_VFPE_CQPERRCODES_CQP_MINOR_CODE_SHIFT 0 +#define I40E_VFPE_CQPERRCODES_CQP_MINOR_CODE_MASK I40E_MASK(0xFFFF, I40E_VFPE_CQPERRCODES_CQP_MINOR_CODE_SHIFT) +#define I40E_VFPE_CQPERRCODES_CQP_MAJOR_CODE_SHIFT 16 +#define I40E_VFPE_CQPERRCODES_CQP_MAJOR_CODE_MASK I40E_MASK(0xFFFF, I40E_VFPE_CQPERRCODES_CQP_MAJOR_CODE_SHIFT) +#define I40E_VFPE_CQPTAIL(_VF) (0x00000400 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */ +#define I40E_VFPE_CQPTAIL_MAX_INDEX 127 +#define I40E_VFPE_CQPTAIL_WQTAIL_SHIFT 0 +#define I40E_VFPE_CQPTAIL_WQTAIL_MASK I40E_MASK(0x7FF, I40E_VFPE_CQPTAIL_WQTAIL_SHIFT) +#define I40E_VFPE_CQPTAIL_CQP_OP_ERR_SHIFT 31 +#define I40E_VFPE_CQPTAIL_CQP_OP_ERR_MASK I40E_MASK(0x1, I40E_VFPE_CQPTAIL_CQP_OP_ERR_SHIFT) +#define I40E_VFPE_IPCONFIG0(_VF) (0x00001400 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */ +#define I40E_VFPE_IPCONFIG0_MAX_INDEX 127 +#define I40E_VFPE_IPCONFIG0_PEIPID_SHIFT 0 +#define I40E_VFPE_IPCONFIG0_PEIPID_MASK I40E_MASK(0xFFFF, I40E_VFPE_IPCONFIG0_PEIPID_SHIFT) +#define I40E_VFPE_IPCONFIG0_USEENTIREIDRANGE_SHIFT 16 +#define I40E_VFPE_IPCONFIG0_USEENTIREIDRANGE_MASK I40E_MASK(0x1, I40E_VFPE_IPCONFIG0_USEENTIREIDRANGE_SHIFT) +#define I40E_VFPE_MRTEIDXMASK(_VF) (0x00003000 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */ +#define I40E_VFPE_MRTEIDXMASK_MAX_INDEX 127 +#define I40E_VFPE_MRTEIDXMASK_MRTEIDXMASKBITS_SHIFT 0 +#define I40E_VFPE_MRTEIDXMASK_MRTEIDXMASKBITS_MASK I40E_MASK(0x1F, I40E_VFPE_MRTEIDXMASK_MRTEIDXMASKBITS_SHIFT) +#define I40E_VFPE_RCVUNEXPECTEDERROR(_VF) (0x00003400 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */ +#define I40E_VFPE_RCVUNEXPECTEDERROR_MAX_INDEX 127 +#define I40E_VFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_SHIFT 0 +#define I40E_VFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_MASK I40E_MASK(0xFFFFFF, I40E_VFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_SHIFT) +#define I40E_VFPE_TCPNOWTIMER(_VF) (0x00002C00 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */ +#define I40E_VFPE_TCPNOWTIMER_MAX_INDEX 127 +#define I40E_VFPE_TCPNOWTIMER_TCP_NOW_SHIFT 0 +#define I40E_VFPE_TCPNOWTIMER_TCP_NOW_MASK I40E_MASK(0xFFFFFFFF, I40E_VFPE_TCPNOWTIMER_TCP_NOW_SHIFT) +#define I40E_VFPE_WQEALLOC(_VF) (0x00138000 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */ +#define I40E_VFPE_WQEALLOC_MAX_INDEX 127 +#define I40E_VFPE_WQEALLOC_PEQPID_SHIFT 0 +#define I40E_VFPE_WQEALLOC_PEQPID_MASK I40E_MASK(0x3FFFF, I40E_VFPE_WQEALLOC_PEQPID_SHIFT) +#define I40E_VFPE_WQEALLOC_WQE_DESC_INDEX_SHIFT 20 +#define I40E_VFPE_WQEALLOC_WQE_DESC_INDEX_MASK I40E_MASK(0xFFF, I40E_VFPE_WQEALLOC_WQE_DESC_INDEX_SHIFT) +#define I40E_GLPES_PFIP4RXDISCARD(_i) (0x00010600 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP4RXDISCARD_MAX_INDEX 15 +#define I40E_GLPES_PFIP4RXDISCARD_IP4RXDISCARD_SHIFT 0 +#define I40E_GLPES_PFIP4RXDISCARD_IP4RXDISCARD_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFIP4RXDISCARD_IP4RXDISCARD_SHIFT) +#define I40E_GLPES_PFIP4RXFRAGSHI(_i) (0x00010804 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP4RXFRAGSHI_MAX_INDEX 15 +#define I40E_GLPES_PFIP4RXFRAGSHI_IP4RXFRAGSHI_SHIFT 0 +#define I40E_GLPES_PFIP4RXFRAGSHI_IP4RXFRAGSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_PFIP4RXFRAGSHI_IP4RXFRAGSHI_SHIFT) +#define I40E_GLPES_PFIP4RXFRAGSLO(_i) (0x00010800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP4RXFRAGSLO_MAX_INDEX 15 +#define I40E_GLPES_PFIP4RXFRAGSLO_IP4RXFRAGSLO_SHIFT 0 +#define I40E_GLPES_PFIP4RXFRAGSLO_IP4RXFRAGSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFIP4RXFRAGSLO_IP4RXFRAGSLO_SHIFT) +#define I40E_GLPES_PFIP4RXMCOCTSHI(_i) (0x00010A04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP4RXMCOCTSHI_MAX_INDEX 15 +#define I40E_GLPES_PFIP4RXMCOCTSHI_IP4RXMCOCTSHI_SHIFT 0 +#define I40E_GLPES_PFIP4RXMCOCTSHI_IP4RXMCOCTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_PFIP4RXMCOCTSHI_IP4RXMCOCTSHI_SHIFT) +#define I40E_GLPES_PFIP4RXMCOCTSLO(_i) (0x00010A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP4RXMCOCTSLO_MAX_INDEX 15 +#define I40E_GLPES_PFIP4RXMCOCTSLO_IP4RXMCOCTSLO_SHIFT 0 +#define I40E_GLPES_PFIP4RXMCOCTSLO_IP4RXMCOCTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFIP4RXMCOCTSLO_IP4RXMCOCTSLO_SHIFT) +#define I40E_GLPES_PFIP4RXMCPKTSHI(_i) (0x00010C04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP4RXMCPKTSHI_MAX_INDEX 15 +#define I40E_GLPES_PFIP4RXMCPKTSHI_IP4RXMCPKTSHI_SHIFT 0 +#define I40E_GLPES_PFIP4RXMCPKTSHI_IP4RXMCPKTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_PFIP4RXMCPKTSHI_IP4RXMCPKTSHI_SHIFT) +#define I40E_GLPES_PFIP4RXMCPKTSLO(_i) (0x00010C00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP4RXMCPKTSLO_MAX_INDEX 15 +#define I40E_GLPES_PFIP4RXMCPKTSLO_IP4RXMCPKTSLO_SHIFT 0 +#define I40E_GLPES_PFIP4RXMCPKTSLO_IP4RXMCPKTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFIP4RXMCPKTSLO_IP4RXMCPKTSLO_SHIFT) +#define I40E_GLPES_PFIP4RXOCTSHI(_i) (0x00010204 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP4RXOCTSHI_MAX_INDEX 15 +#define I40E_GLPES_PFIP4RXOCTSHI_IP4RXOCTSHI_SHIFT 0 +#define I40E_GLPES_PFIP4RXOCTSHI_IP4RXOCTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_PFIP4RXOCTSHI_IP4RXOCTSHI_SHIFT) +#define I40E_GLPES_PFIP4RXOCTSLO(_i) (0x00010200 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP4RXOCTSLO_MAX_INDEX 15 +#define I40E_GLPES_PFIP4RXOCTSLO_IP4RXOCTSLO_SHIFT 0 +#define I40E_GLPES_PFIP4RXOCTSLO_IP4RXOCTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFIP4RXOCTSLO_IP4RXOCTSLO_SHIFT) +#define I40E_GLPES_PFIP4RXPKTSHI(_i) (0x00010404 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP4RXPKTSHI_MAX_INDEX 15 +#define I40E_GLPES_PFIP4RXPKTSHI_IP4RXPKTSHI_SHIFT 0 +#define I40E_GLPES_PFIP4RXPKTSHI_IP4RXPKTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_PFIP4RXPKTSHI_IP4RXPKTSHI_SHIFT) +#define I40E_GLPES_PFIP4RXPKTSLO(_i) (0x00010400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP4RXPKTSLO_MAX_INDEX 15 +#define I40E_GLPES_PFIP4RXPKTSLO_IP4RXPKTSLO_SHIFT 0 +#define I40E_GLPES_PFIP4RXPKTSLO_IP4RXPKTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFIP4RXPKTSLO_IP4RXPKTSLO_SHIFT) +#define I40E_GLPES_PFIP4RXTRUNC(_i) (0x00010700 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP4RXTRUNC_MAX_INDEX 15 +#define I40E_GLPES_PFIP4RXTRUNC_IP4RXTRUNC_SHIFT 0 +#define I40E_GLPES_PFIP4RXTRUNC_IP4RXTRUNC_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFIP4RXTRUNC_IP4RXTRUNC_SHIFT) +#define I40E_GLPES_PFIP4TXFRAGSHI(_i) (0x00011E04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP4TXFRAGSHI_MAX_INDEX 15 +#define I40E_GLPES_PFIP4TXFRAGSHI_IP4TXFRAGSHI_SHIFT 0 +#define I40E_GLPES_PFIP4TXFRAGSHI_IP4TXFRAGSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_PFIP4TXFRAGSHI_IP4TXFRAGSHI_SHIFT) +#define I40E_GLPES_PFIP4TXFRAGSLO(_i) (0x00011E00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP4TXFRAGSLO_MAX_INDEX 15 +#define I40E_GLPES_PFIP4TXFRAGSLO_IP4TXFRAGSLO_SHIFT 0 +#define I40E_GLPES_PFIP4TXFRAGSLO_IP4TXFRAGSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFIP4TXFRAGSLO_IP4TXFRAGSLO_SHIFT) +#define I40E_GLPES_PFIP4TXMCOCTSHI(_i) (0x00012004 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP4TXMCOCTSHI_MAX_INDEX 15 +#define I40E_GLPES_PFIP4TXMCOCTSHI_IP4TXMCOCTSHI_SHIFT 0 +#define I40E_GLPES_PFIP4TXMCOCTSHI_IP4TXMCOCTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_PFIP4TXMCOCTSHI_IP4TXMCOCTSHI_SHIFT) +#define I40E_GLPES_PFIP4TXMCOCTSLO(_i) (0x00012000 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP4TXMCOCTSLO_MAX_INDEX 15 +#define I40E_GLPES_PFIP4TXMCOCTSLO_IP4TXMCOCTSLO_SHIFT 0 +#define I40E_GLPES_PFIP4TXMCOCTSLO_IP4TXMCOCTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFIP4TXMCOCTSLO_IP4TXMCOCTSLO_SHIFT) +#define I40E_GLPES_PFIP4TXMCPKTSHI(_i) (0x00012204 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP4TXMCPKTSHI_MAX_INDEX 15 +#define I40E_GLPES_PFIP4TXMCPKTSHI_IP4TXMCPKTSHI_SHIFT 0 +#define I40E_GLPES_PFIP4TXMCPKTSHI_IP4TXMCPKTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_PFIP4TXMCPKTSHI_IP4TXMCPKTSHI_SHIFT) +#define I40E_GLPES_PFIP4TXMCPKTSLO(_i) (0x00012200 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP4TXMCPKTSLO_MAX_INDEX 15 +#define I40E_GLPES_PFIP4TXMCPKTSLO_IP4TXMCPKTSLO_SHIFT 0 +#define I40E_GLPES_PFIP4TXMCPKTSLO_IP4TXMCPKTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFIP4TXMCPKTSLO_IP4TXMCPKTSLO_SHIFT) +#define I40E_GLPES_PFIP4TXNOROUTE(_i) (0x00012E00 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP4TXNOROUTE_MAX_INDEX 15 +#define I40E_GLPES_PFIP4TXNOROUTE_IP4TXNOROUTE_SHIFT 0 +#define I40E_GLPES_PFIP4TXNOROUTE_IP4TXNOROUTE_MASK I40E_MASK(0xFFFFFF, I40E_GLPES_PFIP4TXNOROUTE_IP4TXNOROUTE_SHIFT) +#define I40E_GLPES_PFIP4TXOCTSHI(_i) (0x00011A04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP4TXOCTSHI_MAX_INDEX 15 +#define I40E_GLPES_PFIP4TXOCTSHI_IP4TXOCTSHI_SHIFT 0 +#define I40E_GLPES_PFIP4TXOCTSHI_IP4TXOCTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_PFIP4TXOCTSHI_IP4TXOCTSHI_SHIFT) +#define I40E_GLPES_PFIP4TXOCTSLO(_i) (0x00011A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP4TXOCTSLO_MAX_INDEX 15 +#define I40E_GLPES_PFIP4TXOCTSLO_IP4TXOCTSLO_SHIFT 0 +#define I40E_GLPES_PFIP4TXOCTSLO_IP4TXOCTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFIP4TXOCTSLO_IP4TXOCTSLO_SHIFT) +#define I40E_GLPES_PFIP4TXPKTSHI(_i) (0x00011C04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP4TXPKTSHI_MAX_INDEX 15 +#define I40E_GLPES_PFIP4TXPKTSHI_IP4TXPKTSHI_SHIFT 0 +#define I40E_GLPES_PFIP4TXPKTSHI_IP4TXPKTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_PFIP4TXPKTSHI_IP4TXPKTSHI_SHIFT) +#define I40E_GLPES_PFIP4TXPKTSLO(_i) (0x00011C00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP4TXPKTSLO_MAX_INDEX 15 +#define I40E_GLPES_PFIP4TXPKTSLO_IP4TXPKTSLO_SHIFT 0 +#define I40E_GLPES_PFIP4TXPKTSLO_IP4TXPKTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFIP4TXPKTSLO_IP4TXPKTSLO_SHIFT) +#define I40E_GLPES_PFIP6RXDISCARD(_i) (0x00011200 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP6RXDISCARD_MAX_INDEX 15 +#define I40E_GLPES_PFIP6RXDISCARD_IP6RXDISCARD_SHIFT 0 +#define I40E_GLPES_PFIP6RXDISCARD_IP6RXDISCARD_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFIP6RXDISCARD_IP6RXDISCARD_SHIFT) +#define I40E_GLPES_PFIP6RXFRAGSHI(_i) (0x00011404 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP6RXFRAGSHI_MAX_INDEX 15 +#define I40E_GLPES_PFIP6RXFRAGSHI_IP6RXFRAGSHI_SHIFT 0 +#define I40E_GLPES_PFIP6RXFRAGSHI_IP6RXFRAGSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_PFIP6RXFRAGSHI_IP6RXFRAGSHI_SHIFT) +#define I40E_GLPES_PFIP6RXFRAGSLO(_i) (0x00011400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP6RXFRAGSLO_MAX_INDEX 15 +#define I40E_GLPES_PFIP6RXFRAGSLO_IP6RXFRAGSLO_SHIFT 0 +#define I40E_GLPES_PFIP6RXFRAGSLO_IP6RXFRAGSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFIP6RXFRAGSLO_IP6RXFRAGSLO_SHIFT) +#define I40E_GLPES_PFIP6RXMCOCTSHI(_i) (0x00011604 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP6RXMCOCTSHI_MAX_INDEX 15 +#define I40E_GLPES_PFIP6RXMCOCTSHI_IP6RXMCOCTSHI_SHIFT 0 +#define I40E_GLPES_PFIP6RXMCOCTSHI_IP6RXMCOCTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_PFIP6RXMCOCTSHI_IP6RXMCOCTSHI_SHIFT) +#define I40E_GLPES_PFIP6RXMCOCTSLO(_i) (0x00011600 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP6RXMCOCTSLO_MAX_INDEX 15 +#define I40E_GLPES_PFIP6RXMCOCTSLO_IP6RXMCOCTSLO_SHIFT 0 +#define I40E_GLPES_PFIP6RXMCOCTSLO_IP6RXMCOCTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFIP6RXMCOCTSLO_IP6RXMCOCTSLO_SHIFT) +#define I40E_GLPES_PFIP6RXMCPKTSHI(_i) (0x00011804 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP6RXMCPKTSHI_MAX_INDEX 15 +#define I40E_GLPES_PFIP6RXMCPKTSHI_IP6RXMCPKTSHI_SHIFT 0 +#define I40E_GLPES_PFIP6RXMCPKTSHI_IP6RXMCPKTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_PFIP6RXMCPKTSHI_IP6RXMCPKTSHI_SHIFT) +#define I40E_GLPES_PFIP6RXMCPKTSLO(_i) (0x00011800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP6RXMCPKTSLO_MAX_INDEX 15 +#define I40E_GLPES_PFIP6RXMCPKTSLO_IP6RXMCPKTSLO_SHIFT 0 +#define I40E_GLPES_PFIP6RXMCPKTSLO_IP6RXMCPKTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFIP6RXMCPKTSLO_IP6RXMCPKTSLO_SHIFT) +#define I40E_GLPES_PFIP6RXOCTSHI(_i) (0x00010E04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP6RXOCTSHI_MAX_INDEX 15 +#define I40E_GLPES_PFIP6RXOCTSHI_IP6RXOCTSHI_SHIFT 0 +#define I40E_GLPES_PFIP6RXOCTSHI_IP6RXOCTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_PFIP6RXOCTSHI_IP6RXOCTSHI_SHIFT) +#define I40E_GLPES_PFIP6RXOCTSLO(_i) (0x00010E00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP6RXOCTSLO_MAX_INDEX 15 +#define I40E_GLPES_PFIP6RXOCTSLO_IP6RXOCTSLO_SHIFT 0 +#define I40E_GLPES_PFIP6RXOCTSLO_IP6RXOCTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFIP6RXOCTSLO_IP6RXOCTSLO_SHIFT) +#define I40E_GLPES_PFIP6RXPKTSHI(_i) (0x00011004 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP6RXPKTSHI_MAX_INDEX 15 +#define I40E_GLPES_PFIP6RXPKTSHI_IP6RXPKTSHI_SHIFT 0 +#define I40E_GLPES_PFIP6RXPKTSHI_IP6RXPKTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_PFIP6RXPKTSHI_IP6RXPKTSHI_SHIFT) +#define I40E_GLPES_PFIP6RXPKTSLO(_i) (0x00011000 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP6RXPKTSLO_MAX_INDEX 15 +#define I40E_GLPES_PFIP6RXPKTSLO_IP6RXPKTSLO_SHIFT 0 +#define I40E_GLPES_PFIP6RXPKTSLO_IP6RXPKTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFIP6RXPKTSLO_IP6RXPKTSLO_SHIFT) +#define I40E_GLPES_PFIP6RXTRUNC(_i) (0x00011300 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP6RXTRUNC_MAX_INDEX 15 +#define I40E_GLPES_PFIP6RXTRUNC_IP6RXTRUNC_SHIFT 0 +#define I40E_GLPES_PFIP6RXTRUNC_IP6RXTRUNC_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFIP6RXTRUNC_IP6RXTRUNC_SHIFT) +#define I40E_GLPES_PFIP6TXFRAGSHI(_i) (0x00012804 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP6TXFRAGSHI_MAX_INDEX 15 +#define I40E_GLPES_PFIP6TXFRAGSHI_IP6TXFRAGSHI_SHIFT 0 +#define I40E_GLPES_PFIP6TXFRAGSHI_IP6TXFRAGSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_PFIP6TXFRAGSHI_IP6TXFRAGSHI_SHIFT) +#define I40E_GLPES_PFIP6TXFRAGSLO(_i) (0x00012800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP6TXFRAGSLO_MAX_INDEX 15 +#define I40E_GLPES_PFIP6TXFRAGSLO_IP6TXFRAGSLO_SHIFT 0 +#define I40E_GLPES_PFIP6TXFRAGSLO_IP6TXFRAGSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFIP6TXFRAGSLO_IP6TXFRAGSLO_SHIFT) +#define I40E_GLPES_PFIP6TXMCOCTSHI(_i) (0x00012A04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP6TXMCOCTSHI_MAX_INDEX 15 +#define I40E_GLPES_PFIP6TXMCOCTSHI_IP6TXMCOCTSHI_SHIFT 0 +#define I40E_GLPES_PFIP6TXMCOCTSHI_IP6TXMCOCTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_PFIP6TXMCOCTSHI_IP6TXMCOCTSHI_SHIFT) +#define I40E_GLPES_PFIP6TXMCOCTSLO(_i) (0x00012A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP6TXMCOCTSLO_MAX_INDEX 15 +#define I40E_GLPES_PFIP6TXMCOCTSLO_IP6TXMCOCTSLO_SHIFT 0 +#define I40E_GLPES_PFIP6TXMCOCTSLO_IP6TXMCOCTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFIP6TXMCOCTSLO_IP6TXMCOCTSLO_SHIFT) +#define I40E_GLPES_PFIP6TXMCPKTSHI(_i) (0x00012C04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP6TXMCPKTSHI_MAX_INDEX 15 +#define I40E_GLPES_PFIP6TXMCPKTSHI_IP6TXMCPKTSHI_SHIFT 0 +#define I40E_GLPES_PFIP6TXMCPKTSHI_IP6TXMCPKTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_PFIP6TXMCPKTSHI_IP6TXMCPKTSHI_SHIFT) +#define I40E_GLPES_PFIP6TXMCPKTSLO(_i) (0x00012C00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP6TXMCPKTSLO_MAX_INDEX 15 +#define I40E_GLPES_PFIP6TXMCPKTSLO_IP6TXMCPKTSLO_SHIFT 0 +#define I40E_GLPES_PFIP6TXMCPKTSLO_IP6TXMCPKTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFIP6TXMCPKTSLO_IP6TXMCPKTSLO_SHIFT) +#define I40E_GLPES_PFIP6TXNOROUTE(_i) (0x00012F00 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP6TXNOROUTE_MAX_INDEX 15 +#define I40E_GLPES_PFIP6TXNOROUTE_IP6TXNOROUTE_SHIFT 0 +#define I40E_GLPES_PFIP6TXNOROUTE_IP6TXNOROUTE_MASK I40E_MASK(0xFFFFFF, I40E_GLPES_PFIP6TXNOROUTE_IP6TXNOROUTE_SHIFT) +#define I40E_GLPES_PFIP6TXOCTSHI(_i) (0x00012404 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP6TXOCTSHI_MAX_INDEX 15 +#define I40E_GLPES_PFIP6TXOCTSHI_IP6TXOCTSHI_SHIFT 0 +#define I40E_GLPES_PFIP6TXOCTSHI_IP6TXOCTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_PFIP6TXOCTSHI_IP6TXOCTSHI_SHIFT) +#define I40E_GLPES_PFIP6TXOCTSLO(_i) (0x00012400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP6TXOCTSLO_MAX_INDEX 15 +#define I40E_GLPES_PFIP6TXOCTSLO_IP6TXOCTSLO_SHIFT 0 +#define I40E_GLPES_PFIP6TXOCTSLO_IP6TXOCTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFIP6TXOCTSLO_IP6TXOCTSLO_SHIFT) +#define I40E_GLPES_PFIP6TXPKTSHI(_i) (0x00012604 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP6TXPKTSHI_MAX_INDEX 15 +#define I40E_GLPES_PFIP6TXPKTSHI_IP6TXPKTSHI_SHIFT 0 +#define I40E_GLPES_PFIP6TXPKTSHI_IP6TXPKTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_PFIP6TXPKTSHI_IP6TXPKTSHI_SHIFT) +#define I40E_GLPES_PFIP6TXPKTSLO(_i) (0x00012600 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFIP6TXPKTSLO_MAX_INDEX 15 +#define I40E_GLPES_PFIP6TXPKTSLO_IP6TXPKTSLO_SHIFT 0 +#define I40E_GLPES_PFIP6TXPKTSLO_IP6TXPKTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFIP6TXPKTSLO_IP6TXPKTSLO_SHIFT) +#define I40E_GLPES_PFRDMARXRDSHI(_i) (0x00013E04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFRDMARXRDSHI_MAX_INDEX 15 +#define I40E_GLPES_PFRDMARXRDSHI_RDMARXRDSHI_SHIFT 0 +#define I40E_GLPES_PFRDMARXRDSHI_RDMARXRDSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_PFRDMARXRDSHI_RDMARXRDSHI_SHIFT) +#define I40E_GLPES_PFRDMARXRDSLO(_i) (0x00013E00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFRDMARXRDSLO_MAX_INDEX 15 +#define I40E_GLPES_PFRDMARXRDSLO_RDMARXRDSLO_SHIFT 0 +#define I40E_GLPES_PFRDMARXRDSLO_RDMARXRDSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFRDMARXRDSLO_RDMARXRDSLO_SHIFT) +#define I40E_GLPES_PFRDMARXSNDSHI(_i) (0x00014004 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFRDMARXSNDSHI_MAX_INDEX 15 +#define I40E_GLPES_PFRDMARXSNDSHI_RDMARXSNDSHI_SHIFT 0 +#define I40E_GLPES_PFRDMARXSNDSHI_RDMARXSNDSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_PFRDMARXSNDSHI_RDMARXSNDSHI_SHIFT) +#define I40E_GLPES_PFRDMARXSNDSLO(_i) (0x00014000 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFRDMARXSNDSLO_MAX_INDEX 15 +#define I40E_GLPES_PFRDMARXSNDSLO_RDMARXSNDSLO_SHIFT 0 +#define I40E_GLPES_PFRDMARXSNDSLO_RDMARXSNDSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFRDMARXSNDSLO_RDMARXSNDSLO_SHIFT) +#define I40E_GLPES_PFRDMARXWRSHI(_i) (0x00013C04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFRDMARXWRSHI_MAX_INDEX 15 +#define I40E_GLPES_PFRDMARXWRSHI_RDMARXWRSHI_SHIFT 0 +#define I40E_GLPES_PFRDMARXWRSHI_RDMARXWRSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_PFRDMARXWRSHI_RDMARXWRSHI_SHIFT) +#define I40E_GLPES_PFRDMARXWRSLO(_i) (0x00013C00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFRDMARXWRSLO_MAX_INDEX 15 +#define I40E_GLPES_PFRDMARXWRSLO_RDMARXWRSLO_SHIFT 0 +#define I40E_GLPES_PFRDMARXWRSLO_RDMARXWRSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFRDMARXWRSLO_RDMARXWRSLO_SHIFT) +#define I40E_GLPES_PFRDMATXRDSHI(_i) (0x00014404 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFRDMATXRDSHI_MAX_INDEX 15 +#define I40E_GLPES_PFRDMATXRDSHI_RDMARXRDSHI_SHIFT 0 +#define I40E_GLPES_PFRDMATXRDSHI_RDMARXRDSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_PFRDMATXRDSHI_RDMARXRDSHI_SHIFT) +#define I40E_GLPES_PFRDMATXRDSLO(_i) (0x00014400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFRDMATXRDSLO_MAX_INDEX 15 +#define I40E_GLPES_PFRDMATXRDSLO_RDMARXRDSLO_SHIFT 0 +#define I40E_GLPES_PFRDMATXRDSLO_RDMARXRDSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFRDMATXRDSLO_RDMARXRDSLO_SHIFT) +#define I40E_GLPES_PFRDMATXSNDSHI(_i) (0x00014604 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFRDMATXSNDSHI_MAX_INDEX 15 +#define I40E_GLPES_PFRDMATXSNDSHI_RDMARXSNDSHI_SHIFT 0 +#define I40E_GLPES_PFRDMATXSNDSHI_RDMARXSNDSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_PFRDMATXSNDSHI_RDMARXSNDSHI_SHIFT) +#define I40E_GLPES_PFRDMATXSNDSLO(_i) (0x00014600 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFRDMATXSNDSLO_MAX_INDEX 15 +#define I40E_GLPES_PFRDMATXSNDSLO_RDMARXSNDSLO_SHIFT 0 +#define I40E_GLPES_PFRDMATXSNDSLO_RDMARXSNDSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFRDMATXSNDSLO_RDMARXSNDSLO_SHIFT) +#define I40E_GLPES_PFRDMATXWRSHI(_i) (0x00014204 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFRDMATXWRSHI_MAX_INDEX 15 +#define I40E_GLPES_PFRDMATXWRSHI_RDMARXWRSHI_SHIFT 0 +#define I40E_GLPES_PFRDMATXWRSHI_RDMARXWRSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_PFRDMATXWRSHI_RDMARXWRSHI_SHIFT) +#define I40E_GLPES_PFRDMATXWRSLO(_i) (0x00014200 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFRDMATXWRSLO_MAX_INDEX 15 +#define I40E_GLPES_PFRDMATXWRSLO_RDMARXWRSLO_SHIFT 0 +#define I40E_GLPES_PFRDMATXWRSLO_RDMARXWRSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFRDMATXWRSLO_RDMARXWRSLO_SHIFT) +#define I40E_GLPES_PFRDMAVBNDHI(_i) (0x00014804 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFRDMAVBNDHI_MAX_INDEX 15 +#define I40E_GLPES_PFRDMAVBNDHI_RDMAVBNDHI_SHIFT 0 +#define I40E_GLPES_PFRDMAVBNDHI_RDMAVBNDHI_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFRDMAVBNDHI_RDMAVBNDHI_SHIFT) +#define I40E_GLPES_PFRDMAVBNDLO(_i) (0x00014800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFRDMAVBNDLO_MAX_INDEX 15 +#define I40E_GLPES_PFRDMAVBNDLO_RDMAVBNDLO_SHIFT 0 +#define I40E_GLPES_PFRDMAVBNDLO_RDMAVBNDLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFRDMAVBNDLO_RDMAVBNDLO_SHIFT) +#define I40E_GLPES_PFRDMAVINVHI(_i) (0x00014A04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFRDMAVINVHI_MAX_INDEX 15 +#define I40E_GLPES_PFRDMAVINVHI_RDMAVINVHI_SHIFT 0 +#define I40E_GLPES_PFRDMAVINVHI_RDMAVINVHI_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFRDMAVINVHI_RDMAVINVHI_SHIFT) +#define I40E_GLPES_PFRDMAVINVLO(_i) (0x00014A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFRDMAVINVLO_MAX_INDEX 15 +#define I40E_GLPES_PFRDMAVINVLO_RDMAVINVLO_SHIFT 0 +#define I40E_GLPES_PFRDMAVINVLO_RDMAVINVLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFRDMAVINVLO_RDMAVINVLO_SHIFT) +#define I40E_GLPES_PFRXVLANERR(_i) (0x00010000 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFRXVLANERR_MAX_INDEX 15 +#define I40E_GLPES_PFRXVLANERR_RXVLANERR_SHIFT 0 +#define I40E_GLPES_PFRXVLANERR_RXVLANERR_MASK I40E_MASK(0xFFFFFF, I40E_GLPES_PFRXVLANERR_RXVLANERR_SHIFT) +#define I40E_GLPES_PFTCPRTXSEG(_i) (0x00013600 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFTCPRTXSEG_MAX_INDEX 15 +#define I40E_GLPES_PFTCPRTXSEG_TCPRTXSEG_SHIFT 0 +#define I40E_GLPES_PFTCPRTXSEG_TCPRTXSEG_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFTCPRTXSEG_TCPRTXSEG_SHIFT) +#define I40E_GLPES_PFTCPRXOPTERR(_i) (0x00013200 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFTCPRXOPTERR_MAX_INDEX 15 +#define I40E_GLPES_PFTCPRXOPTERR_TCPRXOPTERR_SHIFT 0 +#define I40E_GLPES_PFTCPRXOPTERR_TCPRXOPTERR_MASK I40E_MASK(0xFFFFFF, I40E_GLPES_PFTCPRXOPTERR_TCPRXOPTERR_SHIFT) +#define I40E_GLPES_PFTCPRXPROTOERR(_i) (0x00013300 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFTCPRXPROTOERR_MAX_INDEX 15 +#define I40E_GLPES_PFTCPRXPROTOERR_TCPRXPROTOERR_SHIFT 0 +#define I40E_GLPES_PFTCPRXPROTOERR_TCPRXPROTOERR_MASK I40E_MASK(0xFFFFFF, I40E_GLPES_PFTCPRXPROTOERR_TCPRXPROTOERR_SHIFT) +#define I40E_GLPES_PFTCPRXSEGSHI(_i) (0x00013004 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFTCPRXSEGSHI_MAX_INDEX 15 +#define I40E_GLPES_PFTCPRXSEGSHI_TCPRXSEGSHI_SHIFT 0 +#define I40E_GLPES_PFTCPRXSEGSHI_TCPRXSEGSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_PFTCPRXSEGSHI_TCPRXSEGSHI_SHIFT) +#define I40E_GLPES_PFTCPRXSEGSLO(_i) (0x00013000 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFTCPRXSEGSLO_MAX_INDEX 15 +#define I40E_GLPES_PFTCPRXSEGSLO_TCPRXSEGSLO_SHIFT 0 +#define I40E_GLPES_PFTCPRXSEGSLO_TCPRXSEGSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFTCPRXSEGSLO_TCPRXSEGSLO_SHIFT) +#define I40E_GLPES_PFTCPTXSEGHI(_i) (0x00013404 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFTCPTXSEGHI_MAX_INDEX 15 +#define I40E_GLPES_PFTCPTXSEGHI_TCPTXSEGHI_SHIFT 0 +#define I40E_GLPES_PFTCPTXSEGHI_TCPTXSEGHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_PFTCPTXSEGHI_TCPTXSEGHI_SHIFT) +#define I40E_GLPES_PFTCPTXSEGLO(_i) (0x00013400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFTCPTXSEGLO_MAX_INDEX 15 +#define I40E_GLPES_PFTCPTXSEGLO_TCPTXSEGLO_SHIFT 0 +#define I40E_GLPES_PFTCPTXSEGLO_TCPTXSEGLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFTCPTXSEGLO_TCPTXSEGLO_SHIFT) +#define I40E_GLPES_PFUDPRXPKTSHI(_i) (0x00013804 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFUDPRXPKTSHI_MAX_INDEX 15 +#define I40E_GLPES_PFUDPRXPKTSHI_UDPRXPKTSHI_SHIFT 0 +#define I40E_GLPES_PFUDPRXPKTSHI_UDPRXPKTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_PFUDPRXPKTSHI_UDPRXPKTSHI_SHIFT) +#define I40E_GLPES_PFUDPRXPKTSLO(_i) (0x00013800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFUDPRXPKTSLO_MAX_INDEX 15 +#define I40E_GLPES_PFUDPRXPKTSLO_UDPRXPKTSLO_SHIFT 0 +#define I40E_GLPES_PFUDPRXPKTSLO_UDPRXPKTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFUDPRXPKTSLO_UDPRXPKTSLO_SHIFT) +#define I40E_GLPES_PFUDPTXPKTSHI(_i) (0x00013A04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFUDPTXPKTSHI_MAX_INDEX 15 +#define I40E_GLPES_PFUDPTXPKTSHI_UDPTXPKTSHI_SHIFT 0 +#define I40E_GLPES_PFUDPTXPKTSHI_UDPTXPKTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_PFUDPTXPKTSHI_UDPTXPKTSHI_SHIFT) +#define I40E_GLPES_PFUDPTXPKTSLO(_i) (0x00013A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */ +#define I40E_GLPES_PFUDPTXPKTSLO_MAX_INDEX 15 +#define I40E_GLPES_PFUDPTXPKTSLO_UDPTXPKTSLO_SHIFT 0 +#define I40E_GLPES_PFUDPTXPKTSLO_UDPTXPKTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_PFUDPTXPKTSLO_UDPTXPKTSLO_SHIFT) +#define I40E_GLPES_RDMARXMULTFPDUSHI 0x0001E014 /* Reset: PE_CORER */ +#define I40E_GLPES_RDMARXMULTFPDUSHI_RDMARXMULTFPDUSHI_SHIFT 0 +#define I40E_GLPES_RDMARXMULTFPDUSHI_RDMARXMULTFPDUSHI_MASK I40E_MASK(0xFFFFFF, I40E_GLPES_RDMARXMULTFPDUSHI_RDMARXMULTFPDUSHI_SHIFT) +#define I40E_GLPES_RDMARXMULTFPDUSLO 0x0001E010 /* Reset: PE_CORER */ +#define I40E_GLPES_RDMARXMULTFPDUSLO_RDMARXMULTFPDUSLO_SHIFT 0 +#define I40E_GLPES_RDMARXMULTFPDUSLO_RDMARXMULTFPDUSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_RDMARXMULTFPDUSLO_RDMARXMULTFPDUSLO_SHIFT) +#define I40E_GLPES_RDMARXOOODDPHI 0x0001E01C /* Reset: PE_CORER */ +#define I40E_GLPES_RDMARXOOODDPHI_RDMARXOOODDPHI_SHIFT 0 +#define I40E_GLPES_RDMARXOOODDPHI_RDMARXOOODDPHI_MASK I40E_MASK(0xFFFFFF, I40E_GLPES_RDMARXOOODDPHI_RDMARXOOODDPHI_SHIFT) +#define I40E_GLPES_RDMARXOOODDPLO 0x0001E018 /* Reset: PE_CORER */ +#define I40E_GLPES_RDMARXOOODDPLO_RDMARXOOODDPLO_SHIFT 0 +#define I40E_GLPES_RDMARXOOODDPLO_RDMARXOOODDPLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_RDMARXOOODDPLO_RDMARXOOODDPLO_SHIFT) +#define I40E_GLPES_RDMARXOOONOMARK 0x0001E004 /* Reset: PE_CORER */ +#define I40E_GLPES_RDMARXOOONOMARK_RDMAOOONOMARK_SHIFT 0 +#define I40E_GLPES_RDMARXOOONOMARK_RDMAOOONOMARK_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_RDMARXOOONOMARK_RDMAOOONOMARK_SHIFT) +#define I40E_GLPES_RDMARXUNALIGN 0x0001E000 /* Reset: PE_CORER */ +#define I40E_GLPES_RDMARXUNALIGN_RDMRXAUNALIGN_SHIFT 0 +#define I40E_GLPES_RDMARXUNALIGN_RDMRXAUNALIGN_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_RDMARXUNALIGN_RDMRXAUNALIGN_SHIFT) +#define I40E_GLPES_TCPRXFOURHOLEHI 0x0001E044 /* Reset: PE_CORER */ +#define I40E_GLPES_TCPRXFOURHOLEHI_TCPRXFOURHOLEHI_SHIFT 0 +#define I40E_GLPES_TCPRXFOURHOLEHI_TCPRXFOURHOLEHI_MASK I40E_MASK(0xFFFFFF, I40E_GLPES_TCPRXFOURHOLEHI_TCPRXFOURHOLEHI_SHIFT) +#define I40E_GLPES_TCPRXFOURHOLELO 0x0001E040 /* Reset: PE_CORER */ +#define I40E_GLPES_TCPRXFOURHOLELO_TCPRXFOURHOLELO_SHIFT 0 +#define I40E_GLPES_TCPRXFOURHOLELO_TCPRXFOURHOLELO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_TCPRXFOURHOLELO_TCPRXFOURHOLELO_SHIFT) +#define I40E_GLPES_TCPRXONEHOLEHI 0x0001E02C /* Reset: PE_CORER */ +#define I40E_GLPES_TCPRXONEHOLEHI_TCPRXONEHOLEHI_SHIFT 0 +#define I40E_GLPES_TCPRXONEHOLEHI_TCPRXONEHOLEHI_MASK I40E_MASK(0xFFFFFF, I40E_GLPES_TCPRXONEHOLEHI_TCPRXONEHOLEHI_SHIFT) +#define I40E_GLPES_TCPRXONEHOLELO 0x0001E028 /* Reset: PE_CORER */ +#define I40E_GLPES_TCPRXONEHOLELO_TCPRXONEHOLELO_SHIFT 0 +#define I40E_GLPES_TCPRXONEHOLELO_TCPRXONEHOLELO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_TCPRXONEHOLELO_TCPRXONEHOLELO_SHIFT) +#define I40E_GLPES_TCPRXPUREACKHI 0x0001E024 /* Reset: PE_CORER */ +#define I40E_GLPES_TCPRXPUREACKHI_TCPRXPUREACKSHI_SHIFT 0 +#define I40E_GLPES_TCPRXPUREACKHI_TCPRXPUREACKSHI_MASK I40E_MASK(0xFFFFFF, I40E_GLPES_TCPRXPUREACKHI_TCPRXPUREACKSHI_SHIFT) +#define I40E_GLPES_TCPRXPUREACKSLO 0x0001E020 /* Reset: PE_CORER */ +#define I40E_GLPES_TCPRXPUREACKSLO_TCPRXPUREACKLO_SHIFT 0 +#define I40E_GLPES_TCPRXPUREACKSLO_TCPRXPUREACKLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_TCPRXPUREACKSLO_TCPRXPUREACKLO_SHIFT) +#define I40E_GLPES_TCPRXTHREEHOLEHI 0x0001E03C /* Reset: PE_CORER */ +#define I40E_GLPES_TCPRXTHREEHOLEHI_TCPRXTHREEHOLEHI_SHIFT 0 +#define I40E_GLPES_TCPRXTHREEHOLEHI_TCPRXTHREEHOLEHI_MASK I40E_MASK(0xFFFFFF, I40E_GLPES_TCPRXTHREEHOLEHI_TCPRXTHREEHOLEHI_SHIFT) +#define I40E_GLPES_TCPRXTHREEHOLELO 0x0001E038 /* Reset: PE_CORER */ +#define I40E_GLPES_TCPRXTHREEHOLELO_TCPRXTHREEHOLELO_SHIFT 0 +#define I40E_GLPES_TCPRXTHREEHOLELO_TCPRXTHREEHOLELO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_TCPRXTHREEHOLELO_TCPRXTHREEHOLELO_SHIFT) +#define I40E_GLPES_TCPRXTWOHOLEHI 0x0001E034 /* Reset: PE_CORER */ +#define I40E_GLPES_TCPRXTWOHOLEHI_TCPRXTWOHOLEHI_SHIFT 0 +#define I40E_GLPES_TCPRXTWOHOLEHI_TCPRXTWOHOLEHI_MASK I40E_MASK(0xFFFFFF, I40E_GLPES_TCPRXTWOHOLEHI_TCPRXTWOHOLEHI_SHIFT) +#define I40E_GLPES_TCPRXTWOHOLELO 0x0001E030 /* Reset: PE_CORER */ +#define I40E_GLPES_TCPRXTWOHOLELO_TCPRXTWOHOLELO_SHIFT 0 +#define I40E_GLPES_TCPRXTWOHOLELO_TCPRXTWOHOLELO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_TCPRXTWOHOLELO_TCPRXTWOHOLELO_SHIFT) +#define I40E_GLPES_TCPTXRETRANSFASTHI 0x0001E04C /* Reset: PE_CORER */ +#define I40E_GLPES_TCPTXRETRANSFASTHI_TCPTXRETRANSFASTHI_SHIFT 0 +#define I40E_GLPES_TCPTXRETRANSFASTHI_TCPTXRETRANSFASTHI_MASK I40E_MASK(0xFFFFFF, I40E_GLPES_TCPTXRETRANSFASTHI_TCPTXRETRANSFASTHI_SHIFT) +#define I40E_GLPES_TCPTXRETRANSFASTLO 0x0001E048 /* Reset: PE_CORER */ +#define I40E_GLPES_TCPTXRETRANSFASTLO_TCPTXRETRANSFASTLO_SHIFT 0 +#define I40E_GLPES_TCPTXRETRANSFASTLO_TCPTXRETRANSFASTLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_TCPTXRETRANSFASTLO_TCPTXRETRANSFASTLO_SHIFT) +#define I40E_GLPES_TCPTXTOUTSFASTHI 0x0001E054 /* Reset: PE_CORER */ +#define I40E_GLPES_TCPTXTOUTSFASTHI_TCPTXTOUTSFASTHI_SHIFT 0 +#define I40E_GLPES_TCPTXTOUTSFASTHI_TCPTXTOUTSFASTHI_MASK I40E_MASK(0xFFFFFF, I40E_GLPES_TCPTXTOUTSFASTHI_TCPTXTOUTSFASTHI_SHIFT) +#define I40E_GLPES_TCPTXTOUTSFASTLO 0x0001E050 /* Reset: PE_CORER */ +#define I40E_GLPES_TCPTXTOUTSFASTLO_TCPTXTOUTSFASTLO_SHIFT 0 +#define I40E_GLPES_TCPTXTOUTSFASTLO_TCPTXTOUTSFASTLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_TCPTXTOUTSFASTLO_TCPTXTOUTSFASTLO_SHIFT) +#define I40E_GLPES_TCPTXTOUTSHI 0x0001E05C /* Reset: PE_CORER */ +#define I40E_GLPES_TCPTXTOUTSHI_TCPTXTOUTSHI_SHIFT 0 +#define I40E_GLPES_TCPTXTOUTSHI_TCPTXTOUTSHI_MASK I40E_MASK(0xFFFFFF, I40E_GLPES_TCPTXTOUTSHI_TCPTXTOUTSHI_SHIFT) +#define I40E_GLPES_TCPTXTOUTSLO 0x0001E058 /* Reset: PE_CORER */ +#define I40E_GLPES_TCPTXTOUTSLO_TCPTXTOUTSLO_SHIFT 0 +#define I40E_GLPES_TCPTXTOUTSLO_TCPTXTOUTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_TCPTXTOUTSLO_TCPTXTOUTSLO_SHIFT) +#define I40E_GLPES_VFIP4RXDISCARD(_i) (0x00018600 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP4RXDISCARD_MAX_INDEX 31 +#define I40E_GLPES_VFIP4RXDISCARD_IP4RXDISCARD_SHIFT 0 +#define I40E_GLPES_VFIP4RXDISCARD_IP4RXDISCARD_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFIP4RXDISCARD_IP4RXDISCARD_SHIFT) +#define I40E_GLPES_VFIP4RXFRAGSHI(_i) (0x00018804 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP4RXFRAGSHI_MAX_INDEX 31 +#define I40E_GLPES_VFIP4RXFRAGSHI_IP4RXFRAGSHI_SHIFT 0 +#define I40E_GLPES_VFIP4RXFRAGSHI_IP4RXFRAGSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_VFIP4RXFRAGSHI_IP4RXFRAGSHI_SHIFT) +#define I40E_GLPES_VFIP4RXFRAGSLO(_i) (0x00018800 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP4RXFRAGSLO_MAX_INDEX 31 +#define I40E_GLPES_VFIP4RXFRAGSLO_IP4RXFRAGSLO_SHIFT 0 +#define I40E_GLPES_VFIP4RXFRAGSLO_IP4RXFRAGSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFIP4RXFRAGSLO_IP4RXFRAGSLO_SHIFT) +#define I40E_GLPES_VFIP4RXMCOCTSHI(_i) (0x00018A04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP4RXMCOCTSHI_MAX_INDEX 31 +#define I40E_GLPES_VFIP4RXMCOCTSHI_IP4RXMCOCTSHI_SHIFT 0 +#define I40E_GLPES_VFIP4RXMCOCTSHI_IP4RXMCOCTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_VFIP4RXMCOCTSHI_IP4RXMCOCTSHI_SHIFT) +#define I40E_GLPES_VFIP4RXMCOCTSLO(_i) (0x00018A00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP4RXMCOCTSLO_MAX_INDEX 31 +#define I40E_GLPES_VFIP4RXMCOCTSLO_IP4RXMCOCTSLO_SHIFT 0 +#define I40E_GLPES_VFIP4RXMCOCTSLO_IP4RXMCOCTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFIP4RXMCOCTSLO_IP4RXMCOCTSLO_SHIFT) +#define I40E_GLPES_VFIP4RXMCPKTSHI(_i) (0x00018C04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP4RXMCPKTSHI_MAX_INDEX 31 +#define I40E_GLPES_VFIP4RXMCPKTSHI_IP4RXMCPKTSHI_SHIFT 0 +#define I40E_GLPES_VFIP4RXMCPKTSHI_IP4RXMCPKTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_VFIP4RXMCPKTSHI_IP4RXMCPKTSHI_SHIFT) +#define I40E_GLPES_VFIP4RXMCPKTSLO(_i) (0x00018C00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP4RXMCPKTSLO_MAX_INDEX 31 +#define I40E_GLPES_VFIP4RXMCPKTSLO_IP4RXMCPKTSLO_SHIFT 0 +#define I40E_GLPES_VFIP4RXMCPKTSLO_IP4RXMCPKTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFIP4RXMCPKTSLO_IP4RXMCPKTSLO_SHIFT) +#define I40E_GLPES_VFIP4RXOCTSHI(_i) (0x00018204 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP4RXOCTSHI_MAX_INDEX 31 +#define I40E_GLPES_VFIP4RXOCTSHI_IP4RXOCTSHI_SHIFT 0 +#define I40E_GLPES_VFIP4RXOCTSHI_IP4RXOCTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_VFIP4RXOCTSHI_IP4RXOCTSHI_SHIFT) +#define I40E_GLPES_VFIP4RXOCTSLO(_i) (0x00018200 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP4RXOCTSLO_MAX_INDEX 31 +#define I40E_GLPES_VFIP4RXOCTSLO_IP4RXOCTSLO_SHIFT 0 +#define I40E_GLPES_VFIP4RXOCTSLO_IP4RXOCTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFIP4RXOCTSLO_IP4RXOCTSLO_SHIFT) +#define I40E_GLPES_VFIP4RXPKTSHI(_i) (0x00018404 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP4RXPKTSHI_MAX_INDEX 31 +#define I40E_GLPES_VFIP4RXPKTSHI_IP4RXPKTSHI_SHIFT 0 +#define I40E_GLPES_VFIP4RXPKTSHI_IP4RXPKTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_VFIP4RXPKTSHI_IP4RXPKTSHI_SHIFT) +#define I40E_GLPES_VFIP4RXPKTSLO(_i) (0x00018400 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP4RXPKTSLO_MAX_INDEX 31 +#define I40E_GLPES_VFIP4RXPKTSLO_IP4RXPKTSLO_SHIFT 0 +#define I40E_GLPES_VFIP4RXPKTSLO_IP4RXPKTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFIP4RXPKTSLO_IP4RXPKTSLO_SHIFT) +#define I40E_GLPES_VFIP4RXTRUNC(_i) (0x00018700 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP4RXTRUNC_MAX_INDEX 31 +#define I40E_GLPES_VFIP4RXTRUNC_IP4RXTRUNC_SHIFT 0 +#define I40E_GLPES_VFIP4RXTRUNC_IP4RXTRUNC_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFIP4RXTRUNC_IP4RXTRUNC_SHIFT) +#define I40E_GLPES_VFIP4TXFRAGSHI(_i) (0x00019E04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP4TXFRAGSHI_MAX_INDEX 31 +#define I40E_GLPES_VFIP4TXFRAGSHI_IP4TXFRAGSHI_SHIFT 0 +#define I40E_GLPES_VFIP4TXFRAGSHI_IP4TXFRAGSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_VFIP4TXFRAGSHI_IP4TXFRAGSHI_SHIFT) +#define I40E_GLPES_VFIP4TXFRAGSLO(_i) (0x00019E00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP4TXFRAGSLO_MAX_INDEX 31 +#define I40E_GLPES_VFIP4TXFRAGSLO_IP4TXFRAGSLO_SHIFT 0 +#define I40E_GLPES_VFIP4TXFRAGSLO_IP4TXFRAGSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFIP4TXFRAGSLO_IP4TXFRAGSLO_SHIFT) +#define I40E_GLPES_VFIP4TXMCOCTSHI(_i) (0x0001A004 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP4TXMCOCTSHI_MAX_INDEX 31 +#define I40E_GLPES_VFIP4TXMCOCTSHI_IP4TXMCOCTSHI_SHIFT 0 +#define I40E_GLPES_VFIP4TXMCOCTSHI_IP4TXMCOCTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_VFIP4TXMCOCTSHI_IP4TXMCOCTSHI_SHIFT) +#define I40E_GLPES_VFIP4TXMCOCTSLO(_i) (0x0001A000 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP4TXMCOCTSLO_MAX_INDEX 31 +#define I40E_GLPES_VFIP4TXMCOCTSLO_IP4TXMCOCTSLO_SHIFT 0 +#define I40E_GLPES_VFIP4TXMCOCTSLO_IP4TXMCOCTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFIP4TXMCOCTSLO_IP4TXMCOCTSLO_SHIFT) +#define I40E_GLPES_VFIP4TXMCPKTSHI(_i) (0x0001A204 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP4TXMCPKTSHI_MAX_INDEX 31 +#define I40E_GLPES_VFIP4TXMCPKTSHI_IP4TXMCPKTSHI_SHIFT 0 +#define I40E_GLPES_VFIP4TXMCPKTSHI_IP4TXMCPKTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_VFIP4TXMCPKTSHI_IP4TXMCPKTSHI_SHIFT) +#define I40E_GLPES_VFIP4TXMCPKTSLO(_i) (0x0001A200 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP4TXMCPKTSLO_MAX_INDEX 31 +#define I40E_GLPES_VFIP4TXMCPKTSLO_IP4TXMCPKTSLO_SHIFT 0 +#define I40E_GLPES_VFIP4TXMCPKTSLO_IP4TXMCPKTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFIP4TXMCPKTSLO_IP4TXMCPKTSLO_SHIFT) +#define I40E_GLPES_VFIP4TXNOROUTE(_i) (0x0001AE00 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP4TXNOROUTE_MAX_INDEX 31 +#define I40E_GLPES_VFIP4TXNOROUTE_IP4TXNOROUTE_SHIFT 0 +#define I40E_GLPES_VFIP4TXNOROUTE_IP4TXNOROUTE_MASK I40E_MASK(0xFFFFFF, I40E_GLPES_VFIP4TXNOROUTE_IP4TXNOROUTE_SHIFT) +#define I40E_GLPES_VFIP4TXOCTSHI(_i) (0x00019A04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP4TXOCTSHI_MAX_INDEX 31 +#define I40E_GLPES_VFIP4TXOCTSHI_IP4TXOCTSHI_SHIFT 0 +#define I40E_GLPES_VFIP4TXOCTSHI_IP4TXOCTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_VFIP4TXOCTSHI_IP4TXOCTSHI_SHIFT) +#define I40E_GLPES_VFIP4TXOCTSLO(_i) (0x00019A00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP4TXOCTSLO_MAX_INDEX 31 +#define I40E_GLPES_VFIP4TXOCTSLO_IP4TXOCTSLO_SHIFT 0 +#define I40E_GLPES_VFIP4TXOCTSLO_IP4TXOCTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFIP4TXOCTSLO_IP4TXOCTSLO_SHIFT) +#define I40E_GLPES_VFIP4TXPKTSHI(_i) (0x00019C04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP4TXPKTSHI_MAX_INDEX 31 +#define I40E_GLPES_VFIP4TXPKTSHI_IP4TXPKTSHI_SHIFT 0 +#define I40E_GLPES_VFIP4TXPKTSHI_IP4TXPKTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_VFIP4TXPKTSHI_IP4TXPKTSHI_SHIFT) +#define I40E_GLPES_VFIP4TXPKTSLO(_i) (0x00019C00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP4TXPKTSLO_MAX_INDEX 31 +#define I40E_GLPES_VFIP4TXPKTSLO_IP4TXPKTSLO_SHIFT 0 +#define I40E_GLPES_VFIP4TXPKTSLO_IP4TXPKTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFIP4TXPKTSLO_IP4TXPKTSLO_SHIFT) +#define I40E_GLPES_VFIP6RXDISCARD(_i) (0x00019200 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP6RXDISCARD_MAX_INDEX 31 +#define I40E_GLPES_VFIP6RXDISCARD_IP6RXDISCARD_SHIFT 0 +#define I40E_GLPES_VFIP6RXDISCARD_IP6RXDISCARD_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFIP6RXDISCARD_IP6RXDISCARD_SHIFT) +#define I40E_GLPES_VFIP6RXFRAGSHI(_i) (0x00019404 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP6RXFRAGSHI_MAX_INDEX 31 +#define I40E_GLPES_VFIP6RXFRAGSHI_IP6RXFRAGSHI_SHIFT 0 +#define I40E_GLPES_VFIP6RXFRAGSHI_IP6RXFRAGSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_VFIP6RXFRAGSHI_IP6RXFRAGSHI_SHIFT) +#define I40E_GLPES_VFIP6RXFRAGSLO(_i) (0x00019400 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP6RXFRAGSLO_MAX_INDEX 31 +#define I40E_GLPES_VFIP6RXFRAGSLO_IP6RXFRAGSLO_SHIFT 0 +#define I40E_GLPES_VFIP6RXFRAGSLO_IP6RXFRAGSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFIP6RXFRAGSLO_IP6RXFRAGSLO_SHIFT) +#define I40E_GLPES_VFIP6RXMCOCTSHI(_i) (0x00019604 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP6RXMCOCTSHI_MAX_INDEX 31 +#define I40E_GLPES_VFIP6RXMCOCTSHI_IP6RXMCOCTSHI_SHIFT 0 +#define I40E_GLPES_VFIP6RXMCOCTSHI_IP6RXMCOCTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_VFIP6RXMCOCTSHI_IP6RXMCOCTSHI_SHIFT) +#define I40E_GLPES_VFIP6RXMCOCTSLO(_i) (0x00019600 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP6RXMCOCTSLO_MAX_INDEX 31 +#define I40E_GLPES_VFIP6RXMCOCTSLO_IP6RXMCOCTSLO_SHIFT 0 +#define I40E_GLPES_VFIP6RXMCOCTSLO_IP6RXMCOCTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFIP6RXMCOCTSLO_IP6RXMCOCTSLO_SHIFT) +#define I40E_GLPES_VFIP6RXMCPKTSHI(_i) (0x00019804 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP6RXMCPKTSHI_MAX_INDEX 31 +#define I40E_GLPES_VFIP6RXMCPKTSHI_IP6RXMCPKTSHI_SHIFT 0 +#define I40E_GLPES_VFIP6RXMCPKTSHI_IP6RXMCPKTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_VFIP6RXMCPKTSHI_IP6RXMCPKTSHI_SHIFT) +#define I40E_GLPES_VFIP6RXMCPKTSLO(_i) (0x00019800 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP6RXMCPKTSLO_MAX_INDEX 31 +#define I40E_GLPES_VFIP6RXMCPKTSLO_IP6RXMCPKTSLO_SHIFT 0 +#define I40E_GLPES_VFIP6RXMCPKTSLO_IP6RXMCPKTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFIP6RXMCPKTSLO_IP6RXMCPKTSLO_SHIFT) +#define I40E_GLPES_VFIP6RXOCTSHI(_i) (0x00018E04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP6RXOCTSHI_MAX_INDEX 31 +#define I40E_GLPES_VFIP6RXOCTSHI_IP6RXOCTSHI_SHIFT 0 +#define I40E_GLPES_VFIP6RXOCTSHI_IP6RXOCTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_VFIP6RXOCTSHI_IP6RXOCTSHI_SHIFT) +#define I40E_GLPES_VFIP6RXOCTSLO(_i) (0x00018E00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP6RXOCTSLO_MAX_INDEX 31 +#define I40E_GLPES_VFIP6RXOCTSLO_IP6RXOCTSLO_SHIFT 0 +#define I40E_GLPES_VFIP6RXOCTSLO_IP6RXOCTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFIP6RXOCTSLO_IP6RXOCTSLO_SHIFT) +#define I40E_GLPES_VFIP6RXPKTSHI(_i) (0x00019004 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP6RXPKTSHI_MAX_INDEX 31 +#define I40E_GLPES_VFIP6RXPKTSHI_IP6RXPKTSHI_SHIFT 0 +#define I40E_GLPES_VFIP6RXPKTSHI_IP6RXPKTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_VFIP6RXPKTSHI_IP6RXPKTSHI_SHIFT) +#define I40E_GLPES_VFIP6RXPKTSLO(_i) (0x00019000 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP6RXPKTSLO_MAX_INDEX 31 +#define I40E_GLPES_VFIP6RXPKTSLO_IP6RXPKTSLO_SHIFT 0 +#define I40E_GLPES_VFIP6RXPKTSLO_IP6RXPKTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFIP6RXPKTSLO_IP6RXPKTSLO_SHIFT) +#define I40E_GLPES_VFIP6RXTRUNC(_i) (0x00019300 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP6RXTRUNC_MAX_INDEX 31 +#define I40E_GLPES_VFIP6RXTRUNC_IP6RXTRUNC_SHIFT 0 +#define I40E_GLPES_VFIP6RXTRUNC_IP6RXTRUNC_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFIP6RXTRUNC_IP6RXTRUNC_SHIFT) +#define I40E_GLPES_VFIP6TXFRAGSHI(_i) (0x0001A804 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP6TXFRAGSHI_MAX_INDEX 31 +#define I40E_GLPES_VFIP6TXFRAGSHI_IP6TXFRAGSHI_SHIFT 0 +#define I40E_GLPES_VFIP6TXFRAGSHI_IP6TXFRAGSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_VFIP6TXFRAGSHI_IP6TXFRAGSHI_SHIFT) +#define I40E_GLPES_VFIP6TXFRAGSLO(_i) (0x0001A800 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP6TXFRAGSLO_MAX_INDEX 31 +#define I40E_GLPES_VFIP6TXFRAGSLO_IP6TXFRAGSLO_SHIFT 0 +#define I40E_GLPES_VFIP6TXFRAGSLO_IP6TXFRAGSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFIP6TXFRAGSLO_IP6TXFRAGSLO_SHIFT) +#define I40E_GLPES_VFIP6TXMCOCTSHI(_i) (0x0001AA04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP6TXMCOCTSHI_MAX_INDEX 31 +#define I40E_GLPES_VFIP6TXMCOCTSHI_IP6TXMCOCTSHI_SHIFT 0 +#define I40E_GLPES_VFIP6TXMCOCTSHI_IP6TXMCOCTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_VFIP6TXMCOCTSHI_IP6TXMCOCTSHI_SHIFT) +#define I40E_GLPES_VFIP6TXMCOCTSLO(_i) (0x0001AA00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP6TXMCOCTSLO_MAX_INDEX 31 +#define I40E_GLPES_VFIP6TXMCOCTSLO_IP6TXMCOCTSLO_SHIFT 0 +#define I40E_GLPES_VFIP6TXMCOCTSLO_IP6TXMCOCTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFIP6TXMCOCTSLO_IP6TXMCOCTSLO_SHIFT) +#define I40E_GLPES_VFIP6TXMCPKTSHI(_i) (0x0001AC04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP6TXMCPKTSHI_MAX_INDEX 31 +#define I40E_GLPES_VFIP6TXMCPKTSHI_IP6TXMCPKTSHI_SHIFT 0 +#define I40E_GLPES_VFIP6TXMCPKTSHI_IP6TXMCPKTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_VFIP6TXMCPKTSHI_IP6TXMCPKTSHI_SHIFT) +#define I40E_GLPES_VFIP6TXMCPKTSLO(_i) (0x0001AC00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP6TXMCPKTSLO_MAX_INDEX 31 +#define I40E_GLPES_VFIP6TXMCPKTSLO_IP6TXMCPKTSLO_SHIFT 0 +#define I40E_GLPES_VFIP6TXMCPKTSLO_IP6TXMCPKTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFIP6TXMCPKTSLO_IP6TXMCPKTSLO_SHIFT) +#define I40E_GLPES_VFIP6TXNOROUTE(_i) (0x0001AF00 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP6TXNOROUTE_MAX_INDEX 31 +#define I40E_GLPES_VFIP6TXNOROUTE_IP6TXNOROUTE_SHIFT 0 +#define I40E_GLPES_VFIP6TXNOROUTE_IP6TXNOROUTE_MASK I40E_MASK(0xFFFFFF, I40E_GLPES_VFIP6TXNOROUTE_IP6TXNOROUTE_SHIFT) +#define I40E_GLPES_VFIP6TXOCTSHI(_i) (0x0001A404 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP6TXOCTSHI_MAX_INDEX 31 +#define I40E_GLPES_VFIP6TXOCTSHI_IP6TXOCTSHI_SHIFT 0 +#define I40E_GLPES_VFIP6TXOCTSHI_IP6TXOCTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_VFIP6TXOCTSHI_IP6TXOCTSHI_SHIFT) +#define I40E_GLPES_VFIP6TXOCTSLO(_i) (0x0001A400 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP6TXOCTSLO_MAX_INDEX 31 +#define I40E_GLPES_VFIP6TXOCTSLO_IP6TXOCTSLO_SHIFT 0 +#define I40E_GLPES_VFIP6TXOCTSLO_IP6TXOCTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFIP6TXOCTSLO_IP6TXOCTSLO_SHIFT) +#define I40E_GLPES_VFIP6TXPKTSHI(_i) (0x0001A604 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP6TXPKTSHI_MAX_INDEX 31 +#define I40E_GLPES_VFIP6TXPKTSHI_IP6TXPKTSHI_SHIFT 0 +#define I40E_GLPES_VFIP6TXPKTSHI_IP6TXPKTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_VFIP6TXPKTSHI_IP6TXPKTSHI_SHIFT) +#define I40E_GLPES_VFIP6TXPKTSLO(_i) (0x0001A600 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFIP6TXPKTSLO_MAX_INDEX 31 +#define I40E_GLPES_VFIP6TXPKTSLO_IP6TXPKTSLO_SHIFT 0 +#define I40E_GLPES_VFIP6TXPKTSLO_IP6TXPKTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFIP6TXPKTSLO_IP6TXPKTSLO_SHIFT) +#define I40E_GLPES_VFRDMARXRDSHI(_i) (0x0001BE04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFRDMARXRDSHI_MAX_INDEX 31 +#define I40E_GLPES_VFRDMARXRDSHI_RDMARXRDSHI_SHIFT 0 +#define I40E_GLPES_VFRDMARXRDSHI_RDMARXRDSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_VFRDMARXRDSHI_RDMARXRDSHI_SHIFT) +#define I40E_GLPES_VFRDMARXRDSLO(_i) (0x0001BE00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFRDMARXRDSLO_MAX_INDEX 31 +#define I40E_GLPES_VFRDMARXRDSLO_RDMARXRDSLO_SHIFT 0 +#define I40E_GLPES_VFRDMARXRDSLO_RDMARXRDSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFRDMARXRDSLO_RDMARXRDSLO_SHIFT) +#define I40E_GLPES_VFRDMARXSNDSHI(_i) (0x0001C004 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFRDMARXSNDSHI_MAX_INDEX 31 +#define I40E_GLPES_VFRDMARXSNDSHI_RDMARXSNDSHI_SHIFT 0 +#define I40E_GLPES_VFRDMARXSNDSHI_RDMARXSNDSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_VFRDMARXSNDSHI_RDMARXSNDSHI_SHIFT) +#define I40E_GLPES_VFRDMARXSNDSLO(_i) (0x0001C000 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFRDMARXSNDSLO_MAX_INDEX 31 +#define I40E_GLPES_VFRDMARXSNDSLO_RDMARXSNDSLO_SHIFT 0 +#define I40E_GLPES_VFRDMARXSNDSLO_RDMARXSNDSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFRDMARXSNDSLO_RDMARXSNDSLO_SHIFT) +#define I40E_GLPES_VFRDMARXWRSHI(_i) (0x0001BC04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFRDMARXWRSHI_MAX_INDEX 31 +#define I40E_GLPES_VFRDMARXWRSHI_RDMARXWRSHI_SHIFT 0 +#define I40E_GLPES_VFRDMARXWRSHI_RDMARXWRSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_VFRDMARXWRSHI_RDMARXWRSHI_SHIFT) +#define I40E_GLPES_VFRDMARXWRSLO(_i) (0x0001BC00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFRDMARXWRSLO_MAX_INDEX 31 +#define I40E_GLPES_VFRDMARXWRSLO_RDMARXWRSLO_SHIFT 0 +#define I40E_GLPES_VFRDMARXWRSLO_RDMARXWRSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFRDMARXWRSLO_RDMARXWRSLO_SHIFT) +#define I40E_GLPES_VFRDMATXRDSHI(_i) (0x0001C404 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFRDMATXRDSHI_MAX_INDEX 31 +#define I40E_GLPES_VFRDMATXRDSHI_RDMARXRDSHI_SHIFT 0 +#define I40E_GLPES_VFRDMATXRDSHI_RDMARXRDSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_VFRDMATXRDSHI_RDMARXRDSHI_SHIFT) +#define I40E_GLPES_VFRDMATXRDSLO(_i) (0x0001C400 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFRDMATXRDSLO_MAX_INDEX 31 +#define I40E_GLPES_VFRDMATXRDSLO_RDMARXRDSLO_SHIFT 0 +#define I40E_GLPES_VFRDMATXRDSLO_RDMARXRDSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFRDMATXRDSLO_RDMARXRDSLO_SHIFT) +#define I40E_GLPES_VFRDMATXSNDSHI(_i) (0x0001C604 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFRDMATXSNDSHI_MAX_INDEX 31 +#define I40E_GLPES_VFRDMATXSNDSHI_RDMARXSNDSHI_SHIFT 0 +#define I40E_GLPES_VFRDMATXSNDSHI_RDMARXSNDSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_VFRDMATXSNDSHI_RDMARXSNDSHI_SHIFT) +#define I40E_GLPES_VFRDMATXSNDSLO(_i) (0x0001C600 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFRDMATXSNDSLO_MAX_INDEX 31 +#define I40E_GLPES_VFRDMATXSNDSLO_RDMARXSNDSLO_SHIFT 0 +#define I40E_GLPES_VFRDMATXSNDSLO_RDMARXSNDSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFRDMATXSNDSLO_RDMARXSNDSLO_SHIFT) +#define I40E_GLPES_VFRDMATXWRSHI(_i) (0x0001C204 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFRDMATXWRSHI_MAX_INDEX 31 +#define I40E_GLPES_VFRDMATXWRSHI_RDMARXWRSHI_SHIFT 0 +#define I40E_GLPES_VFRDMATXWRSHI_RDMARXWRSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_VFRDMATXWRSHI_RDMARXWRSHI_SHIFT) +#define I40E_GLPES_VFRDMATXWRSLO(_i) (0x0001C200 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFRDMATXWRSLO_MAX_INDEX 31 +#define I40E_GLPES_VFRDMATXWRSLO_RDMARXWRSLO_SHIFT 0 +#define I40E_GLPES_VFRDMATXWRSLO_RDMARXWRSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFRDMATXWRSLO_RDMARXWRSLO_SHIFT) +#define I40E_GLPES_VFRDMAVBNDHI(_i) (0x0001C804 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFRDMAVBNDHI_MAX_INDEX 31 +#define I40E_GLPES_VFRDMAVBNDHI_RDMAVBNDHI_SHIFT 0 +#define I40E_GLPES_VFRDMAVBNDHI_RDMAVBNDHI_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFRDMAVBNDHI_RDMAVBNDHI_SHIFT) +#define I40E_GLPES_VFRDMAVBNDLO(_i) (0x0001C800 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFRDMAVBNDLO_MAX_INDEX 31 +#define I40E_GLPES_VFRDMAVBNDLO_RDMAVBNDLO_SHIFT 0 +#define I40E_GLPES_VFRDMAVBNDLO_RDMAVBNDLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFRDMAVBNDLO_RDMAVBNDLO_SHIFT) +#define I40E_GLPES_VFRDMAVINVHI(_i) (0x0001CA04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFRDMAVINVHI_MAX_INDEX 31 +#define I40E_GLPES_VFRDMAVINVHI_RDMAVINVHI_SHIFT 0 +#define I40E_GLPES_VFRDMAVINVHI_RDMAVINVHI_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFRDMAVINVHI_RDMAVINVHI_SHIFT) +#define I40E_GLPES_VFRDMAVINVLO(_i) (0x0001CA00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFRDMAVINVLO_MAX_INDEX 31 +#define I40E_GLPES_VFRDMAVINVLO_RDMAVINVLO_SHIFT 0 +#define I40E_GLPES_VFRDMAVINVLO_RDMAVINVLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFRDMAVINVLO_RDMAVINVLO_SHIFT) +#define I40E_GLPES_VFRXVLANERR(_i) (0x00018000 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFRXVLANERR_MAX_INDEX 31 +#define I40E_GLPES_VFRXVLANERR_RXVLANERR_SHIFT 0 +#define I40E_GLPES_VFRXVLANERR_RXVLANERR_MASK I40E_MASK(0xFFFFFF, I40E_GLPES_VFRXVLANERR_RXVLANERR_SHIFT) +#define I40E_GLPES_VFTCPRTXSEG(_i) (0x0001B600 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFTCPRTXSEG_MAX_INDEX 31 +#define I40E_GLPES_VFTCPRTXSEG_TCPRTXSEG_SHIFT 0 +#define I40E_GLPES_VFTCPRTXSEG_TCPRTXSEG_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFTCPRTXSEG_TCPRTXSEG_SHIFT) +#define I40E_GLPES_VFTCPRXOPTERR(_i) (0x0001B200 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFTCPRXOPTERR_MAX_INDEX 31 +#define I40E_GLPES_VFTCPRXOPTERR_TCPRXOPTERR_SHIFT 0 +#define I40E_GLPES_VFTCPRXOPTERR_TCPRXOPTERR_MASK I40E_MASK(0xFFFFFF, I40E_GLPES_VFTCPRXOPTERR_TCPRXOPTERR_SHIFT) +#define I40E_GLPES_VFTCPRXPROTOERR(_i) (0x0001B300 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFTCPRXPROTOERR_MAX_INDEX 31 +#define I40E_GLPES_VFTCPRXPROTOERR_TCPRXPROTOERR_SHIFT 0 +#define I40E_GLPES_VFTCPRXPROTOERR_TCPRXPROTOERR_MASK I40E_MASK(0xFFFFFF, I40E_GLPES_VFTCPRXPROTOERR_TCPRXPROTOERR_SHIFT) +#define I40E_GLPES_VFTCPRXSEGSHI(_i) (0x0001B004 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFTCPRXSEGSHI_MAX_INDEX 31 +#define I40E_GLPES_VFTCPRXSEGSHI_TCPRXSEGSHI_SHIFT 0 +#define I40E_GLPES_VFTCPRXSEGSHI_TCPRXSEGSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_VFTCPRXSEGSHI_TCPRXSEGSHI_SHIFT) +#define I40E_GLPES_VFTCPRXSEGSLO(_i) (0x0001B000 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFTCPRXSEGSLO_MAX_INDEX 31 +#define I40E_GLPES_VFTCPRXSEGSLO_TCPRXSEGSLO_SHIFT 0 +#define I40E_GLPES_VFTCPRXSEGSLO_TCPRXSEGSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFTCPRXSEGSLO_TCPRXSEGSLO_SHIFT) +#define I40E_GLPES_VFTCPTXSEGHI(_i) (0x0001B404 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFTCPTXSEGHI_MAX_INDEX 31 +#define I40E_GLPES_VFTCPTXSEGHI_TCPTXSEGHI_SHIFT 0 +#define I40E_GLPES_VFTCPTXSEGHI_TCPTXSEGHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_VFTCPTXSEGHI_TCPTXSEGHI_SHIFT) +#define I40E_GLPES_VFTCPTXSEGLO(_i) (0x0001B400 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFTCPTXSEGLO_MAX_INDEX 31 +#define I40E_GLPES_VFTCPTXSEGLO_TCPTXSEGLO_SHIFT 0 +#define I40E_GLPES_VFTCPTXSEGLO_TCPTXSEGLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFTCPTXSEGLO_TCPTXSEGLO_SHIFT) +#define I40E_GLPES_VFUDPRXPKTSHI(_i) (0x0001B804 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFUDPRXPKTSHI_MAX_INDEX 31 +#define I40E_GLPES_VFUDPRXPKTSHI_UDPRXPKTSHI_SHIFT 0 +#define I40E_GLPES_VFUDPRXPKTSHI_UDPRXPKTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_VFUDPRXPKTSHI_UDPRXPKTSHI_SHIFT) +#define I40E_GLPES_VFUDPRXPKTSLO(_i) (0x0001B800 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFUDPRXPKTSLO_MAX_INDEX 31 +#define I40E_GLPES_VFUDPRXPKTSLO_UDPRXPKTSLO_SHIFT 0 +#define I40E_GLPES_VFUDPRXPKTSLO_UDPRXPKTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFUDPRXPKTSLO_UDPRXPKTSLO_SHIFT) +#define I40E_GLPES_VFUDPTXPKTSHI(_i) (0x0001BA04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFUDPTXPKTSHI_MAX_INDEX 31 +#define I40E_GLPES_VFUDPTXPKTSHI_UDPTXPKTSHI_SHIFT 0 +#define I40E_GLPES_VFUDPTXPKTSHI_UDPTXPKTSHI_MASK I40E_MASK(0xFFFF, I40E_GLPES_VFUDPTXPKTSHI_UDPTXPKTSHI_SHIFT) +#define I40E_GLPES_VFUDPTXPKTSLO(_i) (0x0001BA00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */ +#define I40E_GLPES_VFUDPTXPKTSLO_MAX_INDEX 31 +#define I40E_GLPES_VFUDPTXPKTSLO_UDPTXPKTSLO_SHIFT 0 +#define I40E_GLPES_VFUDPTXPKTSLO_UDPTXPKTSLO_MASK I40E_MASK(0xFFFFFFFF, I40E_GLPES_VFUDPTXPKTSLO_UDPTXPKTSLO_SHIFT) +#define I40E_GLGEN_PME_TO 0x000B81BC /* Reset: POR */ +#define I40E_GLGEN_PME_TO_PME_TO_FOR_PE_SHIFT 0 +#define I40E_GLGEN_PME_TO_PME_TO_FOR_PE_MASK I40E_MASK(0x1, I40E_GLGEN_PME_TO_PME_TO_FOR_PE_SHIFT) +#define I40E_GLQF_APBVT(_i) (0x00260000 + ((_i) * 4)) /* _i=0...2047 */ /* Reset: CORER */ +#define I40E_GLQF_APBVT_MAX_INDEX 2047 +#define I40E_GLQF_APBVT_APBVT_SHIFT 0 +#define I40E_GLQF_APBVT_APBVT_MASK I40E_MASK(0xFFFFFFFF, I40E_GLQF_APBVT_APBVT_SHIFT) +#define I40E_GLQF_FD_PCTYPES(_i) (0x00268000 + ((_i) * 4)) /* _i=0...63 */ /* Reset: POR */ +#define I40E_GLQF_FD_PCTYPES_MAX_INDEX 63 +#define I40E_GLQF_FD_PCTYPES_FD_PCTYPE_SHIFT 0 +#define I40E_GLQF_FD_PCTYPES_FD_PCTYPE_MASK I40E_MASK(0x3F, I40E_GLQF_FD_PCTYPES_FD_PCTYPE_SHIFT) +#define I40E_GLQF_FD_MSK(_i, _j) (0x00267200 + ((_i) * 4 + (_j) * 8)) /* _i=0...1, _j=0...63 */ /* Reset: CORER */ +#define I40E_GLQF_FD_MSK_MAX_INDEX 1 +#define I40E_GLQF_FD_MSK_MASK_SHIFT 0 +#define I40E_GLQF_FD_MSK_MASK_MASK I40E_MASK(0xFFFF, I40E_GLQF_FD_MSK_MASK_SHIFT) +#define I40E_GLQF_FD_MSK_OFFSET_SHIFT 16 +#define I40E_GLQF_FD_MSK_OFFSET_MASK I40E_MASK(0x3F, I40E_GLQF_FD_MSK_OFFSET_SHIFT) +#define I40E_GLQF_HASH_INSET(_i, _j) (0x00267600 + ((_i) * 4 + (_j) * 8)) /* _i=0...1, _j=0...63 */ /* Reset: CORER */ +#define I40E_GLQF_HASH_INSET_MAX_INDEX 1 +#define I40E_GLQF_HASH_INSET_INSET_SHIFT 0 +#define I40E_GLQF_HASH_INSET_INSET_MASK I40E_MASK(0xFFFFFFFF, I40E_GLQF_HASH_INSET_INSET_SHIFT) +#define I40E_GLQF_HASH_MSK(_i, _j) (0x00267A00 + ((_i) * 4 + (_j) * 8)) /* _i=0...1, _j=0...63 */ /* Reset: CORER */ +#define I40E_GLQF_HASH_MSK_MAX_INDEX 1 +#define I40E_GLQF_HASH_MSK_MASK_SHIFT 0 +#define I40E_GLQF_HASH_MSK_MASK_MASK I40E_MASK(0xFFFF, I40E_GLQF_HASH_MSK_MASK_SHIFT) +#define I40E_GLQF_HASH_MSK_OFFSET_SHIFT 16 +#define I40E_GLQF_HASH_MSK_OFFSET_MASK I40E_MASK(0x3F, I40E_GLQF_HASH_MSK_OFFSET_SHIFT) +#define I40E_GLQF_ORT(_i) (0x00268900 + ((_i) * 4)) /* _i=0...63 */ /* Reset: CORER */ +#define I40E_GLQF_ORT_MAX_INDEX 63 +#define I40E_GLQF_ORT_PIT_INDX_SHIFT 0 +#define I40E_GLQF_ORT_PIT_INDX_MASK I40E_MASK(0x1F, I40E_GLQF_ORT_PIT_INDX_SHIFT) +#define I40E_GLQF_ORT_FIELD_CNT_SHIFT 5 +#define I40E_GLQF_ORT_FIELD_CNT_MASK I40E_MASK(0x3, I40E_GLQF_ORT_FIELD_CNT_SHIFT) +#define I40E_GLQF_ORT_FLX_PAYLOAD_SHIFT 7 +#define I40E_GLQF_ORT_FLX_PAYLOAD_MASK I40E_MASK(0x1, I40E_GLQF_ORT_FLX_PAYLOAD_SHIFT) +#define I40E_GLQF_PIT(_i) (0x00268C80 + ((_i) * 4)) /* _i=0...23 */ /* Reset: CORER */ +#define I40E_GLQF_PIT_MAX_INDEX 23 +#define I40E_GLQF_PIT_SOURCE_OFF_SHIFT 0 +#define I40E_GLQF_PIT_SOURCE_OFF_MASK I40E_MASK(0x1F, I40E_GLQF_PIT_SOURCE_OFF_SHIFT) +#define I40E_GLQF_PIT_FSIZE_SHIFT 5 +#define I40E_GLQF_PIT_FSIZE_MASK I40E_MASK(0x1F, I40E_GLQF_PIT_FSIZE_SHIFT) +#define I40E_GLQF_PIT_DEST_OFF_SHIFT 10 +#define I40E_GLQF_PIT_DEST_OFF_MASK I40E_MASK(0x3F, I40E_GLQF_PIT_DEST_OFF_SHIFT) +#define I40E_GLQF_FDEVICTENA(_i) (0x00270384 + ((_i) * 4)) /* _i=0...1 */ /* Reset: CORER */ +#define I40E_GLQF_FDEVICTENA_MAX_INDEX 1 +#define I40E_GLQF_FDEVICTENA_GLQF_FDEVICTENA_SHIFT 0 +#define I40E_GLQF_FDEVICTENA_GLQF_FDEVICTENA_MASK I40E_MASK(0xFFFFFFFF, I40E_GLQF_FDEVICTENA_GLQF_FDEVICTENA_SHIFT) +#define I40E_GLQF_FDEVICTFLAG 0x00270280 /* Reset: CORER */ +#define I40E_GLQF_FDEVICTFLAG_TX_FLAGS_SHIFT 0 +#define I40E_GLQF_FDEVICTFLAG_TX_FLAGS_MASK I40E_MASK(0xFF, I40E_GLQF_FDEVICTFLAG_TX_FLAGS_SHIFT) +#define I40E_GLQF_FDEVICTFLAG_RX_FLAGS_SHIFT 8 +#define I40E_GLQF_FDEVICTFLAG_RX_FLAGS_MASK I40E_MASK(0xFF, I40E_GLQF_FDEVICTFLAG_RX_FLAGS_SHIFT) +#define I40E_PFQF_CTL_2 0x00270300 /* Reset: CORER */ +#define I40E_PFQF_CTL_2_PEHSIZE_SHIFT 0 +#define I40E_PFQF_CTL_2_PEHSIZE_MASK I40E_MASK(0x1F, I40E_PFQF_CTL_2_PEHSIZE_SHIFT) +#define I40E_PFQF_CTL_2_PEDSIZE_SHIFT 5 +#define I40E_PFQF_CTL_2_PEDSIZE_MASK I40E_MASK(0x1F, I40E_PFQF_CTL_2_PEDSIZE_SHIFT) +/* Redefined for X722 family */ +#define I40E_X722_PFQF_HLUT(_i) (0x00240000 + ((_i) * 128)) /* _i=0...127 */ /* Reset: CORER */ +#define I40E_X722_PFQF_HLUT_MAX_INDEX 127 +#define I40E_X722_PFQF_HLUT_LUT0_SHIFT 0 +#define I40E_X722_PFQF_HLUT_LUT0_MASK I40E_MASK(0x7F, I40E_X722_PFQF_HLUT_LUT0_SHIFT) +#define I40E_X722_PFQF_HLUT_LUT1_SHIFT 8 +#define I40E_X722_PFQF_HLUT_LUT1_MASK I40E_MASK(0x7F, I40E_X722_PFQF_HLUT_LUT1_SHIFT) +#define I40E_X722_PFQF_HLUT_LUT2_SHIFT 16 +#define I40E_X722_PFQF_HLUT_LUT2_MASK I40E_MASK(0x7F, I40E_X722_PFQF_HLUT_LUT2_SHIFT) +#define I40E_X722_PFQF_HLUT_LUT3_SHIFT 24 +#define I40E_X722_PFQF_HLUT_LUT3_MASK I40E_MASK(0x7F, I40E_X722_PFQF_HLUT_LUT3_SHIFT) +#define I40E_PFQF_HREGION(_i) (0x00245400 + ((_i) * 128)) /* _i=0...7 */ /* Reset: CORER */ +#define I40E_PFQF_HREGION_MAX_INDEX 7 +#define I40E_PFQF_HREGION_OVERRIDE_ENA_0_SHIFT 0 +#define I40E_PFQF_HREGION_OVERRIDE_ENA_0_MASK I40E_MASK(0x1, I40E_PFQF_HREGION_OVERRIDE_ENA_0_SHIFT) +#define I40E_PFQF_HREGION_REGION_0_SHIFT 1 +#define I40E_PFQF_HREGION_REGION_0_MASK I40E_MASK(0x7, I40E_PFQF_HREGION_REGION_0_SHIFT) +#define I40E_PFQF_HREGION_OVERRIDE_ENA_1_SHIFT 4 +#define I40E_PFQF_HREGION_OVERRIDE_ENA_1_MASK I40E_MASK(0x1, I40E_PFQF_HREGION_OVERRIDE_ENA_1_SHIFT) +#define I40E_PFQF_HREGION_REGION_1_SHIFT 5 +#define I40E_PFQF_HREGION_REGION_1_MASK I40E_MASK(0x7, I40E_PFQF_HREGION_REGION_1_SHIFT) +#define I40E_PFQF_HREGION_OVERRIDE_ENA_2_SHIFT 8 +#define I40E_PFQF_HREGION_OVERRIDE_ENA_2_MASK I40E_MASK(0x1, I40E_PFQF_HREGION_OVERRIDE_ENA_2_SHIFT) +#define I40E_PFQF_HREGION_REGION_2_SHIFT 9 +#define I40E_PFQF_HREGION_REGION_2_MASK I40E_MASK(0x7, I40E_PFQF_HREGION_REGION_2_SHIFT) +#define I40E_PFQF_HREGION_OVERRIDE_ENA_3_SHIFT 12 +#define I40E_PFQF_HREGION_OVERRIDE_ENA_3_MASK I40E_MASK(0x1, I40E_PFQF_HREGION_OVERRIDE_ENA_3_SHIFT) +#define I40E_PFQF_HREGION_REGION_3_SHIFT 13 +#define I40E_PFQF_HREGION_REGION_3_MASK I40E_MASK(0x7, I40E_PFQF_HREGION_REGION_3_SHIFT) +#define I40E_PFQF_HREGION_OVERRIDE_ENA_4_SHIFT 16 +#define I40E_PFQF_HREGION_OVERRIDE_ENA_4_MASK I40E_MASK(0x1, I40E_PFQF_HREGION_OVERRIDE_ENA_4_SHIFT) +#define I40E_PFQF_HREGION_REGION_4_SHIFT 17 +#define I40E_PFQF_HREGION_REGION_4_MASK I40E_MASK(0x7, I40E_PFQF_HREGION_REGION_4_SHIFT) +#define I40E_PFQF_HREGION_OVERRIDE_ENA_5_SHIFT 20 +#define I40E_PFQF_HREGION_OVERRIDE_ENA_5_MASK I40E_MASK(0x1, I40E_PFQF_HREGION_OVERRIDE_ENA_5_SHIFT) +#define I40E_PFQF_HREGION_REGION_5_SHIFT 21 +#define I40E_PFQF_HREGION_REGION_5_MASK I40E_MASK(0x7, I40E_PFQF_HREGION_REGION_5_SHIFT) +#define I40E_PFQF_HREGION_OVERRIDE_ENA_6_SHIFT 24 +#define I40E_PFQF_HREGION_OVERRIDE_ENA_6_MASK I40E_MASK(0x1, I40E_PFQF_HREGION_OVERRIDE_ENA_6_SHIFT) +#define I40E_PFQF_HREGION_REGION_6_SHIFT 25 +#define I40E_PFQF_HREGION_REGION_6_MASK I40E_MASK(0x7, I40E_PFQF_HREGION_REGION_6_SHIFT) +#define I40E_PFQF_HREGION_OVERRIDE_ENA_7_SHIFT 28 +#define I40E_PFQF_HREGION_OVERRIDE_ENA_7_MASK I40E_MASK(0x1, I40E_PFQF_HREGION_OVERRIDE_ENA_7_SHIFT) +#define I40E_PFQF_HREGION_REGION_7_SHIFT 29 +#define I40E_PFQF_HREGION_REGION_7_MASK I40E_MASK(0x7, I40E_PFQF_HREGION_REGION_7_SHIFT) +#define I40E_VSIQF_CTL_RSS_LUT_TYPE_SHIFT 8 +#define I40E_VSIQF_CTL_RSS_LUT_TYPE_MASK I40E_MASK(0x1, I40E_VSIQF_CTL_RSS_LUT_TYPE_SHIFT) +#define I40E_VSIQF_HKEY(_i, _VSI) (0x002A0000 + ((_i) * 2048 + (_VSI) * 4)) /* _i=0...12, _VSI=0...383 */ /* Reset: CORER */ +#define I40E_VSIQF_HKEY_MAX_INDEX 12 +#define I40E_VSIQF_HKEY_KEY_0_SHIFT 0 +#define I40E_VSIQF_HKEY_KEY_0_MASK I40E_MASK(0xFF, I40E_VSIQF_HKEY_KEY_0_SHIFT) +#define I40E_VSIQF_HKEY_KEY_1_SHIFT 8 +#define I40E_VSIQF_HKEY_KEY_1_MASK I40E_MASK(0xFF, I40E_VSIQF_HKEY_KEY_1_SHIFT) +#define I40E_VSIQF_HKEY_KEY_2_SHIFT 16 +#define I40E_VSIQF_HKEY_KEY_2_MASK I40E_MASK(0xFF, I40E_VSIQF_HKEY_KEY_2_SHIFT) +#define I40E_VSIQF_HKEY_KEY_3_SHIFT 24 +#define I40E_VSIQF_HKEY_KEY_3_MASK I40E_MASK(0xFF, I40E_VSIQF_HKEY_KEY_3_SHIFT) +#define I40E_VSIQF_HLUT(_i, _VSI) (0x00220000 + ((_i) * 2048 + (_VSI) * 4)) /* _i=0...15, _VSI=0...383 */ /* Reset: CORER */ +#define I40E_VSIQF_HLUT_MAX_INDEX 15 +#define I40E_VSIQF_HLUT_LUT0_SHIFT 0 +#define I40E_VSIQF_HLUT_LUT0_MASK I40E_MASK(0xF, I40E_VSIQF_HLUT_LUT0_SHIFT) +#define I40E_VSIQF_HLUT_LUT1_SHIFT 8 +#define I40E_VSIQF_HLUT_LUT1_MASK I40E_MASK(0xF, I40E_VSIQF_HLUT_LUT1_SHIFT) +#define I40E_VSIQF_HLUT_LUT2_SHIFT 16 +#define I40E_VSIQF_HLUT_LUT2_MASK I40E_MASK(0xF, I40E_VSIQF_HLUT_LUT2_SHIFT) +#define I40E_VSIQF_HLUT_LUT3_SHIFT 24 +#define I40E_VSIQF_HLUT_LUT3_MASK I40E_MASK(0xF, I40E_VSIQF_HLUT_LUT3_SHIFT) +#define I40E_GLGEN_STAT_CLEAR 0x00390004 /* Reset: CORER */ +#define I40E_GLGEN_STAT_CLEAR_GLGEN_STAT_CLEAR_SHIFT 0 +#define I40E_GLGEN_STAT_CLEAR_GLGEN_STAT_CLEAR_MASK I40E_MASK(0x1, I40E_GLGEN_STAT_CLEAR_GLGEN_STAT_CLEAR_SHIFT) +#define I40E_GLGEN_STAT_HALT 0x00390000 /* Reset: CORER */ +#define I40E_GLGEN_STAT_HALT_HALT_CELLS_SHIFT 0 +#define I40E_GLGEN_STAT_HALT_HALT_CELLS_MASK I40E_MASK(0x3FFFFFFF, I40E_GLGEN_STAT_HALT_HALT_CELLS_SHIFT) +#define I40E_VFINT_DYN_CTL01_WB_ON_ITR_SHIFT 30 +#define I40E_VFINT_DYN_CTL01_WB_ON_ITR_MASK I40E_MASK(0x1, I40E_VFINT_DYN_CTL01_WB_ON_ITR_SHIFT) +#define I40E_VFINT_DYN_CTLN1_WB_ON_ITR_SHIFT 30 +#define I40E_VFINT_DYN_CTLN1_WB_ON_ITR_MASK I40E_MASK(0x1, I40E_VFINT_DYN_CTLN1_WB_ON_ITR_SHIFT) +#define I40E_VFPE_AEQALLOC1 0x0000A400 /* Reset: VFR */ +#define I40E_VFPE_AEQALLOC1_AECOUNT_SHIFT 0 +#define I40E_VFPE_AEQALLOC1_AECOUNT_MASK I40E_MASK(0xFFFFFFFF, I40E_VFPE_AEQALLOC1_AECOUNT_SHIFT) +#define I40E_VFPE_CCQPHIGH1 0x00009800 /* Reset: VFR */ +#define I40E_VFPE_CCQPHIGH1_PECCQPHIGH_SHIFT 0 +#define I40E_VFPE_CCQPHIGH1_PECCQPHIGH_MASK I40E_MASK(0xFFFFFFFF, I40E_VFPE_CCQPHIGH1_PECCQPHIGH_SHIFT) +#define I40E_VFPE_CCQPLOW1 0x0000AC00 /* Reset: VFR */ +#define I40E_VFPE_CCQPLOW1_PECCQPLOW_SHIFT 0 +#define I40E_VFPE_CCQPLOW1_PECCQPLOW_MASK I40E_MASK(0xFFFFFFFF, I40E_VFPE_CCQPLOW1_PECCQPLOW_SHIFT) +#define I40E_VFPE_CCQPSTATUS1 0x0000B800 /* Reset: VFR */ +#define I40E_VFPE_CCQPSTATUS1_CCQP_DONE_SHIFT 0 +#define I40E_VFPE_CCQPSTATUS1_CCQP_DONE_MASK I40E_MASK(0x1, I40E_VFPE_CCQPSTATUS1_CCQP_DONE_SHIFT) +#define I40E_VFPE_CCQPSTATUS1_HMC_PROFILE_SHIFT 4 +#define I40E_VFPE_CCQPSTATUS1_HMC_PROFILE_MASK I40E_MASK(0x7, I40E_VFPE_CCQPSTATUS1_HMC_PROFILE_SHIFT) +#define I40E_VFPE_CCQPSTATUS1_RDMA_EN_VFS_SHIFT 16 +#define I40E_VFPE_CCQPSTATUS1_RDMA_EN_VFS_MASK I40E_MASK(0x3F, I40E_VFPE_CCQPSTATUS1_RDMA_EN_VFS_SHIFT) +#define I40E_VFPE_CCQPSTATUS1_CCQP_ERR_SHIFT 31 +#define I40E_VFPE_CCQPSTATUS1_CCQP_ERR_MASK I40E_MASK(0x1, I40E_VFPE_CCQPSTATUS1_CCQP_ERR_SHIFT) +#define I40E_VFPE_CQACK1 0x0000B000 /* Reset: VFR */ +#define I40E_VFPE_CQACK1_PECQID_SHIFT 0 +#define I40E_VFPE_CQACK1_PECQID_MASK I40E_MASK(0x1FFFF, I40E_VFPE_CQACK1_PECQID_SHIFT) +#define I40E_VFPE_CQARM1 0x0000B400 /* Reset: VFR */ +#define I40E_VFPE_CQARM1_PECQID_SHIFT 0 +#define I40E_VFPE_CQARM1_PECQID_MASK I40E_MASK(0x1FFFF, I40E_VFPE_CQARM1_PECQID_SHIFT) +#define I40E_VFPE_CQPDB1 0x0000BC00 /* Reset: VFR */ +#define I40E_VFPE_CQPDB1_WQHEAD_SHIFT 0 +#define I40E_VFPE_CQPDB1_WQHEAD_MASK I40E_MASK(0x7FF, I40E_VFPE_CQPDB1_WQHEAD_SHIFT) +#define I40E_VFPE_CQPERRCODES1 0x00009C00 /* Reset: VFR */ +#define I40E_VFPE_CQPERRCODES1_CQP_MINOR_CODE_SHIFT 0 +#define I40E_VFPE_CQPERRCODES1_CQP_MINOR_CODE_MASK I40E_MASK(0xFFFF, I40E_VFPE_CQPERRCODES1_CQP_MINOR_CODE_SHIFT) +#define I40E_VFPE_CQPERRCODES1_CQP_MAJOR_CODE_SHIFT 16 +#define I40E_VFPE_CQPERRCODES1_CQP_MAJOR_CODE_MASK I40E_MASK(0xFFFF, I40E_VFPE_CQPERRCODES1_CQP_MAJOR_CODE_SHIFT) +#define I40E_VFPE_CQPTAIL1 0x0000A000 /* Reset: VFR */ +#define I40E_VFPE_CQPTAIL1_WQTAIL_SHIFT 0 +#define I40E_VFPE_CQPTAIL1_WQTAIL_MASK I40E_MASK(0x7FF, I40E_VFPE_CQPTAIL1_WQTAIL_SHIFT) +#define I40E_VFPE_CQPTAIL1_CQP_OP_ERR_SHIFT 31 +#define I40E_VFPE_CQPTAIL1_CQP_OP_ERR_MASK I40E_MASK(0x1, I40E_VFPE_CQPTAIL1_CQP_OP_ERR_SHIFT) +#define I40E_VFPE_IPCONFIG01 0x00008C00 /* Reset: VFR */ +#define I40E_VFPE_IPCONFIG01_PEIPID_SHIFT 0 +#define I40E_VFPE_IPCONFIG01_PEIPID_MASK I40E_MASK(0xFFFF, I40E_VFPE_IPCONFIG01_PEIPID_SHIFT) +#define I40E_VFPE_IPCONFIG01_USEENTIREIDRANGE_SHIFT 16 +#define I40E_VFPE_IPCONFIG01_USEENTIREIDRANGE_MASK I40E_MASK(0x1, I40E_VFPE_IPCONFIG01_USEENTIREIDRANGE_SHIFT) +#define I40E_VFPE_MRTEIDXMASK1 0x00009000 /* Reset: VFR */ +#define I40E_VFPE_MRTEIDXMASK1_MRTEIDXMASKBITS_SHIFT 0 +#define I40E_VFPE_MRTEIDXMASK1_MRTEIDXMASKBITS_MASK I40E_MASK(0x1F, I40E_VFPE_MRTEIDXMASK1_MRTEIDXMASKBITS_SHIFT) +#define I40E_VFPE_RCVUNEXPECTEDERROR1 0x00009400 /* Reset: VFR */ +#define I40E_VFPE_RCVUNEXPECTEDERROR1_TCP_RX_UNEXP_ERR_SHIFT 0 +#define I40E_VFPE_RCVUNEXPECTEDERROR1_TCP_RX_UNEXP_ERR_MASK I40E_MASK(0xFFFFFF, I40E_VFPE_RCVUNEXPECTEDERROR1_TCP_RX_UNEXP_ERR_SHIFT) +#define I40E_VFPE_TCPNOWTIMER1 0x0000A800 /* Reset: VFR */ +#define I40E_VFPE_TCPNOWTIMER1_TCP_NOW_SHIFT 0 +#define I40E_VFPE_TCPNOWTIMER1_TCP_NOW_MASK I40E_MASK(0xFFFFFFFF, I40E_VFPE_TCPNOWTIMER1_TCP_NOW_SHIFT) +#define I40E_VFPE_WQEALLOC1 0x0000C000 /* Reset: VFR */ +#define I40E_VFPE_WQEALLOC1_PEQPID_SHIFT 0 +#define I40E_VFPE_WQEALLOC1_PEQPID_MASK I40E_MASK(0x3FFFF, I40E_VFPE_WQEALLOC1_PEQPID_SHIFT) +#define I40E_VFPE_WQEALLOC1_WQE_DESC_INDEX_SHIFT 20 +#define I40E_VFPE_WQEALLOC1_WQE_DESC_INDEX_MASK I40E_MASK(0xFFF, I40E_VFPE_WQEALLOC1_WQE_DESC_INDEX_SHIFT) + #endif /* _I40E_REGISTER_H_ */ diff --git a/sys/dev/ixl/i40e_type.h b/sys/dev/ixl/i40e_type.h index 57351cb0bb04..fa8c7192e99f 100644 --- a/sys/dev/ixl/i40e_type.h +++ b/sys/dev/ixl/i40e_type.h @@ -43,7 +43,6 @@ #include "i40e_lan_hmc.h" #include "i40e_devids.h" -#define UNREFERENCED_XPARAMETER #define BIT(a) (1UL << (a)) #define BIT_ULL(a) (1ULL << (a)) @@ -147,8 +146,10 @@ enum i40e_debug_mask { #define I40E_PCI_LINK_SPEED_5000 0x2 #define I40E_PCI_LINK_SPEED_8000 0x3 -#define I40E_MDIO_STCODE 0 -#define I40E_MDIO_OPCODE_ADDRESS 0 +#define I40E_MDIO_STCODE I40E_MASK(0, \ + I40E_GLGEN_MSCA_STCODE_SHIFT) +#define I40E_MDIO_OPCODE_ADDRESS I40E_MASK(0, \ + I40E_GLGEN_MSCA_OPCODE_SHIFT) #define I40E_MDIO_OPCODE_WRITE I40E_MASK(1, \ I40E_GLGEN_MSCA_OPCODE_SHIFT) #define I40E_MDIO_OPCODE_READ_INC_ADDR I40E_MASK(2, \ @@ -177,6 +178,7 @@ enum i40e_memcpy_type { I40E_DMA_TO_NONDMA }; +#define I40E_FW_API_VERSION_MINOR_X722 0x0005 #define I40E_FW_API_VERSION_MINOR_X710 0x0005 @@ -193,6 +195,8 @@ enum i40e_mac_type { I40E_MAC_X710, I40E_MAC_XL710, I40E_MAC_VF, + I40E_MAC_X722, + I40E_MAC_X722_VF, I40E_MAC_GENERIC, }; @@ -275,50 +279,61 @@ struct i40e_link_status { #define I40E_MODULE_TYPE_1000BASE_T 0x08 }; -enum i40e_aq_capabilities_phy_type { - I40E_CAP_PHY_TYPE_SGMII = BIT(I40E_PHY_TYPE_SGMII), - I40E_CAP_PHY_TYPE_1000BASE_KX = BIT(I40E_PHY_TYPE_1000BASE_KX), - I40E_CAP_PHY_TYPE_10GBASE_KX4 = BIT(I40E_PHY_TYPE_10GBASE_KX4), - I40E_CAP_PHY_TYPE_10GBASE_KR = BIT(I40E_PHY_TYPE_10GBASE_KR), - I40E_CAP_PHY_TYPE_40GBASE_KR4 = BIT(I40E_PHY_TYPE_40GBASE_KR4), - I40E_CAP_PHY_TYPE_XAUI = BIT(I40E_PHY_TYPE_XAUI), - I40E_CAP_PHY_TYPE_XFI = BIT(I40E_PHY_TYPE_XFI), - I40E_CAP_PHY_TYPE_SFI = BIT(I40E_PHY_TYPE_SFI), - I40E_CAP_PHY_TYPE_XLAUI = BIT(I40E_PHY_TYPE_XLAUI), - I40E_CAP_PHY_TYPE_XLPPI = BIT(I40E_PHY_TYPE_XLPPI), - I40E_CAP_PHY_TYPE_40GBASE_CR4_CU = BIT(I40E_PHY_TYPE_40GBASE_CR4_CU), - I40E_CAP_PHY_TYPE_10GBASE_CR1_CU = BIT(I40E_PHY_TYPE_10GBASE_CR1_CU), - I40E_CAP_PHY_TYPE_10GBASE_AOC = BIT(I40E_PHY_TYPE_10GBASE_AOC), - I40E_CAP_PHY_TYPE_40GBASE_AOC = BIT(I40E_PHY_TYPE_40GBASE_AOC), - I40E_CAP_PHY_TYPE_100BASE_TX = BIT(I40E_PHY_TYPE_100BASE_TX), - I40E_CAP_PHY_TYPE_1000BASE_T = BIT(I40E_PHY_TYPE_1000BASE_T), - I40E_CAP_PHY_TYPE_10GBASE_T = BIT(I40E_PHY_TYPE_10GBASE_T), - I40E_CAP_PHY_TYPE_10GBASE_SR = BIT(I40E_PHY_TYPE_10GBASE_SR), - I40E_CAP_PHY_TYPE_10GBASE_LR = BIT(I40E_PHY_TYPE_10GBASE_LR), - I40E_CAP_PHY_TYPE_10GBASE_SFPP_CU = BIT(I40E_PHY_TYPE_10GBASE_SFPP_CU), - I40E_CAP_PHY_TYPE_10GBASE_CR1 = BIT(I40E_PHY_TYPE_10GBASE_CR1), - I40E_CAP_PHY_TYPE_40GBASE_CR4 = BIT(I40E_PHY_TYPE_40GBASE_CR4), - I40E_CAP_PHY_TYPE_40GBASE_SR4 = BIT(I40E_PHY_TYPE_40GBASE_SR4), - I40E_CAP_PHY_TYPE_40GBASE_LR4 = BIT(I40E_PHY_TYPE_40GBASE_LR4), - I40E_CAP_PHY_TYPE_1000BASE_SX = BIT(I40E_PHY_TYPE_1000BASE_SX), - I40E_CAP_PHY_TYPE_1000BASE_LX = BIT(I40E_PHY_TYPE_1000BASE_LX), - I40E_CAP_PHY_TYPE_1000BASE_T_OPTICAL = BIT(I40E_PHY_TYPE_1000BASE_T_OPTICAL), - I40E_CAP_PHY_TYPE_20GBASE_KR2 = BIT(I40E_PHY_TYPE_20GBASE_KR2) -}; - struct i40e_phy_info { struct i40e_link_status link_info; struct i40e_link_status link_info_old; bool get_link_info; enum i40e_media_type media_type; /* all the phy types the NVM is capable of */ - u32 phy_types; -}; - + u64 phy_types; +}; + +#define I40E_CAP_PHY_TYPE_SGMII BIT_ULL(I40E_PHY_TYPE_SGMII) +#define I40E_CAP_PHY_TYPE_1000BASE_KX BIT_ULL(I40E_PHY_TYPE_1000BASE_KX) +#define I40E_CAP_PHY_TYPE_10GBASE_KX4 BIT_ULL(I40E_PHY_TYPE_10GBASE_KX4) +#define I40E_CAP_PHY_TYPE_10GBASE_KR BIT_ULL(I40E_PHY_TYPE_10GBASE_KR) +#define I40E_CAP_PHY_TYPE_40GBASE_KR4 BIT_ULL(I40E_PHY_TYPE_40GBASE_KR4) +#define I40E_CAP_PHY_TYPE_XAUI BIT_ULL(I40E_PHY_TYPE_XAUI) +#define I40E_CAP_PHY_TYPE_XFI BIT_ULL(I40E_PHY_TYPE_XFI) +#define I40E_CAP_PHY_TYPE_SFI BIT_ULL(I40E_PHY_TYPE_SFI) +#define I40E_CAP_PHY_TYPE_XLAUI BIT_ULL(I40E_PHY_TYPE_XLAUI) +#define I40E_CAP_PHY_TYPE_XLPPI BIT_ULL(I40E_PHY_TYPE_XLPPI) +#define I40E_CAP_PHY_TYPE_40GBASE_CR4_CU BIT_ULL(I40E_PHY_TYPE_40GBASE_CR4_CU) +#define I40E_CAP_PHY_TYPE_10GBASE_CR1_CU BIT_ULL(I40E_PHY_TYPE_10GBASE_CR1_CU) +#define I40E_CAP_PHY_TYPE_10GBASE_AOC BIT_ULL(I40E_PHY_TYPE_10GBASE_AOC) +#define I40E_CAP_PHY_TYPE_40GBASE_AOC BIT_ULL(I40E_PHY_TYPE_40GBASE_AOC) +#define I40E_CAP_PHY_TYPE_100BASE_TX BIT_ULL(I40E_PHY_TYPE_100BASE_TX) +#define I40E_CAP_PHY_TYPE_1000BASE_T BIT_ULL(I40E_PHY_TYPE_1000BASE_T) +#define I40E_CAP_PHY_TYPE_10GBASE_T BIT_ULL(I40E_PHY_TYPE_10GBASE_T) +#define I40E_CAP_PHY_TYPE_10GBASE_SR BIT_ULL(I40E_PHY_TYPE_10GBASE_SR) +#define I40E_CAP_PHY_TYPE_10GBASE_LR BIT_ULL(I40E_PHY_TYPE_10GBASE_LR) +#define I40E_CAP_PHY_TYPE_10GBASE_SFPP_CU BIT_ULL(I40E_PHY_TYPE_10GBASE_SFPP_CU) +#define I40E_CAP_PHY_TYPE_10GBASE_CR1 BIT_ULL(I40E_PHY_TYPE_10GBASE_CR1) +#define I40E_CAP_PHY_TYPE_40GBASE_CR4 BIT_ULL(I40E_PHY_TYPE_40GBASE_CR4) +#define I40E_CAP_PHY_TYPE_40GBASE_SR4 BIT_ULL(I40E_PHY_TYPE_40GBASE_SR4) +#define I40E_CAP_PHY_TYPE_40GBASE_LR4 BIT_ULL(I40E_PHY_TYPE_40GBASE_LR4) +#define I40E_CAP_PHY_TYPE_1000BASE_SX BIT_ULL(I40E_PHY_TYPE_1000BASE_SX) +#define I40E_CAP_PHY_TYPE_1000BASE_LX BIT_ULL(I40E_PHY_TYPE_1000BASE_LX) +#define I40E_CAP_PHY_TYPE_1000BASE_T_OPTICAL \ + BIT_ULL(I40E_PHY_TYPE_1000BASE_T_OPTICAL) +#define I40E_CAP_PHY_TYPE_20GBASE_KR2 BIT_ULL(I40E_PHY_TYPE_20GBASE_KR2) +#define I40E_CAP_PHY_TYPE_25GBASE_KR BIT_ULL(I40E_AQ_PHY_TYPE_EXT_25G_KR + 32) +#define I40E_CAP_PHY_TYPE_25GBASE_CR BIT_ULL(I40E_AQ_PHY_TYPE_EXT_25G_CR + 32) +#define I40E_CAP_PHY_TYPE_25GBASE_SR BIT_ULL(I40E_AQ_PHY_TYPE_EXT_25G_SR + 32) +#define I40E_CAP_PHY_TYPE_25GBASE_LR BIT_ULL(I40E_AQ_PHY_TYPE_EXT_25G_LR + 32) #define I40E_HW_CAP_MAX_GPIO 30 #define I40E_HW_CAP_MDIO_PORT_MODE_MDIO 0 #define I40E_HW_CAP_MDIO_PORT_MODE_I2C 1 +enum i40e_acpi_programming_method { + I40E_ACPI_PROGRAMMING_METHOD_HW_FVL = 0, + I40E_ACPI_PROGRAMMING_METHOD_AQC_FPK = 1 +}; + +#define I40E_WOL_SUPPORT_MASK 1 +#define I40E_ACPI_PROGRAMMING_METHOD_MASK (1 << 1) +#define I40E_PROXY_SUPPORT_MASK (1 << 2) + /* Capabilities of a PF or a VF or the whole device */ struct i40e_hw_capabilities { u32 switch_mode; @@ -348,6 +363,11 @@ struct i40e_hw_capabilities { #define I40E_FLEX10_STATUS_DCC_ERROR 0x1 #define I40E_FLEX10_STATUS_VC_MODE 0x2 + bool sec_rev_disabled; + bool update_disabled; +#define I40E_NVM_MGMT_SEC_REV_DISABLED 0x1 +#define I40E_NVM_MGMT_UPDATE_DISABLED 0x2 + bool mgmt_cem; bool ieee_1588; bool iwarp; @@ -377,6 +397,9 @@ struct i40e_hw_capabilities { u32 enabled_tcmap; u32 maxtc; u64 wr_csr_prot; + bool apm_wol_support; + enum i40e_acpi_programming_method acpi_prog_method; + bool proxy_support; }; struct i40e_mac_info { @@ -622,6 +645,8 @@ struct i40e_hw { enum i40e_nvmupd_state nvmupd_state; struct i40e_aq_desc nvm_wb_desc; struct i40e_virt_mem nvm_buff; + bool nvm_release_on_done; + u16 nvm_wait_opcode; /* HMC info */ struct i40e_hmc_info hmc; /* HMC info struct */ @@ -634,6 +659,10 @@ struct i40e_hw { struct i40e_dcbx_config remote_dcbx_config; /* Peer Cfg */ struct i40e_dcbx_config desired_dcbx_config; /* CEE Desired Cfg */ + /* WoL and proxy support */ + u16 num_wol_proxy_filters; + u16 wol_proxy_vsi_seid; + #define I40E_HW_FLAG_AQ_SRCTL_ACCESS_ENABLE BIT_ULL(0) u64 flags; @@ -644,7 +673,8 @@ struct i40e_hw { static INLINE bool i40e_is_vf(struct i40e_hw *hw) { - return hw->mac.type == I40E_MAC_VF; + return (hw->mac.type == I40E_MAC_VF || + hw->mac.type == I40E_MAC_X722_VF); } struct i40e_driver_version { @@ -748,7 +778,7 @@ enum i40e_rx_desc_status_bits { I40E_RX_DESC_STATUS_CRCP_SHIFT = 4, I40E_RX_DESC_STATUS_TSYNINDX_SHIFT = 5, /* 2 BITS */ I40E_RX_DESC_STATUS_TSYNVALID_SHIFT = 7, - I40E_RX_DESC_STATUS_RESERVED1_SHIFT = 8, + I40E_RX_DESC_STATUS_EXT_UDP_0_SHIFT = 8, I40E_RX_DESC_STATUS_UMBCAST_SHIFT = 9, /* 2 BITS */ I40E_RX_DESC_STATUS_FLM_SHIFT = 11, @@ -756,7 +786,7 @@ enum i40e_rx_desc_status_bits { I40E_RX_DESC_STATUS_LPBK_SHIFT = 14, I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT = 15, I40E_RX_DESC_STATUS_RESERVED2_SHIFT = 16, /* 2 BITS */ - I40E_RX_DESC_STATUS_UDP_0_SHIFT = 18, + I40E_RX_DESC_STATUS_INT_UDP_0_SHIFT = 18, I40E_RX_DESC_STATUS_LAST /* this entry must be last!!! */ }; @@ -1134,6 +1164,8 @@ enum i40e_tx_ctx_desc_eipt_offload { #define I40E_TXD_CTX_QW0_DECTTL_MASK (0xFULL << \ I40E_TXD_CTX_QW0_DECTTL_SHIFT) +#define I40E_TXD_CTX_QW0_L4T_CS_SHIFT 23 +#define I40E_TXD_CTX_QW0_L4T_CS_MASK BIT_ULL(I40E_TXD_CTX_QW0_L4T_CS_SHIFT) struct i40e_nop_desc { __le64 rsvd; __le64 dtype_cmd; @@ -1170,15 +1202,24 @@ struct i40e_filter_program_desc { /* Packet Classifier Types for filters */ enum i40e_filter_pctype { - /* Note: Values 0-30 are reserved for future use */ + /* Note: Values 0-28 are reserved for future use. + * Value 29, 30, 32 are not supported on XL710 and X710. + */ + I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP = 29, + I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP = 30, I40E_FILTER_PCTYPE_NONF_IPV4_UDP = 31, - /* Note: Value 32 is reserved for future use */ + I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK = 32, I40E_FILTER_PCTYPE_NONF_IPV4_TCP = 33, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP = 34, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER = 35, I40E_FILTER_PCTYPE_FRAG_IPV4 = 36, - /* Note: Values 37-40 are reserved for future use */ + /* Note: Values 37-38 are reserved for future use. + * Value 39, 40, 42 are not supported on XL710 and X710. + */ + I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP = 39, + I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP = 40, I40E_FILTER_PCTYPE_NONF_IPV6_UDP = 41, + I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK = 42, I40E_FILTER_PCTYPE_NONF_IPV6_TCP = 43, I40E_FILTER_PCTYPE_NONF_IPV6_SCTP = 44, I40E_FILTER_PCTYPE_NONF_IPV6_OTHER = 45, @@ -1234,6 +1275,10 @@ enum i40e_filter_program_desc_pcmd { #define I40E_TXD_FLTR_QW1_FD_STATUS_MASK (0x3ULL << \ I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT) +#define I40E_TXD_FLTR_QW1_ATR_SHIFT (0xEULL + \ + I40E_TXD_FLTR_QW1_CMD_SHIFT) +#define I40E_TXD_FLTR_QW1_ATR_MASK BIT_ULL(I40E_TXD_FLTR_QW1_ATR_SHIFT) + #define I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT 20 #define I40E_TXD_FLTR_QW1_CNTINDEX_MASK (0x1FFUL << \ I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) @@ -1544,4 +1589,37 @@ struct i40e_lldp_variables { /* RSS Hash Table Size */ #define I40E_PFQF_CTL_0_HASHLUTSIZE_512 0x00010000 + +/* INPUT SET MASK for RSS, flow director, and flexible payload */ +#define I40E_L3_SRC_SHIFT 47 +#define I40E_L3_SRC_MASK (0x3ULL << I40E_L3_SRC_SHIFT) +#define I40E_L3_V6_SRC_SHIFT 43 +#define I40E_L3_V6_SRC_MASK (0xFFULL << I40E_L3_V6_SRC_SHIFT) +#define I40E_L3_DST_SHIFT 35 +#define I40E_L3_DST_MASK (0x3ULL << I40E_L3_DST_SHIFT) +#define I40E_L3_V6_DST_SHIFT 35 +#define I40E_L3_V6_DST_MASK (0xFFULL << I40E_L3_V6_DST_SHIFT) +#define I40E_L4_SRC_SHIFT 34 +#define I40E_L4_SRC_MASK (0x1ULL << I40E_L4_SRC_SHIFT) +#define I40E_L4_DST_SHIFT 33 +#define I40E_L4_DST_MASK (0x1ULL << I40E_L4_DST_SHIFT) +#define I40E_VERIFY_TAG_SHIFT 31 +#define I40E_VERIFY_TAG_MASK (0x3ULL << I40E_VERIFY_TAG_SHIFT) + +#define I40E_FLEX_50_SHIFT 13 +#define I40E_FLEX_50_MASK (0x1ULL << I40E_FLEX_50_SHIFT) +#define I40E_FLEX_51_SHIFT 12 +#define I40E_FLEX_51_MASK (0x1ULL << I40E_FLEX_51_SHIFT) +#define I40E_FLEX_52_SHIFT 11 +#define I40E_FLEX_52_MASK (0x1ULL << I40E_FLEX_52_SHIFT) +#define I40E_FLEX_53_SHIFT 10 +#define I40E_FLEX_53_MASK (0x1ULL << I40E_FLEX_53_SHIFT) +#define I40E_FLEX_54_SHIFT 9 +#define I40E_FLEX_54_MASK (0x1ULL << I40E_FLEX_54_SHIFT) +#define I40E_FLEX_55_SHIFT 8 +#define I40E_FLEX_55_MASK (0x1ULL << I40E_FLEX_55_SHIFT) +#define I40E_FLEX_56_SHIFT 7 +#define I40E_FLEX_56_MASK (0x1ULL << I40E_FLEX_56_SHIFT) +#define I40E_FLEX_57_SHIFT 6 +#define I40E_FLEX_57_MASK (0x1ULL << I40E_FLEX_57_SHIFT) #endif /* _I40E_TYPE_H_ */ diff --git a/sys/dev/ixl/i40e_virtchnl.h b/sys/dev/ixl/i40e_virtchnl.h index 7939a0dc5732..4ebe578d1972 100644 --- a/sys/dev/ixl/i40e_virtchnl.h +++ b/sys/dev/ixl/i40e_virtchnl.h @@ -88,7 +88,12 @@ enum i40e_virtchnl_ops { I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE = 14, I40E_VIRTCHNL_OP_GET_STATS = 15, I40E_VIRTCHNL_OP_FCOE = 16, - I40E_VIRTCHNL_OP_EVENT = 17, + I40E_VIRTCHNL_OP_EVENT = 17, /* must ALWAYS be 17 */ + I40E_VIRTCHNL_OP_CONFIG_RSS_KEY = 23, + I40E_VIRTCHNL_OP_CONFIG_RSS_LUT = 24, + I40E_VIRTCHNL_OP_GET_RSS_HENA_CAPS = 25, + I40E_VIRTCHNL_OP_SET_RSS_HENA = 26, + }; /* Virtual channel message descriptor. This overlays the admin queue @@ -162,6 +167,7 @@ struct i40e_virtchnl_vsi_resource { #define I40E_VIRTCHNL_VF_OFFLOAD_VLAN 0x00010000 #define I40E_VIRTCHNL_VF_OFFLOAD_RX_POLLING 0x00020000 #define I40E_VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2 0x00040000 +#define I40E_VIRTCHNL_VF_OFFLOAD_RSS_PF 0X00080000 struct i40e_virtchnl_vf_resource { u16 num_vsis; @@ -170,8 +176,8 @@ struct i40e_virtchnl_vf_resource { u16 max_mtu; u32 vf_offload_flags; - u32 max_fcoe_contexts; - u32 max_fcoe_filters; + u32 rss_key_size; + u32 rss_lut_size; struct i40e_virtchnl_vsi_resource vsi_res[1]; }; @@ -330,6 +336,39 @@ struct i40e_virtchnl_promisc_info { * PF replies with struct i40e_eth_stats in an external buffer. */ +/* I40E_VIRTCHNL_OP_CONFIG_RSS_KEY + * I40E_VIRTCHNL_OP_CONFIG_RSS_LUT + * VF sends these messages to configure RSS. Only supported if both PF + * and VF drivers set the I40E_VIRTCHNL_VF_OFFLOAD_RSS_PF bit during + * configuration negotiation. If this is the case, then the rss fields in + * the vf resource struct are valid. + * Both the key and LUT are initialized to 0 by the PF, meaning that + * RSS is effectively disabled until set up by the VF. + */ +struct i40e_virtchnl_rss_key { + u16 vsi_id; + u16 key_len; + u8 key[1]; /* RSS hash key, packed bytes */ +}; + +struct i40e_virtchnl_rss_lut { + u16 vsi_id; + u16 lut_entries; + u8 lut[1]; /* RSS lookup table*/ +}; + +/* I40E_VIRTCHNL_OP_GET_RSS_HENA_CAPS + * I40E_VIRTCHNL_OP_SET_RSS_HENA + * VF sends these messages to get and set the hash filter enable bits for RSS. + * By default, the PF sets these to all possible traffic types that the + * hardware supports. The VF can query this value if it wants to change the + * traffic types that are hashed by the hardware. + * Traffic types are defined in the i40e_filter_pctype enum in i40e_type.h + */ +struct i40e_virtchnl_rss_hena { + u64 hena; +}; + /* I40E_VIRTCHNL_OP_EVENT * PF sends this message to inform the VF driver of events that may affect it. * No direct response is expected from the VF, though it may generate other diff --git a/sys/dev/ixl/if_ixl.c b/sys/dev/ixl/if_ixl.c index 8e9ba80987ae..a9221d323a26 100644 --- a/sys/dev/ixl/if_ixl.c +++ b/sys/dev/ixl/if_ixl.c @@ -32,23 +32,17 @@ ******************************************************************************/ /*$FreeBSD$*/ -#ifndef IXL_STANDALONE_BUILD -#include "opt_inet.h" -#include "opt_inet6.h" -#include "opt_rss.h" -#endif - #include "ixl.h" #include "ixl_pf.h" -#ifdef RSS -#include <net/rss_config.h> +#ifdef PCI_IOV +#include "ixl_pf_iov.h" #endif /********************************************************************* * Driver version *********************************************************************/ -char ixl_driver_version[] = "1.4.27-k"; +char ixl_driver_version[] = "1.6.6-k"; /********************************************************************* * PCI Device ID Table @@ -70,6 +64,12 @@ static ixl_vendor_info_t ixl_vendor_info_array[] = {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_C, 0, 0, 0}, {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T, 0, 0, 0}, {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T4, 0, 0, 0}, + {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_X722, 0, 0, 0}, + {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_X722, 0, 0, 0}, + {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_X722, 0, 0, 0}, + {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_1G_BASE_T_X722, 0, 0, 0}, + {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T_X722, 0, 0, 0}, + {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_I_X722, 0, 0, 0}, /* required last entry */ {0, 0, 0, 0, 0} }; @@ -79,7 +79,7 @@ static ixl_vendor_info_t ixl_vendor_info_array[] = *********************************************************************/ static char *ixl_strings[] = { - "Intel(R) Ethernet Connection XL710 Driver" + "Intel(R) Ethernet Connection XL710/X722 Driver" }; @@ -90,146 +90,9 @@ static int ixl_probe(device_t); static int ixl_attach(device_t); static int ixl_detach(device_t); static int ixl_shutdown(device_t); -static int ixl_get_hw_capabilities(struct ixl_pf *); -static void ixl_cap_txcsum_tso(struct ixl_vsi *, struct ifnet *, int); -static int ixl_ioctl(struct ifnet *, u_long, caddr_t); -static void ixl_init(void *); -static void ixl_init_locked(struct ixl_pf *); -static void ixl_stop(struct ixl_pf *); -static void ixl_stop_locked(struct ixl_pf *); -static void ixl_media_status(struct ifnet *, struct ifmediareq *); -static int ixl_media_change(struct ifnet *); -static void ixl_update_link_status(struct ixl_pf *); -static int ixl_allocate_pci_resources(struct ixl_pf *); -static u16 ixl_get_bus_info(struct i40e_hw *, device_t); -static int ixl_setup_stations(struct ixl_pf *); -static int ixl_switch_config(struct ixl_pf *); -static int ixl_initialize_vsi(struct ixl_vsi *); - -static int ixl_setup_adminq_msix(struct ixl_pf *); -static int ixl_setup_adminq_tq(struct ixl_pf *); -static int ixl_setup_queue_msix(struct ixl_vsi *); -static int ixl_setup_queue_tqs(struct ixl_vsi *); -static int ixl_teardown_adminq_msix(struct ixl_pf *); -static int ixl_teardown_queue_msix(struct ixl_vsi *); -static void ixl_configure_intr0_msix(struct ixl_pf *); -static void ixl_configure_queue_intr_msix(struct ixl_pf *); -static void ixl_free_queue_tqs(struct ixl_vsi *); -static void ixl_free_adminq_tq(struct ixl_pf *); - -static int ixl_assign_vsi_legacy(struct ixl_pf *); -static int ixl_init_msix(struct ixl_pf *); -static void ixl_configure_itr(struct ixl_pf *); -static void ixl_configure_legacy(struct ixl_pf *); -static void ixl_free_pci_resources(struct ixl_pf *); -static void ixl_local_timer(void *); -static int ixl_setup_interface(device_t, struct ixl_vsi *); -static void ixl_link_event(struct ixl_pf *, struct i40e_arq_event_info *); -static void ixl_config_rss(struct ixl_vsi *); -static void ixl_set_queue_rx_itr(struct ixl_queue *); -static void ixl_set_queue_tx_itr(struct ixl_queue *); -static int ixl_set_advertised_speeds(struct ixl_pf *, int); -static void ixl_get_initial_advertised_speeds(struct ixl_pf *); - -static int ixl_enable_rings(struct ixl_vsi *); -static int ixl_disable_rings(struct ixl_vsi *); -static void ixl_enable_intr(struct ixl_vsi *); -static void ixl_disable_intr(struct ixl_vsi *); -static void ixl_disable_rings_intr(struct ixl_vsi *); - -static void ixl_enable_adminq(struct i40e_hw *); -static void ixl_disable_adminq(struct i40e_hw *); -static void ixl_enable_queue(struct i40e_hw *, int); -static void ixl_disable_queue(struct i40e_hw *, int); -static void ixl_enable_legacy(struct i40e_hw *); -static void ixl_disable_legacy(struct i40e_hw *); - -static void ixl_set_promisc(struct ixl_vsi *); -static void ixl_add_multi(struct ixl_vsi *); -static void ixl_del_multi(struct ixl_vsi *); -static void ixl_register_vlan(void *, struct ifnet *, u16); -static void ixl_unregister_vlan(void *, struct ifnet *, u16); -static void ixl_setup_vlan_filters(struct ixl_vsi *); - -static void ixl_init_filters(struct ixl_vsi *); -static void ixl_reconfigure_filters(struct ixl_vsi *vsi); -static void ixl_add_filter(struct ixl_vsi *, u8 *, s16 vlan); -static void ixl_del_filter(struct ixl_vsi *, u8 *, s16 vlan); -static void ixl_add_hw_filters(struct ixl_vsi *, int, int); -static void ixl_del_hw_filters(struct ixl_vsi *, int); -static struct ixl_mac_filter * - ixl_find_filter(struct ixl_vsi *, u8 *, s16); -static void ixl_add_mc_filter(struct ixl_vsi *, u8 *); -static void ixl_free_mac_filters(struct ixl_vsi *vsi); -/* Sysctls*/ -static void ixl_add_device_sysctls(struct ixl_pf *); - -static int ixl_set_flowcntl(SYSCTL_HANDLER_ARGS); -static int ixl_set_advertise(SYSCTL_HANDLER_ARGS); -static int ixl_current_speed(SYSCTL_HANDLER_ARGS); -static int ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS); - -#ifdef IXL_DEBUG_SYSCTL -static int ixl_debug_info(SYSCTL_HANDLER_ARGS); -static void ixl_print_debug_info(struct ixl_pf *); - -static int ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS); -static int ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS); -static int ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS); -static int ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS); -static int ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS); -#endif - -/* The MSI/X Interrupt handlers */ -static void ixl_intr(void *); -static void ixl_msix_que(void *); -static void ixl_msix_adminq(void *); -static void ixl_handle_mdd_event(struct ixl_pf *); - -/* Deferred interrupt tasklets */ -static void ixl_do_adminq(void *, int); - -/* Statistics */ -static void ixl_add_hw_stats(struct ixl_pf *); -static void ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *, - struct sysctl_oid_list *, struct i40e_hw_port_stats *); -static void ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *, - struct sysctl_oid_list *, - struct i40e_eth_stats *); -static void ixl_update_stats_counters(struct ixl_pf *); -static void ixl_update_eth_stats(struct ixl_vsi *); -static void ixl_update_vsi_stats(struct ixl_vsi *); -static void ixl_pf_reset_stats(struct ixl_pf *); -static void ixl_vsi_reset_stats(struct ixl_vsi *); -static void ixl_stat_update48(struct i40e_hw *, u32, u32, bool, - u64 *, u64 *); -static void ixl_stat_update32(struct i40e_hw *, u32, bool, - u64 *, u64 *); -/* NVM update */ -static int ixl_handle_nvmupd_cmd(struct ixl_pf *, struct ifdrv *); -static void ixl_handle_empr_reset(struct ixl_pf *); -static int ixl_rebuild_hw_structs_after_reset(struct ixl_pf *); - -/* Debug helper functions */ -#ifdef IXL_DEBUG -static void ixl_print_nvm_cmd(device_t, struct i40e_nvm_access *); -#endif - -#ifdef PCI_IOV -static int ixl_adminq_err_to_errno(enum i40e_admin_queue_err err); - -static int ixl_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t*); -static void ixl_iov_uninit(device_t dev); -static int ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t*); - -static void ixl_handle_vf_msg(struct ixl_pf *, - struct i40e_arq_event_info *); -static void ixl_handle_vflr(void *arg, int pending); - -static void ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf); -static void ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf); -#endif +static int ixl_save_pf_tunables(struct ixl_pf *); +static int ixl_attach_get_link_status(struct ixl_pf *); /********************************************************************* * FreeBSD Device Interface Entry Points @@ -263,11 +126,6 @@ MODULE_DEPEND(ixl, netmap, 1, 1, 1); #endif /* DEV_NETMAP */ /* -** Global reset mutex -*/ -static struct mtx ixl_reset_mtx; - -/* ** TUNEABLE PARAMETERS: */ @@ -287,73 +145,72 @@ SYSCTL_INT(_hw_ixl, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixl_enable_msix, 0, ** Number of descriptors per ring: ** - TX and RX are the same size */ -static int ixl_ringsz = DEFAULT_RING; -TUNABLE_INT("hw.ixl.ringsz", &ixl_ringsz); +static int ixl_ring_size = DEFAULT_RING; +TUNABLE_INT("hw.ixl.ring_size", &ixl_ring_size); SYSCTL_INT(_hw_ixl, OID_AUTO, ring_size, CTLFLAG_RDTUN, - &ixl_ringsz, 0, "Descriptor Ring Size"); + &ixl_ring_size, 0, "Descriptor Ring Size"); /* ** This can be set manually, if left as 0 the ** number of queues will be calculated based ** on cpus and msix vectors available. */ -int ixl_max_queues = 0; +static int ixl_max_queues = 0; TUNABLE_INT("hw.ixl.max_queues", &ixl_max_queues); SYSCTL_INT(_hw_ixl, OID_AUTO, max_queues, CTLFLAG_RDTUN, &ixl_max_queues, 0, "Number of Queues"); +static int ixl_enable_tx_fc_filter = 1; +TUNABLE_INT("hw.ixl.enable_tx_fc_filter", + &ixl_enable_tx_fc_filter); +SYSCTL_INT(_hw_ixl, OID_AUTO, enable_tx_fc_filter, CTLFLAG_RDTUN, + &ixl_enable_tx_fc_filter, 0, + "Filter out packets with Ethertype 0x8808 from being sent out by non-HW sources"); + +static int ixl_core_debug_mask = 0; +TUNABLE_INT("hw.ixl.core_debug_mask", + &ixl_core_debug_mask); +SYSCTL_INT(_hw_ixl, OID_AUTO, core_debug_mask, CTLFLAG_RDTUN, + &ixl_core_debug_mask, 0, + "Display debug statements that are printed in non-shared code"); + +static int ixl_shared_debug_mask = 0; +TUNABLE_INT("hw.ixl.shared_debug_mask", + &ixl_shared_debug_mask); +SYSCTL_INT(_hw_ixl, OID_AUTO, shared_debug_mask, CTLFLAG_RDTUN, + &ixl_shared_debug_mask, 0, + "Display debug statements that are printed in shared code"); + /* ** Controls for Interrupt Throttling ** - true/false for dynamic adjustment ** - default values for static ITR */ -int ixl_dynamic_rx_itr = 1; +static int ixl_dynamic_rx_itr = 1; TUNABLE_INT("hw.ixl.dynamic_rx_itr", &ixl_dynamic_rx_itr); SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_rx_itr, CTLFLAG_RDTUN, &ixl_dynamic_rx_itr, 0, "Dynamic RX Interrupt Rate"); -int ixl_dynamic_tx_itr = 1; +static int ixl_dynamic_tx_itr = 1; TUNABLE_INT("hw.ixl.dynamic_tx_itr", &ixl_dynamic_tx_itr); SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_tx_itr, CTLFLAG_RDTUN, &ixl_dynamic_tx_itr, 0, "Dynamic TX Interrupt Rate"); -int ixl_rx_itr = IXL_ITR_8K; +static int ixl_rx_itr = IXL_ITR_8K; TUNABLE_INT("hw.ixl.rx_itr", &ixl_rx_itr); SYSCTL_INT(_hw_ixl, OID_AUTO, rx_itr, CTLFLAG_RDTUN, &ixl_rx_itr, 0, "RX Interrupt Rate"); -int ixl_tx_itr = IXL_ITR_4K; +static int ixl_tx_itr = IXL_ITR_4K; TUNABLE_INT("hw.ixl.tx_itr", &ixl_tx_itr); SYSCTL_INT(_hw_ixl, OID_AUTO, tx_itr, CTLFLAG_RDTUN, &ixl_tx_itr, 0, "TX Interrupt Rate"); -#ifdef IXL_FDIR -static int ixl_enable_fdir = 1; -TUNABLE_INT("hw.ixl.enable_fdir", &ixl_enable_fdir); -/* Rate at which we sample */ -int ixl_atr_rate = 20; -TUNABLE_INT("hw.ixl.atr_rate", &ixl_atr_rate); -#endif - #ifdef DEV_NETMAP #define NETMAP_IXL_MAIN /* only bring in one part of the netmap code */ #include <dev/netmap/if_ixl_netmap.h> #endif /* DEV_NETMAP */ -static char *ixl_fc_string[6] = { - "None", - "Rx", - "Tx", - "Full", - "Priority", - "Default" -}; - -static MALLOC_DEFINE(M_IXL, "ixl", "ixl driver allocations"); - -static uint8_t ixl_bcast_addr[ETHER_ADDR_LEN] = - {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; - /********************************************************************* * Device identification routine * @@ -371,7 +228,6 @@ ixl_probe(device_t dev) u16 pci_vendor_id, pci_device_id; u16 pci_subvendor_id, pci_subdevice_id; char device_name[256]; - static bool lock_init = FALSE; #if 0 INIT_DEBUGOUT("ixl_probe: begin"); @@ -398,13 +254,6 @@ ixl_probe(device_t dev) ixl_strings[ent->index], ixl_driver_version); device_set_desc_copy(dev, device_name); - /* One shot mutex init */ - if (lock_init == FALSE) { - lock_init = TRUE; - mtx_init(&ixl_reset_mtx, - "ixl_reset", - "IXL RESET Lock", MTX_DEF); - } return (BUS_PROBE_DEFAULT); } ent++; @@ -412,6 +261,64 @@ ixl_probe(device_t dev) return (ENXIO); } +static int +ixl_attach_get_link_status(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + device_t dev = pf->dev; + int error = 0; + + if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) || + (hw->aq.fw_maj_ver < 4)) { + i40e_msec_delay(75); + error = i40e_aq_set_link_restart_an(hw, TRUE, NULL); + if (error) { + device_printf(dev, "link restart failed, aq_err=%d\n", + pf->hw.aq.asq_last_status); + return error; + } + } + + /* Determine link state */ + hw->phy.get_link_info = TRUE; + i40e_get_link_status(hw, &pf->link_up); + return (0); +} + +/* + * Sanity check and save off tunable values. + */ +static int +ixl_save_pf_tunables(struct ixl_pf *pf) +{ + device_t dev = pf->dev; + + /* Save tunable information */ + pf->enable_msix = ixl_enable_msix; + pf->max_queues = ixl_max_queues; + pf->ringsz = ixl_ring_size; + pf->enable_tx_fc_filter = ixl_enable_tx_fc_filter; + pf->dynamic_rx_itr = ixl_dynamic_rx_itr; + pf->dynamic_tx_itr = ixl_dynamic_tx_itr; + pf->tx_itr = ixl_tx_itr; + pf->rx_itr = ixl_rx_itr; + pf->dbg_mask = ixl_core_debug_mask; + pf->hw.debug_mask = ixl_shared_debug_mask; + + if (ixl_ring_size < IXL_MIN_RING + || ixl_ring_size > IXL_MAX_RING + || ixl_ring_size % IXL_RING_INCREMENT != 0) { + device_printf(dev, "Invalid ring_size value of %d set!\n", + ixl_ring_size); + device_printf(dev, "ring_size must be between %d and %d, " + "inclusive, and must be a multiple of %d\n", + IXL_MIN_RING, IXL_MAX_RING, IXL_RING_INCREMENT); + return (EINVAL); + } + + return (0); +} + /********************************************************************* * Device initialization routine * @@ -428,12 +335,8 @@ ixl_attach(device_t dev) struct ixl_pf *pf; struct i40e_hw *hw; struct ixl_vsi *vsi; - u16 bus; + enum i40e_status_code status; int error = 0; -#ifdef PCI_IOV - nvlist_t *pf_schema, *vf_schema; - int iov_error; -#endif INIT_DEBUGOUT("ixl_attach: begin"); @@ -449,26 +352,17 @@ ixl_attach(device_t dev) vsi = &pf->vsi; vsi->dev = pf->dev; + /* Save tunable values */ + error = ixl_save_pf_tunables(pf); + if (error) + return (error); + /* Core Lock Init*/ IXL_PF_LOCK_INIT(pf, device_get_nameunit(dev)); /* Set up the timer callout */ callout_init_mtx(&pf->timer, &pf->pf_mtx, 0); - /* Save off the PCI information */ - hw->vendor_id = pci_get_vendor(dev); - hw->device_id = pci_get_device(dev); - hw->revision_id = pci_read_config(dev, PCIR_REVID, 1); - hw->subsystem_vendor_id = - pci_read_config(dev, PCIR_SUBVEND_0, 2); - hw->subsystem_device_id = - pci_read_config(dev, PCIR_SUBDEV_0, 2); - - hw->bus.device = pci_get_slot(dev); - hw->bus.func = pci_get_function(dev); - - pf->vc_debug_lvl = 1; - /* Do PCI setup - map BAR0, etc */ if (ixl_allocate_pci_resources(pf)) { device_printf(dev, "Allocation of PCI resources failed\n"); @@ -478,42 +372,45 @@ ixl_attach(device_t dev) /* Establish a clean starting point */ i40e_clear_hw(hw); - error = i40e_pf_reset(hw); - if (error) { - device_printf(dev, "PF reset failure %d\n", error); + status = i40e_pf_reset(hw); + if (status) { + device_printf(dev, "PF reset failure %s\n", + i40e_stat_str(hw, status)); error = EIO; goto err_out; } - /* Set admin queue parameters */ - hw->aq.num_arq_entries = IXL_AQ_LEN; - hw->aq.num_asq_entries = IXL_AQ_LEN; - hw->aq.arq_buf_size = IXL_AQ_BUFSZ; - hw->aq.asq_buf_size = IXL_AQ_BUFSZ; - - /* Initialize mac filter list for VSI */ - SLIST_INIT(&vsi->ftl); - /* Initialize the shared code */ - error = i40e_init_shared_code(hw); - if (error) { - device_printf(dev, "Unable to initialize shared code, error %d\n", - error); + status = i40e_init_shared_code(hw); + if (status) { + device_printf(dev, "Unable to initialize shared code, error %s\n", + i40e_stat_str(hw, status)); error = EIO; goto err_out; } + /* + * Allocate interrupts and figure out number of queues to use + * for PF interface + */ + pf->msix = ixl_init_msix(pf); + /* Set up the admin queue */ - error = i40e_init_adminq(hw); - if (error != 0 && error != I40E_ERR_FIRMWARE_API_VERSION) { - device_printf(dev, "Unable to initialize Admin Queue, error %d\n", - error); + hw->aq.num_arq_entries = IXL_AQ_LEN; + hw->aq.num_asq_entries = IXL_AQ_LEN; + hw->aq.arq_buf_size = IXL_AQ_BUF_SZ; + hw->aq.asq_buf_size = IXL_AQ_BUF_SZ; + + status = i40e_init_adminq(hw); + if (status != 0 && status != I40E_ERR_FIRMWARE_API_VERSION) { + device_printf(dev, "Unable to initialize Admin Queue, error %s\n", + i40e_stat_str(hw, status)); error = EIO; goto err_out; } ixl_print_nvm_version(pf); - if (error == I40E_ERR_FIRMWARE_API_VERSION) { + if (status == I40E_ERR_FIRMWARE_API_VERSION) { device_printf(dev, "The driver for the device stopped " "because the NVM image is newer than expected.\n" "You must install the most recent version of " @@ -544,24 +441,44 @@ ixl_attach(device_t dev) } /* Set up host memory cache */ - error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, + status = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, hw->func_caps.num_rx_qp, 0, 0); - if (error) { - device_printf(dev, "init_lan_hmc failed: %d\n", error); + if (status) { + device_printf(dev, "init_lan_hmc failed: %s\n", + i40e_stat_str(hw, status)); goto err_get_cap; } - error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); + status = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); + if (status) { + device_printf(dev, "configure_lan_hmc failed: %s\n", + i40e_stat_str(hw, status)); + goto err_mac_hmc; + } + + /* Init queue allocation manager */ + error = ixl_pf_qmgr_init(&pf->qmgr, hw->func_caps.num_tx_qp); + if (error) { + device_printf(dev, "Failed to init queue manager for PF queues, error %d\n", + error); + goto err_mac_hmc; + } + /* reserve a contiguous allocation for the PF's VSI */ + error = ixl_pf_qmgr_alloc_contiguous(&pf->qmgr, vsi->num_queues, &pf->qtag); if (error) { - device_printf(dev, "configure_lan_hmc failed: %d\n", error); + device_printf(dev, "Failed to reserve queues for PF LAN VSI, error %d\n", + error); goto err_mac_hmc; } + device_printf(dev, "Allocating %d queues for PF LAN VSI; %d queues active\n", + pf->qtag.num_allocated, pf->qtag.num_active); /* Disable LLDP from the firmware for certain NVM versions */ if (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) || (pf->hw.aq.fw_maj_ver < 4)) i40e_aq_stop_lldp(hw, TRUE, NULL); + /* Get MAC addresses from hardware */ i40e_get_mac_addr(hw, hw->mac.addr); error = i40e_validate_mac_addr(hw->mac.addr); if (error) { @@ -571,35 +488,29 @@ ixl_attach(device_t dev) bcopy(hw->mac.addr, hw->mac.perm_addr, ETHER_ADDR_LEN); i40e_get_port_mac_addr(hw, hw->mac.port_addr); - /* Set up VSI and queues */ - if (ixl_setup_stations(pf) != 0) { + /* Initialize mac filter list for VSI */ + SLIST_INIT(&vsi->ftl); + + /* Set up SW VSI and allocate queue memory and rings */ + if (ixl_setup_stations(pf)) { device_printf(dev, "setup stations failed!\n"); error = ENOMEM; goto err_mac_hmc; } - if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) || - (hw->aq.fw_maj_ver < 4)) { - i40e_msec_delay(75); - error = i40e_aq_set_link_restart_an(hw, TRUE, NULL); - if (error) { - device_printf(dev, "link restart failed, aq_err=%d\n", - pf->hw.aq.asq_last_status); - goto err_late; - } - } - - /* Determine link state */ - hw->phy.get_link_info = TRUE; - i40e_get_link_status(hw, &pf->link_up); - /* Setup OS network interface / ifnet */ - if (ixl_setup_interface(dev, vsi) != 0) { + if (ixl_setup_interface(dev, vsi)) { device_printf(dev, "interface setup failed!\n"); error = EIO; goto err_late; } + /* Determine link state */ + if (ixl_attach_get_link_status(pf)) { + error = EINVAL; + goto err_late; + } + error = ixl_switch_config(pf); if (error) { device_printf(dev, "Initial ixl_switch_config() failed: %d\n", @@ -608,17 +519,17 @@ ixl_attach(device_t dev) } /* Limit PHY interrupts to link, autoneg, and modules failure */ - error = i40e_aq_set_phy_int_mask(hw, IXL_DEFAULT_PHY_INT_MASK, + status = i40e_aq_set_phy_int_mask(hw, IXL_DEFAULT_PHY_INT_MASK, NULL); - if (error) { - device_printf(dev, "i40e_aq_set_phy_mask() failed: err %d," - " aq_err %d\n", error, hw->aq.asq_last_status); + if (status) { + device_printf(dev, "i40e_aq_set_phy_mask() failed: err %s," + " aq_err %s\n", i40e_stat_str(hw, status), + i40e_aq_str(hw, hw->aq.asq_last_status)); goto err_late; } /* Get the bus configuration and set the shared code's config */ - bus = ixl_get_bus_info(hw, dev); - i40e_set_pci_config_data(hw, bus); + ixl_get_bus_info(hw, dev); /* * In MSI-X mode, initialize the Admin Queue interrupt, @@ -656,26 +567,7 @@ ixl_attach(device_t dev) ixl_unregister_vlan, vsi, EVENTHANDLER_PRI_FIRST); #ifdef PCI_IOV - /* SR-IOV is only supported when MSI-X is in use. */ - if (pf->msix > 1) { - pf_schema = pci_iov_schema_alloc_node(); - vf_schema = pci_iov_schema_alloc_node(); - pci_iov_schema_add_unicast_mac(vf_schema, "mac-addr", 0, NULL); - pci_iov_schema_add_bool(vf_schema, "mac-anti-spoof", - IOV_SCHEMA_HASDEFAULT, TRUE); - pci_iov_schema_add_bool(vf_schema, "allow-set-mac", - IOV_SCHEMA_HASDEFAULT, FALSE); - pci_iov_schema_add_bool(vf_schema, "allow-promisc", - IOV_SCHEMA_HASDEFAULT, FALSE); - - iov_error = pci_iov_attach(dev, pf_schema, vf_schema); - if (iov_error != 0) { - device_printf(dev, - "Failed to initialize SR-IOV (error=%d)\n", - iov_error); - } else - device_printf(dev, "SR-IOV ready\n"); - } + ixl_initialize_sriov(pf); #endif #ifdef DEV_NETMAP @@ -685,8 +577,10 @@ ixl_attach(device_t dev) return (0); err_late: - if (vsi->ifp != NULL) + if (vsi->ifp != NULL) { + ether_ifdetach(vsi->ifp); if_free(vsi->ifp); + } err_mac_hmc: i40e_shutdown_lan_hmc(hw); err_get_cap: @@ -766,6 +660,7 @@ ixl_detach(device_t dev) #ifdef DEV_NETMAP netmap_detach(vsi->ifp); #endif /* DEV_NETMAP */ + ixl_pf_qmgr_destroy(&pf->qmgr); ixl_free_pci_resources(pf); bus_generic_detach(dev); if_free(vsi->ifp); @@ -788,6692 +683,3 @@ ixl_shutdown(device_t dev) return (0); } - -/********************************************************************* - * - * Get the hardware capabilities - * - **********************************************************************/ - -static int -ixl_get_hw_capabilities(struct ixl_pf *pf) -{ - struct i40e_aqc_list_capabilities_element_resp *buf; - struct i40e_hw *hw = &pf->hw; - device_t dev = pf->dev; - int error, len; - u16 needed; - bool again = TRUE; - - len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp); -retry: - if (!(buf = (struct i40e_aqc_list_capabilities_element_resp *) - malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO))) { - device_printf(dev, "Unable to allocate cap memory\n"); - return (ENOMEM); - } - - /* This populates the hw struct */ - error = i40e_aq_discover_capabilities(hw, buf, len, - &needed, i40e_aqc_opc_list_func_capabilities, NULL); - free(buf, M_DEVBUF); - if ((pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) && - (again == TRUE)) { - /* retry once with a larger buffer */ - again = FALSE; - len = needed; - goto retry; - } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) { - device_printf(dev, "capability discovery failed: %d\n", - pf->hw.aq.asq_last_status); - return (ENODEV); - } - - /* Capture this PF's starting queue pair */ - pf->qbase = hw->func_caps.base_queue; - -#ifdef IXL_DEBUG - device_printf(dev, "pf_id=%d, num_vfs=%d, msix_pf=%d, " - "msix_vf=%d, fd_g=%d, fd_b=%d, tx_qp=%d rx_qp=%d qbase=%d\n", - hw->pf_id, hw->func_caps.num_vfs, - hw->func_caps.num_msix_vectors, - hw->func_caps.num_msix_vectors_vf, - hw->func_caps.fd_filters_guaranteed, - hw->func_caps.fd_filters_best_effort, - hw->func_caps.num_tx_qp, - hw->func_caps.num_rx_qp, - hw->func_caps.base_queue); -#endif - return (error); -} - -static void -ixl_cap_txcsum_tso(struct ixl_vsi *vsi, struct ifnet *ifp, int mask) -{ - device_t dev = vsi->dev; - - /* Enable/disable TXCSUM/TSO4 */ - if (!(ifp->if_capenable & IFCAP_TXCSUM) - && !(ifp->if_capenable & IFCAP_TSO4)) { - if (mask & IFCAP_TXCSUM) { - ifp->if_capenable |= IFCAP_TXCSUM; - /* enable TXCSUM, restore TSO if previously enabled */ - if (vsi->flags & IXL_FLAGS_KEEP_TSO4) { - vsi->flags &= ~IXL_FLAGS_KEEP_TSO4; - ifp->if_capenable |= IFCAP_TSO4; - } - } - else if (mask & IFCAP_TSO4) { - ifp->if_capenable |= (IFCAP_TXCSUM | IFCAP_TSO4); - vsi->flags &= ~IXL_FLAGS_KEEP_TSO4; - device_printf(dev, - "TSO4 requires txcsum, enabling both...\n"); - } - } else if((ifp->if_capenable & IFCAP_TXCSUM) - && !(ifp->if_capenable & IFCAP_TSO4)) { - if (mask & IFCAP_TXCSUM) - ifp->if_capenable &= ~IFCAP_TXCSUM; - else if (mask & IFCAP_TSO4) - ifp->if_capenable |= IFCAP_TSO4; - } else if((ifp->if_capenable & IFCAP_TXCSUM) - && (ifp->if_capenable & IFCAP_TSO4)) { - if (mask & IFCAP_TXCSUM) { - vsi->flags |= IXL_FLAGS_KEEP_TSO4; - ifp->if_capenable &= ~(IFCAP_TXCSUM | IFCAP_TSO4); - device_printf(dev, - "TSO4 requires txcsum, disabling both...\n"); - } else if (mask & IFCAP_TSO4) - ifp->if_capenable &= ~IFCAP_TSO4; - } - - /* Enable/disable TXCSUM_IPV6/TSO6 */ - if (!(ifp->if_capenable & IFCAP_TXCSUM_IPV6) - && !(ifp->if_capenable & IFCAP_TSO6)) { - if (mask & IFCAP_TXCSUM_IPV6) { - ifp->if_capenable |= IFCAP_TXCSUM_IPV6; - if (vsi->flags & IXL_FLAGS_KEEP_TSO6) { - vsi->flags &= ~IXL_FLAGS_KEEP_TSO6; - ifp->if_capenable |= IFCAP_TSO6; - } - } else if (mask & IFCAP_TSO6) { - ifp->if_capenable |= (IFCAP_TXCSUM_IPV6 | IFCAP_TSO6); - vsi->flags &= ~IXL_FLAGS_KEEP_TSO6; - device_printf(dev, - "TSO6 requires txcsum6, enabling both...\n"); - } - } else if((ifp->if_capenable & IFCAP_TXCSUM_IPV6) - && !(ifp->if_capenable & IFCAP_TSO6)) { - if (mask & IFCAP_TXCSUM_IPV6) - ifp->if_capenable &= ~IFCAP_TXCSUM_IPV6; - else if (mask & IFCAP_TSO6) - ifp->if_capenable |= IFCAP_TSO6; - } else if ((ifp->if_capenable & IFCAP_TXCSUM_IPV6) - && (ifp->if_capenable & IFCAP_TSO6)) { - if (mask & IFCAP_TXCSUM_IPV6) { - vsi->flags |= IXL_FLAGS_KEEP_TSO6; - ifp->if_capenable &= ~(IFCAP_TXCSUM_IPV6 | IFCAP_TSO6); - device_printf(dev, - "TSO6 requires txcsum6, disabling both...\n"); - } else if (mask & IFCAP_TSO6) - ifp->if_capenable &= ~IFCAP_TSO6; - } -} - -/********************************************************************* - * Ioctl entry point - * - * ixl_ioctl is called when the user wants to configure the - * interface. - * - * return 0 on success, positive on failure - **********************************************************************/ - -static int -ixl_ioctl(struct ifnet * ifp, u_long command, caddr_t data) -{ - struct ixl_vsi *vsi = ifp->if_softc; - struct ixl_pf *pf = vsi->back; - struct ifreq *ifr = (struct ifreq *)data; - struct ifdrv *ifd = (struct ifdrv *)data; -#if defined(INET) || defined(INET6) - struct ifaddr *ifa = (struct ifaddr *)data; - bool avoid_reset = FALSE; -#endif - int error = 0; - - switch (command) { - - case SIOCSIFADDR: -#ifdef INET - if (ifa->ifa_addr->sa_family == AF_INET) - avoid_reset = TRUE; -#endif -#ifdef INET6 - if (ifa->ifa_addr->sa_family == AF_INET6) - avoid_reset = TRUE; -#endif -#if defined(INET) || defined(INET6) - /* - ** Calling init results in link renegotiation, - ** so we avoid doing it when possible. - */ - if (avoid_reset) { - ifp->if_flags |= IFF_UP; - if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) - ixl_init(pf); -#ifdef INET - if (!(ifp->if_flags & IFF_NOARP)) - arp_ifinit(ifp, ifa); -#endif - } else - error = ether_ioctl(ifp, command, data); - break; -#endif - case SIOCSIFMTU: - IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); - if (ifr->ifr_mtu > IXL_MAX_FRAME - - ETHER_HDR_LEN - ETHER_CRC_LEN - ETHER_VLAN_ENCAP_LEN) { - error = EINVAL; - } else { - IXL_PF_LOCK(pf); - ifp->if_mtu = ifr->ifr_mtu; - vsi->max_frame_size = - ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN - + ETHER_VLAN_ENCAP_LEN; - if (ifp->if_drv_flags & IFF_DRV_RUNNING) - ixl_init_locked(pf); - IXL_PF_UNLOCK(pf); - } - break; - case SIOCSIFFLAGS: - IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); - IXL_PF_LOCK(pf); - if (ifp->if_flags & IFF_UP) { - if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { - if ((ifp->if_flags ^ pf->if_flags) & - (IFF_PROMISC | IFF_ALLMULTI)) { - ixl_set_promisc(vsi); - } - } else { - IXL_PF_UNLOCK(pf); - ixl_init(pf); - IXL_PF_LOCK(pf); - } - } else { - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { - IXL_PF_UNLOCK(pf); - ixl_stop(pf); - IXL_PF_LOCK(pf); - } - } - pf->if_flags = ifp->if_flags; - IXL_PF_UNLOCK(pf); - break; - case SIOCSDRVSPEC: - case SIOCGDRVSPEC: - IOCTL_DEBUGOUT("ioctl: SIOCxDRVSPEC (Get/Set Driver-specific " - "Info)\n"); - - /* NVM update command */ - if (ifd->ifd_cmd == I40E_NVM_ACCESS) - error = ixl_handle_nvmupd_cmd(pf, ifd); - else - error = EINVAL; - break; - case SIOCADDMULTI: - IOCTL_DEBUGOUT("ioctl: SIOCADDMULTI"); - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { - IXL_PF_LOCK(pf); - ixl_disable_intr(vsi); - ixl_add_multi(vsi); - ixl_enable_intr(vsi); - IXL_PF_UNLOCK(pf); - } - break; - case SIOCDELMULTI: - IOCTL_DEBUGOUT("ioctl: SIOCDELMULTI"); - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { - IXL_PF_LOCK(pf); - ixl_disable_intr(vsi); - ixl_del_multi(vsi); - ixl_enable_intr(vsi); - IXL_PF_UNLOCK(pf); - } - break; - case SIOCSIFMEDIA: - case SIOCGIFMEDIA: -#ifdef IFM_ETH_XTYPE - case SIOCGIFXMEDIA: -#endif - IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); - error = ifmedia_ioctl(ifp, ifr, &vsi->media, command); - break; - case SIOCSIFCAP: - { - int mask = ifr->ifr_reqcap ^ ifp->if_capenable; - IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); - - ixl_cap_txcsum_tso(vsi, ifp, mask); - - if (mask & IFCAP_RXCSUM) - ifp->if_capenable ^= IFCAP_RXCSUM; - if (mask & IFCAP_RXCSUM_IPV6) - ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; - if (mask & IFCAP_LRO) - ifp->if_capenable ^= IFCAP_LRO; - if (mask & IFCAP_VLAN_HWTAGGING) - ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; - if (mask & IFCAP_VLAN_HWFILTER) - ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; - if (mask & IFCAP_VLAN_HWTSO) - ifp->if_capenable ^= IFCAP_VLAN_HWTSO; - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { - IXL_PF_LOCK(pf); - ixl_init_locked(pf); - IXL_PF_UNLOCK(pf); - } - VLAN_CAPABILITIES(ifp); - - break; - } - - default: - IOCTL_DEBUGOUT("ioctl: UNKNOWN (0x%X)\n", (int)command); - error = ether_ioctl(ifp, command, data); - break; - } - - return (error); -} - - -/********************************************************************* - * Init entry point - * - * This routine is used in two ways. It is used by the stack as - * init entry point in network interface structure. It is also used - * by the driver as a hw/sw initialization routine to get to a - * consistent state. - * - * return 0 on success, positive on failure - **********************************************************************/ - -static void -ixl_init_locked(struct ixl_pf *pf) -{ - struct i40e_hw *hw = &pf->hw; - struct ixl_vsi *vsi = &pf->vsi; - struct ifnet *ifp = vsi->ifp; - device_t dev = pf->dev; - struct i40e_filter_control_settings filter; - u8 tmpaddr[ETHER_ADDR_LEN]; - int ret; - - mtx_assert(&pf->pf_mtx, MA_OWNED); - INIT_DEBUGOUT("ixl_init_locked: begin"); - - ixl_stop_locked(pf); - - /* Get the latest mac address... User might use a LAA */ - bcopy(IF_LLADDR(vsi->ifp), tmpaddr, - I40E_ETH_LENGTH_OF_ADDRESS); - if (!cmp_etheraddr(hw->mac.addr, tmpaddr) && - (i40e_validate_mac_addr(tmpaddr) == I40E_SUCCESS)) { - ixl_del_filter(vsi, hw->mac.addr, IXL_VLAN_ANY); - bcopy(tmpaddr, hw->mac.addr, - I40E_ETH_LENGTH_OF_ADDRESS); - ret = i40e_aq_mac_address_write(hw, - I40E_AQC_WRITE_TYPE_LAA_ONLY, - hw->mac.addr, NULL); - if (ret) { - device_printf(dev, "LLA address" - "change failed!!\n"); - return; - } - } - - ixl_add_filter(vsi, hw->mac.addr, IXL_VLAN_ANY); - - /* Set the various hardware offload abilities */ - ifp->if_hwassist = 0; - if (ifp->if_capenable & IFCAP_TSO) - ifp->if_hwassist |= CSUM_TSO; - if (ifp->if_capenable & IFCAP_TXCSUM) - ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); - if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) - ifp->if_hwassist |= (CSUM_TCP_IPV6 | CSUM_UDP_IPV6); - - /* Set up the device filtering */ - bzero(&filter, sizeof(filter)); - filter.enable_ethtype = TRUE; - filter.enable_macvlan = TRUE; -#ifdef IXL_FDIR - filter.enable_fdir = TRUE; -#endif - filter.hash_lut_size = I40E_HASH_LUT_SIZE_512; - if (i40e_set_filter_control(hw, &filter)) - device_printf(dev, "i40e_set_filter_control() failed\n"); - - /* Set up RSS */ - ixl_config_rss(vsi); - - /* Prepare the VSI: rings, hmc contexts, etc... */ - if (ixl_initialize_vsi(vsi)) { - device_printf(dev, "initialize vsi failed!!\n"); - return; - } - - /* Add protocol filters to list */ - ixl_init_filters(vsi); - - /* Setup vlan's if needed */ - ixl_setup_vlan_filters(vsi); - - /* Set up MSI/X routing and the ITR settings */ - if (ixl_enable_msix) { - ixl_configure_queue_intr_msix(pf); - ixl_configure_itr(pf); - } else - ixl_configure_legacy(pf); - - ixl_enable_rings(vsi); - - i40e_aq_set_default_vsi(hw, vsi->seid, NULL); - - ixl_reconfigure_filters(vsi); - - /* And now turn on interrupts */ - ixl_enable_intr(vsi); - - /* Get link info */ - hw->phy.get_link_info = TRUE; - i40e_get_link_status(hw, &pf->link_up); - ixl_update_link_status(pf); - - /* Set initial advertised speed sysctl value */ - ixl_get_initial_advertised_speeds(pf); - - /* Start the local timer */ - callout_reset(&pf->timer, hz, ixl_local_timer, pf); - - /* Now inform the stack we're ready */ - ifp->if_drv_flags |= IFF_DRV_RUNNING; - - return; -} - -/* For the set_advertise sysctl */ -static void -ixl_get_initial_advertised_speeds(struct ixl_pf *pf) -{ - struct i40e_hw *hw = &pf->hw; - device_t dev = pf->dev; - enum i40e_status_code status; - struct i40e_aq_get_phy_abilities_resp abilities; - - /* Set initial sysctl values */ - status = i40e_aq_get_phy_capabilities(hw, FALSE, false, &abilities, - NULL); - if (status) { - /* Non-fatal error */ - device_printf(dev, "%s: i40e_aq_get_phy_capabilities() error %d\n", - __func__, status); - return; - } - - if (abilities.link_speed & I40E_LINK_SPEED_40GB) - pf->advertised_speed |= 0x10; - if (abilities.link_speed & I40E_LINK_SPEED_20GB) - pf->advertised_speed |= 0x8; - if (abilities.link_speed & I40E_LINK_SPEED_10GB) - pf->advertised_speed |= 0x4; - if (abilities.link_speed & I40E_LINK_SPEED_1GB) - pf->advertised_speed |= 0x2; - if (abilities.link_speed & I40E_LINK_SPEED_100MB) - pf->advertised_speed |= 0x1; -} - -static int -ixl_teardown_hw_structs(struct ixl_pf *pf) -{ - enum i40e_status_code status = 0; - struct i40e_hw *hw = &pf->hw; - device_t dev = pf->dev; - - /* Shutdown LAN HMC */ - if (hw->hmc.hmc_obj) { - status = i40e_shutdown_lan_hmc(hw); - if (status) { - device_printf(dev, - "init: LAN HMC shutdown failure; status %d\n", status); - goto err_out; - } - } - - // XXX: This gets called when we know the adminq is inactive; - // so we already know it's setup when we get here. - - /* Shutdown admin queue */ - status = i40e_shutdown_adminq(hw); - if (status) - device_printf(dev, - "init: Admin Queue shutdown failure; status %d\n", status); - -err_out: - return (status); -} - -static int -ixl_reset(struct ixl_pf *pf) -{ - struct i40e_hw *hw = &pf->hw; - device_t dev = pf->dev; - u8 set_fc_err_mask; - int error = 0; - - // XXX: clear_hw() actually writes to hw registers -- maybe this isn't necessary - i40e_clear_hw(hw); - error = i40e_pf_reset(hw); - if (error) { - device_printf(dev, "init: PF reset failure"); - error = EIO; - goto err_out; - } - - error = i40e_init_adminq(hw); - if (error) { - device_printf(dev, "init: Admin queue init failure;" - " status code %d", error); - error = EIO; - goto err_out; - } - - i40e_clear_pxe_mode(hw); - - error = ixl_get_hw_capabilities(pf); - if (error) { - device_printf(dev, "init: Error retrieving HW capabilities;" - " status code %d\n", error); - goto err_out; - } - - error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, - hw->func_caps.num_rx_qp, 0, 0); - if (error) { - device_printf(dev, "init: LAN HMC init failed; status code %d\n", - error); - error = EIO; - goto err_out; - } - - error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); - if (error) { - device_printf(dev, "init: LAN HMC config failed; status code %d\n", - error); - error = EIO; - goto err_out; - } - - // XXX: possible fix for panic, but our failure recovery is still broken - error = ixl_switch_config(pf); - if (error) { - device_printf(dev, "init: ixl_switch_config() failed: %d\n", - error); - goto err_out; - } - - error = i40e_aq_set_phy_int_mask(hw, IXL_DEFAULT_PHY_INT_MASK, - NULL); - if (error) { - device_printf(dev, "init: i40e_aq_set_phy_mask() failed: err %d," - " aq_err %d\n", error, hw->aq.asq_last_status); - error = EIO; - goto err_out; - } - - error = i40e_set_fc(hw, &set_fc_err_mask, true); - if (error) { - device_printf(dev, "init: setting link flow control failed; retcode %d," - " fc_err_mask 0x%02x\n", error, set_fc_err_mask); - goto err_out; - } - - // XXX: (Rebuild VSIs?) - - /* Firmware delay workaround */ - if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) || - (hw->aq.fw_maj_ver < 4)) { - i40e_msec_delay(75); - error = i40e_aq_set_link_restart_an(hw, TRUE, NULL); - if (error) { - device_printf(dev, "init: link restart failed, aq_err %d\n", - hw->aq.asq_last_status); - goto err_out; - } - } - - -err_out: - return (error); -} - -static void -ixl_init(void *arg) -{ - struct ixl_pf *pf = arg; - struct ixl_vsi *vsi = &pf->vsi; - device_t dev = pf->dev; - int error = 0; - - /* - * If the aq is dead here, it probably means something outside of the driver - * did something to the adapter, like a PF reset. - * So rebuild the driver's state here if that occurs. - */ - if (!i40e_check_asq_alive(&pf->hw)) { - device_printf(dev, "Admin Queue is down; resetting...\n"); - IXL_PF_LOCK(pf); - ixl_teardown_hw_structs(pf); - ixl_reset(pf); - IXL_PF_UNLOCK(pf); - } - - /* - * Set up LAN queue interrupts here. - * Kernel interrupt setup functions cannot be called while holding a lock, - * so this is done outside of init_locked(). - */ - if (pf->msix > 1) { - /* Teardown existing interrupts, if they exist */ - ixl_teardown_queue_msix(vsi); - ixl_free_queue_tqs(vsi); - /* Then set them up again */ - error = ixl_setup_queue_msix(vsi); - if (error) - device_printf(dev, "ixl_setup_queue_msix() error: %d\n", - error); - error = ixl_setup_queue_tqs(vsi); - if (error) - device_printf(dev, "ixl_setup_queue_tqs() error: %d\n", - error); - } else - // possibly broken - error = ixl_assign_vsi_legacy(pf); - if (error) { - device_printf(pf->dev, "assign_vsi_msix/legacy error: %d\n", error); - return; - } - - IXL_PF_LOCK(pf); - ixl_init_locked(pf); - IXL_PF_UNLOCK(pf); -} - -/* -** MSIX Interrupt Handlers and Tasklets -*/ -static void -ixl_handle_que(void *context, int pending) -{ - struct ixl_queue *que = context; - struct ixl_vsi *vsi = que->vsi; - struct i40e_hw *hw = vsi->hw; - struct tx_ring *txr = &que->txr; - struct ifnet *ifp = vsi->ifp; - bool more; - - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { - more = ixl_rxeof(que, IXL_RX_LIMIT); - IXL_TX_LOCK(txr); - ixl_txeof(que); - if (!drbr_empty(ifp, txr->br)) - ixl_mq_start_locked(ifp, txr); - IXL_TX_UNLOCK(txr); - if (more) { - taskqueue_enqueue(que->tq, &que->task); - return; - } - } - - /* Reenable this interrupt - hmmm */ - ixl_enable_queue(hw, que->me); - return; -} - - -/********************************************************************* - * - * Legacy Interrupt Service routine - * - **********************************************************************/ -void -ixl_intr(void *arg) -{ - struct ixl_pf *pf = arg; - struct i40e_hw *hw = &pf->hw; - struct ixl_vsi *vsi = &pf->vsi; - struct ixl_queue *que = vsi->queues; - struct ifnet *ifp = vsi->ifp; - struct tx_ring *txr = &que->txr; - u32 reg, icr0, mask; - bool more_tx, more_rx; - - ++que->irqs; - - /* Protect against spurious interrupts */ - if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) - return; - - icr0 = rd32(hw, I40E_PFINT_ICR0); - - reg = rd32(hw, I40E_PFINT_DYN_CTL0); - reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK; - wr32(hw, I40E_PFINT_DYN_CTL0, reg); - - mask = rd32(hw, I40E_PFINT_ICR0_ENA); - -#ifdef PCI_IOV - if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) - taskqueue_enqueue(pf->tq, &pf->vflr_task); -#endif - - if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) { - taskqueue_enqueue(pf->tq, &pf->adminq); - return; - } - - more_rx = ixl_rxeof(que, IXL_RX_LIMIT); - - IXL_TX_LOCK(txr); - more_tx = ixl_txeof(que); - if (!drbr_empty(vsi->ifp, txr->br)) - more_tx = 1; - IXL_TX_UNLOCK(txr); - - /* re-enable other interrupt causes */ - wr32(hw, I40E_PFINT_ICR0_ENA, mask); - - /* And now the queues */ - reg = rd32(hw, I40E_QINT_RQCTL(0)); - reg |= I40E_QINT_RQCTL_CAUSE_ENA_MASK; - wr32(hw, I40E_QINT_RQCTL(0), reg); - - reg = rd32(hw, I40E_QINT_TQCTL(0)); - reg |= I40E_QINT_TQCTL_CAUSE_ENA_MASK; - reg &= ~I40E_PFINT_ICR0_INTEVENT_MASK; - wr32(hw, I40E_QINT_TQCTL(0), reg); - - ixl_enable_legacy(hw); - - return; -} - - -/********************************************************************* - * - * MSIX VSI Interrupt Service routine - * - **********************************************************************/ -void -ixl_msix_que(void *arg) -{ - struct ixl_queue *que = arg; - struct ixl_vsi *vsi = que->vsi; - struct i40e_hw *hw = vsi->hw; - struct tx_ring *txr = &que->txr; - bool more_tx, more_rx; - - /* Protect against spurious interrupts */ - if (!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING)) - return; - - ++que->irqs; - - more_rx = ixl_rxeof(que, IXL_RX_LIMIT); - - IXL_TX_LOCK(txr); - more_tx = ixl_txeof(que); - /* - ** Make certain that if the stack - ** has anything queued the task gets - ** scheduled to handle it. - */ - if (!drbr_empty(vsi->ifp, txr->br)) - more_tx = 1; - IXL_TX_UNLOCK(txr); - - ixl_set_queue_rx_itr(que); - ixl_set_queue_tx_itr(que); - - if (more_tx || more_rx) - taskqueue_enqueue(que->tq, &que->task); - else - ixl_enable_queue(hw, que->me); - - return; -} - - -/********************************************************************* - * - * MSIX Admin Queue Interrupt Service routine - * - **********************************************************************/ -static void -ixl_msix_adminq(void *arg) -{ - struct ixl_pf *pf = arg; - struct i40e_hw *hw = &pf->hw; - u32 reg, mask, rstat_reg; - bool do_task = FALSE; - - ++pf->admin_irq; - - reg = rd32(hw, I40E_PFINT_ICR0); - mask = rd32(hw, I40E_PFINT_ICR0_ENA); - - /* Check on the cause */ - if (reg & I40E_PFINT_ICR0_ADMINQ_MASK) { - mask &= ~I40E_PFINT_ICR0_ADMINQ_MASK; - do_task = TRUE; - } - - if (reg & I40E_PFINT_ICR0_MAL_DETECT_MASK) { - ixl_handle_mdd_event(pf); - mask &= ~I40E_PFINT_ICR0_MAL_DETECT_MASK; - } - - if (reg & I40E_PFINT_ICR0_GRST_MASK) { - device_printf(pf->dev, "Reset Requested!\n"); - rstat_reg = rd32(hw, I40E_GLGEN_RSTAT); - rstat_reg = (rstat_reg & I40E_GLGEN_RSTAT_RESET_TYPE_MASK) - >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; - device_printf(pf->dev, "Reset type: "); - switch (rstat_reg) { - /* These others might be handled similarly to an EMPR reset */ - case I40E_RESET_CORER: - printf("CORER\n"); - break; - case I40E_RESET_GLOBR: - printf("GLOBR\n"); - break; - case I40E_RESET_EMPR: - printf("EMPR\n"); - atomic_set_int(&pf->state, IXL_PF_STATE_EMPR_RESETTING); - break; - default: - printf("?\n"); - break; - } - // overload admin queue task to check reset progress? - do_task = TRUE; - } - - if (reg & I40E_PFINT_ICR0_ECC_ERR_MASK) { - device_printf(pf->dev, "ECC Error detected!\n"); - } - - if (reg & I40E_PFINT_ICR0_HMC_ERR_MASK) { - device_printf(pf->dev, "HMC Error detected!\n"); - } - - if (reg & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) { - device_printf(pf->dev, "PCI Exception detected!\n"); - } - -#ifdef PCI_IOV - if (reg & I40E_PFINT_ICR0_VFLR_MASK) { - mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK; - taskqueue_enqueue(pf->tq, &pf->vflr_task); - } -#endif - - reg = rd32(hw, I40E_PFINT_DYN_CTL0); - reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK; - wr32(hw, I40E_PFINT_DYN_CTL0, reg); - - if (do_task) - taskqueue_enqueue(pf->tq, &pf->adminq); -} - -/********************************************************************* - * - * Media Ioctl callback - * - * This routine is called whenever the user queries the status of - * the interface using ifconfig. - * - **********************************************************************/ -static void -ixl_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) -{ - struct ixl_vsi *vsi = ifp->if_softc; - struct ixl_pf *pf = vsi->back; - struct i40e_hw *hw = &pf->hw; - - INIT_DEBUGOUT("ixl_media_status: begin"); - IXL_PF_LOCK(pf); - - hw->phy.get_link_info = TRUE; - i40e_get_link_status(hw, &pf->link_up); - ixl_update_link_status(pf); - - ifmr->ifm_status = IFM_AVALID; - ifmr->ifm_active = IFM_ETHER; - - if (!pf->link_up) { - IXL_PF_UNLOCK(pf); - return; - } - - ifmr->ifm_status |= IFM_ACTIVE; - - /* Hardware always does full-duplex */ - ifmr->ifm_active |= IFM_FDX; - - switch (hw->phy.link_info.phy_type) { - /* 100 M */ - case I40E_PHY_TYPE_100BASE_TX: - ifmr->ifm_active |= IFM_100_TX; - break; - /* 1 G */ - case I40E_PHY_TYPE_1000BASE_T: - ifmr->ifm_active |= IFM_1000_T; - break; - case I40E_PHY_TYPE_1000BASE_SX: - ifmr->ifm_active |= IFM_1000_SX; - break; - case I40E_PHY_TYPE_1000BASE_LX: - ifmr->ifm_active |= IFM_1000_LX; - break; - case I40E_PHY_TYPE_1000BASE_T_OPTICAL: - ifmr->ifm_active |= IFM_OTHER; - break; - /* 10 G */ - case I40E_PHY_TYPE_10GBASE_SFPP_CU: - ifmr->ifm_active |= IFM_10G_TWINAX; - break; - case I40E_PHY_TYPE_10GBASE_SR: - ifmr->ifm_active |= IFM_10G_SR; - break; - case I40E_PHY_TYPE_10GBASE_LR: - ifmr->ifm_active |= IFM_10G_LR; - break; - case I40E_PHY_TYPE_10GBASE_T: - ifmr->ifm_active |= IFM_10G_T; - break; - case I40E_PHY_TYPE_XAUI: - case I40E_PHY_TYPE_XFI: - case I40E_PHY_TYPE_10GBASE_AOC: - ifmr->ifm_active |= IFM_OTHER; - break; - /* 40 G */ - case I40E_PHY_TYPE_40GBASE_CR4: - case I40E_PHY_TYPE_40GBASE_CR4_CU: - ifmr->ifm_active |= IFM_40G_CR4; - break; - case I40E_PHY_TYPE_40GBASE_SR4: - ifmr->ifm_active |= IFM_40G_SR4; - break; - case I40E_PHY_TYPE_40GBASE_LR4: - ifmr->ifm_active |= IFM_40G_LR4; - break; - case I40E_PHY_TYPE_XLAUI: - ifmr->ifm_active |= IFM_OTHER; - break; -#ifndef IFM_ETH_XTYPE - case I40E_PHY_TYPE_1000BASE_KX: - ifmr->ifm_active |= IFM_1000_CX; - break; - case I40E_PHY_TYPE_SGMII: - ifmr->ifm_active |= IFM_OTHER; - break; - case I40E_PHY_TYPE_10GBASE_CR1_CU: - case I40E_PHY_TYPE_10GBASE_CR1: - ifmr->ifm_active |= IFM_10G_TWINAX; - break; - case I40E_PHY_TYPE_10GBASE_KX4: - ifmr->ifm_active |= IFM_10G_CX4; - break; - case I40E_PHY_TYPE_10GBASE_KR: - ifmr->ifm_active |= IFM_10G_SR; - break; - case I40E_PHY_TYPE_SFI: - ifmr->ifm_active |= IFM_OTHER; - break; - case I40E_PHY_TYPE_40GBASE_KR4: - case I40E_PHY_TYPE_XLPPI: - case I40E_PHY_TYPE_40GBASE_AOC: - ifmr->ifm_active |= IFM_40G_SR4; - break; -#else - case I40E_PHY_TYPE_1000BASE_KX: - ifmr->ifm_active |= IFM_1000_KX; - break; - case I40E_PHY_TYPE_SGMII: - ifmr->ifm_active |= IFM_1000_SGMII; - break; - /* ERJ: What's the difference between these? */ - case I40E_PHY_TYPE_10GBASE_CR1_CU: - case I40E_PHY_TYPE_10GBASE_CR1: - ifmr->ifm_active |= IFM_10G_CR1; - break; - case I40E_PHY_TYPE_10GBASE_KX4: - ifmr->ifm_active |= IFM_10G_KX4; - break; - case I40E_PHY_TYPE_10GBASE_KR: - ifmr->ifm_active |= IFM_10G_KR; - break; - case I40E_PHY_TYPE_SFI: - ifmr->ifm_active |= IFM_10G_SFI; - break; - /* Our single 20G media type */ - case I40E_PHY_TYPE_20GBASE_KR2: - ifmr->ifm_active |= IFM_20G_KR2; - break; - case I40E_PHY_TYPE_40GBASE_KR4: - ifmr->ifm_active |= IFM_40G_KR4; - break; - case I40E_PHY_TYPE_XLPPI: - case I40E_PHY_TYPE_40GBASE_AOC: - ifmr->ifm_active |= IFM_40G_XLPPI; - break; -#endif - /* Unknown to driver */ - default: - ifmr->ifm_active |= IFM_UNKNOWN; - break; - } - /* Report flow control status as well */ - if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX) - ifmr->ifm_active |= IFM_ETH_TXPAUSE; - if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX) - ifmr->ifm_active |= IFM_ETH_RXPAUSE; - - IXL_PF_UNLOCK(pf); - - return; -} - -/* - * NOTE: Fortville does not support forcing media speeds. Instead, - * use the set_advertise sysctl to set the speeds Fortville - * will advertise or be allowed to operate at. - */ -static int -ixl_media_change(struct ifnet * ifp) -{ - struct ixl_vsi *vsi = ifp->if_softc; - struct ifmedia *ifm = &vsi->media; - - INIT_DEBUGOUT("ixl_media_change: begin"); - - if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) - return (EINVAL); - - if_printf(ifp, "Media change is not supported.\n"); - - return (ENODEV); -} - - -#ifdef IXL_FDIR -/* -** ATR: Application Targetted Receive - creates a filter -** based on TX flow info that will keep the receive -** portion of the flow on the same queue. Based on the -** implementation this is only available for TCP connections -*/ -void -ixl_atr(struct ixl_queue *que, struct tcphdr *th, int etype) -{ - struct ixl_vsi *vsi = que->vsi; - struct tx_ring *txr = &que->txr; - struct i40e_filter_program_desc *FDIR; - u32 ptype, dtype; - int idx; - - /* check if ATR is enabled and sample rate */ - if ((!ixl_enable_fdir) || (!txr->atr_rate)) - return; - /* - ** We sample all TCP SYN/FIN packets, - ** or at the selected sample rate - */ - txr->atr_count++; - if (((th->th_flags & (TH_FIN | TH_SYN)) == 0) && - (txr->atr_count < txr->atr_rate)) - return; - txr->atr_count = 0; - - /* Get a descriptor to use */ - idx = txr->next_avail; - FDIR = (struct i40e_filter_program_desc *) &txr->base[idx]; - if (++idx == que->num_desc) - idx = 0; - txr->avail--; - txr->next_avail = idx; - - ptype = (que->me << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) & - I40E_TXD_FLTR_QW0_QINDEX_MASK; - - ptype |= (etype == ETHERTYPE_IP) ? - (I40E_FILTER_PCTYPE_NONF_IPV4_TCP << - I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) : - (I40E_FILTER_PCTYPE_NONF_IPV6_TCP << - I40E_TXD_FLTR_QW0_PCTYPE_SHIFT); - - ptype |= vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT; - - dtype = I40E_TX_DESC_DTYPE_FILTER_PROG; - - /* - ** We use the TCP TH_FIN as a trigger to remove - ** the filter, otherwise its an update. - */ - dtype |= (th->th_flags & TH_FIN) ? - (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE << - I40E_TXD_FLTR_QW1_PCMD_SHIFT) : - (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE << - I40E_TXD_FLTR_QW1_PCMD_SHIFT); - - dtype |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX << - I40E_TXD_FLTR_QW1_DEST_SHIFT; - - dtype |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID << - I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT; - - FDIR->qindex_flex_ptype_vsi = htole32(ptype); - FDIR->dtype_cmd_cntindex = htole32(dtype); - return; -} -#endif - - -static void -ixl_set_promisc(struct ixl_vsi *vsi) -{ - struct ifnet *ifp = vsi->ifp; - struct i40e_hw *hw = vsi->hw; - int err, mcnt = 0; - bool uni = FALSE, multi = FALSE; - - if (ifp->if_flags & IFF_ALLMULTI) - multi = TRUE; - else { /* Need to count the multicast addresses */ - struct ifmultiaddr *ifma; - if_maddr_rlock(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { - if (ifma->ifma_addr->sa_family != AF_LINK) - continue; - if (mcnt == MAX_MULTICAST_ADDR) - break; - mcnt++; - } - if_maddr_runlock(ifp); - } - - if (mcnt >= MAX_MULTICAST_ADDR) - multi = TRUE; - if (ifp->if_flags & IFF_PROMISC) - uni = TRUE; - - err = i40e_aq_set_vsi_unicast_promiscuous(hw, - vsi->seid, uni, NULL); - err = i40e_aq_set_vsi_multicast_promiscuous(hw, - vsi->seid, multi, NULL); - return; -} - -/********************************************************************* - * Filter Routines - * - * Routines for multicast and vlan filter management. - * - *********************************************************************/ -static void -ixl_add_multi(struct ixl_vsi *vsi) -{ - struct ifmultiaddr *ifma; - struct ifnet *ifp = vsi->ifp; - struct i40e_hw *hw = vsi->hw; - int mcnt = 0, flags; - - IOCTL_DEBUGOUT("ixl_add_multi: begin"); - - if_maddr_rlock(ifp); - /* - ** First just get a count, to decide if we - ** we simply use multicast promiscuous. - */ - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { - if (ifma->ifma_addr->sa_family != AF_LINK) - continue; - mcnt++; - } - if_maddr_runlock(ifp); - - if (__predict_false(mcnt >= MAX_MULTICAST_ADDR)) { - /* delete existing MC filters */ - ixl_del_hw_filters(vsi, mcnt); - i40e_aq_set_vsi_multicast_promiscuous(hw, - vsi->seid, TRUE, NULL); - return; - } - - mcnt = 0; - if_maddr_rlock(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { - if (ifma->ifma_addr->sa_family != AF_LINK) - continue; - ixl_add_mc_filter(vsi, - (u8*)LLADDR((struct sockaddr_dl *) ifma->ifma_addr)); - mcnt++; - } - if_maddr_runlock(ifp); - if (mcnt > 0) { - flags = (IXL_FILTER_ADD | IXL_FILTER_USED | IXL_FILTER_MC); - ixl_add_hw_filters(vsi, flags, mcnt); - } - - IOCTL_DEBUGOUT("ixl_add_multi: end"); - return; -} - -static void -ixl_del_multi(struct ixl_vsi *vsi) -{ - struct ifnet *ifp = vsi->ifp; - struct ifmultiaddr *ifma; - struct ixl_mac_filter *f; - int mcnt = 0; - bool match = FALSE; - - IOCTL_DEBUGOUT("ixl_del_multi: begin"); - - /* Search for removed multicast addresses */ - if_maddr_rlock(ifp); - SLIST_FOREACH(f, &vsi->ftl, next) { - if ((f->flags & IXL_FILTER_USED) && (f->flags & IXL_FILTER_MC)) { - match = FALSE; - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { - if (ifma->ifma_addr->sa_family != AF_LINK) - continue; - u8 *mc_addr = (u8 *)LLADDR((struct sockaddr_dl *)ifma->ifma_addr); - if (cmp_etheraddr(f->macaddr, mc_addr)) { - match = TRUE; - break; - } - } - if (match == FALSE) { - f->flags |= IXL_FILTER_DEL; - mcnt++; - } - } - } - if_maddr_runlock(ifp); - - if (mcnt > 0) - ixl_del_hw_filters(vsi, mcnt); -} - - -/********************************************************************* - * Timer routine - * - * This routine checks for link status,updates statistics, - * and runs the watchdog check. - * - * Only runs when the driver is configured UP and RUNNING. - * - **********************************************************************/ - -static void -ixl_local_timer(void *arg) -{ - struct ixl_pf *pf = arg; - struct i40e_hw *hw = &pf->hw; - struct ixl_vsi *vsi = &pf->vsi; - struct ixl_queue *que = vsi->queues; - device_t dev = pf->dev; - int hung = 0; - u32 mask; - - mtx_assert(&pf->pf_mtx, MA_OWNED); - - /* Fire off the adminq task */ - taskqueue_enqueue(pf->tq, &pf->adminq); - - /* Update stats */ - ixl_update_stats_counters(pf); - - /* - ** Check status of the queues - */ - mask = (I40E_PFINT_DYN_CTLN_INTENA_MASK | - I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK); - - for (int i = 0; i < vsi->num_queues; i++, que++) { - /* Any queues with outstanding work get a sw irq */ - if (que->busy) - wr32(hw, I40E_PFINT_DYN_CTLN(que->me), mask); - /* - ** Each time txeof runs without cleaning, but there - ** are uncleaned descriptors it increments busy. If - ** we get to 5 we declare it hung. - */ - if (que->busy == IXL_QUEUE_HUNG) { - ++hung; - /* Mark the queue as inactive */ - vsi->active_queues &= ~((u64)1 << que->me); - continue; - } else { - /* Check if we've come back from hung */ - if ((vsi->active_queues & ((u64)1 << que->me)) == 0) - vsi->active_queues |= ((u64)1 << que->me); - } - if (que->busy >= IXL_MAX_TX_BUSY) { -#ifdef IXL_DEBUG - device_printf(dev,"Warning queue %d " - "appears to be hung!\n", i); -#endif - que->busy = IXL_QUEUE_HUNG; - ++hung; - } - } - /* Only reinit if all queues show hung */ - if (hung == vsi->num_queues) - goto hung; - - callout_reset(&pf->timer, hz, ixl_local_timer, pf); - return; - -hung: - device_printf(dev, "Local Timer: HANG DETECT - Resetting!!\n"); - ixl_init_locked(pf); -} - -/* -** Note: this routine updates the OS on the link state -** the real check of the hardware only happens with -** a link interrupt. -*/ -static void -ixl_update_link_status(struct ixl_pf *pf) -{ - struct ixl_vsi *vsi = &pf->vsi; - struct i40e_hw *hw = &pf->hw; - struct ifnet *ifp = vsi->ifp; - device_t dev = pf->dev; - - if (pf->link_up) { - if (vsi->link_active == FALSE) { - pf->fc = hw->fc.current_mode; - if (bootverbose) { - device_printf(dev,"Link is up %d Gbps %s," - " Flow Control: %s\n", - ((pf->link_speed == - I40E_LINK_SPEED_40GB)? 40:10), - "Full Duplex", ixl_fc_string[pf->fc]); - } - vsi->link_active = TRUE; - /* - ** Warn user if link speed on NPAR enabled - ** partition is not at least 10GB - */ - if (hw->func_caps.npar_enable && - (hw->phy.link_info.link_speed == - I40E_LINK_SPEED_1GB || - hw->phy.link_info.link_speed == - I40E_LINK_SPEED_100MB)) - device_printf(dev, "The partition detected" - "link speed that is less than 10Gbps\n"); - if_link_state_change(ifp, LINK_STATE_UP); - } - } else { /* Link down */ - if (vsi->link_active == TRUE) { - if (bootverbose) - device_printf(dev, "Link is Down\n"); - if_link_state_change(ifp, LINK_STATE_DOWN); - vsi->link_active = FALSE; - } - } - - return; -} - -static void -ixl_stop(struct ixl_pf *pf) -{ - IXL_PF_LOCK(pf); - ixl_stop_locked(pf); - IXL_PF_UNLOCK(pf); - - ixl_teardown_queue_msix(&pf->vsi); - ixl_free_queue_tqs(&pf->vsi); -} - -/********************************************************************* - * - * This routine disables all traffic on the adapter by issuing a - * global reset on the MAC and deallocates TX/RX buffers. - * - **********************************************************************/ - -static void -ixl_stop_locked(struct ixl_pf *pf) -{ - struct ixl_vsi *vsi = &pf->vsi; - struct ifnet *ifp = vsi->ifp; - - INIT_DEBUGOUT("ixl_stop: begin\n"); - - IXL_PF_LOCK_ASSERT(pf); - - /* Stop the local timer */ - callout_stop(&pf->timer); - - ixl_disable_rings_intr(vsi); - ixl_disable_rings(vsi); - - /* Tell the stack that the interface is no longer active */ - ifp->if_drv_flags &= ~(IFF_DRV_RUNNING); -} - - -/********************************************************************* - * - * Setup MSIX Interrupt resources and handlers for the VSI - * - **********************************************************************/ -static int -ixl_assign_vsi_legacy(struct ixl_pf *pf) -{ - device_t dev = pf->dev; - struct ixl_vsi *vsi = &pf->vsi; - struct ixl_queue *que = vsi->queues; - int error, rid = 0; - - if (pf->msix == 1) - rid = 1; - pf->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, - &rid, RF_SHAREABLE | RF_ACTIVE); - if (pf->res == NULL) { - device_printf(dev, "Unable to allocate" - " bus resource: vsi legacy/msi interrupt\n"); - return (ENXIO); - } - - /* Set the handler function */ - error = bus_setup_intr(dev, pf->res, - INTR_TYPE_NET | INTR_MPSAFE, NULL, - ixl_intr, pf, &pf->tag); - if (error) { - pf->res = NULL; - device_printf(dev, "Failed to register legacy/msi handler\n"); - return (error); - } - bus_describe_intr(dev, pf->res, pf->tag, "irq0"); - TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que); - TASK_INIT(&que->task, 0, ixl_handle_que, que); - que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT, - taskqueue_thread_enqueue, &que->tq); - taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que", - device_get_nameunit(dev)); - TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf); - - pf->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT, - taskqueue_thread_enqueue, &pf->tq); - taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s adminq", - device_get_nameunit(dev)); - - return (0); -} - -static int -ixl_setup_adminq_tq(struct ixl_pf *pf) -{ - device_t dev = pf->dev; - int error = 0; - - /* Tasklet for Admin Queue interrupts */ - TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf); -#ifdef PCI_IOV - /* VFLR Tasklet */ - TASK_INIT(&pf->vflr_task, 0, ixl_handle_vflr, pf); -#endif - /* Create and start Admin Queue taskqueue */ - pf->tq = taskqueue_create_fast("ixl_aq", M_NOWAIT, - taskqueue_thread_enqueue, &pf->tq); - if (!pf->tq) { - device_printf(dev, "taskqueue_create_fast (for AQ) returned NULL!\n"); - return (ENOMEM); - } - error = taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s aq", - device_get_nameunit(dev)); - if (error) { - device_printf(dev, "taskqueue_start_threads (for AQ) error: %d\n", - error); - taskqueue_free(pf->tq); - return (error); - } - return (0); -} - -static int -ixl_setup_queue_tqs(struct ixl_vsi *vsi) -{ - struct ixl_queue *que = vsi->queues; - device_t dev = vsi->dev; -#ifdef RSS - cpuset_t cpu_mask; - int cpu_id; -#endif - - /* Create queue tasks and start queue taskqueues */ - for (int i = 0; i < vsi->num_queues; i++, que++) { - TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que); - TASK_INIT(&que->task, 0, ixl_handle_que, que); - que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT, - taskqueue_thread_enqueue, &que->tq); -#ifdef RSS - cpu_id = rss_getcpu(i % rss_getnumbuckets()); - CPU_SETOF(cpu_id, &cpu_mask); - taskqueue_start_threads_cpuset(&que->tq, 1, PI_NET, - &cpu_mask, "%s (bucket %d)", - device_get_nameunit(dev), cpu_id); -#else - taskqueue_start_threads(&que->tq, 1, PI_NET, - "%s (que %d)", device_get_nameunit(dev), que->me); -#endif - } - - return (0); -} - -static void -ixl_free_adminq_tq(struct ixl_pf *pf) -{ - if (pf->tq) { - taskqueue_free(pf->tq); - pf->tq = NULL; - } -} - -static void -ixl_free_queue_tqs(struct ixl_vsi *vsi) -{ - struct ixl_queue *que = vsi->queues; - - for (int i = 0; i < vsi->num_queues; i++, que++) { - if (que->tq) { - taskqueue_free(que->tq); - que->tq = NULL; - } - } -} - -static int -ixl_setup_adminq_msix(struct ixl_pf *pf) -{ - device_t dev = pf->dev; - int rid, error = 0; - - /* Admin IRQ rid is 1, vector is 0 */ - rid = 1; - /* Get interrupt resource from bus */ - pf->res = bus_alloc_resource_any(dev, - SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); - if (!pf->res) { - device_printf(dev, "bus_alloc_resource_any() for Admin Queue" - " interrupt failed [rid=%d]\n", rid); - return (ENXIO); - } - /* Then associate interrupt with handler */ - error = bus_setup_intr(dev, pf->res, - INTR_TYPE_NET | INTR_MPSAFE, NULL, - ixl_msix_adminq, pf, &pf->tag); - if (error) { - pf->res = NULL; - device_printf(dev, "bus_setup_intr() for Admin Queue" - " interrupt handler failed, error %d\n", error); - return (ENXIO); - } - error = bus_describe_intr(dev, pf->res, pf->tag, "aq"); - if (error) { - /* Probably non-fatal? */ - device_printf(dev, "bus_describe_intr() for Admin Queue" - " interrupt name failed, error %d\n", error); - } - pf->admvec = 0; - - return (0); -} - -/* - * Allocate interrupt resources from bus and associate an interrupt handler - * to those for the VSI's queues. - */ -static int -ixl_setup_queue_msix(struct ixl_vsi *vsi) -{ - device_t dev = vsi->dev; - struct ixl_queue *que = vsi->queues; - struct tx_ring *txr; - int error, rid, vector = 1; - - /* Queue interrupt vector numbers start at 1 (adminq intr is 0) */ - for (int i = 0; i < vsi->num_queues; i++, vector++, que++) { - int cpu_id = i; - rid = vector + 1; - txr = &que->txr; - que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, - RF_SHAREABLE | RF_ACTIVE); - if (!que->res) { - device_printf(dev, "bus_alloc_resource_any() for" - " Queue %d interrupt failed [rid=%d]\n", - que->me, rid); - return (ENXIO); - } - /* Set the handler function */ - error = bus_setup_intr(dev, que->res, - INTR_TYPE_NET | INTR_MPSAFE, NULL, - ixl_msix_que, que, &que->tag); - if (error) { - device_printf(dev, "bus_setup_intr() for Queue %d" - " interrupt handler failed, error %d\n", - que->me, error); - // TODO: Check for error from this? - bus_release_resource(dev, SYS_RES_IRQ, rid, que->res); - return (error); - } - error = bus_describe_intr(dev, que->res, que->tag, "q%d", i); - if (error) { - device_printf(dev, "bus_describe_intr() for Queue %d" - " interrupt name failed, error %d\n", - que->me, error); - } - /* Bind the vector to a CPU */ -#ifdef RSS - cpu_id = rss_getcpu(i % rss_getnumbuckets()); -#endif - error = bus_bind_intr(dev, que->res, cpu_id); - if (error) { - device_printf(dev, "bus_bind_intr() for Queue %d" - " to CPU %d failed, error %d\n", - que->me, cpu_id, error); - } - que->msix = vector; - } - - return (0); -} - - -/* - * Allocate MSI/X vectors - */ -static int -ixl_init_msix(struct ixl_pf *pf) -{ - device_t dev = pf->dev; - int rid, want, vectors, queues, available; - - /* Override by tuneable */ - if (ixl_enable_msix == 0) - goto no_msix; - - /* - ** When used in a virtualized environment - ** PCI BUSMASTER capability may not be set - ** so explicity set it here and rewrite - ** the ENABLE in the MSIX control register - ** at this point to cause the host to - ** successfully initialize us. - */ - { - u16 pci_cmd_word; - int msix_ctrl; - pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); - pci_cmd_word |= PCIM_CMD_BUSMASTEREN; - pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2); - pci_find_cap(dev, PCIY_MSIX, &rid); - rid += PCIR_MSIX_CTRL; - msix_ctrl = pci_read_config(dev, rid, 2); - msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE; - pci_write_config(dev, rid, msix_ctrl, 2); - } - - /* First try MSI/X */ - rid = PCIR_BAR(IXL_BAR); - pf->msix_mem = bus_alloc_resource_any(dev, - SYS_RES_MEMORY, &rid, RF_ACTIVE); - if (!pf->msix_mem) { - /* May not be enabled */ - device_printf(pf->dev, - "Unable to map MSIX table\n"); - goto no_msix; - } - - available = pci_msix_count(dev); - if (available == 0) { /* system has msix disabled */ - bus_release_resource(dev, SYS_RES_MEMORY, - rid, pf->msix_mem); - pf->msix_mem = NULL; - goto no_msix; - } - - /* Figure out a reasonable auto config value */ - queues = (mp_ncpus > (available - 1)) ? (available - 1) : mp_ncpus; - - /* Override with tunable value if tunable is less than autoconfig count */ - if ((ixl_max_queues != 0) && (ixl_max_queues <= queues)) - queues = ixl_max_queues; - else if ((ixl_max_queues != 0) && (ixl_max_queues > queues)) - device_printf(dev, "ixl_max_queues > # of cpus, using " - "autoconfig amount...\n"); - /* Or limit maximum auto-configured queues to 8 */ - else if ((ixl_max_queues == 0) && (queues > 8)) - queues = 8; - -#ifdef RSS - /* If we're doing RSS, clamp at the number of RSS buckets */ - if (queues > rss_getnumbuckets()) - queues = rss_getnumbuckets(); -#endif - - /* - ** Want one vector (RX/TX pair) per queue - ** plus an additional for the admin queue. - */ - want = queues + 1; - if (want <= available) /* Have enough */ - vectors = want; - else { - device_printf(pf->dev, - "MSIX Configuration Problem, " - "%d vectors available but %d wanted!\n", - available, want); - return (0); /* Will go to Legacy setup */ - } - - if (pci_alloc_msix(dev, &vectors) == 0) { - device_printf(pf->dev, - "Using MSIX interrupts with %d vectors\n", vectors); - pf->msix = vectors; - pf->vsi.num_queues = queues; -#ifdef RSS - /* - * If we're doing RSS, the number of queues needs to - * match the number of RSS buckets that are configured. - * - * + If there's more queues than RSS buckets, we'll end - * up with queues that get no traffic. - * - * + If there's more RSS buckets than queues, we'll end - * up having multiple RSS buckets map to the same queue, - * so there'll be some contention. - */ - if (queues != rss_getnumbuckets()) { - device_printf(dev, - "%s: queues (%d) != RSS buckets (%d)" - "; performance will be impacted.\n", - __func__, queues, rss_getnumbuckets()); - } -#endif - return (vectors); - } -no_msix: - vectors = pci_msi_count(dev); - pf->vsi.num_queues = 1; - ixl_max_queues = 1; - ixl_enable_msix = 0; - if (vectors == 1 && pci_alloc_msi(dev, &vectors) == 0) - device_printf(pf->dev, "Using an MSI interrupt\n"); - else { - vectors = 0; - device_printf(pf->dev, "Using a Legacy interrupt\n"); - } - return (vectors); -} - -/* - * Configure admin queue/misc interrupt cause registers in hardware. - */ -static void -ixl_configure_intr0_msix(struct ixl_pf *pf) -{ - struct i40e_hw *hw = &pf->hw; - u32 reg; - - /* First set up the adminq - vector 0 */ - wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */ - rd32(hw, I40E_PFINT_ICR0); /* read to clear */ - - reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | - I40E_PFINT_ICR0_ENA_GRST_MASK | - I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | - I40E_PFINT_ICR0_ENA_ADMINQ_MASK | - I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | - I40E_PFINT_ICR0_ENA_VFLR_MASK | - I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK; - wr32(hw, I40E_PFINT_ICR0_ENA, reg); - - /* - * 0x7FF is the end of the queue list. - * This means we won't use MSI-X vector 0 for a queue interrupt - * in MSIX mode. - */ - wr32(hw, I40E_PFINT_LNKLST0, 0x7FF); - /* Value is in 2 usec units, so 0x3E is 62*2 = 124 usecs. */ - wr32(hw, I40E_PFINT_ITR0(IXL_RX_ITR), 0x3E); - - wr32(hw, I40E_PFINT_DYN_CTL0, - I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK | - I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK); - - wr32(hw, I40E_PFINT_STAT_CTL0, 0); -} - -/* - * Configure queue interrupt cause registers in hardware. - */ -static void -ixl_configure_queue_intr_msix(struct ixl_pf *pf) -{ - struct i40e_hw *hw = &pf->hw; - struct ixl_vsi *vsi = &pf->vsi; - u32 reg; - u16 vector = 1; - - for (int i = 0; i < vsi->num_queues; i++, vector++) { - wr32(hw, I40E_PFINT_DYN_CTLN(i), 0); - /* First queue type is RX / 0 */ - wr32(hw, I40E_PFINT_LNKLSTN(i), i); - - reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK | - (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | - (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | - (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | - (I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); - wr32(hw, I40E_QINT_RQCTL(i), reg); - - reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK | - (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | - (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | - (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | - (I40E_QUEUE_TYPE_RX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); - wr32(hw, I40E_QINT_TQCTL(i), reg); - } -} - -/* - * Configure for MSI single vector operation - */ -static void -ixl_configure_legacy(struct ixl_pf *pf) -{ - struct i40e_hw *hw = &pf->hw; - u32 reg; - - wr32(hw, I40E_PFINT_ITR0(0), 0); - wr32(hw, I40E_PFINT_ITR0(1), 0); - - /* Setup "other" causes */ - reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK - | I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK - | I40E_PFINT_ICR0_ENA_GRST_MASK - | I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK - | I40E_PFINT_ICR0_ENA_GPIO_MASK - | I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK - | I40E_PFINT_ICR0_ENA_HMC_ERR_MASK - | I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK - | I40E_PFINT_ICR0_ENA_VFLR_MASK - | I40E_PFINT_ICR0_ENA_ADMINQ_MASK - ; - wr32(hw, I40E_PFINT_ICR0_ENA, reg); - - /* SW_ITR_IDX = 0, but don't change INTENA */ - wr32(hw, I40E_PFINT_DYN_CTL0, - I40E_PFINT_DYN_CTLN_SW_ITR_INDX_MASK | - I40E_PFINT_DYN_CTLN_INTENA_MSK_MASK); - /* SW_ITR_IDX = 0, OTHER_ITR_IDX = 0 */ - wr32(hw, I40E_PFINT_STAT_CTL0, 0); - - /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */ - wr32(hw, I40E_PFINT_LNKLST0, 0); - - /* Associate the queue pair to the vector and enable the q int */ - reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK - | (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) - | (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); - wr32(hw, I40E_QINT_RQCTL(0), reg); - - reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK - | (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) - | (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); - wr32(hw, I40E_QINT_TQCTL(0), reg); -} - - -/* - * Get initial ITR values from tunable values. - */ -static void -ixl_configure_itr(struct ixl_pf *pf) -{ - struct i40e_hw *hw = &pf->hw; - struct ixl_vsi *vsi = &pf->vsi; - struct ixl_queue *que = vsi->queues; - - vsi->rx_itr_setting = ixl_rx_itr; - vsi->tx_itr_setting = ixl_tx_itr; - - for (int i = 0; i < vsi->num_queues; i++, que++) { - struct tx_ring *txr = &que->txr; - struct rx_ring *rxr = &que->rxr; - - wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, i), - vsi->rx_itr_setting); - rxr->itr = vsi->rx_itr_setting; - rxr->latency = IXL_AVE_LATENCY; - - wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, i), - vsi->tx_itr_setting); - txr->itr = vsi->tx_itr_setting; - txr->latency = IXL_AVE_LATENCY; - } -} - - -static int -ixl_allocate_pci_resources(struct ixl_pf *pf) -{ - int rid; - device_t dev = pf->dev; - - rid = PCIR_BAR(0); - pf->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, - &rid, RF_ACTIVE); - - if (!(pf->pci_mem)) { - device_printf(dev, "Unable to allocate bus resource: PCI memory\n"); - return (ENXIO); - } - - pf->osdep.mem_bus_space_tag = - rman_get_bustag(pf->pci_mem); - pf->osdep.mem_bus_space_handle = - rman_get_bushandle(pf->pci_mem); - pf->osdep.mem_bus_space_size = rman_get_size(pf->pci_mem); - pf->osdep.flush_reg = I40E_GLGEN_STAT; - pf->hw.hw_addr = (u8 *) &pf->osdep.mem_bus_space_handle; - - pf->hw.back = &pf->osdep; - - /* - ** Now setup MSI or MSI/X, should - ** return us the number of supported - ** vectors. (Will be 1 for MSI) - */ - pf->msix = ixl_init_msix(pf); - return (0); -} - -/* - * Teardown and release the admin queue/misc vector - * interrupt. - */ -static int -ixl_teardown_adminq_msix(struct ixl_pf *pf) -{ - device_t dev = pf->dev; - int rid; - - if (pf->admvec) /* we are doing MSIX */ - rid = pf->admvec + 1; - else - (pf->msix != 0) ? (rid = 1):(rid = 0); - - // TODO: Check for errors from bus_teardown_intr - // TODO: Check for errors from bus_release_resource - if (pf->tag != NULL) { - bus_teardown_intr(dev, pf->res, pf->tag); - pf->tag = NULL; - } - if (pf->res != NULL) { - bus_release_resource(dev, SYS_RES_IRQ, rid, pf->res); - pf->res = NULL; - } - - return (0); -} - -static int -ixl_teardown_queue_msix(struct ixl_vsi *vsi) -{ - struct ixl_queue *que = vsi->queues; - device_t dev = vsi->dev; - int rid, error = 0; - - /* We may get here before stations are setup */ - if ((!ixl_enable_msix) || (que == NULL)) - return (0); - - /* Release all MSIX queue resources */ - for (int i = 0; i < vsi->num_queues; i++, que++) { - rid = que->msix + 1; - if (que->tag != NULL) { - error = bus_teardown_intr(dev, que->res, que->tag); - if (error) { - device_printf(dev, "bus_teardown_intr() for" - " Queue %d interrupt failed\n", - que->me); - // return (ENXIO); - } - que->tag = NULL; - } - if (que->res != NULL) { - error = bus_release_resource(dev, SYS_RES_IRQ, rid, que->res); - if (error) { - device_printf(dev, "bus_release_resource() for" - " Queue %d interrupt failed [rid=%d]\n", - que->me, rid); - // return (ENXIO); - } - que->res = NULL; - } - } - - return (0); -} - -static void -ixl_free_pci_resources(struct ixl_pf *pf) -{ - device_t dev = pf->dev; - int memrid; - - ixl_teardown_queue_msix(&pf->vsi); - ixl_teardown_adminq_msix(pf); - - if (pf->msix) - pci_release_msi(dev); - - memrid = PCIR_BAR(IXL_BAR); - - if (pf->msix_mem != NULL) - bus_release_resource(dev, SYS_RES_MEMORY, - memrid, pf->msix_mem); - - if (pf->pci_mem != NULL) - bus_release_resource(dev, SYS_RES_MEMORY, - PCIR_BAR(0), pf->pci_mem); - - return; -} - -static void -ixl_add_ifmedia(struct ixl_vsi *vsi, u32 phy_type) -{ - /* Display supported media types */ - if (phy_type & (1 << I40E_PHY_TYPE_100BASE_TX)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_100_TX, 0, NULL); - - if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_T)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_T, 0, NULL); - if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_SX)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_SX, 0, NULL); - if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_LX)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_LX, 0, NULL); - - if (phy_type & (1 << I40E_PHY_TYPE_XAUI) || - phy_type & (1 << I40E_PHY_TYPE_XFI) || - phy_type & (1 << I40E_PHY_TYPE_10GBASE_SFPP_CU)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL); - - if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_SR)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL); - if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_LR)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_LR, 0, NULL); - if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_T)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_T, 0, NULL); - - if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4) || - phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4_CU) || - phy_type & (1 << I40E_PHY_TYPE_40GBASE_AOC) || - phy_type & (1 << I40E_PHY_TYPE_XLAUI) || - phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL); - if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_SR4)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL); - if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_LR4)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_LR4, 0, NULL); - -#ifndef IFM_ETH_XTYPE - if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_KX)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_CX, 0, NULL); - - if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1_CU) || - phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1) || - phy_type & (1 << I40E_PHY_TYPE_10GBASE_AOC) || - phy_type & (1 << I40E_PHY_TYPE_SFI)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL); - if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KX4)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_CX4, 0, NULL); - if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KR)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL); - - if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL); - if (phy_type & (1 << I40E_PHY_TYPE_XLPPI)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL); -#else - if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_KX)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_KX, 0, NULL); - - if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1_CU) - || phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_CR1, 0, NULL); - if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_AOC)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX_LONG, 0, NULL); - if (phy_type & (1 << I40E_PHY_TYPE_SFI)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SFI, 0, NULL); - if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KX4)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_KX4, 0, NULL); - if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KR)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_KR, 0, NULL); - - if (phy_type & (1 << I40E_PHY_TYPE_20GBASE_KR2)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_20G_KR2, 0, NULL); - - if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_KR4, 0, NULL); - if (phy_type & (1 << I40E_PHY_TYPE_XLPPI)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_XLPPI, 0, NULL); -#endif -} - -/********************************************************************* - * - * Setup networking device structure and register an interface. - * - **********************************************************************/ -static int -ixl_setup_interface(device_t dev, struct ixl_vsi *vsi) -{ - struct ifnet *ifp; - struct i40e_hw *hw = vsi->hw; - struct ixl_queue *que = vsi->queues; - struct i40e_aq_get_phy_abilities_resp abilities; - enum i40e_status_code aq_error = 0; - - INIT_DEBUGOUT("ixl_setup_interface: begin"); - - ifp = vsi->ifp = if_alloc(IFT_ETHER); - if (ifp == NULL) { - device_printf(dev, "can not allocate ifnet structure\n"); - return (-1); - } - if_initname(ifp, device_get_name(dev), device_get_unit(dev)); - ifp->if_mtu = ETHERMTU; - ifp->if_baudrate = IF_Gbps(40); - ifp->if_init = ixl_init; - ifp->if_softc = vsi; - ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; - ifp->if_ioctl = ixl_ioctl; - -#if __FreeBSD_version >= 1100036 - if_setgetcounterfn(ifp, ixl_get_counter); -#endif - - ifp->if_transmit = ixl_mq_start; - - ifp->if_qflush = ixl_qflush; - - ifp->if_snd.ifq_maxlen = que->num_desc - 2; - - vsi->max_frame_size = - ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN - + ETHER_VLAN_ENCAP_LEN; - - /* - * Tell the upper layer(s) we support long frames. - */ - ifp->if_hdrlen = sizeof(struct ether_vlan_header); - - ifp->if_capabilities |= IFCAP_HWCSUM; - ifp->if_capabilities |= IFCAP_HWCSUM_IPV6; - ifp->if_capabilities |= IFCAP_TSO; - ifp->if_capabilities |= IFCAP_JUMBO_MTU; - ifp->if_capabilities |= IFCAP_LRO; - - /* VLAN capabilties */ - ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING - | IFCAP_VLAN_HWTSO - | IFCAP_VLAN_MTU - | IFCAP_VLAN_HWCSUM; - ifp->if_capenable = ifp->if_capabilities; - - /* - ** Don't turn this on by default, if vlans are - ** created on another pseudo device (eg. lagg) - ** then vlan events are not passed thru, breaking - ** operation, but with HW FILTER off it works. If - ** using vlans directly on the ixl driver you can - ** enable this and get full hardware tag filtering. - */ - ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; - - /* - * Specify the media types supported by this adapter and register - * callbacks to update media and link information - */ - ifmedia_init(&vsi->media, IFM_IMASK, ixl_media_change, - ixl_media_status); - - aq_error = i40e_aq_get_phy_capabilities(hw, - FALSE, TRUE, &abilities, NULL); - /* May need delay to detect fiber correctly */ - if (aq_error == I40E_ERR_UNKNOWN_PHY) { - i40e_msec_delay(200); - aq_error = i40e_aq_get_phy_capabilities(hw, FALSE, - TRUE, &abilities, NULL); - } - if (aq_error) { - if (aq_error == I40E_ERR_UNKNOWN_PHY) - device_printf(dev, "Unknown PHY type detected!\n"); - else - device_printf(dev, - "Error getting supported media types, err %d," - " AQ error %d\n", aq_error, hw->aq.asq_last_status); - return (0); - } - - ixl_add_ifmedia(vsi, abilities.phy_type); - - /* Use autoselect media by default */ - ifmedia_add(&vsi->media, IFM_ETHER | IFM_AUTO, 0, NULL); - ifmedia_set(&vsi->media, IFM_ETHER | IFM_AUTO); - - ether_ifattach(ifp, hw->mac.addr); - - return (0); -} - -/* -** Run when the Admin Queue gets a link state change interrupt. -*/ -static void -ixl_link_event(struct ixl_pf *pf, struct i40e_arq_event_info *e) -{ - struct i40e_hw *hw = &pf->hw; - device_t dev = pf->dev; - struct i40e_aqc_get_link_status *status = - (struct i40e_aqc_get_link_status *)&e->desc.params.raw; - - /* Request link status from adapter */ - hw->phy.get_link_info = TRUE; - i40e_get_link_status(hw, &pf->link_up); - - /* Print out message if an unqualified module is found */ - if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) && - (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) && - (!(status->link_info & I40E_AQ_LINK_UP))) - device_printf(dev, "Link failed because " - "an unqualified module was detected!\n"); - - /* Update OS link info */ - ixl_update_link_status(pf); -} - -/********************************************************************* - * - * Get Firmware Switch configuration - * - this will need to be more robust when more complex - * switch configurations are enabled. - * - **********************************************************************/ -static int -ixl_switch_config(struct ixl_pf *pf) -{ - struct i40e_hw *hw = &pf->hw; - struct ixl_vsi *vsi = &pf->vsi; - device_t dev = vsi->dev; - struct i40e_aqc_get_switch_config_resp *sw_config; - u8 aq_buf[I40E_AQ_LARGE_BUF]; - int ret; - u16 next = 0; - - memset(&aq_buf, 0, sizeof(aq_buf)); - sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; - ret = i40e_aq_get_switch_config(hw, sw_config, - sizeof(aq_buf), &next, NULL); - if (ret) { - device_printf(dev, "aq_get_switch_config() failed, error %d," - " aq_error %d\n", ret, pf->hw.aq.asq_last_status); - return (ret); - } -#ifdef IXL_DEBUG - device_printf(dev, - "Switch config: header reported: %d in structure, %d total\n", - sw_config->header.num_reported, sw_config->header.num_total); - for (int i = 0; i < sw_config->header.num_reported; i++) { - device_printf(dev, - "%d: type=%d seid=%d uplink=%d downlink=%d\n", i, - sw_config->element[i].element_type, - sw_config->element[i].seid, - sw_config->element[i].uplink_seid, - sw_config->element[i].downlink_seid); - } -#endif - /* Simplified due to a single VSI at the moment */ - vsi->uplink_seid = sw_config->element[0].uplink_seid; - vsi->downlink_seid = sw_config->element[0].downlink_seid; - vsi->seid = sw_config->element[0].seid; - return (ret); -} - -/********************************************************************* - * - * Initialize the VSI: this handles contexts, which means things - * like the number of descriptors, buffer size, - * plus we init the rings thru this function. - * - **********************************************************************/ -static int -ixl_initialize_vsi(struct ixl_vsi *vsi) -{ - struct ixl_pf *pf = vsi->back; - struct ixl_queue *que = vsi->queues; - device_t dev = vsi->dev; - struct i40e_hw *hw = vsi->hw; - struct i40e_vsi_context ctxt; - int err = 0; - - memset(&ctxt, 0, sizeof(ctxt)); - ctxt.seid = vsi->seid; - if (pf->veb_seid != 0) - ctxt.uplink_seid = pf->veb_seid; - ctxt.pf_num = hw->pf_id; - err = i40e_aq_get_vsi_params(hw, &ctxt, NULL); - if (err) { - device_printf(dev, "i40e_aq_get_vsi_params() failed, error %d" - " aq_error %d\n", err, hw->aq.asq_last_status); - return (err); - } -#ifdef IXL_DEBUG - device_printf(dev, "get_vsi_params: seid: %d, uplinkseid: %d, vsi_number: %d, " - "vsis_allocated: %d, vsis_unallocated: %d, flags: 0x%x, " - "pfnum: %d, vfnum: %d, stat idx: %d, enabled: %d\n", ctxt.seid, - ctxt.uplink_seid, ctxt.vsi_number, - ctxt.vsis_allocated, ctxt.vsis_unallocated, - ctxt.flags, ctxt.pf_num, ctxt.vf_num, - ctxt.info.stat_counter_idx, ctxt.info.up_enable_bits); -#endif - /* - ** Set the queue and traffic class bits - ** - when multiple traffic classes are supported - ** this will need to be more robust. - */ - ctxt.info.valid_sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; - ctxt.info.mapping_flags |= I40E_AQ_VSI_QUE_MAP_CONTIG; - /* In contig mode, que_mapping[0] is first queue index used by this VSI */ - ctxt.info.queue_mapping[0] = 0; - /* - * This VSI will only use traffic class 0; start traffic class 0's - * queue allocation at queue 0, and assign it 64 (2^6) queues (though - * the driver may not use all of them). - */ - ctxt.info.tc_mapping[0] = ((0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) - & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) | - ((6 << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT) - & I40E_AQ_VSI_TC_QUE_NUMBER_MASK); - - /* Set VLAN receive stripping mode */ - ctxt.info.valid_sections |= I40E_AQ_VSI_PROP_VLAN_VALID; - ctxt.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL; - if (vsi->ifp->if_capenable & IFCAP_VLAN_HWTAGGING) - ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; - else - ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_NOTHING; - - /* Keep copy of VSI info in VSI for statistic counters */ - memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info)); - - /* Reset VSI statistics */ - ixl_vsi_reset_stats(vsi); - vsi->hw_filters_add = 0; - vsi->hw_filters_del = 0; - - ctxt.flags = htole16(I40E_AQ_VSI_TYPE_PF); - - err = i40e_aq_update_vsi_params(hw, &ctxt, NULL); - if (err) { - device_printf(dev, "i40e_aq_update_vsi_params() failed, error %d, aq_error %d\n", - err, hw->aq.asq_last_status); - return (err); - } - - for (int i = 0; i < vsi->num_queues; i++, que++) { - struct tx_ring *txr = &que->txr; - struct rx_ring *rxr = &que->rxr; - struct i40e_hmc_obj_txq tctx; - struct i40e_hmc_obj_rxq rctx; - u32 txctl; - u16 size; - - /* Setup the HMC TX Context */ - size = que->num_desc * sizeof(struct i40e_tx_desc); - memset(&tctx, 0, sizeof(struct i40e_hmc_obj_txq)); - tctx.new_context = 1; - tctx.base = (txr->dma.pa/IXL_TX_CTX_BASE_UNITS); - tctx.qlen = que->num_desc; - tctx.fc_ena = 0; - tctx.rdylist = vsi->info.qs_handle[0]; /* index is TC */ - /* Enable HEAD writeback */ - tctx.head_wb_ena = 1; - tctx.head_wb_addr = txr->dma.pa + - (que->num_desc * sizeof(struct i40e_tx_desc)); - tctx.rdylist_act = 0; - err = i40e_clear_lan_tx_queue_context(hw, i); - if (err) { - device_printf(dev, "Unable to clear TX context\n"); - break; - } - err = i40e_set_lan_tx_queue_context(hw, i, &tctx); - if (err) { - device_printf(dev, "Unable to set TX context\n"); - break; - } - /* Associate the ring with this PF */ - txctl = I40E_QTX_CTL_PF_QUEUE; - txctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) & - I40E_QTX_CTL_PF_INDX_MASK); - wr32(hw, I40E_QTX_CTL(i), txctl); - ixl_flush(hw); - - /* Do ring (re)init */ - ixl_init_tx_ring(que); - - /* Next setup the HMC RX Context */ - if (vsi->max_frame_size <= MCLBYTES) - rxr->mbuf_sz = MCLBYTES; - else - rxr->mbuf_sz = MJUMPAGESIZE; - - u16 max_rxmax = rxr->mbuf_sz * hw->func_caps.rx_buf_chain_len; - - /* Set up an RX context for the HMC */ - memset(&rctx, 0, sizeof(struct i40e_hmc_obj_rxq)); - rctx.dbuff = rxr->mbuf_sz >> I40E_RXQ_CTX_DBUFF_SHIFT; - /* ignore header split for now */ - rctx.hbuff = 0 >> I40E_RXQ_CTX_HBUFF_SHIFT; - rctx.rxmax = (vsi->max_frame_size < max_rxmax) ? - vsi->max_frame_size : max_rxmax; - rctx.dtype = 0; - rctx.dsize = 1; /* do 32byte descriptors */ - rctx.hsplit_0 = 0; /* no HDR split initially */ - rctx.base = (rxr->dma.pa/IXL_RX_CTX_BASE_UNITS); - rctx.qlen = que->num_desc; - rctx.tphrdesc_ena = 1; - rctx.tphwdesc_ena = 1; - rctx.tphdata_ena = 0; - rctx.tphhead_ena = 0; - rctx.lrxqthresh = 2; - rctx.crcstrip = 1; - rctx.l2tsel = 1; - rctx.showiv = 1; - rctx.fc_ena = 0; - rctx.prefena = 1; - - err = i40e_clear_lan_rx_queue_context(hw, i); - if (err) { - device_printf(dev, - "Unable to clear RX context %d\n", i); - break; - } - err = i40e_set_lan_rx_queue_context(hw, i, &rctx); - if (err) { - device_printf(dev, "Unable to set RX context %d\n", i); - break; - } - err = ixl_init_rx_ring(que); - if (err) { - device_printf(dev, "Fail in init_rx_ring %d\n", i); - break; - } -#ifdef DEV_NETMAP - /* preserve queue */ - if (vsi->ifp->if_capenable & IFCAP_NETMAP) { - struct netmap_adapter *na = NA(vsi->ifp); - struct netmap_kring *kring = &na->rx_rings[i]; - int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring); - wr32(vsi->hw, I40E_QRX_TAIL(que->me), t); - } else -#endif /* DEV_NETMAP */ - wr32(vsi->hw, I40E_QRX_TAIL(que->me), que->num_desc - 1); - } - return (err); -} - - -/********************************************************************* - * - * Free all VSI structs. - * - **********************************************************************/ -void -ixl_free_vsi(struct ixl_vsi *vsi) -{ - struct ixl_pf *pf = (struct ixl_pf *)vsi->back; - struct ixl_queue *que = vsi->queues; - - /* Free station queues */ - if (!vsi->queues) - goto free_filters; - - for (int i = 0; i < vsi->num_queues; i++, que++) { - struct tx_ring *txr = &que->txr; - struct rx_ring *rxr = &que->rxr; - - if (!mtx_initialized(&txr->mtx)) /* uninitialized */ - continue; - IXL_TX_LOCK(txr); - ixl_free_que_tx(que); - if (txr->base) - i40e_free_dma_mem(&pf->hw, &txr->dma); - IXL_TX_UNLOCK(txr); - IXL_TX_LOCK_DESTROY(txr); - - if (!mtx_initialized(&rxr->mtx)) /* uninitialized */ - continue; - IXL_RX_LOCK(rxr); - ixl_free_que_rx(que); - if (rxr->base) - i40e_free_dma_mem(&pf->hw, &rxr->dma); - IXL_RX_UNLOCK(rxr); - IXL_RX_LOCK_DESTROY(rxr); - - } - free(vsi->queues, M_DEVBUF); - -free_filters: - /* Free VSI filter list */ - ixl_free_mac_filters(vsi); -} - -static void -ixl_free_mac_filters(struct ixl_vsi *vsi) -{ - struct ixl_mac_filter *f; - - while (!SLIST_EMPTY(&vsi->ftl)) { - f = SLIST_FIRST(&vsi->ftl); - SLIST_REMOVE_HEAD(&vsi->ftl, next); - free(f, M_DEVBUF); - } -} - - -/********************************************************************* - * - * Allocate memory for the VSI (virtual station interface) and their - * associated queues, rings and the descriptors associated with each, - * called only once at attach. - * - **********************************************************************/ -static int -ixl_setup_stations(struct ixl_pf *pf) -{ - device_t dev = pf->dev; - struct ixl_vsi *vsi; - struct ixl_queue *que; - struct tx_ring *txr; - struct rx_ring *rxr; - int rsize, tsize; - int error = I40E_SUCCESS; - - vsi = &pf->vsi; - vsi->back = (void *)pf; - vsi->hw = &pf->hw; - vsi->id = 0; - vsi->num_vlans = 0; - vsi->back = pf; - - /* Get memory for the station queues */ - if (!(vsi->queues = - (struct ixl_queue *) malloc(sizeof(struct ixl_queue) * - vsi->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { - device_printf(dev, "Unable to allocate queue memory\n"); - error = ENOMEM; - goto early; - } - - for (int i = 0; i < vsi->num_queues; i++) { - que = &vsi->queues[i]; - que->num_desc = ixl_ringsz; - que->me = i; - que->vsi = vsi; - /* mark the queue as active */ - vsi->active_queues |= (u64)1 << que->me; - txr = &que->txr; - txr->que = que; - txr->tail = I40E_QTX_TAIL(que->me); - - /* Initialize the TX lock */ - snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", - device_get_nameunit(dev), que->me); - mtx_init(&txr->mtx, txr->mtx_name, NULL, MTX_DEF); - /* Create the TX descriptor ring */ - tsize = roundup2((que->num_desc * - sizeof(struct i40e_tx_desc)) + - sizeof(u32), DBA_ALIGN); - if (i40e_allocate_dma_mem(&pf->hw, - &txr->dma, i40e_mem_reserved, tsize, DBA_ALIGN)) { - device_printf(dev, - "Unable to allocate TX Descriptor memory\n"); - error = ENOMEM; - goto fail; - } - txr->base = (struct i40e_tx_desc *)txr->dma.va; - bzero((void *)txr->base, tsize); - /* Now allocate transmit soft structs for the ring */ - if (ixl_allocate_tx_data(que)) { - device_printf(dev, - "Critical Failure setting up TX structures\n"); - error = ENOMEM; - goto fail; - } - /* Allocate a buf ring */ - txr->br = buf_ring_alloc(4096, M_DEVBUF, - M_NOWAIT, &txr->mtx); - if (txr->br == NULL) { - device_printf(dev, - "Critical Failure setting up TX buf ring\n"); - error = ENOMEM; - goto fail; - } - - /* - * Next the RX queues... - */ - rsize = roundup2(que->num_desc * - sizeof(union i40e_rx_desc), DBA_ALIGN); - rxr = &que->rxr; - rxr->que = que; - rxr->tail = I40E_QRX_TAIL(que->me); - - /* Initialize the RX side lock */ - snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", - device_get_nameunit(dev), que->me); - mtx_init(&rxr->mtx, rxr->mtx_name, NULL, MTX_DEF); - - if (i40e_allocate_dma_mem(&pf->hw, - &rxr->dma, i40e_mem_reserved, rsize, 4096)) { - device_printf(dev, - "Unable to allocate RX Descriptor memory\n"); - error = ENOMEM; - goto fail; - } - rxr->base = (union i40e_rx_desc *)rxr->dma.va; - bzero((void *)rxr->base, rsize); - - /* Allocate receive soft structs for the ring*/ - if (ixl_allocate_rx_data(que)) { - device_printf(dev, - "Critical Failure setting up receive structs\n"); - error = ENOMEM; - goto fail; - } - } - - return (0); - -fail: - for (int i = 0; i < vsi->num_queues; i++) { - que = &vsi->queues[i]; - rxr = &que->rxr; - txr = &que->txr; - if (rxr->base) - i40e_free_dma_mem(&pf->hw, &rxr->dma); - if (txr->base) - i40e_free_dma_mem(&pf->hw, &txr->dma); - } - -early: - return (error); -} - -/* -** Provide a update to the queue RX -** interrupt moderation value. -*/ -static void -ixl_set_queue_rx_itr(struct ixl_queue *que) -{ - struct ixl_vsi *vsi = que->vsi; - struct i40e_hw *hw = vsi->hw; - struct rx_ring *rxr = &que->rxr; - u16 rx_itr; - u16 rx_latency = 0; - int rx_bytes; - - /* Idle, do nothing */ - if (rxr->bytes == 0) - return; - - if (ixl_dynamic_rx_itr) { - rx_bytes = rxr->bytes/rxr->itr; - rx_itr = rxr->itr; - - /* Adjust latency range */ - switch (rxr->latency) { - case IXL_LOW_LATENCY: - if (rx_bytes > 10) { - rx_latency = IXL_AVE_LATENCY; - rx_itr = IXL_ITR_20K; - } - break; - case IXL_AVE_LATENCY: - if (rx_bytes > 20) { - rx_latency = IXL_BULK_LATENCY; - rx_itr = IXL_ITR_8K; - } else if (rx_bytes <= 10) { - rx_latency = IXL_LOW_LATENCY; - rx_itr = IXL_ITR_100K; - } - break; - case IXL_BULK_LATENCY: - if (rx_bytes <= 20) { - rx_latency = IXL_AVE_LATENCY; - rx_itr = IXL_ITR_20K; - } - break; - } - - rxr->latency = rx_latency; - - if (rx_itr != rxr->itr) { - /* do an exponential smoothing */ - rx_itr = (10 * rx_itr * rxr->itr) / - ((9 * rx_itr) + rxr->itr); - rxr->itr = rx_itr & IXL_MAX_ITR; - wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, - que->me), rxr->itr); - } - } else { /* We may have have toggled to non-dynamic */ - if (vsi->rx_itr_setting & IXL_ITR_DYNAMIC) - vsi->rx_itr_setting = ixl_rx_itr; - /* Update the hardware if needed */ - if (rxr->itr != vsi->rx_itr_setting) { - rxr->itr = vsi->rx_itr_setting; - wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, - que->me), rxr->itr); - } - } - rxr->bytes = 0; - rxr->packets = 0; - return; -} - - -/* -** Provide a update to the queue TX -** interrupt moderation value. -*/ -static void -ixl_set_queue_tx_itr(struct ixl_queue *que) -{ - struct ixl_vsi *vsi = que->vsi; - struct i40e_hw *hw = vsi->hw; - struct tx_ring *txr = &que->txr; - u16 tx_itr; - u16 tx_latency = 0; - int tx_bytes; - - - /* Idle, do nothing */ - if (txr->bytes == 0) - return; - - if (ixl_dynamic_tx_itr) { - tx_bytes = txr->bytes/txr->itr; - tx_itr = txr->itr; - - switch (txr->latency) { - case IXL_LOW_LATENCY: - if (tx_bytes > 10) { - tx_latency = IXL_AVE_LATENCY; - tx_itr = IXL_ITR_20K; - } - break; - case IXL_AVE_LATENCY: - if (tx_bytes > 20) { - tx_latency = IXL_BULK_LATENCY; - tx_itr = IXL_ITR_8K; - } else if (tx_bytes <= 10) { - tx_latency = IXL_LOW_LATENCY; - tx_itr = IXL_ITR_100K; - } - break; - case IXL_BULK_LATENCY: - if (tx_bytes <= 20) { - tx_latency = IXL_AVE_LATENCY; - tx_itr = IXL_ITR_20K; - } - break; - } - - txr->latency = tx_latency; - - if (tx_itr != txr->itr) { - /* do an exponential smoothing */ - tx_itr = (10 * tx_itr * txr->itr) / - ((9 * tx_itr) + txr->itr); - txr->itr = tx_itr & IXL_MAX_ITR; - wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, - que->me), txr->itr); - } - - } else { /* We may have have toggled to non-dynamic */ - if (vsi->tx_itr_setting & IXL_ITR_DYNAMIC) - vsi->tx_itr_setting = ixl_tx_itr; - /* Update the hardware if needed */ - if (txr->itr != vsi->tx_itr_setting) { - txr->itr = vsi->tx_itr_setting; - wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, - que->me), txr->itr); - } - } - txr->bytes = 0; - txr->packets = 0; - return; -} - -#define QUEUE_NAME_LEN 32 - -static void -ixl_add_vsi_sysctls(struct ixl_pf *pf, struct ixl_vsi *vsi, - struct sysctl_ctx_list *ctx, const char *sysctl_name) -{ - struct sysctl_oid *tree; - struct sysctl_oid_list *child; - struct sysctl_oid_list *vsi_list; - - tree = device_get_sysctl_tree(pf->dev); - child = SYSCTL_CHILDREN(tree); - vsi->vsi_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, sysctl_name, - CTLFLAG_RD, NULL, "VSI Number"); - vsi_list = SYSCTL_CHILDREN(vsi->vsi_node); - - ixl_add_sysctls_eth_stats(ctx, vsi_list, &vsi->eth_stats); -} - -#ifdef IXL_DEBUG -/** - * ixl_sysctl_qtx_tail_handler - * Retrieves I40E_QTX_TAIL value from hardware - * for a sysctl. - */ -static int -ixl_sysctl_qtx_tail_handler(SYSCTL_HANDLER_ARGS) -{ - struct ixl_queue *que; - int error; - u32 val; - - que = ((struct ixl_queue *)oidp->oid_arg1); - if (!que) return 0; - - val = rd32(que->vsi->hw, que->txr.tail); - error = sysctl_handle_int(oidp, &val, 0, req); - if (error || !req->newptr) - return error; - return (0); -} - -/** - * ixl_sysctl_qrx_tail_handler - * Retrieves I40E_QRX_TAIL value from hardware - * for a sysctl. - */ -static int -ixl_sysctl_qrx_tail_handler(SYSCTL_HANDLER_ARGS) -{ - struct ixl_queue *que; - int error; - u32 val; - - que = ((struct ixl_queue *)oidp->oid_arg1); - if (!que) return 0; - - val = rd32(que->vsi->hw, que->rxr.tail); - error = sysctl_handle_int(oidp, &val, 0, req); - if (error || !req->newptr) - return error; - return (0); -} -#endif - -static void -ixl_add_hw_stats(struct ixl_pf *pf) -{ - device_t dev = pf->dev; - struct ixl_vsi *vsi = &pf->vsi; - struct ixl_queue *queues = vsi->queues; - struct i40e_hw_port_stats *pf_stats = &pf->stats; - - struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); - struct sysctl_oid *tree = device_get_sysctl_tree(dev); - struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); - struct sysctl_oid_list *vsi_list; - - struct sysctl_oid *queue_node; - struct sysctl_oid_list *queue_list; - - struct tx_ring *txr; - struct rx_ring *rxr; - char queue_namebuf[QUEUE_NAME_LEN]; - - /* Driver statistics */ - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events", - CTLFLAG_RD, &pf->watchdog_events, - "Watchdog timeouts"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "admin_irq", - CTLFLAG_RD, &pf->admin_irq, - "Admin Queue IRQ Handled"); - - ixl_add_vsi_sysctls(pf, &pf->vsi, ctx, "pf"); - vsi_list = SYSCTL_CHILDREN(pf->vsi.vsi_node); - - /* Queue statistics */ - for (int q = 0; q < vsi->num_queues; q++) { - snprintf(queue_namebuf, QUEUE_NAME_LEN, "que%d", q); - queue_node = SYSCTL_ADD_NODE(ctx, vsi_list, - OID_AUTO, queue_namebuf, CTLFLAG_RD, NULL, "Queue #"); - queue_list = SYSCTL_CHILDREN(queue_node); - - txr = &(queues[q].txr); - rxr = &(queues[q].rxr); - - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "mbuf_defrag_failed", - CTLFLAG_RD, &(queues[q].mbuf_defrag_failed), - "m_defrag() failed"); - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs", - CTLFLAG_RD, &(queues[q].irqs), - "irqs on this queue"); - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso_tx", - CTLFLAG_RD, &(queues[q].tso), - "TSO"); - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_dma_setup", - CTLFLAG_RD, &(queues[q].tx_dma_setup), - "Driver tx dma failure in xmit"); - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail", - CTLFLAG_RD, &(txr->no_desc), - "Queue No Descriptor Available"); - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets", - CTLFLAG_RD, &(txr->total_packets), - "Queue Packets Transmitted"); - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_bytes", - CTLFLAG_RD, &(txr->tx_bytes), - "Queue Bytes Transmitted"); - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets", - CTLFLAG_RD, &(rxr->rx_packets), - "Queue Packets Received"); - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes", - CTLFLAG_RD, &(rxr->rx_bytes), - "Queue Bytes Received"); - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_desc_err", - CTLFLAG_RD, &(rxr->desc_errs), - "Queue Rx Descriptor Errors"); - SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rx_itr", - CTLFLAG_RD, &(rxr->itr), 0, - "Queue Rx ITR Interval"); - SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "tx_itr", - CTLFLAG_RD, &(txr->itr), 0, - "Queue Tx ITR Interval"); - // Not actual latency; just a calculated value to put in a register - // TODO: Put in better descriptions here - SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rx_latency", - CTLFLAG_RD, &(rxr->latency), 0, - "Queue Rx ITRL Average Interval"); - SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "tx_latency", - CTLFLAG_RD, &(txr->latency), 0, - "Queue Tx ITRL Average Interval"); - -#ifdef IXL_DEBUG - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_not_done", - CTLFLAG_RD, &(rxr->not_done), - "Queue Rx Descriptors not Done"); - SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rx_next_refresh", - CTLFLAG_RD, &(rxr->next_refresh), 0, - "Queue Rx Descriptors not Done"); - SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rx_next_check", - CTLFLAG_RD, &(rxr->next_check), 0, - "Queue Rx Descriptors not Done"); - SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "qtx_tail", - CTLTYPE_UINT | CTLFLAG_RD, &queues[q], - sizeof(struct ixl_queue), - ixl_sysctl_qtx_tail_handler, "IU", - "Queue Transmit Descriptor Tail"); - SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "qrx_tail", - CTLTYPE_UINT | CTLFLAG_RD, &queues[q], - sizeof(struct ixl_queue), - ixl_sysctl_qrx_tail_handler, "IU", - "Queue Receive Descriptor Tail"); -#endif - } - - /* MAC stats */ - ixl_add_sysctls_mac_stats(ctx, child, pf_stats); -} - -static void -ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *ctx, - struct sysctl_oid_list *child, - struct i40e_eth_stats *eth_stats) -{ - struct ixl_sysctl_info ctls[] = - { - {ð_stats->rx_bytes, "good_octets_rcvd", "Good Octets Received"}, - {ð_stats->rx_unicast, "ucast_pkts_rcvd", - "Unicast Packets Received"}, - {ð_stats->rx_multicast, "mcast_pkts_rcvd", - "Multicast Packets Received"}, - {ð_stats->rx_broadcast, "bcast_pkts_rcvd", - "Broadcast Packets Received"}, - {ð_stats->rx_discards, "rx_discards", "Discarded RX packets"}, - {ð_stats->tx_bytes, "good_octets_txd", "Good Octets Transmitted"}, - {ð_stats->tx_unicast, "ucast_pkts_txd", "Unicast Packets Transmitted"}, - {ð_stats->tx_multicast, "mcast_pkts_txd", - "Multicast Packets Transmitted"}, - {ð_stats->tx_broadcast, "bcast_pkts_txd", - "Broadcast Packets Transmitted"}, - // end - {0,0,0} - }; - - struct ixl_sysctl_info *entry = ctls; - while (entry->stat != NULL) - { - SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, entry->name, - CTLFLAG_RD, entry->stat, - entry->description); - entry++; - } -} - -static void -ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *ctx, - struct sysctl_oid_list *child, - struct i40e_hw_port_stats *stats) -{ - struct sysctl_oid *stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac", - CTLFLAG_RD, NULL, "Mac Statistics"); - struct sysctl_oid_list *stat_list = SYSCTL_CHILDREN(stat_node); - - struct i40e_eth_stats *eth_stats = &stats->eth; - ixl_add_sysctls_eth_stats(ctx, stat_list, eth_stats); - - struct ixl_sysctl_info ctls[] = - { - {&stats->crc_errors, "crc_errors", "CRC Errors"}, - {&stats->illegal_bytes, "illegal_bytes", "Illegal Byte Errors"}, - {&stats->mac_local_faults, "local_faults", "MAC Local Faults"}, - {&stats->mac_remote_faults, "remote_faults", "MAC Remote Faults"}, - {&stats->rx_length_errors, "rx_length_errors", "Receive Length Errors"}, - /* Packet Reception Stats */ - {&stats->rx_size_64, "rx_frames_64", "64 byte frames received"}, - {&stats->rx_size_127, "rx_frames_65_127", "65-127 byte frames received"}, - {&stats->rx_size_255, "rx_frames_128_255", "128-255 byte frames received"}, - {&stats->rx_size_511, "rx_frames_256_511", "256-511 byte frames received"}, - {&stats->rx_size_1023, "rx_frames_512_1023", "512-1023 byte frames received"}, - {&stats->rx_size_1522, "rx_frames_1024_1522", "1024-1522 byte frames received"}, - {&stats->rx_size_big, "rx_frames_big", "1523-9522 byte frames received"}, - {&stats->rx_undersize, "rx_undersize", "Undersized packets received"}, - {&stats->rx_fragments, "rx_fragmented", "Fragmented packets received"}, - {&stats->rx_oversize, "rx_oversized", "Oversized packets received"}, - {&stats->rx_jabber, "rx_jabber", "Received Jabber"}, - {&stats->checksum_error, "checksum_errors", "Checksum Errors"}, - /* Packet Transmission Stats */ - {&stats->tx_size_64, "tx_frames_64", "64 byte frames transmitted"}, - {&stats->tx_size_127, "tx_frames_65_127", "65-127 byte frames transmitted"}, - {&stats->tx_size_255, "tx_frames_128_255", "128-255 byte frames transmitted"}, - {&stats->tx_size_511, "tx_frames_256_511", "256-511 byte frames transmitted"}, - {&stats->tx_size_1023, "tx_frames_512_1023", "512-1023 byte frames transmitted"}, - {&stats->tx_size_1522, "tx_frames_1024_1522", "1024-1522 byte frames transmitted"}, - {&stats->tx_size_big, "tx_frames_big", "1523-9522 byte frames transmitted"}, - /* Flow control */ - {&stats->link_xon_tx, "xon_txd", "Link XON transmitted"}, - {&stats->link_xon_rx, "xon_recvd", "Link XON received"}, - {&stats->link_xoff_tx, "xoff_txd", "Link XOFF transmitted"}, - {&stats->link_xoff_rx, "xoff_recvd", "Link XOFF received"}, - /* End */ - {0,0,0} - }; - - struct ixl_sysctl_info *entry = ctls; - while (entry->stat != NULL) - { - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, entry->name, - CTLFLAG_RD, entry->stat, - entry->description); - entry++; - } -} - - -/* -** ixl_config_rss - setup RSS -** - note this is done for the single vsi -*/ -static void -ixl_config_rss(struct ixl_vsi *vsi) -{ - struct ixl_pf *pf = (struct ixl_pf *)vsi->back; - struct i40e_hw *hw = vsi->hw; - u32 lut = 0; - u64 set_hena = 0, hena; - int i, j, que_id; -#ifdef RSS - u32 rss_hash_config; - u32 rss_seed[IXL_KEYSZ]; -#else - u32 rss_seed[IXL_KEYSZ] = {0x41b01687, - 0x183cfd8c, 0xce880440, 0x580cbc3c, - 0x35897377, 0x328b25e1, 0x4fa98922, - 0xb7d90c14, 0xd5bad70d, 0xcd15a2c1}; -#endif - -#ifdef RSS - /* Fetch the configured RSS key */ - rss_getkey((uint8_t *) &rss_seed); -#endif - - /* Fill out hash function seed */ - for (i = 0; i < IXL_KEYSZ; i++) - i40e_write_rx_ctl(hw, I40E_PFQF_HKEY(i), rss_seed[i]); - - /* Enable PCTYPES for RSS: */ -#ifdef RSS - rss_hash_config = rss_gethashconfig(); - if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) - set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER); - if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) - set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP); - if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) - set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP); - if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) - set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER); - if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) - set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6); - if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) - set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP); - if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) - set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP); -#else - set_hena = - ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | - ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP) | - ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) | - ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) | - ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV4) | - ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP) | - ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP) | - ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) | - ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) | - ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6) | - ((u64)1 << I40E_FILTER_PCTYPE_L2_PAYLOAD); -#endif - hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) | - ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32); - hena |= set_hena; - i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena); - i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32)); - - /* Populate the LUT with max no. of queues in round robin fashion */ - for (i = j = 0; i < pf->hw.func_caps.rss_table_size; i++, j++) { - if (j == vsi->num_queues) - j = 0; -#ifdef RSS - /* - * Fetch the RSS bucket id for the given indirection entry. - * Cap it at the number of configured buckets (which is - * num_queues.) - */ - que_id = rss_get_indirection_to_bucket(i); - que_id = que_id % vsi->num_queues; -#else - que_id = j; -#endif - /* lut = 4-byte sliding window of 4 lut entries */ - lut = (lut << 8) | (que_id & - ((0x1 << pf->hw.func_caps.rss_table_entry_width) - 1)); - /* On i = 3, we have 4 entries in lut; write to the register */ - if ((i & 3) == 3) - wr32(hw, I40E_PFQF_HLUT(i >> 2), lut); - } - ixl_flush(hw); -} - - -/* -** This routine is run via an vlan config EVENT, -** it enables us to use the HW Filter table since -** we can get the vlan id. This just creates the -** entry in the soft version of the VFTA, init will -** repopulate the real table. -*/ -static void -ixl_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) -{ - struct ixl_vsi *vsi = ifp->if_softc; - struct i40e_hw *hw = vsi->hw; - struct ixl_pf *pf = (struct ixl_pf *)vsi->back; - - if (ifp->if_softc != arg) /* Not our event */ - return; - - if ((vtag == 0) || (vtag > 4095)) /* Invalid */ - return; - - IXL_PF_LOCK(pf); - ++vsi->num_vlans; - ixl_add_filter(vsi, hw->mac.addr, vtag); - IXL_PF_UNLOCK(pf); -} - -/* -** This routine is run via an vlan -** unconfig EVENT, remove our entry -** in the soft vfta. -*/ -static void -ixl_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) -{ - struct ixl_vsi *vsi = ifp->if_softc; - struct i40e_hw *hw = vsi->hw; - struct ixl_pf *pf = (struct ixl_pf *)vsi->back; - - if (ifp->if_softc != arg) - return; - - if ((vtag == 0) || (vtag > 4095)) /* Invalid */ - return; - - IXL_PF_LOCK(pf); - --vsi->num_vlans; - ixl_del_filter(vsi, hw->mac.addr, vtag); - IXL_PF_UNLOCK(pf); -} - -/* -** This routine updates vlan filters, called by init -** it scans the filter table and then updates the hw -** after a soft reset. -*/ -static void -ixl_setup_vlan_filters(struct ixl_vsi *vsi) -{ - struct ixl_mac_filter *f; - int cnt = 0, flags; - - if (vsi->num_vlans == 0) - return; - /* - ** Scan the filter list for vlan entries, - ** mark them for addition and then call - ** for the AQ update. - */ - SLIST_FOREACH(f, &vsi->ftl, next) { - if (f->flags & IXL_FILTER_VLAN) { - f->flags |= - (IXL_FILTER_ADD | - IXL_FILTER_USED); - cnt++; - } - } - if (cnt == 0) { - printf("setup vlan: no filters found!\n"); - return; - } - flags = IXL_FILTER_VLAN; - flags |= (IXL_FILTER_ADD | IXL_FILTER_USED); - ixl_add_hw_filters(vsi, flags, cnt); - return; -} - -/* -** Initialize filter list and add filters that the hardware -** needs to know about. -** -** Requires VSI's filter list & seid to be set before calling. -*/ -static void -ixl_init_filters(struct ixl_vsi *vsi) -{ - /* Add broadcast address */ - ixl_add_filter(vsi, ixl_bcast_addr, IXL_VLAN_ANY); - - /* - * Prevent Tx flow control frames from being sent out by - * non-firmware transmitters. - */ - i40e_add_filter_to_drop_tx_flow_control_frames(vsi->hw, vsi->seid); -} - -/* -** This routine adds mulicast filters -*/ -static void -ixl_add_mc_filter(struct ixl_vsi *vsi, u8 *macaddr) -{ - struct ixl_mac_filter *f; - - /* Does one already exist */ - f = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY); - if (f != NULL) - return; - - f = ixl_get_filter(vsi); - if (f == NULL) { - printf("WARNING: no filter available!!\n"); - return; - } - bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN); - f->vlan = IXL_VLAN_ANY; - f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED - | IXL_FILTER_MC); - - return; -} - -static void -ixl_reconfigure_filters(struct ixl_vsi *vsi) -{ - - ixl_add_hw_filters(vsi, IXL_FILTER_USED, vsi->num_macs); -} - -/* -** This routine adds macvlan filters -*/ -static void -ixl_add_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan) -{ - struct ixl_mac_filter *f, *tmp; - struct ixl_pf *pf; - device_t dev; - - DEBUGOUT("ixl_add_filter: begin"); - - pf = vsi->back; - dev = pf->dev; - - /* Does one already exist */ - f = ixl_find_filter(vsi, macaddr, vlan); - if (f != NULL) - return; - /* - ** Is this the first vlan being registered, if so we - ** need to remove the ANY filter that indicates we are - ** not in a vlan, and replace that with a 0 filter. - */ - if ((vlan != IXL_VLAN_ANY) && (vsi->num_vlans == 1)) { - tmp = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY); - if (tmp != NULL) { - ixl_del_filter(vsi, macaddr, IXL_VLAN_ANY); - ixl_add_filter(vsi, macaddr, 0); - } - } - - f = ixl_get_filter(vsi); - if (f == NULL) { - device_printf(dev, "WARNING: no filter available!!\n"); - return; - } - bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN); - f->vlan = vlan; - f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED); - if (f->vlan != IXL_VLAN_ANY) - f->flags |= IXL_FILTER_VLAN; - else - vsi->num_macs++; - - ixl_add_hw_filters(vsi, f->flags, 1); - return; -} - -static void -ixl_del_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan) -{ - struct ixl_mac_filter *f; - - f = ixl_find_filter(vsi, macaddr, vlan); - if (f == NULL) - return; - - f->flags |= IXL_FILTER_DEL; - ixl_del_hw_filters(vsi, 1); - vsi->num_macs--; - - /* Check if this is the last vlan removal */ - if (vlan != IXL_VLAN_ANY && vsi->num_vlans == 0) { - /* Switch back to a non-vlan filter */ - ixl_del_filter(vsi, macaddr, 0); - ixl_add_filter(vsi, macaddr, IXL_VLAN_ANY); - } - return; -} - -/* -** Find the filter with both matching mac addr and vlan id -*/ -static struct ixl_mac_filter * -ixl_find_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan) -{ - struct ixl_mac_filter *f; - bool match = FALSE; - - SLIST_FOREACH(f, &vsi->ftl, next) { - if (!cmp_etheraddr(f->macaddr, macaddr)) - continue; - if (f->vlan == vlan) { - match = TRUE; - break; - } - } - - if (!match) - f = NULL; - return (f); -} - -/* -** This routine takes additions to the vsi filter -** table and creates an Admin Queue call to create -** the filters in the hardware. -*/ -static void -ixl_add_hw_filters(struct ixl_vsi *vsi, int flags, int cnt) -{ - struct i40e_aqc_add_macvlan_element_data *a, *b; - struct ixl_mac_filter *f; - struct ixl_pf *pf; - struct i40e_hw *hw; - device_t dev; - int err, j = 0; - - pf = vsi->back; - dev = pf->dev; - hw = &pf->hw; - IXL_PF_LOCK_ASSERT(pf); - - a = malloc(sizeof(struct i40e_aqc_add_macvlan_element_data) * cnt, - M_DEVBUF, M_NOWAIT | M_ZERO); - if (a == NULL) { - device_printf(dev, "add_hw_filters failed to get memory\n"); - return; - } - - /* - ** Scan the filter list, each time we find one - ** we add it to the admin queue array and turn off - ** the add bit. - */ - SLIST_FOREACH(f, &vsi->ftl, next) { - if (f->flags == flags) { - b = &a[j]; // a pox on fvl long names :) - bcopy(f->macaddr, b->mac_addr, ETHER_ADDR_LEN); - if (f->vlan == IXL_VLAN_ANY) { - b->vlan_tag = 0; - b->flags = I40E_AQC_MACVLAN_ADD_IGNORE_VLAN; - } else { - b->vlan_tag = f->vlan; - b->flags = 0; - } - b->flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; - f->flags &= ~IXL_FILTER_ADD; - j++; - } - if (j == cnt) - break; - } - if (j > 0) { - err = i40e_aq_add_macvlan(hw, vsi->seid, a, j, NULL); - if (err) - device_printf(dev, "aq_add_macvlan err %d, " - "aq_error %d\n", err, hw->aq.asq_last_status); - else - vsi->hw_filters_add += j; - } - free(a, M_DEVBUF); - return; -} - -/* -** This routine takes removals in the vsi filter -** table and creates an Admin Queue call to delete -** the filters in the hardware. -*/ -static void -ixl_del_hw_filters(struct ixl_vsi *vsi, int cnt) -{ - struct i40e_aqc_remove_macvlan_element_data *d, *e; - struct ixl_pf *pf; - struct i40e_hw *hw; - device_t dev; - struct ixl_mac_filter *f, *f_temp; - int err, j = 0; - - DEBUGOUT("ixl_del_hw_filters: begin\n"); - - pf = vsi->back; - hw = &pf->hw; - dev = pf->dev; - - d = malloc(sizeof(struct i40e_aqc_remove_macvlan_element_data) * cnt, - M_DEVBUF, M_NOWAIT | M_ZERO); - if (d == NULL) { - printf("del hw filter failed to get memory\n"); - return; - } - - SLIST_FOREACH_SAFE(f, &vsi->ftl, next, f_temp) { - if (f->flags & IXL_FILTER_DEL) { - e = &d[j]; // a pox on fvl long names :) - bcopy(f->macaddr, e->mac_addr, ETHER_ADDR_LEN); - e->vlan_tag = (f->vlan == IXL_VLAN_ANY ? 0 : f->vlan); - e->flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; - /* delete entry from vsi list */ - SLIST_REMOVE(&vsi->ftl, f, ixl_mac_filter, next); - free(f, M_DEVBUF); - j++; - } - if (j == cnt) - break; - } - if (j > 0) { - err = i40e_aq_remove_macvlan(hw, vsi->seid, d, j, NULL); - /* NOTE: returns ENOENT every time but seems to work fine, - so we'll ignore that specific error. */ - // TODO: Does this still occur on current firmwares? - if (err && hw->aq.asq_last_status != I40E_AQ_RC_ENOENT) { - int sc = 0; - for (int i = 0; i < j; i++) - sc += (!d[i].error_code); - vsi->hw_filters_del += sc; - device_printf(dev, - "Failed to remove %d/%d filters, aq error %d\n", - j - sc, j, hw->aq.asq_last_status); - } else - vsi->hw_filters_del += j; - } - free(d, M_DEVBUF); - - DEBUGOUT("ixl_del_hw_filters: end\n"); - return; -} - -static int -ixl_enable_rings(struct ixl_vsi *vsi) -{ - struct ixl_pf *pf = vsi->back; - struct i40e_hw *hw = &pf->hw; - int index, error; - u32 reg; - - error = 0; - for (int i = 0; i < vsi->num_queues; i++) { - index = vsi->first_queue + i; - i40e_pre_tx_queue_cfg(hw, index, TRUE); - - reg = rd32(hw, I40E_QTX_ENA(index)); - reg |= I40E_QTX_ENA_QENA_REQ_MASK | - I40E_QTX_ENA_QENA_STAT_MASK; - wr32(hw, I40E_QTX_ENA(index), reg); - /* Verify the enable took */ - for (int j = 0; j < 10; j++) { - reg = rd32(hw, I40E_QTX_ENA(index)); - if (reg & I40E_QTX_ENA_QENA_STAT_MASK) - break; - i40e_msec_delay(10); - } - if ((reg & I40E_QTX_ENA_QENA_STAT_MASK) == 0) { - device_printf(pf->dev, "TX queue %d disabled!\n", - index); - error = ETIMEDOUT; - } - - reg = rd32(hw, I40E_QRX_ENA(index)); - reg |= I40E_QRX_ENA_QENA_REQ_MASK | - I40E_QRX_ENA_QENA_STAT_MASK; - wr32(hw, I40E_QRX_ENA(index), reg); - /* Verify the enable took */ - for (int j = 0; j < 10; j++) { - reg = rd32(hw, I40E_QRX_ENA(index)); - if (reg & I40E_QRX_ENA_QENA_STAT_MASK) - break; - i40e_msec_delay(10); - } - if ((reg & I40E_QRX_ENA_QENA_STAT_MASK) == 0) { - device_printf(pf->dev, "RX queue %d disabled!\n", - index); - error = ETIMEDOUT; - } - } - - return (error); -} - -static int -ixl_disable_rings(struct ixl_vsi *vsi) -{ - struct ixl_pf *pf = vsi->back; - struct i40e_hw *hw = &pf->hw; - int index, error; - u32 reg; - - error = 0; - for (int i = 0; i < vsi->num_queues; i++) { - index = vsi->first_queue + i; - - i40e_pre_tx_queue_cfg(hw, index, FALSE); - i40e_usec_delay(500); - - reg = rd32(hw, I40E_QTX_ENA(index)); - reg &= ~I40E_QTX_ENA_QENA_REQ_MASK; - wr32(hw, I40E_QTX_ENA(index), reg); - /* Verify the disable took */ - for (int j = 0; j < 10; j++) { - reg = rd32(hw, I40E_QTX_ENA(index)); - if (!(reg & I40E_QTX_ENA_QENA_STAT_MASK)) - break; - i40e_msec_delay(10); - } - if (reg & I40E_QTX_ENA_QENA_STAT_MASK) { - device_printf(pf->dev, "TX queue %d still enabled!\n", - index); - error = ETIMEDOUT; - } - - reg = rd32(hw, I40E_QRX_ENA(index)); - reg &= ~I40E_QRX_ENA_QENA_REQ_MASK; - wr32(hw, I40E_QRX_ENA(index), reg); - /* Verify the disable took */ - for (int j = 0; j < 10; j++) { - reg = rd32(hw, I40E_QRX_ENA(index)); - if (!(reg & I40E_QRX_ENA_QENA_STAT_MASK)) - break; - i40e_msec_delay(10); - } - if (reg & I40E_QRX_ENA_QENA_STAT_MASK) { - device_printf(pf->dev, "RX queue %d still enabled!\n", - index); - error = ETIMEDOUT; - } - } - - return (error); -} - -/** - * ixl_handle_mdd_event - * - * Called from interrupt handler to identify possibly malicious vfs - * (But also detects events from the PF, as well) - **/ -static void -ixl_handle_mdd_event(struct ixl_pf *pf) -{ - struct i40e_hw *hw = &pf->hw; - device_t dev = pf->dev; - bool mdd_detected = false; - bool pf_mdd_detected = false; - u32 reg; - - /* find what triggered the MDD event */ - reg = rd32(hw, I40E_GL_MDET_TX); - if (reg & I40E_GL_MDET_TX_VALID_MASK) { - u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >> - I40E_GL_MDET_TX_PF_NUM_SHIFT; - u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >> - I40E_GL_MDET_TX_EVENT_SHIFT; - u8 queue = (reg & I40E_GL_MDET_TX_QUEUE_MASK) >> - I40E_GL_MDET_TX_QUEUE_SHIFT; - device_printf(dev, - "Malicious Driver Detection event 0x%02x" - " on TX queue %d pf number 0x%02x\n", - event, queue, pf_num); - wr32(hw, I40E_GL_MDET_TX, 0xffffffff); - mdd_detected = true; - } - reg = rd32(hw, I40E_GL_MDET_RX); - if (reg & I40E_GL_MDET_RX_VALID_MASK) { - u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >> - I40E_GL_MDET_RX_FUNCTION_SHIFT; - u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >> - I40E_GL_MDET_RX_EVENT_SHIFT; - u8 queue = (reg & I40E_GL_MDET_RX_QUEUE_MASK) >> - I40E_GL_MDET_RX_QUEUE_SHIFT; - device_printf(dev, - "Malicious Driver Detection event 0x%02x" - " on RX queue %d of function 0x%02x\n", - event, queue, func); - wr32(hw, I40E_GL_MDET_RX, 0xffffffff); - mdd_detected = true; - } - - if (mdd_detected) { - reg = rd32(hw, I40E_PF_MDET_TX); - if (reg & I40E_PF_MDET_TX_VALID_MASK) { - wr32(hw, I40E_PF_MDET_TX, 0xFFFF); - device_printf(dev, - "MDD TX event is for this function 0x%08x", - reg); - pf_mdd_detected = true; - } - reg = rd32(hw, I40E_PF_MDET_RX); - if (reg & I40E_PF_MDET_RX_VALID_MASK) { - wr32(hw, I40E_PF_MDET_RX, 0xFFFF); - device_printf(dev, - "MDD RX event is for this function 0x%08x", - reg); - pf_mdd_detected = true; - } - } - - /* re-enable mdd interrupt cause */ - reg = rd32(hw, I40E_PFINT_ICR0_ENA); - reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; - wr32(hw, I40E_PFINT_ICR0_ENA, reg); - ixl_flush(hw); -} - -static void -ixl_enable_intr(struct ixl_vsi *vsi) -{ - struct i40e_hw *hw = vsi->hw; - struct ixl_queue *que = vsi->queues; - - if (ixl_enable_msix) { - for (int i = 0; i < vsi->num_queues; i++, que++) - ixl_enable_queue(hw, que->me); - } else - ixl_enable_legacy(hw); -} - -static void -ixl_disable_rings_intr(struct ixl_vsi *vsi) -{ - struct i40e_hw *hw = vsi->hw; - struct ixl_queue *que = vsi->queues; - - for (int i = 0; i < vsi->num_queues; i++, que++) - ixl_disable_queue(hw, que->me); -} - -static void -ixl_disable_intr(struct ixl_vsi *vsi) -{ - struct i40e_hw *hw = vsi->hw; - - if (ixl_enable_msix) - ixl_disable_adminq(hw); - else - ixl_disable_legacy(hw); -} - -static void -ixl_enable_adminq(struct i40e_hw *hw) -{ - u32 reg; - - reg = I40E_PFINT_DYN_CTL0_INTENA_MASK | - I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | - (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); - wr32(hw, I40E_PFINT_DYN_CTL0, reg); - ixl_flush(hw); -} - -static void -ixl_disable_adminq(struct i40e_hw *hw) -{ - u32 reg; - - reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT; - wr32(hw, I40E_PFINT_DYN_CTL0, reg); - ixl_flush(hw); -} - -static void -ixl_enable_queue(struct i40e_hw *hw, int id) -{ - u32 reg; - - reg = I40E_PFINT_DYN_CTLN_INTENA_MASK | - I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | - (IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT); - wr32(hw, I40E_PFINT_DYN_CTLN(id), reg); -} - -static void -ixl_disable_queue(struct i40e_hw *hw, int id) -{ - u32 reg; - - reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT; - wr32(hw, I40E_PFINT_DYN_CTLN(id), reg); -} - -static void -ixl_enable_legacy(struct i40e_hw *hw) -{ - u32 reg; - reg = I40E_PFINT_DYN_CTL0_INTENA_MASK | - I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | - (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); - wr32(hw, I40E_PFINT_DYN_CTL0, reg); -} - -static void -ixl_disable_legacy(struct i40e_hw *hw) -{ - u32 reg; - - reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT; - wr32(hw, I40E_PFINT_DYN_CTL0, reg); -} - -static void -ixl_update_stats_counters(struct ixl_pf *pf) -{ - struct i40e_hw *hw = &pf->hw; - struct ixl_vsi *vsi = &pf->vsi; - struct ixl_vf *vf; - - struct i40e_hw_port_stats *nsd = &pf->stats; - struct i40e_hw_port_stats *osd = &pf->stats_offsets; - - /* Update hw stats */ - ixl_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port), - pf->stat_offsets_loaded, - &osd->crc_errors, &nsd->crc_errors); - ixl_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port), - pf->stat_offsets_loaded, - &osd->illegal_bytes, &nsd->illegal_bytes); - ixl_stat_update48(hw, I40E_GLPRT_GORCH(hw->port), - I40E_GLPRT_GORCL(hw->port), - pf->stat_offsets_loaded, - &osd->eth.rx_bytes, &nsd->eth.rx_bytes); - ixl_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port), - I40E_GLPRT_GOTCL(hw->port), - pf->stat_offsets_loaded, - &osd->eth.tx_bytes, &nsd->eth.tx_bytes); - ixl_stat_update32(hw, I40E_GLPRT_RDPC(hw->port), - pf->stat_offsets_loaded, - &osd->eth.rx_discards, - &nsd->eth.rx_discards); - ixl_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port), - I40E_GLPRT_UPRCL(hw->port), - pf->stat_offsets_loaded, - &osd->eth.rx_unicast, - &nsd->eth.rx_unicast); - ixl_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port), - I40E_GLPRT_UPTCL(hw->port), - pf->stat_offsets_loaded, - &osd->eth.tx_unicast, - &nsd->eth.tx_unicast); - ixl_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port), - I40E_GLPRT_MPRCL(hw->port), - pf->stat_offsets_loaded, - &osd->eth.rx_multicast, - &nsd->eth.rx_multicast); - ixl_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port), - I40E_GLPRT_MPTCL(hw->port), - pf->stat_offsets_loaded, - &osd->eth.tx_multicast, - &nsd->eth.tx_multicast); - ixl_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port), - I40E_GLPRT_BPRCL(hw->port), - pf->stat_offsets_loaded, - &osd->eth.rx_broadcast, - &nsd->eth.rx_broadcast); - ixl_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port), - I40E_GLPRT_BPTCL(hw->port), - pf->stat_offsets_loaded, - &osd->eth.tx_broadcast, - &nsd->eth.tx_broadcast); - - ixl_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port), - pf->stat_offsets_loaded, - &osd->tx_dropped_link_down, - &nsd->tx_dropped_link_down); - ixl_stat_update32(hw, I40E_GLPRT_MLFC(hw->port), - pf->stat_offsets_loaded, - &osd->mac_local_faults, - &nsd->mac_local_faults); - ixl_stat_update32(hw, I40E_GLPRT_MRFC(hw->port), - pf->stat_offsets_loaded, - &osd->mac_remote_faults, - &nsd->mac_remote_faults); - ixl_stat_update32(hw, I40E_GLPRT_RLEC(hw->port), - pf->stat_offsets_loaded, - &osd->rx_length_errors, - &nsd->rx_length_errors); - - /* Flow control (LFC) stats */ - ixl_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port), - pf->stat_offsets_loaded, - &osd->link_xon_rx, &nsd->link_xon_rx); - ixl_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port), - pf->stat_offsets_loaded, - &osd->link_xon_tx, &nsd->link_xon_tx); - ixl_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port), - pf->stat_offsets_loaded, - &osd->link_xoff_rx, &nsd->link_xoff_rx); - ixl_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port), - pf->stat_offsets_loaded, - &osd->link_xoff_tx, &nsd->link_xoff_tx); - - /* Packet size stats rx */ - ixl_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port), - I40E_GLPRT_PRC64L(hw->port), - pf->stat_offsets_loaded, - &osd->rx_size_64, &nsd->rx_size_64); - ixl_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port), - I40E_GLPRT_PRC127L(hw->port), - pf->stat_offsets_loaded, - &osd->rx_size_127, &nsd->rx_size_127); - ixl_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port), - I40E_GLPRT_PRC255L(hw->port), - pf->stat_offsets_loaded, - &osd->rx_size_255, &nsd->rx_size_255); - ixl_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port), - I40E_GLPRT_PRC511L(hw->port), - pf->stat_offsets_loaded, - &osd->rx_size_511, &nsd->rx_size_511); - ixl_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port), - I40E_GLPRT_PRC1023L(hw->port), - pf->stat_offsets_loaded, - &osd->rx_size_1023, &nsd->rx_size_1023); - ixl_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port), - I40E_GLPRT_PRC1522L(hw->port), - pf->stat_offsets_loaded, - &osd->rx_size_1522, &nsd->rx_size_1522); - ixl_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port), - I40E_GLPRT_PRC9522L(hw->port), - pf->stat_offsets_loaded, - &osd->rx_size_big, &nsd->rx_size_big); - - /* Packet size stats tx */ - ixl_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port), - I40E_GLPRT_PTC64L(hw->port), - pf->stat_offsets_loaded, - &osd->tx_size_64, &nsd->tx_size_64); - ixl_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port), - I40E_GLPRT_PTC127L(hw->port), - pf->stat_offsets_loaded, - &osd->tx_size_127, &nsd->tx_size_127); - ixl_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port), - I40E_GLPRT_PTC255L(hw->port), - pf->stat_offsets_loaded, - &osd->tx_size_255, &nsd->tx_size_255); - ixl_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port), - I40E_GLPRT_PTC511L(hw->port), - pf->stat_offsets_loaded, - &osd->tx_size_511, &nsd->tx_size_511); - ixl_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port), - I40E_GLPRT_PTC1023L(hw->port), - pf->stat_offsets_loaded, - &osd->tx_size_1023, &nsd->tx_size_1023); - ixl_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port), - I40E_GLPRT_PTC1522L(hw->port), - pf->stat_offsets_loaded, - &osd->tx_size_1522, &nsd->tx_size_1522); - ixl_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port), - I40E_GLPRT_PTC9522L(hw->port), - pf->stat_offsets_loaded, - &osd->tx_size_big, &nsd->tx_size_big); - - ixl_stat_update32(hw, I40E_GLPRT_RUC(hw->port), - pf->stat_offsets_loaded, - &osd->rx_undersize, &nsd->rx_undersize); - ixl_stat_update32(hw, I40E_GLPRT_RFC(hw->port), - pf->stat_offsets_loaded, - &osd->rx_fragments, &nsd->rx_fragments); - ixl_stat_update32(hw, I40E_GLPRT_ROC(hw->port), - pf->stat_offsets_loaded, - &osd->rx_oversize, &nsd->rx_oversize); - ixl_stat_update32(hw, I40E_GLPRT_RJC(hw->port), - pf->stat_offsets_loaded, - &osd->rx_jabber, &nsd->rx_jabber); - pf->stat_offsets_loaded = true; - /* End hw stats */ - - /* Update vsi stats */ - ixl_update_vsi_stats(vsi); - - for (int i = 0; i < pf->num_vfs; i++) { - vf = &pf->vfs[i]; - if (vf->vf_flags & VF_FLAG_ENABLED) - ixl_update_eth_stats(&pf->vfs[i].vsi); - } -} - -static int -ixl_rebuild_hw_structs_after_reset(struct ixl_pf *pf) -{ - struct i40e_hw *hw = &pf->hw; - struct ixl_vsi *vsi = &pf->vsi; - device_t dev = pf->dev; - bool is_up = false; - int error = 0; - - is_up = !!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING); - - /* Teardown */ - if (is_up) - ixl_stop(pf); - error = i40e_shutdown_lan_hmc(hw); - if (error) - device_printf(dev, - "Shutdown LAN HMC failed with code %d\n", error); - ixl_disable_adminq(hw); - ixl_teardown_adminq_msix(pf); - error = i40e_shutdown_adminq(hw); - if (error) - device_printf(dev, - "Shutdown Admin queue failed with code %d\n", error); - - /* Setup */ - error = i40e_init_adminq(hw); - if (error != 0 && error != I40E_ERR_FIRMWARE_API_VERSION) { - device_printf(dev, "Unable to initialize Admin Queue, error %d\n", - error); - } - error = ixl_setup_adminq_msix(pf); - if (error) { - device_printf(dev, "ixl_setup_adminq_msix error: %d\n", - error); - } - ixl_configure_intr0_msix(pf); - ixl_enable_adminq(hw); - error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, - hw->func_caps.num_rx_qp, 0, 0); - if (error) { - device_printf(dev, "init_lan_hmc failed: %d\n", error); - } - error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); - if (error) { - device_printf(dev, "configure_lan_hmc failed: %d\n", error); - } - if (is_up) - ixl_init(pf); - - return (0); -} - -static void -ixl_handle_empr_reset(struct ixl_pf *pf) -{ - struct i40e_hw *hw = &pf->hw; - device_t dev = pf->dev; - int count = 0; - u32 reg; - - /* Typically finishes within 3-4 seconds */ - while (count++ < 100) { - reg = rd32(hw, I40E_GLGEN_RSTAT) - & I40E_GLGEN_RSTAT_DEVSTATE_MASK; - if (reg) - i40e_msec_delay(100); - else - break; - } -#ifdef IXL_DEBUG - // Reset-related - device_printf(dev, "EMPR reset wait count: %d\n", count); -#endif - - device_printf(dev, "Rebuilding driver state...\n"); - ixl_rebuild_hw_structs_after_reset(pf); - device_printf(dev, "Rebuilding driver state done.\n"); - - atomic_clear_int(&pf->state, IXL_PF_STATE_EMPR_RESETTING); -} - -/* -** Tasklet handler for MSIX Adminq interrupts -** - do outside interrupt since it might sleep -*/ -static void -ixl_do_adminq(void *context, int pending) -{ - struct ixl_pf *pf = context; - struct i40e_hw *hw = &pf->hw; - struct i40e_arq_event_info event; - i40e_status ret; - device_t dev = pf->dev; - u32 loop = 0; - u16 opcode, result; - - if (pf->state & IXL_PF_STATE_EMPR_RESETTING) { - /* Flag cleared at end of this function */ - ixl_handle_empr_reset(pf); - return; - } - - /* Admin Queue handling */ - event.buf_len = IXL_AQ_BUF_SZ; - event.msg_buf = malloc(event.buf_len, - M_DEVBUF, M_NOWAIT | M_ZERO); - if (!event.msg_buf) { - device_printf(dev, "%s: Unable to allocate memory for Admin" - " Queue event!\n", __func__); - return; - } - - IXL_PF_LOCK(pf); - /* clean and process any events */ - do { - ret = i40e_clean_arq_element(hw, &event, &result); - if (ret) - break; - opcode = LE16_TO_CPU(event.desc.opcode); -#ifdef IXL_DEBUG - device_printf(dev, "%s: Admin Queue event: %#06x\n", __func__, - opcode); -#endif - switch (opcode) { - case i40e_aqc_opc_get_link_status: - ixl_link_event(pf, &event); - break; - case i40e_aqc_opc_send_msg_to_pf: -#ifdef PCI_IOV - ixl_handle_vf_msg(pf, &event); -#endif - break; - case i40e_aqc_opc_event_lan_overflow: - default: - break; - } - - } while (result && (loop++ < IXL_ADM_LIMIT)); - - free(event.msg_buf, M_DEVBUF); - - /* - * If there are still messages to process, reschedule ourselves. - * Otherwise, re-enable our interrupt and go to sleep. - */ - if (result > 0) - taskqueue_enqueue(pf->tq, &pf->adminq); - else - ixl_enable_adminq(hw); - - IXL_PF_UNLOCK(pf); -} - -/** - * Update VSI-specific ethernet statistics counters. - **/ -void -ixl_update_eth_stats(struct ixl_vsi *vsi) -{ - struct ixl_pf *pf = (struct ixl_pf *)vsi->back; - struct i40e_hw *hw = &pf->hw; - struct i40e_eth_stats *es; - struct i40e_eth_stats *oes; - struct i40e_hw_port_stats *nsd; - u16 stat_idx = vsi->info.stat_counter_idx; - - es = &vsi->eth_stats; - oes = &vsi->eth_stats_offsets; - nsd = &pf->stats; - - /* Gather up the stats that the hw collects */ - ixl_stat_update32(hw, I40E_GLV_TEPC(stat_idx), - vsi->stat_offsets_loaded, - &oes->tx_errors, &es->tx_errors); - ixl_stat_update32(hw, I40E_GLV_RDPC(stat_idx), - vsi->stat_offsets_loaded, - &oes->rx_discards, &es->rx_discards); - - ixl_stat_update48(hw, I40E_GLV_GORCH(stat_idx), - I40E_GLV_GORCL(stat_idx), - vsi->stat_offsets_loaded, - &oes->rx_bytes, &es->rx_bytes); - ixl_stat_update48(hw, I40E_GLV_UPRCH(stat_idx), - I40E_GLV_UPRCL(stat_idx), - vsi->stat_offsets_loaded, - &oes->rx_unicast, &es->rx_unicast); - ixl_stat_update48(hw, I40E_GLV_MPRCH(stat_idx), - I40E_GLV_MPRCL(stat_idx), - vsi->stat_offsets_loaded, - &oes->rx_multicast, &es->rx_multicast); - ixl_stat_update48(hw, I40E_GLV_BPRCH(stat_idx), - I40E_GLV_BPRCL(stat_idx), - vsi->stat_offsets_loaded, - &oes->rx_broadcast, &es->rx_broadcast); - - ixl_stat_update48(hw, I40E_GLV_GOTCH(stat_idx), - I40E_GLV_GOTCL(stat_idx), - vsi->stat_offsets_loaded, - &oes->tx_bytes, &es->tx_bytes); - ixl_stat_update48(hw, I40E_GLV_UPTCH(stat_idx), - I40E_GLV_UPTCL(stat_idx), - vsi->stat_offsets_loaded, - &oes->tx_unicast, &es->tx_unicast); - ixl_stat_update48(hw, I40E_GLV_MPTCH(stat_idx), - I40E_GLV_MPTCL(stat_idx), - vsi->stat_offsets_loaded, - &oes->tx_multicast, &es->tx_multicast); - ixl_stat_update48(hw, I40E_GLV_BPTCH(stat_idx), - I40E_GLV_BPTCL(stat_idx), - vsi->stat_offsets_loaded, - &oes->tx_broadcast, &es->tx_broadcast); - vsi->stat_offsets_loaded = true; -} - -static void -ixl_update_vsi_stats(struct ixl_vsi *vsi) -{ - struct ixl_pf *pf; - struct ifnet *ifp; - struct i40e_eth_stats *es; - u64 tx_discards; - - struct i40e_hw_port_stats *nsd; - - pf = vsi->back; - ifp = vsi->ifp; - es = &vsi->eth_stats; - nsd = &pf->stats; - - ixl_update_eth_stats(vsi); - - tx_discards = es->tx_discards + nsd->tx_dropped_link_down; - for (int i = 0; i < vsi->num_queues; i++) - tx_discards += vsi->queues[i].txr.br->br_drops; - - /* Update ifnet stats */ - IXL_SET_IPACKETS(vsi, es->rx_unicast + - es->rx_multicast + - es->rx_broadcast); - IXL_SET_OPACKETS(vsi, es->tx_unicast + - es->tx_multicast + - es->tx_broadcast); - IXL_SET_IBYTES(vsi, es->rx_bytes); - IXL_SET_OBYTES(vsi, es->tx_bytes); - IXL_SET_IMCASTS(vsi, es->rx_multicast); - IXL_SET_OMCASTS(vsi, es->tx_multicast); - - IXL_SET_IERRORS(vsi, nsd->crc_errors + nsd->illegal_bytes + - nsd->rx_undersize + nsd->rx_oversize + nsd->rx_fragments + - nsd->rx_jabber); - IXL_SET_OERRORS(vsi, es->tx_errors); - IXL_SET_IQDROPS(vsi, es->rx_discards + nsd->eth.rx_discards); - IXL_SET_OQDROPS(vsi, tx_discards); - IXL_SET_NOPROTO(vsi, es->rx_unknown_protocol); - IXL_SET_COLLISIONS(vsi, 0); -} - -/** - * Reset all of the stats for the given pf - **/ -void ixl_pf_reset_stats(struct ixl_pf *pf) -{ - bzero(&pf->stats, sizeof(struct i40e_hw_port_stats)); - bzero(&pf->stats_offsets, sizeof(struct i40e_hw_port_stats)); - pf->stat_offsets_loaded = false; -} - -/** - * Resets all stats of the given vsi - **/ -void ixl_vsi_reset_stats(struct ixl_vsi *vsi) -{ - bzero(&vsi->eth_stats, sizeof(struct i40e_eth_stats)); - bzero(&vsi->eth_stats_offsets, sizeof(struct i40e_eth_stats)); - vsi->stat_offsets_loaded = false; -} - -/** - * Read and update a 48 bit stat from the hw - * - * Since the device stats are not reset at PFReset, they likely will not - * be zeroed when the driver starts. We'll save the first values read - * and use them as offsets to be subtracted from the raw values in order - * to report stats that count from zero. - **/ -static void -ixl_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg, - bool offset_loaded, u64 *offset, u64 *stat) -{ - u64 new_data; - -#if defined(__FreeBSD__) && (__FreeBSD_version >= 1000000) && defined(__amd64__) - new_data = rd64(hw, loreg); -#else - /* - * Use two rd32's instead of one rd64; FreeBSD versions before - * 10 don't support 8 byte bus reads/writes. - */ - new_data = rd32(hw, loreg); - new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32; -#endif - - if (!offset_loaded) - *offset = new_data; - if (new_data >= *offset) - *stat = new_data - *offset; - else - *stat = (new_data + ((u64)1 << 48)) - *offset; - *stat &= 0xFFFFFFFFFFFFULL; -} - -/** - * Read and update a 32 bit stat from the hw - **/ -static void -ixl_stat_update32(struct i40e_hw *hw, u32 reg, - bool offset_loaded, u64 *offset, u64 *stat) -{ - u32 new_data; - - new_data = rd32(hw, reg); - if (!offset_loaded) - *offset = new_data; - if (new_data >= *offset) - *stat = (u32)(new_data - *offset); - else - *stat = (u32)((new_data + ((u64)1 << 32)) - *offset); -} - -static void -ixl_add_device_sysctls(struct ixl_pf *pf) -{ - device_t dev = pf->dev; - - /* Set up sysctls */ - SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW, - pf, 0, ixl_set_flowcntl, "I", IXL_SYSCTL_HELP_FC); - - SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW, - pf, 0, ixl_set_advertise, "I", IXL_SYSCTL_HELP_SET_ADVERTISE); - - SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "current_speed", CTLTYPE_STRING | CTLFLAG_RD, - pf, 0, ixl_current_speed, "A", "Current Port Speed"); - - SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD, - pf, 0, ixl_sysctl_show_fw, "A", "Firmware version"); - -#if 0 - SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "rx_itr", CTLFLAG_RW, - &ixl_rx_itr, 0, "RX ITR"); - - SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "dynamic_rx_itr", CTLFLAG_RW, - &ixl_dynamic_rx_itr, 0, "Dynamic RX ITR"); - - SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "tx_itr", CTLFLAG_RW, - &ixl_tx_itr, 0, "TX ITR"); - - SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "dynamic_tx_itr", CTLFLAG_RW, - &ixl_dynamic_tx_itr, 0, "Dynamic TX ITR"); -#endif - -#ifdef IXL_DEBUG_SYSCTL - SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, pf, 0, - ixl_debug_info, "I", "Debug Information"); - - /* Shared-code debug message level */ - SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "debug_mask", CTLFLAG_RW, - &pf->hw.debug_mask, 0, "Debug Message Level"); - - SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "link_status", CTLTYPE_STRING | CTLFLAG_RD, - pf, 0, ixl_sysctl_link_status, "A", IXL_SYSCTL_HELP_LINK_STATUS); - - SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "phy_abilities", CTLTYPE_STRING | CTLFLAG_RD, - pf, 0, ixl_sysctl_phy_abilities, "A", "PHY Abilities"); - - SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "filter_list", CTLTYPE_STRING | CTLFLAG_RD, - pf, 0, ixl_sysctl_sw_filter_list, "A", "SW Filter List"); - - SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "hw_res_alloc", CTLTYPE_STRING | CTLFLAG_RD, - pf, 0, ixl_sysctl_hw_res_alloc, "A", "HW Resource Allocation"); - - SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "switch_config", CTLTYPE_STRING | CTLFLAG_RD, - pf, 0, ixl_sysctl_switch_config, "A", "HW Switch Configuration"); - -#ifdef PCI_IOV - SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "vc_debug_level", CTLFLAG_RW, &pf->vc_debug_lvl, - 0, "PF/VF Virtual Channel debug level"); -#endif -#endif -} - -/* -** Set flow control using sysctl: -** 0 - off -** 1 - rx pause -** 2 - tx pause -** 3 - full -*/ -static int -ixl_set_flowcntl(SYSCTL_HANDLER_ARGS) -{ - /* - * TODO: ensure tx CRC by hardware should be enabled - * if tx flow control is enabled. - * ^ N/A for 40G ports - */ - struct ixl_pf *pf = (struct ixl_pf *)arg1; - struct i40e_hw *hw = &pf->hw; - device_t dev = pf->dev; - int requested_fc, error = 0; - enum i40e_status_code aq_error = 0; - u8 fc_aq_err = 0; - - /* Get request */ - requested_fc = pf->fc; - error = sysctl_handle_int(oidp, &requested_fc, 0, req); - if ((error) || (req->newptr == NULL)) - return (error); - if (requested_fc < 0 || requested_fc > 3) { - device_printf(dev, - "Invalid fc mode; valid modes are 0 through 3\n"); - return (EINVAL); - } - - /* Set fc ability for port */ - hw->fc.requested_mode = requested_fc; - aq_error = i40e_set_fc(hw, &fc_aq_err, TRUE); - if (aq_error) { - device_printf(dev, - "%s: Error setting new fc mode %d; fc_err %#x\n", - __func__, aq_error, fc_aq_err); - return (EIO); - } - pf->fc = requested_fc; - - /* Get new link state */ - i40e_msec_delay(250); - hw->phy.get_link_info = TRUE; - i40e_get_link_status(hw, &pf->link_up); - - return (0); -} - -static int -ixl_current_speed(SYSCTL_HANDLER_ARGS) -{ - struct ixl_pf *pf = (struct ixl_pf *)arg1; - struct i40e_hw *hw = &pf->hw; - int error = 0, index = 0; - - char *speeds[] = { - "Unknown", - "100M", - "1G", - "10G", - "40G", - "20G" - }; - - ixl_update_link_status(pf); - - switch (hw->phy.link_info.link_speed) { - case I40E_LINK_SPEED_100MB: - index = 1; - break; - case I40E_LINK_SPEED_1GB: - index = 2; - break; - case I40E_LINK_SPEED_10GB: - index = 3; - break; - case I40E_LINK_SPEED_40GB: - index = 4; - break; - case I40E_LINK_SPEED_20GB: - index = 5; - break; - case I40E_LINK_SPEED_UNKNOWN: - default: - index = 0; - break; - } - - error = sysctl_handle_string(oidp, speeds[index], - strlen(speeds[index]), req); - return (error); -} - -static int -ixl_set_advertised_speeds(struct ixl_pf *pf, int speeds) -{ - struct i40e_hw *hw = &pf->hw; - device_t dev = pf->dev; - struct i40e_aq_get_phy_abilities_resp abilities; - struct i40e_aq_set_phy_config config; - enum i40e_status_code aq_error = 0; - - /* Get current capability information */ - aq_error = i40e_aq_get_phy_capabilities(hw, - FALSE, FALSE, &abilities, NULL); - if (aq_error) { - device_printf(dev, - "%s: Error getting phy capabilities %d," - " aq error: %d\n", __func__, aq_error, - hw->aq.asq_last_status); - return (EAGAIN); - } - - /* Prepare new config */ - bzero(&config, sizeof(config)); - config.phy_type = abilities.phy_type; - config.abilities = abilities.abilities - | I40E_AQ_PHY_ENABLE_ATOMIC_LINK; - config.eee_capability = abilities.eee_capability; - config.eeer = abilities.eeer_val; - config.low_power_ctrl = abilities.d3_lpan; - /* Translate into aq cmd link_speed */ - if (speeds & 0x10) - config.link_speed |= I40E_LINK_SPEED_40GB; - if (speeds & 0x8) - config.link_speed |= I40E_LINK_SPEED_20GB; - if (speeds & 0x4) - config.link_speed |= I40E_LINK_SPEED_10GB; - if (speeds & 0x2) - config.link_speed |= I40E_LINK_SPEED_1GB; - if (speeds & 0x1) - config.link_speed |= I40E_LINK_SPEED_100MB; - - /* Do aq command & restart link */ - aq_error = i40e_aq_set_phy_config(hw, &config, NULL); - if (aq_error) { - device_printf(dev, - "%s: Error setting new phy config %d," - " aq error: %d\n", __func__, aq_error, - hw->aq.asq_last_status); - return (EAGAIN); - } - - /* - ** This seems a bit heavy handed, but we - ** need to get a reinit on some devices - */ - IXL_PF_LOCK(pf); - ixl_stop_locked(pf); - ixl_init_locked(pf); - IXL_PF_UNLOCK(pf); - - return (0); -} - -/* -** Control link advertise speed: -** Flags: -** 0x1 - advertise 100 Mb -** 0x2 - advertise 1G -** 0x4 - advertise 10G -** 0x8 - advertise 20G -** 0x10 - advertise 40G -** -** Set to 0 to disable link -*/ -static int -ixl_set_advertise(SYSCTL_HANDLER_ARGS) -{ - struct ixl_pf *pf = (struct ixl_pf *)arg1; - struct i40e_hw *hw = &pf->hw; - device_t dev = pf->dev; - int requested_ls = 0; - int error = 0; - - /* Read in new mode */ - requested_ls = pf->advertised_speed; - error = sysctl_handle_int(oidp, &requested_ls, 0, req); - if ((error) || (req->newptr == NULL)) - return (error); - /* Check for sane value */ - if (requested_ls > 0x10) { - device_printf(dev, "Invalid advertised speed; " - "valid modes are 0x1 through 0x10\n"); - return (EINVAL); - } - /* Then check for validity based on adapter type */ - switch (hw->device_id) { - case I40E_DEV_ID_10G_BASE_T: - case I40E_DEV_ID_10G_BASE_T4: - /* BaseT */ - if (requested_ls & ~(0x7)) { - device_printf(dev, - "Only 100M/1G/10G speeds supported on this device.\n"); - return (EINVAL); - } - break; - case I40E_DEV_ID_20G_KR2: - case I40E_DEV_ID_20G_KR2_A: - /* 20G */ - if (requested_ls & ~(0xE)) { - device_printf(dev, - "Only 1G/10G/20G speeds supported on this device.\n"); - return (EINVAL); - } - break; - case I40E_DEV_ID_KX_B: - case I40E_DEV_ID_QSFP_A: - case I40E_DEV_ID_QSFP_B: - /* 40G */ - if (requested_ls & ~(0x10)) { - device_printf(dev, - "Only 40G speeds supported on this device.\n"); - return (EINVAL); - } - break; - default: - /* 10G (1G) */ - if (requested_ls & ~(0x6)) { - device_printf(dev, - "Only 1/10Gbs speeds are supported on this device.\n"); - return (EINVAL); - } - break; - } - - /* Exit if no change */ - if (pf->advertised_speed == requested_ls) - return (0); - - error = ixl_set_advertised_speeds(pf, requested_ls); - if (error) - return (error); - - pf->advertised_speed = requested_ls; - ixl_update_link_status(pf); - return (0); -} - -/* -** Get the width and transaction speed of -** the bus this adapter is plugged into. -*/ -static u16 -ixl_get_bus_info(struct i40e_hw *hw, device_t dev) -{ - u16 link; - u32 offset; - - /* Get the PCI Express Capabilities offset */ - pci_find_cap(dev, PCIY_EXPRESS, &offset); - - /* ...and read the Link Status Register */ - link = pci_read_config(dev, offset + PCIER_LINK_STA, 2); - - switch (link & I40E_PCI_LINK_WIDTH) { - case I40E_PCI_LINK_WIDTH_1: - hw->bus.width = i40e_bus_width_pcie_x1; - break; - case I40E_PCI_LINK_WIDTH_2: - hw->bus.width = i40e_bus_width_pcie_x2; - break; - case I40E_PCI_LINK_WIDTH_4: - hw->bus.width = i40e_bus_width_pcie_x4; - break; - case I40E_PCI_LINK_WIDTH_8: - hw->bus.width = i40e_bus_width_pcie_x8; - break; - default: - hw->bus.width = i40e_bus_width_unknown; - break; - } - - switch (link & I40E_PCI_LINK_SPEED) { - case I40E_PCI_LINK_SPEED_2500: - hw->bus.speed = i40e_bus_speed_2500; - break; - case I40E_PCI_LINK_SPEED_5000: - hw->bus.speed = i40e_bus_speed_5000; - break; - case I40E_PCI_LINK_SPEED_8000: - hw->bus.speed = i40e_bus_speed_8000; - break; - default: - hw->bus.speed = i40e_bus_speed_unknown; - break; - } - - device_printf(dev,"PCI Express Bus: Speed %s %s\n", - ((hw->bus.speed == i40e_bus_speed_8000) ? "8.0GT/s": - (hw->bus.speed == i40e_bus_speed_5000) ? "5.0GT/s": - (hw->bus.speed == i40e_bus_speed_2500) ? "2.5GT/s":"Unknown"), - (hw->bus.width == i40e_bus_width_pcie_x8) ? "Width x8" : - (hw->bus.width == i40e_bus_width_pcie_x4) ? "Width x4" : - (hw->bus.width == i40e_bus_width_pcie_x1) ? "Width x1" : - ("Unknown")); - - if ((hw->bus.width <= i40e_bus_width_pcie_x8) && - (hw->bus.speed < i40e_bus_speed_8000)) { - device_printf(dev, "PCI-Express bandwidth available" - " for this device\n may be insufficient for" - " optimal performance.\n"); - device_printf(dev, "For expected performance a x8 " - "PCIE Gen3 slot is required.\n"); - } - - return (link); -} - -static int -ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS) -{ - struct ixl_pf *pf = (struct ixl_pf *)arg1; - struct i40e_hw *hw = &pf->hw; - struct sbuf *sbuf; - - sbuf = sbuf_new_for_sysctl(NULL, NULL, 128, req); - ixl_nvm_version_str(hw, sbuf); - sbuf_finish(sbuf); - sbuf_delete(sbuf); - - return 0; -} - -#ifdef IXL_DEBUG -static void -ixl_print_nvm_cmd(device_t dev, struct i40e_nvm_access *nvma) -{ - if ((nvma->command == I40E_NVM_READ) && - ((nvma->config & 0xFF) == 0xF) && - (((nvma->config & 0xF00) >> 8) == 0xF) && - (nvma->offset == 0) && - (nvma->data_size == 1)) { - // device_printf(dev, "- Get Driver Status Command\n"); - } - else if (nvma->command == I40E_NVM_READ) { - - } - else { - switch (nvma->command) { - case 0xB: - device_printf(dev, "- command: I40E_NVM_READ\n"); - break; - case 0xC: - device_printf(dev, "- command: I40E_NVM_WRITE\n"); - break; - default: - device_printf(dev, "- command: unknown 0x%08x\n", nvma->command); - break; - } - - device_printf(dev, "- config (ptr) : 0x%02x\n", nvma->config & 0xFF); - device_printf(dev, "- config (flags): 0x%01x\n", (nvma->config & 0xF00) >> 8); - device_printf(dev, "- offset : 0x%08x\n", nvma->offset); - device_printf(dev, "- data_s : 0x%08x\n", nvma->data_size); - } -} -#endif - -static int -ixl_handle_nvmupd_cmd(struct ixl_pf *pf, struct ifdrv *ifd) -{ - struct i40e_hw *hw = &pf->hw; - struct i40e_nvm_access *nvma; - device_t dev = pf->dev; - enum i40e_status_code status = 0; - int perrno; - - DEBUGFUNC("ixl_handle_nvmupd_cmd"); - - /* Sanity checks */ - if (ifd->ifd_len < sizeof(struct i40e_nvm_access) || - ifd->ifd_data == NULL) { - device_printf(dev, "%s: incorrect ifdrv length or data pointer\n", - __func__); - device_printf(dev, "%s: ifdrv length: %lu, sizeof(struct i40e_nvm_access): %lu\n", - __func__, ifd->ifd_len, sizeof(struct i40e_nvm_access)); - device_printf(dev, "%s: data pointer: %p\n", __func__, - ifd->ifd_data); - return (EINVAL); - } - - nvma = (struct i40e_nvm_access *)ifd->ifd_data; - -#ifdef IXL_DEBUG - ixl_print_nvm_cmd(dev, nvma); -#endif - - if (pf->state & IXL_PF_STATE_EMPR_RESETTING) { - int count = 0; - while (count++ < 100) { - i40e_msec_delay(100); - if (!(pf->state & IXL_PF_STATE_EMPR_RESETTING)) - break; - } - } - - if (!(pf->state & IXL_PF_STATE_EMPR_RESETTING)) { - IXL_PF_LOCK(pf); - status = i40e_nvmupd_command(hw, nvma, nvma->data, &perrno); - IXL_PF_UNLOCK(pf); - } else { - perrno = -EBUSY; - } - - if (status) - device_printf(dev, "i40e_nvmupd_command status %d, perrno %d\n", - status, perrno); - - /* - * -EPERM is actually ERESTART, which the kernel interprets as it needing - * to run this ioctl again. So use -EACCES for -EPERM instead. - */ - if (perrno == -EPERM) - return (-EACCES); - else - return (perrno); -} - -#ifdef IXL_DEBUG_SYSCTL -static int -ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS) -{ - struct ixl_pf *pf = (struct ixl_pf *)arg1; - struct i40e_hw *hw = &pf->hw; - struct i40e_link_status link_status; - char buf[512]; - - enum i40e_status_code aq_error = 0; - - aq_error = i40e_aq_get_link_info(hw, TRUE, &link_status, NULL); - if (aq_error) { - printf("i40e_aq_get_link_info() error %d\n", aq_error); - return (EPERM); - } - - sprintf(buf, "\n" - "PHY Type : %#04x\n" - "Speed : %#04x\n" - "Link info: %#04x\n" - "AN info : %#04x\n" - "Ext info : %#04x\n" - "Max Frame: %d\n" - "Pacing : %#04x\n" - "CRC En? : %d", - link_status.phy_type, link_status.link_speed, - link_status.link_info, link_status.an_info, - link_status.ext_info, link_status.max_frame_size, - link_status.pacing, link_status.crc_enable); - - return (sysctl_handle_string(oidp, buf, strlen(buf), req)); -} - -static int -ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS) -{ - struct ixl_pf *pf = (struct ixl_pf *)arg1; - struct i40e_hw *hw = &pf->hw; - char buf[512]; - enum i40e_status_code aq_error = 0; - - struct i40e_aq_get_phy_abilities_resp abilities; - - aq_error = i40e_aq_get_phy_capabilities(hw, - TRUE, FALSE, &abilities, NULL); - if (aq_error) { - printf("i40e_aq_get_phy_capabilities() error %d\n", aq_error); - return (EPERM); - } - - sprintf(buf, "\n" - "PHY Type : %#010x\n" - "Speed : %#04x\n" - "Abilities: %#04x\n" - "EEE cap : %#06x\n" - "EEER reg : %#010x\n" - "D3 Lpan : %#04x", - abilities.phy_type, abilities.link_speed, - abilities.abilities, abilities.eee_capability, - abilities.eeer_val, abilities.d3_lpan); - - return (sysctl_handle_string(oidp, buf, strlen(buf), req)); -} - -static int -ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS) -{ - struct ixl_pf *pf = (struct ixl_pf *)arg1; - struct ixl_vsi *vsi = &pf->vsi; - struct ixl_mac_filter *f; - char *buf, *buf_i; - - int error = 0; - int ftl_len = 0; - int ftl_counter = 0; - int buf_len = 0; - int entry_len = 42; - - SLIST_FOREACH(f, &vsi->ftl, next) { - ftl_len++; - } - - if (ftl_len < 1) { - sysctl_handle_string(oidp, "(none)", 6, req); - return (0); - } - - buf_len = sizeof(char) * (entry_len + 1) * ftl_len + 2; - buf = buf_i = malloc(buf_len, M_DEVBUF, M_NOWAIT); - - sprintf(buf_i++, "\n"); - SLIST_FOREACH(f, &vsi->ftl, next) { - sprintf(buf_i, - MAC_FORMAT ", vlan %4d, flags %#06x", - MAC_FORMAT_ARGS(f->macaddr), f->vlan, f->flags); - buf_i += entry_len; - /* don't print '\n' for last entry */ - if (++ftl_counter != ftl_len) { - sprintf(buf_i, "\n"); - buf_i++; - } - } - - error = sysctl_handle_string(oidp, buf, strlen(buf), req); - if (error) - printf("sysctl error: %d\n", error); - free(buf, M_DEVBUF); - return error; -} - -#define IXL_SW_RES_SIZE 0x14 -static int -ixl_res_alloc_cmp(const void *a, const void *b) -{ - const struct i40e_aqc_switch_resource_alloc_element_resp *one, *two; - one = (const struct i40e_aqc_switch_resource_alloc_element_resp *)a; - two = (const struct i40e_aqc_switch_resource_alloc_element_resp *)b; - - return ((int)one->resource_type - (int)two->resource_type); -} - -/* - * Longest string length: 25 - */ -static char * -ixl_switch_res_type_string(u8 type) -{ - static char * ixl_switch_res_type_strings[0x14] = { - "VEB", - "VSI", - "Perfect Match MAC address", - "S-tag", - "(Reserved)", - "Multicast hash entry", - "Unicast hash entry", - "VLAN", - "VSI List entry", - "(Reserved)", - "VLAN Statistic Pool", - "Mirror Rule", - "Queue Set", - "Inner VLAN Forward filter", - "(Reserved)", - "Inner MAC", - "IP", - "GRE/VN1 Key", - "VN2 Key", - "Tunneling Port" - }; - - if (type < 0x14) - return ixl_switch_res_type_strings[type]; - else - return "(Reserved)"; -} - -static int -ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS) -{ - struct ixl_pf *pf = (struct ixl_pf *)arg1; - struct i40e_hw *hw = &pf->hw; - device_t dev = pf->dev; - struct sbuf *buf; - int error = 0; - - u8 num_entries; - struct i40e_aqc_switch_resource_alloc_element_resp resp[IXL_SW_RES_SIZE]; - - buf = sbuf_new_for_sysctl(NULL, NULL, 128, req); - if (!buf) { - device_printf(dev, "Could not allocate sbuf for output.\n"); - return (ENOMEM); - } - - bzero(resp, sizeof(resp)); - error = i40e_aq_get_switch_resource_alloc(hw, &num_entries, - resp, - IXL_SW_RES_SIZE, - NULL); - if (error) { - device_printf(dev, - "%s: get_switch_resource_alloc() error %d, aq error %d\n", - __func__, error, hw->aq.asq_last_status); - sbuf_delete(buf); - return error; - } - - /* Sort entries by type for display */ - qsort(resp, num_entries, - sizeof(struct i40e_aqc_switch_resource_alloc_element_resp), - &ixl_res_alloc_cmp); - - sbuf_cat(buf, "\n"); - sbuf_printf(buf, "# of entries: %d\n", num_entries); - sbuf_printf(buf, - " Type | Guaranteed | Total | Used | Un-allocated\n" - " | (this) | (all) | (this) | (all) \n"); - for (int i = 0; i < num_entries; i++) { - sbuf_printf(buf, - "%25s | %10d %5d %6d %12d", - ixl_switch_res_type_string(resp[i].resource_type), - resp[i].guaranteed, - resp[i].total, - resp[i].used, - resp[i].total_unalloced); - if (i < num_entries - 1) - sbuf_cat(buf, "\n"); - } - - error = sbuf_finish(buf); - if (error) - device_printf(dev, "Error finishing sbuf: %d\n", error); - - sbuf_delete(buf); - return error; -} - -/* -** Caller must init and delete sbuf; this function will clear and -** finish it for caller. -** -** XXX: Cannot use the SEID for this, since there is no longer a -** fixed mapping between SEID and element type. -*/ -static char * -ixl_switch_element_string(struct sbuf *s, - struct i40e_aqc_switch_config_element_resp *element) -{ - sbuf_clear(s); - - switch (element->element_type) { - case I40E_AQ_SW_ELEM_TYPE_MAC: - sbuf_printf(s, "MAC %3d", element->element_info); - break; - case I40E_AQ_SW_ELEM_TYPE_PF: - sbuf_printf(s, "PF %3d", element->element_info); - break; - case I40E_AQ_SW_ELEM_TYPE_VF: - sbuf_printf(s, "VF %3d", element->element_info); - break; - case I40E_AQ_SW_ELEM_TYPE_EMP: - sbuf_cat(s, "EMP"); - break; - case I40E_AQ_SW_ELEM_TYPE_BMC: - sbuf_cat(s, "BMC"); - break; - case I40E_AQ_SW_ELEM_TYPE_PV: - sbuf_cat(s, "PV"); - break; - case I40E_AQ_SW_ELEM_TYPE_VEB: - sbuf_cat(s, "VEB"); - break; - case I40E_AQ_SW_ELEM_TYPE_PA: - sbuf_cat(s, "PA"); - break; - case I40E_AQ_SW_ELEM_TYPE_VSI: - sbuf_printf(s, "VSI %3d", element->element_info); - break; - default: - sbuf_cat(s, "?"); - break; - } - - sbuf_finish(s); - return sbuf_data(s); -} - -static int -ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS) -{ - struct ixl_pf *pf = (struct ixl_pf *)arg1; - struct i40e_hw *hw = &pf->hw; - device_t dev = pf->dev; - struct sbuf *buf; - struct sbuf *nmbuf; - int error = 0; - u16 next = 0; - u8 aq_buf[I40E_AQ_LARGE_BUF]; - - struct i40e_aqc_get_switch_config_resp *sw_config; - sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; - - buf = sbuf_new_for_sysctl(NULL, NULL, 128, req); - if (!buf) { - device_printf(dev, "Could not allocate sbuf for sysctl output.\n"); - return (ENOMEM); - } - - error = i40e_aq_get_switch_config(hw, sw_config, - sizeof(aq_buf), &next, NULL); - if (error) { - device_printf(dev, - "%s: aq_get_switch_config() error %d, aq error %d\n", - __func__, error, hw->aq.asq_last_status); - sbuf_delete(buf); - return error; - } - if (next) - device_printf(dev, "%s: TODO: get more config with SEID %d\n", - __func__, next); - - nmbuf = sbuf_new_auto(); - if (!nmbuf) { - device_printf(dev, "Could not allocate sbuf for name output.\n"); - sbuf_delete(buf); - return (ENOMEM); - } - - sbuf_cat(buf, "\n"); - // Assuming <= 255 elements in switch - sbuf_printf(buf, "# of reported elements: %d\n", sw_config->header.num_reported); - sbuf_printf(buf, "total # of elements: %d\n", sw_config->header.num_total); - /* Exclude: - ** Revision -- all elements are revision 1 for now - */ - sbuf_printf(buf, - "SEID ( Name ) | Uplink | Downlink | Conn Type\n" - " | | | (uplink)\n"); - for (int i = 0; i < sw_config->header.num_reported; i++) { - // "%4d (%8s) | %8s %8s %#8x", - sbuf_printf(buf, "%4d", sw_config->element[i].seid); - sbuf_cat(buf, " "); - sbuf_printf(buf, "(%8s)", ixl_switch_element_string(nmbuf, - &sw_config->element[i])); - sbuf_cat(buf, " | "); - sbuf_printf(buf, "%8d", sw_config->element[i].uplink_seid); - sbuf_cat(buf, " "); - sbuf_printf(buf, "%8d", sw_config->element[i].downlink_seid); - sbuf_cat(buf, " "); - sbuf_printf(buf, "%#8x", sw_config->element[i].connection_type); - if (i < sw_config->header.num_reported - 1) - sbuf_cat(buf, "\n"); - } - sbuf_delete(nmbuf); - - error = sbuf_finish(buf); - if (error) - device_printf(dev, "Error finishing sbuf: %d\n", error); - - sbuf_delete(buf); - - return (error); -} - -static int -ixl_debug_info(SYSCTL_HANDLER_ARGS) -{ - struct ixl_pf *pf; - int error, input = 0; - - error = sysctl_handle_int(oidp, &input, 0, req); - - if (error || !req->newptr) - return (error); - - if (input == 1) { - pf = (struct ixl_pf *)arg1; - ixl_print_debug_info(pf); - } - - return (error); -} - -static void -ixl_print_debug_info(struct ixl_pf *pf) -{ - struct i40e_hw *hw = &pf->hw; - struct ixl_vsi *vsi = &pf->vsi; - struct ixl_queue *que = vsi->queues; - struct rx_ring *rxr = &que->rxr; - struct tx_ring *txr = &que->txr; - u32 reg; - - - printf("Queue irqs = %jx\n", (uintmax_t)que->irqs); - printf("AdminQ irqs = %jx\n", (uintmax_t)pf->admin_irq); - printf("RX next check = %x\n", rxr->next_check); - printf("RX not ready = %jx\n", (uintmax_t)rxr->not_done); - printf("RX packets = %jx\n", (uintmax_t)rxr->rx_packets); - printf("TX desc avail = %x\n", txr->avail); - - reg = rd32(hw, I40E_GLV_GORCL(0xc)); - printf("RX Bytes = %x\n", reg); - reg = rd32(hw, I40E_GLPRT_GORCL(hw->port)); - printf("Port RX Bytes = %x\n", reg); - reg = rd32(hw, I40E_GLV_RDPC(0xc)); - printf("RX discard = %x\n", reg); - reg = rd32(hw, I40E_GLPRT_RDPC(hw->port)); - printf("Port RX discard = %x\n", reg); - - reg = rd32(hw, I40E_GLV_TEPC(0xc)); - printf("TX errors = %x\n", reg); - reg = rd32(hw, I40E_GLV_GOTCL(0xc)); - printf("TX Bytes = %x\n", reg); - - reg = rd32(hw, I40E_GLPRT_RUC(hw->port)); - printf("RX undersize = %x\n", reg); - reg = rd32(hw, I40E_GLPRT_RFC(hw->port)); - printf("RX fragments = %x\n", reg); - reg = rd32(hw, I40E_GLPRT_ROC(hw->port)); - printf("RX oversize = %x\n", reg); - reg = rd32(hw, I40E_GLPRT_RLEC(hw->port)); - printf("RX length error = %x\n", reg); - reg = rd32(hw, I40E_GLPRT_MRFC(hw->port)); - printf("mac remote fault = %x\n", reg); - reg = rd32(hw, I40E_GLPRT_MLFC(hw->port)); - printf("mac local fault = %x\n", reg); -} - -#endif /* IXL_DEBUG_SYSCTL */ - -#ifdef PCI_IOV -static int -ixl_vf_alloc_vsi(struct ixl_pf *pf, struct ixl_vf *vf) -{ - struct i40e_hw *hw; - struct ixl_vsi *vsi; - struct i40e_vsi_context vsi_ctx; - int i; - uint16_t first_queue; - enum i40e_status_code code; - - hw = &pf->hw; - vsi = &pf->vsi; - - vsi_ctx.pf_num = hw->pf_id; - vsi_ctx.uplink_seid = pf->veb_seid; - vsi_ctx.connection_type = IXL_VSI_DATA_PORT; - vsi_ctx.vf_num = hw->func_caps.vf_base_id + vf->vf_num; - vsi_ctx.flags = I40E_AQ_VSI_TYPE_VF; - - bzero(&vsi_ctx.info, sizeof(vsi_ctx.info)); - - vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_SWITCH_VALID); - vsi_ctx.info.switch_id = htole16(0); - - vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_SECURITY_VALID); - vsi_ctx.info.sec_flags = 0; - if (vf->vf_flags & VF_FLAG_MAC_ANTI_SPOOF) - vsi_ctx.info.sec_flags |= I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK; - - /* TODO: If a port VLAN is set, then this needs to be changed */ - vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_VLAN_VALID); - vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | - I40E_AQ_VSI_PVLAN_EMOD_NOTHING; - - vsi_ctx.info.valid_sections |= - htole16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID); - vsi_ctx.info.mapping_flags = htole16(I40E_AQ_VSI_QUE_MAP_NONCONTIG); - first_queue = vsi->num_queues + vf->vf_num * IXLV_MAX_QUEUES; - for (i = 0; i < IXLV_MAX_QUEUES; i++) - vsi_ctx.info.queue_mapping[i] = htole16(first_queue + i); - for (; i < nitems(vsi_ctx.info.queue_mapping); i++) - vsi_ctx.info.queue_mapping[i] = htole16(I40E_AQ_VSI_QUEUE_MASK); - - vsi_ctx.info.tc_mapping[0] = htole16( - (0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | - (1 << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT)); - - code = i40e_aq_add_vsi(hw, &vsi_ctx, NULL); - if (code != I40E_SUCCESS) - return (ixl_adminq_err_to_errno(hw->aq.asq_last_status)); - vf->vsi.seid = vsi_ctx.seid; - vf->vsi.vsi_num = vsi_ctx.vsi_number; - vf->vsi.first_queue = first_queue; - vf->vsi.num_queues = IXLV_MAX_QUEUES; - - code = i40e_aq_get_vsi_params(hw, &vsi_ctx, NULL); - if (code != I40E_SUCCESS) - return (ixl_adminq_err_to_errno(hw->aq.asq_last_status)); - - code = i40e_aq_config_vsi_bw_limit(hw, vf->vsi.seid, 0, 0, NULL); - if (code != I40E_SUCCESS) { - device_printf(pf->dev, "Failed to disable BW limit: %d\n", - ixl_adminq_err_to_errno(hw->aq.asq_last_status)); - return (ixl_adminq_err_to_errno(hw->aq.asq_last_status)); - } - - memcpy(&vf->vsi.info, &vsi_ctx.info, sizeof(vf->vsi.info)); - return (0); -} - -static int -ixl_vf_setup_vsi(struct ixl_pf *pf, struct ixl_vf *vf) -{ - struct i40e_hw *hw; - int error; - - hw = &pf->hw; - - error = ixl_vf_alloc_vsi(pf, vf); - if (error != 0) - return (error); - - vf->vsi.hw_filters_add = 0; - vf->vsi.hw_filters_del = 0; - ixl_add_filter(&vf->vsi, ixl_bcast_addr, IXL_VLAN_ANY); - ixl_reconfigure_filters(&vf->vsi); - - return (0); -} - -static void -ixl_vf_map_vsi_queue(struct i40e_hw *hw, struct ixl_vf *vf, int qnum, - uint32_t val) -{ - uint32_t qtable; - int index, shift; - - /* - * Two queues are mapped in a single register, so we have to do some - * gymnastics to convert the queue number into a register index and - * shift. - */ - index = qnum / 2; - shift = (qnum % 2) * I40E_VSILAN_QTABLE_QINDEX_1_SHIFT; - - qtable = i40e_read_rx_ctl(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num)); - qtable &= ~(I40E_VSILAN_QTABLE_QINDEX_0_MASK << shift); - qtable |= val << shift; - i40e_write_rx_ctl(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num), qtable); -} - -static void -ixl_vf_map_queues(struct ixl_pf *pf, struct ixl_vf *vf) -{ - struct i40e_hw *hw; - uint32_t qtable; - int i; - - hw = &pf->hw; - - /* - * Contiguous mappings aren't actually supported by the hardware, - * so we have to use non-contiguous mappings. - */ - i40e_write_rx_ctl(hw, I40E_VSILAN_QBASE(vf->vsi.vsi_num), - I40E_VSILAN_QBASE_VSIQTABLE_ENA_MASK); - - wr32(hw, I40E_VPLAN_MAPENA(vf->vf_num), - I40E_VPLAN_MAPENA_TXRX_ENA_MASK); - - for (i = 0; i < vf->vsi.num_queues; i++) { - qtable = (vf->vsi.first_queue + i) << - I40E_VPLAN_QTABLE_QINDEX_SHIFT; - - wr32(hw, I40E_VPLAN_QTABLE(i, vf->vf_num), qtable); - } - - /* Map queues allocated to VF to its VSI. */ - for (i = 0; i < vf->vsi.num_queues; i++) - ixl_vf_map_vsi_queue(hw, vf, i, vf->vsi.first_queue + i); - - /* Set rest of VSI queues as unused. */ - for (; i < IXL_MAX_VSI_QUEUES; i++) - ixl_vf_map_vsi_queue(hw, vf, i, - I40E_VSILAN_QTABLE_QINDEX_0_MASK); - - ixl_flush(hw); -} - -static void -ixl_vf_vsi_release(struct ixl_pf *pf, struct ixl_vsi *vsi) -{ - struct i40e_hw *hw; - - hw = &pf->hw; - - if (vsi->seid == 0) - return; - - i40e_aq_delete_element(hw, vsi->seid, NULL); -} - -static void -ixl_vf_disable_queue_intr(struct i40e_hw *hw, uint32_t vfint_reg) -{ - - wr32(hw, vfint_reg, I40E_VFINT_DYN_CTLN_CLEARPBA_MASK); - ixl_flush(hw); -} - -static void -ixl_vf_unregister_intr(struct i40e_hw *hw, uint32_t vpint_reg) -{ - - wr32(hw, vpint_reg, I40E_VPINT_LNKLSTN_FIRSTQ_TYPE_MASK | - I40E_VPINT_LNKLSTN_FIRSTQ_INDX_MASK); - ixl_flush(hw); -} - -static void -ixl_vf_release_resources(struct ixl_pf *pf, struct ixl_vf *vf) -{ - struct i40e_hw *hw; - uint32_t vfint_reg, vpint_reg; - int i; - - hw = &pf->hw; - - ixl_vf_vsi_release(pf, &vf->vsi); - - /* Index 0 has a special register. */ - ixl_vf_disable_queue_intr(hw, I40E_VFINT_DYN_CTL0(vf->vf_num)); - - for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) { - vfint_reg = IXL_VFINT_DYN_CTLN_REG(hw, i , vf->vf_num); - ixl_vf_disable_queue_intr(hw, vfint_reg); - } - - /* Index 0 has a special register. */ - ixl_vf_unregister_intr(hw, I40E_VPINT_LNKLST0(vf->vf_num)); - - for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) { - vpint_reg = IXL_VPINT_LNKLSTN_REG(hw, i, vf->vf_num); - ixl_vf_unregister_intr(hw, vpint_reg); - } - - vf->vsi.num_queues = 0; -} - -static int -ixl_flush_pcie(struct ixl_pf *pf, struct ixl_vf *vf) -{ - struct i40e_hw *hw; - int i; - uint16_t global_vf_num; - uint32_t ciad; - - hw = &pf->hw; - global_vf_num = hw->func_caps.vf_base_id + vf->vf_num; - - wr32(hw, I40E_PF_PCI_CIAA, IXL_PF_PCI_CIAA_VF_DEVICE_STATUS | - (global_vf_num << I40E_PF_PCI_CIAA_VF_NUM_SHIFT)); - for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) { - ciad = rd32(hw, I40E_PF_PCI_CIAD); - if ((ciad & IXL_PF_PCI_CIAD_VF_TRANS_PENDING_MASK) == 0) - return (0); - DELAY(1); - } - - return (ETIMEDOUT); -} - -static void -ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf) -{ - struct i40e_hw *hw; - uint32_t vfrtrig; - - hw = &pf->hw; - - vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num)); - vfrtrig |= I40E_VPGEN_VFRTRIG_VFSWR_MASK; - wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig); - ixl_flush(hw); - - ixl_reinit_vf(pf, vf); -} - -static void -ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf) -{ - struct i40e_hw *hw; - uint32_t vfrstat, vfrtrig; - int i, error; - - hw = &pf->hw; - - error = ixl_flush_pcie(pf, vf); - if (error != 0) - device_printf(pf->dev, - "Timed out waiting for PCIe activity to stop on VF-%d\n", - vf->vf_num); - - for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) { - DELAY(10); - - vfrstat = rd32(hw, I40E_VPGEN_VFRSTAT(vf->vf_num)); - if (vfrstat & I40E_VPGEN_VFRSTAT_VFRD_MASK) - break; - } - - if (i == IXL_VF_RESET_TIMEOUT) - device_printf(pf->dev, "VF %d failed to reset\n", vf->vf_num); - - wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_COMPLETED); - - vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num)); - vfrtrig &= ~I40E_VPGEN_VFRTRIG_VFSWR_MASK; - wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig); - - if (vf->vsi.seid != 0) - ixl_disable_rings(&vf->vsi); - - ixl_vf_release_resources(pf, vf); - ixl_vf_setup_vsi(pf, vf); - ixl_vf_map_queues(pf, vf); - - wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_VFACTIVE); - ixl_flush(hw); -} - -static const char * -ixl_vc_opcode_str(uint16_t op) -{ - - switch (op) { - case I40E_VIRTCHNL_OP_VERSION: - return ("VERSION"); - case I40E_VIRTCHNL_OP_RESET_VF: - return ("RESET_VF"); - case I40E_VIRTCHNL_OP_GET_VF_RESOURCES: - return ("GET_VF_RESOURCES"); - case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE: - return ("CONFIG_TX_QUEUE"); - case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE: - return ("CONFIG_RX_QUEUE"); - case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES: - return ("CONFIG_VSI_QUEUES"); - case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP: - return ("CONFIG_IRQ_MAP"); - case I40E_VIRTCHNL_OP_ENABLE_QUEUES: - return ("ENABLE_QUEUES"); - case I40E_VIRTCHNL_OP_DISABLE_QUEUES: - return ("DISABLE_QUEUES"); - case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS: - return ("ADD_ETHER_ADDRESS"); - case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS: - return ("DEL_ETHER_ADDRESS"); - case I40E_VIRTCHNL_OP_ADD_VLAN: - return ("ADD_VLAN"); - case I40E_VIRTCHNL_OP_DEL_VLAN: - return ("DEL_VLAN"); - case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE: - return ("CONFIG_PROMISCUOUS_MODE"); - case I40E_VIRTCHNL_OP_GET_STATS: - return ("GET_STATS"); - case I40E_VIRTCHNL_OP_FCOE: - return ("FCOE"); - case I40E_VIRTCHNL_OP_EVENT: - return ("EVENT"); - default: - return ("UNKNOWN"); - } -} - -static int -ixl_vc_opcode_level(uint16_t opcode) -{ - switch (opcode) { - case I40E_VIRTCHNL_OP_GET_STATS: - return (10); - default: - return (5); - } -} - -static void -ixl_send_vf_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op, - enum i40e_status_code status, void *msg, uint16_t len) -{ - struct i40e_hw *hw; - int global_vf_id; - - hw = &pf->hw; - global_vf_id = hw->func_caps.vf_base_id + vf->vf_num; - - I40E_VC_DEBUG(pf, ixl_vc_opcode_level(op), - "Sending msg (op=%s[%d], status=%d) to VF-%d\n", - ixl_vc_opcode_str(op), op, status, vf->vf_num); - - i40e_aq_send_msg_to_vf(hw, global_vf_id, op, status, msg, len, NULL); -} - -static void -ixl_send_vf_ack(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op) -{ - - ixl_send_vf_msg(pf, vf, op, I40E_SUCCESS, NULL, 0); -} - -static void -ixl_send_vf_nack_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op, - enum i40e_status_code status, const char *file, int line) -{ - - I40E_VC_DEBUG(pf, 1, - "Sending NACK (op=%s[%d], err=%d) to VF-%d from %s:%d\n", - ixl_vc_opcode_str(op), op, status, vf->vf_num, file, line); - ixl_send_vf_msg(pf, vf, op, status, NULL, 0); -} - -static void -ixl_vf_version_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, - uint16_t msg_size) -{ - struct i40e_virtchnl_version_info reply; - - if (msg_size != sizeof(struct i40e_virtchnl_version_info)) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_VERSION, - I40E_ERR_PARAM); - return; - } - - vf->version = ((struct i40e_virtchnl_version_info *)msg)->minor; - - reply.major = I40E_VIRTCHNL_VERSION_MAJOR; - reply.minor = I40E_VIRTCHNL_VERSION_MINOR; - ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_VERSION, I40E_SUCCESS, &reply, - sizeof(reply)); -} - -static void -ixl_vf_reset_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, - uint16_t msg_size) -{ - - if (msg_size != 0) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_RESET_VF, - I40E_ERR_PARAM); - return; - } - - ixl_reset_vf(pf, vf); - - /* No response to a reset message. */ -} - -static void -ixl_vf_get_resources_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, - uint16_t msg_size) -{ - struct i40e_virtchnl_vf_resource reply; - - if ((vf->version == 0 && msg_size != 0) || - (vf->version == 1 && msg_size != 4)) { - device_printf(pf->dev, "Invalid GET_VF_RESOURCES message size," - " for VF version %d.%d\n", I40E_VIRTCHNL_VERSION_MAJOR, - vf->version); - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES, - I40E_ERR_PARAM); - return; - } - - bzero(&reply, sizeof(reply)); - - if (vf->version == I40E_VIRTCHNL_VERSION_MINOR_NO_VF_CAPS) - reply.vf_offload_flags = I40E_VIRTCHNL_VF_OFFLOAD_L2 | - I40E_VIRTCHNL_VF_OFFLOAD_RSS_REG | - I40E_VIRTCHNL_VF_OFFLOAD_VLAN; - else - reply.vf_offload_flags = *(u32 *)msg; - - reply.num_vsis = 1; - reply.num_queue_pairs = vf->vsi.num_queues; - reply.max_vectors = pf->hw.func_caps.num_msix_vectors_vf; - reply.vsi_res[0].vsi_id = vf->vsi.vsi_num; - reply.vsi_res[0].vsi_type = I40E_VSI_SRIOV; - reply.vsi_res[0].num_queue_pairs = vf->vsi.num_queues; - memcpy(reply.vsi_res[0].default_mac_addr, vf->mac, ETHER_ADDR_LEN); - - ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES, - I40E_SUCCESS, &reply, sizeof(reply)); -} - -static int -ixl_vf_config_tx_queue(struct ixl_pf *pf, struct ixl_vf *vf, - struct i40e_virtchnl_txq_info *info) -{ - struct i40e_hw *hw; - struct i40e_hmc_obj_txq txq; - uint16_t global_queue_num, global_vf_num; - enum i40e_status_code status; - uint32_t qtx_ctl; - - hw = &pf->hw; - global_queue_num = vf->vsi.first_queue + info->queue_id; - global_vf_num = hw->func_caps.vf_base_id + vf->vf_num; - bzero(&txq, sizeof(txq)); - - status = i40e_clear_lan_tx_queue_context(hw, global_queue_num); - if (status != I40E_SUCCESS) - return (EINVAL); - - txq.base = info->dma_ring_addr / IXL_TX_CTX_BASE_UNITS; - - txq.head_wb_ena = info->headwb_enabled; - txq.head_wb_addr = info->dma_headwb_addr; - txq.qlen = info->ring_len; - txq.rdylist = le16_to_cpu(vf->vsi.info.qs_handle[0]); - txq.rdylist_act = 0; - - status = i40e_set_lan_tx_queue_context(hw, global_queue_num, &txq); - if (status != I40E_SUCCESS) - return (EINVAL); - - qtx_ctl = I40E_QTX_CTL_VF_QUEUE | - (hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) | - (global_vf_num << I40E_QTX_CTL_VFVM_INDX_SHIFT); - wr32(hw, I40E_QTX_CTL(global_queue_num), qtx_ctl); - ixl_flush(hw); - - return (0); -} - -static int -ixl_vf_config_rx_queue(struct ixl_pf *pf, struct ixl_vf *vf, - struct i40e_virtchnl_rxq_info *info) -{ - struct i40e_hw *hw; - struct i40e_hmc_obj_rxq rxq; - uint16_t global_queue_num; - enum i40e_status_code status; - - hw = &pf->hw; - global_queue_num = vf->vsi.first_queue + info->queue_id; - bzero(&rxq, sizeof(rxq)); - - if (info->databuffer_size > IXL_VF_MAX_BUFFER) - return (EINVAL); - - if (info->max_pkt_size > IXL_VF_MAX_FRAME || - info->max_pkt_size < ETHER_MIN_LEN) - return (EINVAL); - - if (info->splithdr_enabled) { - if (info->hdr_size > IXL_VF_MAX_HDR_BUFFER) - return (EINVAL); - - rxq.hsplit_0 = info->rx_split_pos & - (I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_L2 | - I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_IP | - I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_TCP_UDP | - I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_SCTP); - rxq.hbuff = info->hdr_size >> I40E_RXQ_CTX_HBUFF_SHIFT; - - rxq.dtype = 2; - } - - status = i40e_clear_lan_rx_queue_context(hw, global_queue_num); - if (status != I40E_SUCCESS) - return (EINVAL); - - rxq.base = info->dma_ring_addr / IXL_RX_CTX_BASE_UNITS; - rxq.qlen = info->ring_len; - - rxq.dbuff = info->databuffer_size >> I40E_RXQ_CTX_DBUFF_SHIFT; - - rxq.dsize = 1; - rxq.crcstrip = 1; - rxq.l2tsel = 1; - - rxq.rxmax = info->max_pkt_size; - rxq.tphrdesc_ena = 1; - rxq.tphwdesc_ena = 1; - rxq.tphdata_ena = 1; - rxq.tphhead_ena = 1; - rxq.lrxqthresh = 2; - rxq.prefena = 1; - - status = i40e_set_lan_rx_queue_context(hw, global_queue_num, &rxq); - if (status != I40E_SUCCESS) - return (EINVAL); - - return (0); -} - -static void -ixl_vf_config_vsi_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, - uint16_t msg_size) -{ - struct i40e_virtchnl_vsi_queue_config_info *info; - struct i40e_virtchnl_queue_pair_info *pair; - int i; - - if (msg_size < sizeof(*info)) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, - I40E_ERR_PARAM); - return; - } - - info = msg; - if (info->num_queue_pairs == 0) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, - I40E_ERR_PARAM); - return; - } - - if (msg_size != sizeof(*info) + info->num_queue_pairs * sizeof(*pair)) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, - I40E_ERR_PARAM); - return; - } - - if (info->vsi_id != vf->vsi.vsi_num) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, - I40E_ERR_PARAM); - return; - } - - for (i = 0; i < info->num_queue_pairs; i++) { - pair = &info->qpair[i]; - - if (pair->txq.vsi_id != vf->vsi.vsi_num || - pair->rxq.vsi_id != vf->vsi.vsi_num || - pair->txq.queue_id != pair->rxq.queue_id || - pair->txq.queue_id >= vf->vsi.num_queues) { - - i40e_send_vf_nack(pf, vf, - I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM); - return; - } - - if (ixl_vf_config_tx_queue(pf, vf, &pair->txq) != 0) { - i40e_send_vf_nack(pf, vf, - I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM); - return; - } - - if (ixl_vf_config_rx_queue(pf, vf, &pair->rxq) != 0) { - i40e_send_vf_nack(pf, vf, - I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM); - return; - } - } - - ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES); -} - -static void -ixl_vf_set_qctl(struct ixl_pf *pf, - const struct i40e_virtchnl_vector_map *vector, - enum i40e_queue_type cur_type, uint16_t cur_queue, - enum i40e_queue_type *last_type, uint16_t *last_queue) -{ - uint32_t offset, qctl; - uint16_t itr_indx; - - if (cur_type == I40E_QUEUE_TYPE_RX) { - offset = I40E_QINT_RQCTL(cur_queue); - itr_indx = vector->rxitr_idx; - } else { - offset = I40E_QINT_TQCTL(cur_queue); - itr_indx = vector->txitr_idx; - } - - qctl = htole32((vector->vector_id << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | - (*last_type << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) | - (*last_queue << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | - I40E_QINT_RQCTL_CAUSE_ENA_MASK | - (itr_indx << I40E_QINT_RQCTL_ITR_INDX_SHIFT)); - - wr32(&pf->hw, offset, qctl); - - *last_type = cur_type; - *last_queue = cur_queue; -} - -static void -ixl_vf_config_vector(struct ixl_pf *pf, struct ixl_vf *vf, - const struct i40e_virtchnl_vector_map *vector) -{ - struct i40e_hw *hw; - u_int qindex; - enum i40e_queue_type type, last_type; - uint32_t lnklst_reg; - uint16_t rxq_map, txq_map, cur_queue, last_queue; - - hw = &pf->hw; - - rxq_map = vector->rxq_map; - txq_map = vector->txq_map; - - last_queue = IXL_END_OF_INTR_LNKLST; - last_type = I40E_QUEUE_TYPE_RX; - - /* - * The datasheet says to optimize performance, RX queues and TX queues - * should be interleaved in the interrupt linked list, so we process - * both at once here. - */ - while ((rxq_map != 0) || (txq_map != 0)) { - if (txq_map != 0) { - qindex = ffs(txq_map) - 1; - type = I40E_QUEUE_TYPE_TX; - cur_queue = vf->vsi.first_queue + qindex; - ixl_vf_set_qctl(pf, vector, type, cur_queue, - &last_type, &last_queue); - txq_map &= ~(1 << qindex); - } - - if (rxq_map != 0) { - qindex = ffs(rxq_map) - 1; - type = I40E_QUEUE_TYPE_RX; - cur_queue = vf->vsi.first_queue + qindex; - ixl_vf_set_qctl(pf, vector, type, cur_queue, - &last_type, &last_queue); - rxq_map &= ~(1 << qindex); - } - } - - if (vector->vector_id == 0) - lnklst_reg = I40E_VPINT_LNKLST0(vf->vf_num); - else - lnklst_reg = IXL_VPINT_LNKLSTN_REG(hw, vector->vector_id, - vf->vf_num); - wr32(hw, lnklst_reg, - (last_queue << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT) | - (last_type << I40E_VPINT_LNKLST0_FIRSTQ_TYPE_SHIFT)); - - ixl_flush(hw); -} - -static void -ixl_vf_config_irq_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, - uint16_t msg_size) -{ - struct i40e_virtchnl_irq_map_info *map; - struct i40e_virtchnl_vector_map *vector; - struct i40e_hw *hw; - int i, largest_txq, largest_rxq; - - hw = &pf->hw; - - if (msg_size < sizeof(*map)) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, - I40E_ERR_PARAM); - return; - } - - map = msg; - if (map->num_vectors == 0) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, - I40E_ERR_PARAM); - return; - } - - if (msg_size != sizeof(*map) + map->num_vectors * sizeof(*vector)) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, - I40E_ERR_PARAM); - return; - } - - for (i = 0; i < map->num_vectors; i++) { - vector = &map->vecmap[i]; - - if ((vector->vector_id >= hw->func_caps.num_msix_vectors_vf) || - vector->vsi_id != vf->vsi.vsi_num) { - i40e_send_vf_nack(pf, vf, - I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, I40E_ERR_PARAM); - return; - } - - if (vector->rxq_map != 0) { - largest_rxq = fls(vector->rxq_map) - 1; - if (largest_rxq >= vf->vsi.num_queues) { - i40e_send_vf_nack(pf, vf, - I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, - I40E_ERR_PARAM); - return; - } - } - - if (vector->txq_map != 0) { - largest_txq = fls(vector->txq_map) - 1; - if (largest_txq >= vf->vsi.num_queues) { - i40e_send_vf_nack(pf, vf, - I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, - I40E_ERR_PARAM); - return; - } - } - - if (vector->rxitr_idx > IXL_MAX_ITR_IDX || - vector->txitr_idx > IXL_MAX_ITR_IDX) { - i40e_send_vf_nack(pf, vf, - I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, - I40E_ERR_PARAM); - return; - } - - ixl_vf_config_vector(pf, vf, vector); - } - - ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP); -} - -static void -ixl_vf_enable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, - uint16_t msg_size) -{ - struct i40e_virtchnl_queue_select *select; - int error; - - if (msg_size != sizeof(*select)) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES, - I40E_ERR_PARAM); - return; - } - - select = msg; - if (select->vsi_id != vf->vsi.vsi_num || - select->rx_queues == 0 || select->tx_queues == 0) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES, - I40E_ERR_PARAM); - return; - } - - error = ixl_enable_rings(&vf->vsi); - if (error) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES, - I40E_ERR_TIMEOUT); - return; - } - - ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES); -} - -static void -ixl_vf_disable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf, - void *msg, uint16_t msg_size) -{ - struct i40e_virtchnl_queue_select *select; - int error; - - if (msg_size != sizeof(*select)) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES, - I40E_ERR_PARAM); - return; - } - - select = msg; - if (select->vsi_id != vf->vsi.vsi_num || - select->rx_queues == 0 || select->tx_queues == 0) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES, - I40E_ERR_PARAM); - return; - } - - error = ixl_disable_rings(&vf->vsi); - if (error) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES, - I40E_ERR_TIMEOUT); - return; - } - - ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES); -} - -static boolean_t -ixl_zero_mac(const uint8_t *addr) -{ - uint8_t zero[ETHER_ADDR_LEN] = {0, 0, 0, 0, 0, 0}; - - return (cmp_etheraddr(addr, zero)); -} - -static boolean_t -ixl_bcast_mac(const uint8_t *addr) -{ - - return (cmp_etheraddr(addr, ixl_bcast_addr)); -} - -static int -ixl_vf_mac_valid(struct ixl_vf *vf, const uint8_t *addr) -{ - - if (ixl_zero_mac(addr) || ixl_bcast_mac(addr)) - return (EINVAL); - - /* - * If the VF is not allowed to change its MAC address, don't let it - * set a MAC filter for an address that is not a multicast address and - * is not its assigned MAC. - */ - if (!(vf->vf_flags & VF_FLAG_SET_MAC_CAP) && - !(ETHER_IS_MULTICAST(addr) || cmp_etheraddr(addr, vf->mac))) - return (EPERM); - - return (0); -} - -static void -ixl_vf_add_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, - uint16_t msg_size) -{ - struct i40e_virtchnl_ether_addr_list *addr_list; - struct i40e_virtchnl_ether_addr *addr; - struct ixl_vsi *vsi; - int i; - size_t expected_size; - - vsi = &vf->vsi; - - if (msg_size < sizeof(*addr_list)) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, - I40E_ERR_PARAM); - return; - } - - addr_list = msg; - expected_size = sizeof(*addr_list) + - addr_list->num_elements * sizeof(*addr); - - if (addr_list->num_elements == 0 || - addr_list->vsi_id != vsi->vsi_num || - msg_size != expected_size) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, - I40E_ERR_PARAM); - return; - } - - for (i = 0; i < addr_list->num_elements; i++) { - if (ixl_vf_mac_valid(vf, addr_list->list[i].addr) != 0) { - i40e_send_vf_nack(pf, vf, - I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM); - return; - } - } - - for (i = 0; i < addr_list->num_elements; i++) { - addr = &addr_list->list[i]; - ixl_add_filter(vsi, addr->addr, IXL_VLAN_ANY); - } - - ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS); -} - -static void -ixl_vf_del_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, - uint16_t msg_size) -{ - struct i40e_virtchnl_ether_addr_list *addr_list; - struct i40e_virtchnl_ether_addr *addr; - size_t expected_size; - int i; - - if (msg_size < sizeof(*addr_list)) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, - I40E_ERR_PARAM); - return; - } - - addr_list = msg; - expected_size = sizeof(*addr_list) + - addr_list->num_elements * sizeof(*addr); - - if (addr_list->num_elements == 0 || - addr_list->vsi_id != vf->vsi.vsi_num || - msg_size != expected_size) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, - I40E_ERR_PARAM); - return; - } - - for (i = 0; i < addr_list->num_elements; i++) { - addr = &addr_list->list[i]; - if (ixl_zero_mac(addr->addr) || ixl_bcast_mac(addr->addr)) { - i40e_send_vf_nack(pf, vf, - I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM); - return; - } - } - - for (i = 0; i < addr_list->num_elements; i++) { - addr = &addr_list->list[i]; - ixl_del_filter(&vf->vsi, addr->addr, IXL_VLAN_ANY); - } - - ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS); -} - -static enum i40e_status_code -ixl_vf_enable_vlan_strip(struct ixl_pf *pf, struct ixl_vf *vf) -{ - struct i40e_vsi_context vsi_ctx; - - vsi_ctx.seid = vf->vsi.seid; - - bzero(&vsi_ctx.info, sizeof(vsi_ctx.info)); - vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_VLAN_VALID); - vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | - I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; - return (i40e_aq_update_vsi_params(&pf->hw, &vsi_ctx, NULL)); -} - -static void -ixl_vf_add_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, - uint16_t msg_size) -{ - struct i40e_virtchnl_vlan_filter_list *filter_list; - enum i40e_status_code code; - size_t expected_size; - int i; - - if (msg_size < sizeof(*filter_list)) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, - I40E_ERR_PARAM); - return; - } - - filter_list = msg; - expected_size = sizeof(*filter_list) + - filter_list->num_elements * sizeof(uint16_t); - if (filter_list->num_elements == 0 || - filter_list->vsi_id != vf->vsi.vsi_num || - msg_size != expected_size) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, - I40E_ERR_PARAM); - return; - } - - if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, - I40E_ERR_PARAM); - return; - } - - for (i = 0; i < filter_list->num_elements; i++) { - if (filter_list->vlan_id[i] > EVL_VLID_MASK) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, - I40E_ERR_PARAM); - return; - } - } - - code = ixl_vf_enable_vlan_strip(pf, vf); - if (code != I40E_SUCCESS) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, - I40E_ERR_PARAM); - } - - for (i = 0; i < filter_list->num_elements; i++) - ixl_add_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]); - - ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN); -} - -static void -ixl_vf_del_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, - uint16_t msg_size) -{ - struct i40e_virtchnl_vlan_filter_list *filter_list; - int i; - size_t expected_size; - - if (msg_size < sizeof(*filter_list)) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN, - I40E_ERR_PARAM); - return; - } - - filter_list = msg; - expected_size = sizeof(*filter_list) + - filter_list->num_elements * sizeof(uint16_t); - if (filter_list->num_elements == 0 || - filter_list->vsi_id != vf->vsi.vsi_num || - msg_size != expected_size) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN, - I40E_ERR_PARAM); - return; - } - - for (i = 0; i < filter_list->num_elements; i++) { - if (filter_list->vlan_id[i] > EVL_VLID_MASK) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, - I40E_ERR_PARAM); - return; - } - } - - if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, - I40E_ERR_PARAM); - return; - } - - for (i = 0; i < filter_list->num_elements; i++) - ixl_del_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]); - - ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN); -} - -static void -ixl_vf_config_promisc_msg(struct ixl_pf *pf, struct ixl_vf *vf, - void *msg, uint16_t msg_size) -{ - struct i40e_virtchnl_promisc_info *info; - enum i40e_status_code code; - - if (msg_size != sizeof(*info)) { - i40e_send_vf_nack(pf, vf, - I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM); - return; - } - - if (!(vf->vf_flags & VF_FLAG_PROMISC_CAP)) { - i40e_send_vf_nack(pf, vf, - I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM); - return; - } - - info = msg; - if (info->vsi_id != vf->vsi.vsi_num) { - i40e_send_vf_nack(pf, vf, - I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM); - return; - } - - code = i40e_aq_set_vsi_unicast_promiscuous(&pf->hw, info->vsi_id, - info->flags & I40E_FLAG_VF_UNICAST_PROMISC, NULL); - if (code != I40E_SUCCESS) { - i40e_send_vf_nack(pf, vf, - I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code); - return; - } - - code = i40e_aq_set_vsi_multicast_promiscuous(&pf->hw, info->vsi_id, - info->flags & I40E_FLAG_VF_MULTICAST_PROMISC, NULL); - if (code != I40E_SUCCESS) { - i40e_send_vf_nack(pf, vf, - I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code); - return; - } - - ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE); -} - -static void -ixl_vf_get_stats_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, - uint16_t msg_size) -{ - struct i40e_virtchnl_queue_select *queue; - - if (msg_size != sizeof(*queue)) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS, - I40E_ERR_PARAM); - return; - } - - queue = msg; - if (queue->vsi_id != vf->vsi.vsi_num) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS, - I40E_ERR_PARAM); - return; - } - - ixl_update_eth_stats(&vf->vsi); - - ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_STATS, - I40E_SUCCESS, &vf->vsi.eth_stats, sizeof(vf->vsi.eth_stats)); -} - -static void -ixl_handle_vf_msg(struct ixl_pf *pf, struct i40e_arq_event_info *event) -{ - struct ixl_vf *vf; - void *msg; - uint16_t vf_num, msg_size; - uint32_t opcode; - - vf_num = le16toh(event->desc.retval) - pf->hw.func_caps.vf_base_id; - opcode = le32toh(event->desc.cookie_high); - - if (vf_num >= pf->num_vfs) { - device_printf(pf->dev, "Got msg from illegal VF: %d\n", vf_num); - return; - } - - vf = &pf->vfs[vf_num]; - msg = event->msg_buf; - msg_size = event->msg_len; - - I40E_VC_DEBUG(pf, ixl_vc_opcode_level(opcode), - "Got msg %s(%d) from VF-%d of size %d\n", - ixl_vc_opcode_str(opcode), opcode, vf_num, msg_size); - - switch (opcode) { - case I40E_VIRTCHNL_OP_VERSION: - ixl_vf_version_msg(pf, vf, msg, msg_size); - break; - case I40E_VIRTCHNL_OP_RESET_VF: - ixl_vf_reset_msg(pf, vf, msg, msg_size); - break; - case I40E_VIRTCHNL_OP_GET_VF_RESOURCES: - ixl_vf_get_resources_msg(pf, vf, msg, msg_size); - break; - case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES: - ixl_vf_config_vsi_msg(pf, vf, msg, msg_size); - break; - case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP: - ixl_vf_config_irq_msg(pf, vf, msg, msg_size); - break; - case I40E_VIRTCHNL_OP_ENABLE_QUEUES: - ixl_vf_enable_queues_msg(pf, vf, msg, msg_size); - break; - case I40E_VIRTCHNL_OP_DISABLE_QUEUES: - ixl_vf_disable_queues_msg(pf, vf, msg, msg_size); - break; - case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS: - ixl_vf_add_mac_msg(pf, vf, msg, msg_size); - break; - case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS: - ixl_vf_del_mac_msg(pf, vf, msg, msg_size); - break; - case I40E_VIRTCHNL_OP_ADD_VLAN: - ixl_vf_add_vlan_msg(pf, vf, msg, msg_size); - break; - case I40E_VIRTCHNL_OP_DEL_VLAN: - ixl_vf_del_vlan_msg(pf, vf, msg, msg_size); - break; - case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE: - ixl_vf_config_promisc_msg(pf, vf, msg, msg_size); - break; - case I40E_VIRTCHNL_OP_GET_STATS: - ixl_vf_get_stats_msg(pf, vf, msg, msg_size); - break; - - /* These two opcodes have been superseded by CONFIG_VSI_QUEUES. */ - case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE: - case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE: - default: - i40e_send_vf_nack(pf, vf, opcode, I40E_ERR_NOT_IMPLEMENTED); - break; - } -} - -/* Handle any VFs that have reset themselves via a Function Level Reset(FLR). */ -static void -ixl_handle_vflr(void *arg, int pending) -{ - struct ixl_pf *pf; - struct i40e_hw *hw; - uint16_t global_vf_num; - uint32_t vflrstat_index, vflrstat_mask, vflrstat, icr0; - int i; - - pf = arg; - hw = &pf->hw; - - IXL_PF_LOCK(pf); - for (i = 0; i < pf->num_vfs; i++) { - global_vf_num = hw->func_caps.vf_base_id + i; - - vflrstat_index = IXL_GLGEN_VFLRSTAT_INDEX(global_vf_num); - vflrstat_mask = IXL_GLGEN_VFLRSTAT_MASK(global_vf_num); - vflrstat = rd32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index)); - if (vflrstat & vflrstat_mask) { - wr32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index), - vflrstat_mask); - - ixl_reinit_vf(pf, &pf->vfs[i]); - } - } - - icr0 = rd32(hw, I40E_PFINT_ICR0_ENA); - icr0 |= I40E_PFINT_ICR0_ENA_VFLR_MASK; - wr32(hw, I40E_PFINT_ICR0_ENA, icr0); - ixl_flush(hw); - - IXL_PF_UNLOCK(pf); -} - -static int -ixl_adminq_err_to_errno(enum i40e_admin_queue_err err) -{ - - switch (err) { - case I40E_AQ_RC_EPERM: - return (EPERM); - case I40E_AQ_RC_ENOENT: - return (ENOENT); - case I40E_AQ_RC_ESRCH: - return (ESRCH); - case I40E_AQ_RC_EINTR: - return (EINTR); - case I40E_AQ_RC_EIO: - return (EIO); - case I40E_AQ_RC_ENXIO: - return (ENXIO); - case I40E_AQ_RC_E2BIG: - return (E2BIG); - case I40E_AQ_RC_EAGAIN: - return (EAGAIN); - case I40E_AQ_RC_ENOMEM: - return (ENOMEM); - case I40E_AQ_RC_EACCES: - return (EACCES); - case I40E_AQ_RC_EFAULT: - return (EFAULT); - case I40E_AQ_RC_EBUSY: - return (EBUSY); - case I40E_AQ_RC_EEXIST: - return (EEXIST); - case I40E_AQ_RC_EINVAL: - return (EINVAL); - case I40E_AQ_RC_ENOTTY: - return (ENOTTY); - case I40E_AQ_RC_ENOSPC: - return (ENOSPC); - case I40E_AQ_RC_ENOSYS: - return (ENOSYS); - case I40E_AQ_RC_ERANGE: - return (ERANGE); - case I40E_AQ_RC_EFLUSHED: - return (EINVAL); /* No exact equivalent in errno.h */ - case I40E_AQ_RC_BAD_ADDR: - return (EFAULT); - case I40E_AQ_RC_EMODE: - return (EPERM); - case I40E_AQ_RC_EFBIG: - return (EFBIG); - default: - return (EINVAL); - } -} - -static int -ixl_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *params) -{ - struct ixl_pf *pf; - struct i40e_hw *hw; - struct ixl_vsi *pf_vsi; - enum i40e_status_code ret; - int i, error; - - pf = device_get_softc(dev); - hw = &pf->hw; - pf_vsi = &pf->vsi; - - IXL_PF_LOCK(pf); - pf->vfs = malloc(sizeof(struct ixl_vf) * num_vfs, M_IXL, M_NOWAIT | - M_ZERO); - - if (pf->vfs == NULL) { - error = ENOMEM; - goto fail; - } - - for (i = 0; i < num_vfs; i++) - sysctl_ctx_init(&pf->vfs[i].ctx); - - ret = i40e_aq_add_veb(hw, pf_vsi->uplink_seid, pf_vsi->seid, - 1, FALSE, &pf->veb_seid, FALSE, NULL); - if (ret != I40E_SUCCESS) { - error = ixl_adminq_err_to_errno(hw->aq.asq_last_status); - device_printf(dev, "add_veb failed; code=%d error=%d", ret, - error); - goto fail; - } - - // TODO: [Configure MSI-X here] - ixl_enable_adminq(hw); - - pf->num_vfs = num_vfs; - IXL_PF_UNLOCK(pf); - return (0); - -fail: - free(pf->vfs, M_IXL); - pf->vfs = NULL; - IXL_PF_UNLOCK(pf); - return (error); -} - -static void -ixl_iov_uninit(device_t dev) -{ - struct ixl_pf *pf; - struct i40e_hw *hw; - struct ixl_vsi *vsi; - struct ifnet *ifp; - struct ixl_vf *vfs; - int i, num_vfs; - - pf = device_get_softc(dev); - hw = &pf->hw; - vsi = &pf->vsi; - ifp = vsi->ifp; - - IXL_PF_LOCK(pf); - for (i = 0; i < pf->num_vfs; i++) { - if (pf->vfs[i].vsi.seid != 0) - i40e_aq_delete_element(hw, pf->vfs[i].vsi.seid, NULL); - } - - if (pf->veb_seid != 0) { - i40e_aq_delete_element(hw, pf->veb_seid, NULL); - pf->veb_seid = 0; - } - - if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) - ixl_disable_intr(vsi); - - vfs = pf->vfs; - num_vfs = pf->num_vfs; - - pf->vfs = NULL; - pf->num_vfs = 0; - IXL_PF_UNLOCK(pf); - - /* Do this after the unlock as sysctl_ctx_free might sleep. */ - for (i = 0; i < num_vfs; i++) - sysctl_ctx_free(&vfs[i].ctx); - free(vfs, M_IXL); -} - -static int -ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *params) -{ - char sysctl_name[QUEUE_NAME_LEN]; - struct ixl_pf *pf; - struct ixl_vf *vf; - const void *mac; - size_t size; - int error; - - pf = device_get_softc(dev); - vf = &pf->vfs[vfnum]; - - IXL_PF_LOCK(pf); - vf->vf_num = vfnum; - - vf->vsi.back = pf; - vf->vf_flags = VF_FLAG_ENABLED; - SLIST_INIT(&vf->vsi.ftl); - - error = ixl_vf_setup_vsi(pf, vf); - if (error != 0) - goto out; - - if (nvlist_exists_binary(params, "mac-addr")) { - mac = nvlist_get_binary(params, "mac-addr", &size); - bcopy(mac, vf->mac, ETHER_ADDR_LEN); - - if (nvlist_get_bool(params, "allow-set-mac")) - vf->vf_flags |= VF_FLAG_SET_MAC_CAP; - } else - /* - * If the administrator has not specified a MAC address then - * we must allow the VF to choose one. - */ - vf->vf_flags |= VF_FLAG_SET_MAC_CAP; - - if (nvlist_get_bool(params, "mac-anti-spoof")) - vf->vf_flags |= VF_FLAG_MAC_ANTI_SPOOF; - - if (nvlist_get_bool(params, "allow-promisc")) - vf->vf_flags |= VF_FLAG_PROMISC_CAP; - - /* TODO: Get VLAN that PF has set for the VF */ - - vf->vf_flags |= VF_FLAG_VLAN_CAP; - - ixl_reset_vf(pf, vf); -out: - IXL_PF_UNLOCK(pf); - if (error == 0) { - snprintf(sysctl_name, sizeof(sysctl_name), "vf%d", vfnum); - ixl_add_vsi_sysctls(pf, &vf->vsi, &vf->ctx, sysctl_name); - } - - return (error); -} -#endif /* PCI_IOV */ diff --git a/sys/dev/ixl/if_ixlv.c b/sys/dev/ixl/if_ixlv.c index 9be8a36e8816..9c70d79816c0 100644 --- a/sys/dev/ixl/if_ixlv.c +++ b/sys/dev/ixl/if_ixlv.c @@ -32,23 +32,13 @@ ******************************************************************************/ /*$FreeBSD$*/ -#ifndef IXL_STANDALONE_BUILD -#include "opt_inet.h" -#include "opt_inet6.h" -#include "opt_rss.h" -#endif - #include "ixl.h" #include "ixlv.h" -#ifdef RSS -#include <net/rss_config.h> -#endif - /********************************************************************* * Driver version *********************************************************************/ -char ixlv_driver_version[] = "1.2.11-k"; +char ixlv_driver_version[] = "1.4.6-k"; /********************************************************************* * PCI Device ID Table @@ -64,6 +54,9 @@ static ixl_vendor_info_t ixlv_vendor_info_array[] = { {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_VF, 0, 0, 0}, {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_VF_HV, 0, 0, 0}, + {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_X722_VF, 0, 0, 0}, + {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_X722_A0_VF, 0, 0, 0}, + {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_X722_VF_HV, 0, 0, 0}, /* required last entry */ {0, 0, 0, 0, 0} }; @@ -73,7 +66,7 @@ static ixl_vendor_info_t ixlv_vendor_info_array[] = *********************************************************************/ static char *ixlv_strings[] = { - "Intel(R) Ethernet Connection XL710 VF Driver" + "Intel(R) Ethernet Connection XL710/X722 VF Driver" }; @@ -119,6 +112,7 @@ static void ixlv_set_queue_rx_itr(struct ixl_queue *); static void ixlv_set_queue_tx_itr(struct ixl_queue *); static void ixl_init_cmd_complete(struct ixl_vc_cmd *, void *, enum i40e_status_code); +static void ixlv_configure_itr(struct ixlv_sc *); static void ixlv_enable_adminq_irq(struct i40e_hw *); static void ixlv_disable_adminq_irq(struct i40e_hw *); @@ -137,8 +131,10 @@ static void ixlv_cap_txcsum_tso(struct ixl_vsi *, struct ifnet *, int); static void ixlv_add_sysctls(struct ixlv_sc *); +#ifdef IXL_DEBUG static int ixlv_sysctl_qtx_tail_handler(SYSCTL_HANDLER_ARGS); static int ixlv_sysctl_qrx_tail_handler(SYSCTL_HANDLER_ARGS); +#endif /********************************************************************* * FreeBSD Device Interface Entry Points @@ -306,7 +302,7 @@ ixlv_attach(device_t dev) /* Allocate filter lists */ ixlv_init_filters(sc); - /* Core Lock Init*/ + /* Core Lock Init */ mtx_init(&sc->mtx, device_get_nameunit(dev), "IXL SC Lock", MTX_DEF); @@ -365,14 +361,16 @@ ixlv_attach(device_t dev) goto err_aq; } - INIT_DBG_DEV(dev, "VF config from PF:"); - INIT_DBG_DEV(dev, "VSIs %d, Queues %d, Max Vectors %d, Max MTU %d", + device_printf(dev, "VSIs %d, QPs %d, MSIX %d, RSS sizes: key %d lut %d\n", sc->vf_res->num_vsis, sc->vf_res->num_queue_pairs, sc->vf_res->max_vectors, - sc->vf_res->max_mtu); - INIT_DBG_DEV(dev, "Offload flags: %#010x", - sc->vf_res->vf_offload_flags); + sc->vf_res->rss_key_size, + sc->vf_res->rss_lut_size); +#ifdef IXL_DEBUG + device_printf(dev, "Offload flags: 0x%b\n", + sc->vf_res->vf_offload_flags, IXLV_PRINTF_VF_OFFLOAD_FLAGS); +#endif /* got VF config message back from PF, now we can parse it */ for (int i = 0; i < sc->vf_res->num_vsis; i++) { @@ -396,6 +394,14 @@ ixlv_attach(device_t dev) bcopy(addr, hw->mac.addr, sizeof(addr)); } + /* Now that the number of queues for this VF is known, set up interrupts */ + sc->msix = ixlv_init_msix(sc); + /* We fail without MSIX support */ + if (sc->msix == 0) { + error = ENXIO; + goto err_res_buf; + } + vsi->id = sc->vsi_res->vsi_id; vsi->back = (void *)sc; sc->link_up = TRUE; @@ -419,7 +425,12 @@ ixlv_attach(device_t dev) INIT_DBG_DEV(dev, "Queue memory and interface setup"); /* Do queue interrupt setup */ - ixlv_assign_msix(sc); + if (ixlv_assign_msix(sc) != 0) { + device_printf(dev, "%s: allocating queue interrupts failed!\n", + __func__); + error = ENXIO; + goto out; + } /* Start AdminQ taskqueue */ ixlv_init_taskqueue(sc); @@ -829,8 +840,8 @@ ixl_init_cmd_complete(struct ixl_vc_cmd *cmd, void *arg, */ if (code != I40E_SUCCESS && code != I40E_ERR_ADAPTER_STOPPED) { if_printf(sc->vsi.ifp, - "Error %d waiting for PF to complete operation %d\n", - code, cmd->request); + "Error %s waiting for PF to complete operation %d\n", + i40e_stat_str(&sc->hw, code), cmd->request); } } @@ -901,6 +912,9 @@ ixlv_init_locked(struct ixlv_sc *sc) ixl_init_rx_ring(que); } + /* Set initial ITR values */ + ixlv_configure_itr(sc); + /* Configure queues */ ixl_vc_enqueue(&sc->vc_mgr, &sc->config_queues_cmd, IXLV_FLAG_AQ_CONFIGURE_QUEUES, ixl_init_cmd_complete, sc); @@ -936,18 +950,31 @@ ixlv_init(void *arg) struct ixlv_sc *sc = vsi->back; int retries = 0; + /* Prevent init from running again while waiting for AQ calls + * made in init_locked() to complete. */ mtx_lock(&sc->mtx); + if (sc->init_in_progress) { + mtx_unlock(&sc->mtx); + return; + } else + sc->init_in_progress = true; + ixlv_init_locked(sc); mtx_unlock(&sc->mtx); /* Wait for init_locked to finish */ while (!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING) && ++retries < IXLV_AQ_MAX_ERR) { - i40e_msec_delay(25); + i40e_msec_pause(25); } - if (retries >= IXLV_AQ_MAX_ERR) + if (retries >= IXLV_AQ_MAX_ERR) { if_printf(vsi->ifp, "Init failed to complete in allotted time!\n"); + } + + mtx_lock(&sc->mtx); + sc->init_in_progress = false; + mtx_unlock(&sc->mtx); } /* @@ -990,8 +1017,8 @@ ixlv_setup_vc(struct ixlv_sc *sc) /* Need to set these AQ paramters before initializing AQ */ hw->aq.num_arq_entries = IXL_AQ_LEN; hw->aq.num_asq_entries = IXL_AQ_LEN; - hw->aq.arq_buf_size = IXL_AQ_BUFSZ; - hw->aq.asq_buf_size = IXL_AQ_BUFSZ; + hw->aq.arq_buf_size = IXL_AQ_BUF_SZ; + hw->aq.asq_buf_size = IXL_AQ_BUF_SZ; for (int i = 0; i < IXLV_AQ_MAX_ERR; i++) { /* Initialize admin queue */ @@ -1021,13 +1048,13 @@ retry_send: while (!i40e_asq_done(hw)) { if (++asq_retries > IXLV_AQ_MAX_ERR) { i40e_shutdown_adminq(hw); - DDPRINTF(dev, "Admin Queue timeout " - "(waiting for send_api_ver), %d more retries...", + device_printf(dev, "Admin Queue timeout " + "(waiting for send_api_ver), %d more tries...\n", IXLV_AQ_MAX_ERR - (i + 1)); ret_error = 3; break; } - i40e_msec_delay(10); + i40e_msec_pause(10); } if (asq_retries > IXLV_AQ_MAX_ERR) continue; @@ -1055,7 +1082,7 @@ retry_send: if (error) { device_printf(dev, "%s: Unable to verify API version," - " error %d\n", __func__, error); + " error %s\n", __func__, i40e_stat_str(hw, error)); ret_error = 5; } break; @@ -1096,7 +1123,7 @@ retry_config: ret_error = 3; goto fail; } - i40e_msec_delay(10); + i40e_msec_pause(10); } INIT_DBG_DEV(dev, "Sent VF config message to PF, attempt %d", @@ -1149,6 +1176,7 @@ ixlv_init_msix(struct ixlv_sc *sc) { device_t dev = sc->dev; int rid, want, vectors, queues, available; + int auto_max_queues; rid = PCIR_BAR(IXL_BAR); sc->msix_mem = bus_alloc_resource_any(dev, @@ -1156,7 +1184,7 @@ ixlv_init_msix(struct ixlv_sc *sc) if (!sc->msix_mem) { /* May not be enabled */ device_printf(sc->dev, - "Unable to map MSIX table \n"); + "Unable to map MSIX table\n"); goto fail; } @@ -1168,20 +1196,30 @@ ixlv_init_msix(struct ixlv_sc *sc) goto fail; } - /* Figure out a reasonable auto config value */ - queues = (mp_ncpus > (available - 1)) ? (available - 1) : mp_ncpus; + /* Clamp queues to number of CPUs and # of MSI-X vectors available */ + auto_max_queues = min(mp_ncpus, available - 1); + /* Clamp queues to # assigned to VF by PF */ + auto_max_queues = min(auto_max_queues, sc->vf_res->num_queue_pairs); - /* Override with hardcoded value if sane */ - if ((ixlv_max_queues != 0) && (ixlv_max_queues <= queues)) + /* Override with tunable value if tunable is less than autoconfig count */ + if ((ixlv_max_queues != 0) && (ixlv_max_queues <= auto_max_queues)) queues = ixlv_max_queues; + /* Use autoconfig amount if that's lower */ + else if ((ixlv_max_queues != 0) && (ixlv_max_queues > auto_max_queues)) { + device_printf(dev, "ixlv_max_queues (%d) is too large, using " + "autoconfig amount (%d)...\n", + ixlv_max_queues, auto_max_queues); + queues = auto_max_queues; + } + /* Limit maximum auto-configured queues to 8 if no user value is set */ + else + queues = min(auto_max_queues, 8); + #ifdef RSS /* If we're doing RSS, clamp at the number of RSS buckets */ if (queues > rss_getnumbuckets()) queues = rss_getnumbuckets(); #endif - /* Enforce the VF max value */ - if (queues > IXLV_MAX_QUEUES) - queues = IXLV_MAX_QUEUES; /* ** Want one vector (RX/TX pair) per queue @@ -1225,25 +1263,6 @@ ixlv_init_msix(struct ixlv_sc *sc) sc->vsi.num_queues = queues; } - /* - ** Explicitly set the guest PCI BUSMASTER capability - ** and we must rewrite the ENABLE in the MSIX control - ** register again at this point to cause the host to - ** successfully initialize us. - */ - { - u16 pci_cmd_word; - int msix_ctrl; - pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); - pci_cmd_word |= PCIM_CMD_BUSMASTEREN; - pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2); - pci_find_cap(dev, PCIY_MSIX, &rid); - rid += PCIR_MSIX_CTRL; - msix_ctrl = pci_read_config(dev, rid, 2); - msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE; - pci_write_config(dev, rid, msix_ctrl, 2); - } - /* Next we need to setup the vector for the Admin Queue */ rid = 1; // zero vector + 1 sc->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, @@ -1280,7 +1299,7 @@ ixlv_allocate_pci_resources(struct ixlv_sc *sc) &rid, RF_ACTIVE); if (!(sc->pci_mem)) { - device_printf(dev,"Unable to allocate bus resource: memory\n"); + device_printf(dev, "Unable to allocate bus resource: memory\n"); return (ENXIO); } @@ -1294,18 +1313,29 @@ ixlv_allocate_pci_resources(struct ixlv_sc *sc) sc->hw.back = &sc->osdep; - /* Disable adminq interrupts */ - ixlv_disable_adminq_irq(&sc->hw); - /* - ** Now setup MSI/X, it will return - ** us the number of supported vectors + ** Explicitly set the guest PCI BUSMASTER capability + ** and we must rewrite the ENABLE in the MSIX control + ** register again at this point to cause the host to + ** successfully initialize us. + ** + ** This must be set before accessing any registers. */ - sc->msix = ixlv_init_msix(sc); + { + u16 pci_cmd_word; + int msix_ctrl; + pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); + pci_cmd_word |= PCIM_CMD_BUSMASTEREN; + pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2); + pci_find_cap(dev, PCIY_MSIX, &rid); + rid += PCIR_MSIX_CTRL; + msix_ctrl = pci_read_config(dev, rid, 2); + msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE; + pci_write_config(dev, rid, msix_ctrl, 2); + } - /* We fail without MSIX support */ - if (sc->msix == 0) - return (ENXIO); + /* Disable adminq interrupts (just in case) */ + ixlv_disable_adminq_irq(&sc->hw); return (0); } @@ -1330,8 +1360,10 @@ ixlv_free_pci_resources(struct ixlv_sc *sc) bus_teardown_intr(dev, que->res, que->tag); que->tag = NULL; } - if (que->res != NULL) + if (que->res != NULL) { bus_release_resource(dev, SYS_RES_IRQ, rid, que->res); + que->res = NULL; + } } early: @@ -1340,8 +1372,10 @@ early: bus_teardown_intr(dev, sc->res, sc->tag); sc->tag = NULL; } - if (sc->res != NULL) + if (sc->res != NULL) { bus_release_resource(dev, SYS_RES_IRQ, 1, sc->res); + sc->res = NULL; + } pci_release_msi(dev); @@ -1352,8 +1386,6 @@ early: if (sc->pci_mem != NULL) bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(0), sc->pci_mem); - - return; } /* @@ -1418,7 +1450,6 @@ ixlv_assign_msix(struct ixlv_sc *sc) #endif bus_bind_intr(dev, que->res, cpu_id); que->msix = vector; - vsi->que_mask |= (u64)(1 << que->msix); TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que); TASK_INIT(&que->task, 0, ixlv_handle_que, que); que->tq = taskqueue_create_fast("ixlv_que", M_NOWAIT, @@ -1454,7 +1485,7 @@ ixlv_reset(struct ixlv_sc *sc) if (sc->init_state != IXLV_RESET_PENDING) ixlv_request_reset(sc); - i40e_msec_delay(100); + i40e_msec_pause(100); error = ixlv_reset_complete(hw); if (error) { device_printf(dev, "%s: VF reset failed\n", @@ -1484,6 +1515,7 @@ ixlv_reset_complete(struct i40e_hw *hw) { u32 reg; + /* Wait up to ~10 seconds */ for (int i = 0; i < 100; i++) { reg = rd32(hw, I40E_VFGEN_RSTAT) & I40E_VFGEN_RSTAT_VFR_STATE_MASK; @@ -1491,7 +1523,7 @@ ixlv_reset_complete(struct i40e_hw *hw) if ((reg == I40E_VFR_VFACTIVE) || (reg == I40E_VFR_COMPLETED)) return (0); - i40e_msec_delay(100); + i40e_msec_pause(100); } return (EBUSY); @@ -1522,7 +1554,7 @@ ixlv_setup_interface(device_t dev, struct ixlv_sc *sc) if_initname(ifp, device_get_name(dev), device_get_unit(dev)); ifp->if_mtu = ETHERMTU; - ifp->if_baudrate = 4000000000; // ?? + ifp->if_baudrate = IF_Gbps(40); ifp->if_init = ixlv_init; ifp->if_softc = vsi; ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; @@ -1686,7 +1718,7 @@ ixlv_setup_queues(struct ixlv_sc *sc) rxr->base = (union i40e_rx_desc *)rxr->dma.va; bzero((void *)rxr->base, rsize); - /* Allocate receive soft structs for the ring*/ + /* Allocate receive soft structs for the ring */ if (ixl_allocate_rx_data(que)) { device_printf(dev, "Critical Failure setting up receive structs\n"); @@ -1896,18 +1928,48 @@ ixlv_enable_queue_irq(struct i40e_hw *hw, int id) u32 reg; reg = I40E_VFINT_DYN_CTLN1_INTENA_MASK | - I40E_VFINT_DYN_CTLN1_CLEARPBA_MASK; + I40E_VFINT_DYN_CTLN1_CLEARPBA_MASK | + I40E_VFINT_DYN_CTLN1_ITR_INDX_MASK; wr32(hw, I40E_VFINT_DYN_CTLN1(id), reg); } static void ixlv_disable_queue_irq(struct i40e_hw *hw, int id) { - wr32(hw, I40E_VFINT_DYN_CTLN1(id), 0); + wr32(hw, I40E_VFINT_DYN_CTLN1(id), + I40E_VFINT_DYN_CTLN1_ITR_INDX_MASK); rd32(hw, I40E_VFGEN_RSTAT); return; } +/* + * Get initial ITR values from tunable values. + */ +static void +ixlv_configure_itr(struct ixlv_sc *sc) +{ + struct i40e_hw *hw = &sc->hw; + struct ixl_vsi *vsi = &sc->vsi; + struct ixl_queue *que = vsi->queues; + + vsi->rx_itr_setting = ixlv_rx_itr; + vsi->tx_itr_setting = ixlv_tx_itr; + + for (int i = 0; i < vsi->num_queues; i++, que++) { + struct tx_ring *txr = &que->txr; + struct rx_ring *rxr = &que->rxr; + + wr32(hw, I40E_VFINT_ITRN1(IXL_RX_ITR, i), + vsi->rx_itr_setting); + rxr->itr = vsi->rx_itr_setting; + rxr->latency = IXL_AVE_LATENCY; + + wr32(hw, I40E_VFINT_ITRN1(IXL_TX_ITR, i), + vsi->tx_itr_setting); + txr->itr = vsi->tx_itr_setting; + txr->latency = IXL_AVE_LATENCY; + } +} /* ** Provide a update to the queue RX @@ -2251,7 +2313,7 @@ ixlv_add_multi(struct ixl_vsi *vsi) } if_maddr_runlock(ifp); - // TODO: Remove -- cannot set promiscuous mode in a VF + /* TODO: Remove -- cannot set promiscuous mode in a VF */ if (__predict_false(mcnt >= MAX_MULTICAST_ADDR)) { /* delete all multicast filters */ ixlv_init_multi(vsi); @@ -2381,7 +2443,8 @@ ixlv_local_timer(void *arg) ** Check status on the queues for a hang */ mask = (I40E_VFINT_DYN_CTLN1_INTENA_MASK | - I40E_VFINT_DYN_CTLN1_SWINT_TRIG_MASK); + I40E_VFINT_DYN_CTLN1_SWINT_TRIG_MASK | + I40E_VFINT_DYN_CTLN1_ITR_INDX_MASK); for (int i = 0; i < vsi->num_queues; i++,que++) { /* Any queues with outstanding work get a sw irq */ @@ -2522,28 +2585,17 @@ ixlv_free_queues(struct ixl_vsi *vsi) free(vsi->queues, M_DEVBUF); } - -/* -** ixlv_config_rss - setup RSS -** -** RSS keys and table are cleared on VF reset. -*/ static void -ixlv_config_rss(struct ixlv_sc *sc) +ixlv_config_rss_reg(struct ixlv_sc *sc) { struct i40e_hw *hw = &sc->hw; struct ixl_vsi *vsi = &sc->vsi; u32 lut = 0; u64 set_hena = 0, hena; int i, j, que_id; + u32 rss_seed[IXL_RSS_KEY_SIZE_REG]; #ifdef RSS u32 rss_hash_config; - u32 rss_seed[IXL_KEYSZ]; -#else - u32 rss_seed[IXL_KEYSZ] = {0x41b01687, - 0x183cfd8c, 0xce880440, 0x580cbc3c, - 0x35897377, 0x328b25e1, 0x4fa98922, - 0xb7d90c14, 0xd5bad70d, 0xcd15a2c1}; #endif /* Don't set up RSS if using a single queue */ @@ -2557,9 +2609,12 @@ ixlv_config_rss(struct ixlv_sc *sc) #ifdef RSS /* Fetch the configured RSS key */ rss_getkey((uint8_t *) &rss_seed); +#else + ixl_get_default_rss_key(rss_seed); #endif + /* Fill out hash function seed */ - for (i = 0; i <= IXL_KEYSZ; i++) + for (i = 0; i < IXL_RSS_KEY_SIZE_REG; i++) wr32(hw, I40E_VFQF_HKEY(i), rss_seed[i]); /* Enable PCTYPES for RSS: */ @@ -2580,18 +2635,7 @@ ixlv_config_rss(struct ixlv_sc *sc) if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP); #else - set_hena = - ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | - ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP) | - ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) | - ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) | - ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV4) | - ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP) | - ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP) | - ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) | - ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) | - ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6) | - ((u64)1 << I40E_FILTER_PCTYPE_L2_PAYLOAD); + set_hena = IXL_DEFAULT_RSS_HENA; #endif hena = (u64)rd32(hw, I40E_VFQF_HENA(0)) | ((u64)rd32(hw, I40E_VFQF_HENA(1)) << 32); @@ -2599,9 +2643,8 @@ ixlv_config_rss(struct ixlv_sc *sc) wr32(hw, I40E_VFQF_HENA(0), (u32)hena); wr32(hw, I40E_VFQF_HENA(1), (u32)(hena >> 32)); - // TODO: Fix -- only 3,7,11,15 are filled out, instead of all 16 registers /* Populate the LUT with max no. of queues in round robin fashion */ - for (i = 0, j = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++, j++) { + for (i = 0, j = 0; i < IXL_RSS_VSI_LUT_SIZE; i++, j++) { if (j == vsi->num_queues) j = 0; #ifdef RSS @@ -2616,16 +2659,46 @@ ixlv_config_rss(struct ixlv_sc *sc) que_id = j; #endif /* lut = 4-byte sliding window of 4 lut entries */ - lut = (lut << 8) | (que_id & 0xF); + lut = (lut << 8) | (que_id & IXL_RSS_VF_LUT_ENTRY_MASK); /* On i = 3, we have 4 entries in lut; write to the register */ if ((i & 3) == 3) { - wr32(hw, I40E_VFQF_HLUT(i), lut); + wr32(hw, I40E_VFQF_HLUT(i >> 2), lut); DDPRINTF(sc->dev, "HLUT(%2d): %#010x", i, lut); } } ixl_flush(hw); } +static void +ixlv_config_rss_pf(struct ixlv_sc *sc) +{ + ixl_vc_enqueue(&sc->vc_mgr, &sc->config_rss_key_cmd, + IXLV_FLAG_AQ_CONFIG_RSS_KEY, ixl_init_cmd_complete, sc); + + ixl_vc_enqueue(&sc->vc_mgr, &sc->set_rss_hena_cmd, + IXLV_FLAG_AQ_SET_RSS_HENA, ixl_init_cmd_complete, sc); + + ixl_vc_enqueue(&sc->vc_mgr, &sc->config_rss_lut_cmd, + IXLV_FLAG_AQ_CONFIG_RSS_LUT, ixl_init_cmd_complete, sc); +} + +/* +** ixlv_config_rss - setup RSS +** +** RSS keys and table are cleared on VF reset. +*/ +static void +ixlv_config_rss(struct ixlv_sc *sc) +{ + if (sc->vf_res->vf_offload_flags & I40E_VIRTCHNL_VF_OFFLOAD_RSS_REG) { + DDPRINTF(sc->dev, "Setting up RSS using VF registers..."); + ixlv_config_rss_reg(sc); + } else if (sc->vf_res->vf_offload_flags & I40E_VIRTCHNL_VF_OFFLOAD_RSS_PF) { + DDPRINTF(sc->dev, "Setting up RSS using messages to PF..."); + ixlv_config_rss_pf(sc); + } else + device_printf(sc->dev, "VF does not support RSS capability sent by PF.\n"); +} /* ** This routine refreshes vlan filters, called by init @@ -2868,8 +2941,8 @@ ixlv_add_sysctls(struct ixlv_sc *sc) SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "tso_tx", CTLFLAG_RD, &(queues[q].tso), "TSO"); - SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "tx_dma_setup", - CTLFLAG_RD, &(queues[q].tx_dma_setup), + SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "tx_dmamap_failed", + CTLFLAG_RD, &(queues[q].tx_dmamap_failed), "Driver tx dma failure in xmit"); SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "no_desc_avail", CTLFLAG_RD, &(txr->no_desc), @@ -2886,7 +2959,14 @@ ixlv_add_sysctls(struct ixlv_sc *sc) SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_bytes", CTLFLAG_RD, &(rxr->rx_bytes), "Queue Bytes Received"); - + SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rx_itr", + CTLFLAG_RD, &(rxr->itr), 0, + "Queue Rx ITR Interval"); + SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "tx_itr", + CTLFLAG_RD, &(txr->itr), 0, + "Queue Tx ITR Interval"); + +#ifdef IXL_DEBUG /* Examine queue state */ SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "qtx_head", CTLTYPE_UINT | CTLFLAG_RD, &queues[q], @@ -2898,6 +2978,7 @@ ixlv_add_sysctls(struct ixlv_sc *sc) sizeof(struct ixl_queue), ixlv_sysctl_qrx_tail_handler, "IU", "Queue Receive Descriptor Tail"); +#endif } } @@ -2932,6 +3013,7 @@ ixlv_free_filters(struct ixlv_sc *sc) return; } +#ifdef IXL_DEBUG /** * ixlv_sysctl_qtx_tail_handler * Retrieves I40E_QTX_TAIL1 value from hardware @@ -2975,4 +3057,5 @@ ixlv_sysctl_qrx_tail_handler(SYSCTL_HANDLER_ARGS) return error; return (0); } +#endif diff --git a/sys/dev/ixl/ixl.h b/sys/dev/ixl/ixl.h index 68fd3f18c4b6..1a92edbc15e7 100644 --- a/sys/dev/ixl/ixl.h +++ b/sys/dev/ixl/ixl.h @@ -36,6 +36,9 @@ #ifndef _IXL_H_ #define _IXL_H_ +#include "opt_inet.h" +#include "opt_inet6.h" +#include "opt_rss.h" #include <sys/param.h> #include <sys/systm.h> @@ -90,29 +93,23 @@ #include <sys/smp.h> #include <sys/sbuf.h> #include <machine/smp.h> +#include <machine/stdarg.h> -#ifdef PCI_IOV -#include <sys/nv.h> -#include <sys/iov_schema.h> -#include <dev/pci/pci_iov.h> +#ifdef RSS +#include <net/rss_config.h> +#include <netinet/in_rss.h> #endif #include "i40e_type.h" #include "i40e_prototype.h" -#if defined(IXL_DEBUG) || defined(IXL_DEBUG_SYSCTL) #define MAC_FORMAT "%02x:%02x:%02x:%02x:%02x:%02x" #define MAC_FORMAT_ARGS(mac_addr) \ (mac_addr)[0], (mac_addr)[1], (mac_addr)[2], (mac_addr)[3], \ (mac_addr)[4], (mac_addr)[5] #define ON_OFF_STR(is_set) ((is_set) ? "On" : "Off") -#endif /* IXL_DEBUG || IXL_DEBUG_SYSCTL */ #ifdef IXL_DEBUG -/* Enable debug sysctls */ -#ifndef IXL_DEBUG_SYSCTL -#define IXL_DEBUG_SYSCTL 1 -#endif #define _DBG_PRINTF(S, ...) printf("%s: " S "\n", __func__, ##__VA_ARGS__) #define _DEV_DBG_PRINTF(dev, S, ...) device_printf(dev, "%s: " S "\n", __func__, ##__VA_ARGS__) @@ -157,6 +154,26 @@ #define HW_DEBUGOUT(...) #endif /* IXL_DEBUG */ +enum ixl_dbg_mask { + IXL_DBG_INFO = 0x00000001, + IXL_DBG_EN_DIS = 0x00000002, + IXL_DBG_AQ = 0x00000004, + IXL_DBG_NVMUPD = 0x00000008, + + IXL_DBG_IOCTL_KNOWN = 0x00000010, + IXL_DBG_IOCTL_UNKNOWN = 0x00000020, + IXL_DBG_IOCTL_ALL = 0x00000030, + + I40E_DEBUG_RSS = 0x00000100, + + IXL_DBG_IOV = 0x00001000, + IXL_DBG_IOV_VC = 0x00002000, + + IXL_DBG_SWITCH_INFO = 0x00010000, + + IXL_DBG_ALL = 0xFFFFFFFF +}; + /* Tunables */ /* @@ -167,27 +184,28 @@ * Tx descriptors are always 16 bytes, but Rx descriptors can be 32 bytes. * The driver currently always uses 32 byte Rx descriptors. */ -#define DEFAULT_RING 1024 -#define PERFORM_RING 2048 -#define MAX_RING 4096 -#define MIN_RING 32 +#define DEFAULT_RING 1024 +#define IXL_MAX_RING 8160 +#define IXL_MIN_RING 32 +#define IXL_RING_INCREMENT 32 + +#define IXL_AQ_LEN 256 +#define IXL_AQ_LEN_MAX 1024 /* ** Default number of entries in Tx queue buf_ring. */ -#define SMALL_TXBRSZ 4096 -/* This may require mbuf cluster tuning */ -#define DEFAULT_TXBRSZ (SMALL_TXBRSZ * SMALL_TXBRSZ) +#define DEFAULT_TXBRSZ 4096 /* Alignment for rings */ -#define DBA_ALIGN 128 +#define DBA_ALIGN 128 /* * This is the max watchdog interval, ie. the time that can * pass between any two TX clean operations, such only happening * when the TX hardware is functioning. */ -#define IXL_WATCHDOG (10 * hz) +#define IXL_WATCHDOG (10 * hz) /* * This parameters control when the driver calls the routine to reclaim @@ -196,11 +214,6 @@ #define IXL_TX_CLEANUP_THRESHOLD (que->num_desc / 8) #define IXL_TX_OP_THRESHOLD (que->num_desc / 32) -/* Flow control constants */ -#define IXL_FC_PAUSE 0xFFFF -#define IXL_FC_HI 0x20000 -#define IXL_FC_LO 0x10000 - #define MAX_MULTICAST_ADDR 128 #define IXL_BAR 3 @@ -208,10 +221,6 @@ #define IXL_TSO_SIZE 65535 #define IXL_AQ_BUF_SZ ((u32) 4096) #define IXL_RX_HDR 128 -/* Controls the length of the Admin Queue */ -#define IXL_AQ_LEN 256 -#define IXL_AQ_LEN_MAX 1024 -#define IXL_AQ_BUFSZ 4096 #define IXL_RX_LIMIT 512 #define IXL_RX_ITR 0 #define IXL_TX_ITR 1 @@ -219,25 +228,29 @@ #define IXL_QUEUE_EOL 0x7FF #define IXL_MAX_FRAME 9728 #define IXL_MAX_TX_SEGS 8 -#define IXL_MAX_TSO_SEGS 66 +#define IXL_MAX_TSO_SEGS 128 #define IXL_SPARSE_CHAIN 6 #define IXL_QUEUE_HUNG 0x80000000 -#define IXL_KEYSZ 10 + +#define IXL_RSS_KEY_SIZE_REG 13 +#define IXL_RSS_KEY_SIZE (IXL_RSS_KEY_SIZE_REG * 4) +#define IXL_RSS_VSI_LUT_SIZE 64 /* X722 -> VSI, X710 -> VF */ +#define IXL_RSS_VSI_LUT_ENTRY_MASK 0x3F +#define IXL_RSS_VF_LUT_ENTRY_MASK 0xF #define IXL_VF_MAX_BUFFER 0x3F80 #define IXL_VF_MAX_HDR_BUFFER 0x840 #define IXL_VF_MAX_FRAME 0x3FFF -/* ERJ: hardware can support ~1.5k filters between all functions */ -#define IXL_MAX_FILTERS 256 -#define IXL_MAX_TX_BUSY 10 +/* ERJ: hardware can support ~2k (SW5+) filters between all functions */ +#define IXL_MAX_FILTERS 256 +#define IXL_MAX_TX_BUSY 10 #define IXL_NVM_VERSION_LO_SHIFT 0 #define IXL_NVM_VERSION_LO_MASK (0xff << IXL_NVM_VERSION_LO_SHIFT) #define IXL_NVM_VERSION_HI_SHIFT 12 #define IXL_NVM_VERSION_HI_MASK (0xf << IXL_NVM_VERSION_HI_SHIFT) - /* * Interrupt Moderation parameters */ @@ -298,6 +311,19 @@ #define IXL_END_OF_INTR_LNKLST 0x7FF +#define IXL_DEFAULT_RSS_HENA (\ + BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | \ + BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP) | \ + BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) | \ + BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) | \ + BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV4) | \ + BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_UDP) | \ + BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP) | \ + BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) | \ + BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) | \ + BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV6) | \ + BIT_ULL(I40E_FILTER_PCTYPE_L2_PAYLOAD)) + #define IXL_TX_LOCK(_sc) mtx_lock(&(_sc)->mtx) #define IXL_TX_UNLOCK(_sc) mtx_unlock(&(_sc)->mtx) #define IXL_TX_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->mtx) @@ -337,11 +363,6 @@ #define IXL_SET_NOPROTO(vsi, count) (vsi)->noproto = (count) #endif -/* Pre-10.2 media type compatibility */ -#if __FreeBSD_version < 1002000 -#define IFM_OTHER IFM_UNKNOWN -#endif - /* ***************************************************************************** * vendor_info_array @@ -480,45 +501,40 @@ struct ixl_queue { u64 mbuf_defrag_failed; u64 mbuf_hdr_failed; u64 mbuf_pkt_failed; - u64 tx_map_avail; - u64 tx_dma_setup; + u64 tx_dmamap_failed; u64 dropped_pkts; }; /* -** Virtual Station interface: -** there would be one of these per traffic class/type -** for now just one, and its embedded in the pf +** Virtual Station Interface */ SLIST_HEAD(ixl_ftl_head, ixl_mac_filter); struct ixl_vsi { void *back; struct ifnet *ifp; - struct device *dev; + device_t dev; struct i40e_hw *hw; struct ifmedia media; enum i40e_vsi_type type; - u64 que_mask; int id; - u16 vsi_num; - u16 msix_base; /* station base MSIX vector */ - u16 first_queue; u16 num_queues; u32 rx_itr_setting; u32 tx_itr_setting; + u16 max_frame_size; + struct ixl_queue *queues; /* head of queues */ + + u16 vsi_num; bool link_active; u16 seid; u16 uplink_seid; u16 downlink_seid; - u16 max_frame_size; - u16 rss_table_size; - u16 rss_size; /* MAC/VLAN Filter list */ struct ixl_ftl_head ftl; u16 num_macs; + /* Contains readylist & stat counter id */ struct i40e_aqc_vsi_properties_data info; eventhandler_tag vlan_attach; @@ -565,7 +581,7 @@ ixl_rx_unrefreshed(struct ixl_queue *que) else return ((que->num_desc + rxr->next_check) - rxr->next_refresh - 1); -} +} /* ** Find the next available unused filter @@ -601,6 +617,28 @@ cmp_etheraddr(const u8 *ea1, const u8 *ea2) } /* + * Return next largest power of 2, unsigned + * + * Public domain, from Bit Twiddling Hacks + */ +static inline u32 +next_power_of_two(u32 n) +{ + n--; + n |= n >> 1; + n |= n >> 2; + n |= n >> 4; + n |= n >> 8; + n |= n >> 16; + n++; + + /* Next power of two > 0 is 1 */ + n += (n == 0); + + return (n); +} + +/* * Info for stats sysctls */ struct ixl_sysctl_info { @@ -609,7 +647,8 @@ struct ixl_sysctl_info { char *description; }; -extern int ixl_atr_rate; +static uint8_t ixl_bcast_addr[ETHER_ADDR_LEN] = + {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; /********************************************************************* * TXRX Function prototypes @@ -620,18 +659,18 @@ void ixl_init_tx_ring(struct ixl_queue *); int ixl_init_rx_ring(struct ixl_queue *); bool ixl_rxeof(struct ixl_queue *, int); bool ixl_txeof(struct ixl_queue *); +void ixl_free_que_tx(struct ixl_queue *); +void ixl_free_que_rx(struct ixl_queue *); + int ixl_mq_start(struct ifnet *, struct mbuf *); int ixl_mq_start_locked(struct ifnet *, struct tx_ring *); void ixl_deferred_mq_start(void *, int); -void ixl_qflush(struct ifnet *); void ixl_free_vsi(struct ixl_vsi *); -void ixl_free_que_tx(struct ixl_queue *); -void ixl_free_que_rx(struct ixl_queue *); -#ifdef IXL_FDIR -void ixl_atr(struct ixl_queue *, struct tcphdr *, int); -#endif +void ixl_qflush(struct ifnet *); + +/* Common function prototypes between PF/VF driver */ #if __FreeBSD_version >= 1100000 uint64_t ixl_get_counter(if_t ifp, ift_counter cnt); #endif - +void ixl_get_default_rss_key(u32 *); #endif /* _IXL_H_ */ diff --git a/sys/dev/ixl/ixl_pf.h b/sys/dev/ixl/ixl_pf.h index 65915bea1544..107e30fded5d 100644 --- a/sys/dev/ixl/ixl_pf.h +++ b/sys/dev/ixl/ixl_pf.h @@ -36,6 +36,9 @@ #ifndef _IXL_PF_H_ #define _IXL_PF_H_ +#include "ixl.h" +#include "ixl_pf_qmgr.h" + #define VF_FLAG_ENABLED 0x01 #define VF_FLAG_SET_MAC_CAP 0x02 #define VF_FLAG_VLAN_CAP 0x04 @@ -52,6 +55,7 @@ struct ixl_vf { uint16_t vf_num; uint32_t version; + struct ixl_pf_qtag qtag; struct sysctl_ctx_list ctx; }; @@ -59,7 +63,8 @@ struct ixl_vf { struct ixl_pf { struct i40e_hw hw; struct i40e_osdep osdep; - struct device *dev; + device_t dev; + struct ixl_vsi vsi; struct resource *pci_mem; struct resource *msix_mem; @@ -77,6 +82,19 @@ struct ixl_pf { int if_flags; int state; + struct ixl_pf_qmgr qmgr; + struct ixl_pf_qtag qtag; + + /* Tunable values */ + bool enable_msix; + int max_queues; + int ringsz; + bool enable_tx_fc_filter; + int dynamic_rx_itr; + int dynamic_tx_itr; + int tx_itr; + int rx_itr; + struct mtx pf_mtx; u32 qbase; @@ -87,17 +105,8 @@ struct ixl_pf { bool link_up; u32 link_speed; int advertised_speed; - int fc; /* local flow ctrl setting */ - - /* - ** Network interfaces - ** These are the traffic class holders, and - ** will have a stack interface and queues - ** associated with them. - ** NOTE: The PF has only a single interface, - ** so it is embedded in the PF struct. - */ - struct ixl_vsi vsi; + int fc; /* link flow ctrl setting */ + enum ixl_dbg_mask dbg_mask; /* Misc stats maintained by the driver */ u64 watchdog_events; @@ -108,6 +117,7 @@ struct ixl_pf { struct i40e_hw_port_stats stats_offsets; bool stat_offsets_loaded; + /* SR-IOV */ struct ixl_vf *vfs; int num_vfs; uint16_t veb_seid; @@ -149,42 +159,18 @@ struct ixl_pf { "\nExecutes a \"Get Link Status\" command on the Admin Queue, and displays" \ " the response." \ -/*** Functions / Macros ***/ +static char *ixl_fc_string[6] = { + "None", + "Rx", + "Tx", + "Full", + "Priority", + "Default" +}; -/* -** Put the NVM, EEtrackID, and OEM version information into a string -*/ -static void -ixl_nvm_version_str(struct i40e_hw *hw, struct sbuf *buf) -{ - u8 oem_ver = (u8)(hw->nvm.oem_ver >> 24); - u16 oem_build = (u16)((hw->nvm.oem_ver >> 16) & 0xFFFF); - u8 oem_patch = (u8)(hw->nvm.oem_ver & 0xFF); - - sbuf_printf(buf, - "nvm %x.%02x etid %08x oem %d.%d.%d", - (hw->nvm.version & IXL_NVM_VERSION_HI_MASK) >> - IXL_NVM_VERSION_HI_SHIFT, - (hw->nvm.version & IXL_NVM_VERSION_LO_MASK) >> - IXL_NVM_VERSION_LO_SHIFT, - hw->nvm.eetrack, - oem_ver, oem_build, oem_patch); -} - -static void -ixl_print_nvm_version(struct ixl_pf *pf) -{ - struct i40e_hw *hw = &pf->hw; - device_t dev = pf->dev; - struct sbuf *sbuf; - - sbuf = sbuf_new_auto(); - ixl_nvm_version_str(hw, sbuf); - sbuf_finish(sbuf); - device_printf(dev, "%s\n", sbuf_data(sbuf)); - sbuf_delete(sbuf); -} +static MALLOC_DEFINE(M_IXL, "ixl", "ixl driver allocations"); +/*** Functions / Macros ***/ #define I40E_VC_DEBUG(pf, level, ...) \ do { \ if ((pf)->vc_debug_lvl >= (level)) \ @@ -201,4 +187,136 @@ ixl_print_nvm_version(struct ixl_pf *pf) #define IXL_PF_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->pf_mtx) #define IXL_PF_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->pf_mtx, MA_OWNED) +/* For stats sysctl naming */ +#define QUEUE_NAME_LEN 32 + +/* + * PF-only function declarations + */ + +void ixl_set_busmaster(device_t); +int ixl_setup_interface(device_t, struct ixl_vsi *); +void ixl_print_nvm_cmd(device_t, struct i40e_nvm_access *); + +void ixl_handle_que(void *context, int pending); + +void ixl_init(void *); +void ixl_local_timer(void *); +void ixl_register_vlan(void *, struct ifnet *, u16); +void ixl_unregister_vlan(void *, struct ifnet *, u16); +void ixl_intr(void *); +void ixl_msix_que(void *); +void ixl_msix_adminq(void *); +void ixl_do_adminq(void *, int); + +int ixl_res_alloc_cmp(const void *, const void *); +char * ixl_switch_res_type_string(u8); +char * ixl_switch_element_string(struct sbuf *, + struct i40e_aqc_switch_config_element_resp *); +void ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *, + struct sysctl_oid_list *, struct i40e_hw_port_stats *); +void ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *, + struct sysctl_oid_list *, + struct i40e_eth_stats *); + +void ixl_media_status(struct ifnet *, struct ifmediareq *); +int ixl_media_change(struct ifnet *); +int ixl_ioctl(struct ifnet *, u_long, caddr_t); + +void ixl_enable_adminq(struct i40e_hw *); +void ixl_get_bus_info(struct i40e_hw *, device_t); +void ixl_disable_adminq(struct i40e_hw *); +void ixl_enable_queue(struct i40e_hw *, int); +void ixl_disable_queue(struct i40e_hw *, int); +void ixl_enable_legacy(struct i40e_hw *); +void ixl_disable_legacy(struct i40e_hw *); +void ixl_nvm_version_str(struct i40e_hw *hw, struct sbuf *buf); +void ixl_stat_update48(struct i40e_hw *, u32, u32, bool, + u64 *, u64 *); +void ixl_stat_update32(struct i40e_hw *, u32, bool, + u64 *, u64 *); + +void ixl_stop(struct ixl_pf *); +void ixl_add_vsi_sysctls(struct ixl_pf *pf, struct ixl_vsi *vsi, struct sysctl_ctx_list *ctx, const char *sysctl_name); +int ixl_get_hw_capabilities(struct ixl_pf *); +void ixl_update_link_status(struct ixl_pf *); +int ixl_allocate_pci_resources(struct ixl_pf *); +int ixl_setup_stations(struct ixl_pf *); +int ixl_switch_config(struct ixl_pf *); +void ixl_stop_locked(struct ixl_pf *); +int ixl_teardown_hw_structs(struct ixl_pf *); +int ixl_reset(struct ixl_pf *); +void ixl_init_locked(struct ixl_pf *); +void ixl_set_rss_key(struct ixl_pf *); +void ixl_set_rss_pctypes(struct ixl_pf *); +void ixl_set_rss_hlut(struct ixl_pf *); +int ixl_setup_adminq_msix(struct ixl_pf *); +int ixl_setup_adminq_tq(struct ixl_pf *); +int ixl_teardown_adminq_msix(struct ixl_pf *); +void ixl_configure_intr0_msix(struct ixl_pf *); +void ixl_configure_queue_intr_msix(struct ixl_pf *); +void ixl_free_adminq_tq(struct ixl_pf *); +int ixl_assign_vsi_legacy(struct ixl_pf *); +int ixl_init_msix(struct ixl_pf *); +void ixl_configure_itr(struct ixl_pf *); +void ixl_configure_legacy(struct ixl_pf *); +void ixl_free_pci_resources(struct ixl_pf *); +void ixl_link_event(struct ixl_pf *, struct i40e_arq_event_info *); +void ixl_config_rss(struct ixl_pf *); +int ixl_set_advertised_speeds(struct ixl_pf *, int); +void ixl_get_initial_advertised_speeds(struct ixl_pf *); +void ixl_print_nvm_version(struct ixl_pf *pf); +void ixl_add_device_sysctls(struct ixl_pf *); +void ixl_handle_mdd_event(struct ixl_pf *); +void ixl_add_hw_stats(struct ixl_pf *); +void ixl_update_stats_counters(struct ixl_pf *); +void ixl_pf_reset_stats(struct ixl_pf *); +void ixl_dbg(struct ixl_pf *, enum ixl_dbg_mask, char *, ...); + +int ixl_handle_nvmupd_cmd(struct ixl_pf *, struct ifdrv *); +void ixl_handle_empr_reset(struct ixl_pf *); +int ixl_rebuild_hw_structs_after_reset(struct ixl_pf *); + +void ixl_set_queue_rx_itr(struct ixl_queue *); +void ixl_set_queue_tx_itr(struct ixl_queue *); + +void ixl_add_filter(struct ixl_vsi *, u8 *, s16 vlan); +void ixl_del_filter(struct ixl_vsi *, u8 *, s16 vlan); +void ixl_reconfigure_filters(struct ixl_vsi *vsi); + +int ixl_disable_rings(struct ixl_vsi *); +int ixl_disable_tx_ring(struct ixl_pf *, struct ixl_pf_qtag *, u16); +int ixl_disable_rx_ring(struct ixl_pf *, struct ixl_pf_qtag *, u16); +int ixl_disable_ring(struct ixl_pf *pf, struct ixl_pf_qtag *, u16); + +int ixl_enable_rings(struct ixl_vsi *); +int ixl_enable_tx_ring(struct ixl_pf *, struct ixl_pf_qtag *, u16); +int ixl_enable_rx_ring(struct ixl_pf *, struct ixl_pf_qtag *, u16); +int ixl_enable_ring(struct ixl_pf *pf, struct ixl_pf_qtag *, u16); + +void ixl_update_eth_stats(struct ixl_vsi *); +void ixl_disable_intr(struct ixl_vsi *); +void ixl_cap_txcsum_tso(struct ixl_vsi *, struct ifnet *, int); +int ixl_initialize_vsi(struct ixl_vsi *); +void ixl_add_ifmedia(struct ixl_vsi *, u32); +int ixl_setup_queue_msix(struct ixl_vsi *); +int ixl_setup_queue_tqs(struct ixl_vsi *); +int ixl_teardown_queue_msix(struct ixl_vsi *); +void ixl_free_queue_tqs(struct ixl_vsi *); +void ixl_enable_intr(struct ixl_vsi *); +void ixl_disable_rings_intr(struct ixl_vsi *); +void ixl_set_promisc(struct ixl_vsi *); +void ixl_add_multi(struct ixl_vsi *); +void ixl_del_multi(struct ixl_vsi *); +void ixl_setup_vlan_filters(struct ixl_vsi *); +void ixl_init_filters(struct ixl_vsi *); +void ixl_add_hw_filters(struct ixl_vsi *, int, int); +void ixl_del_hw_filters(struct ixl_vsi *, int); +struct ixl_mac_filter * + ixl_find_filter(struct ixl_vsi *, u8 *, s16); +void ixl_add_mc_filter(struct ixl_vsi *, u8 *); +void ixl_free_mac_filters(struct ixl_vsi *vsi); +void ixl_update_vsi_stats(struct ixl_vsi *); +void ixl_vsi_reset_stats(struct ixl_vsi *); + #endif /* _IXL_PF_H_ */ diff --git a/sys/dev/ixl/ixl_pf_iov.c b/sys/dev/ixl/ixl_pf_iov.c new file mode 100644 index 000000000000..a8c8b29cc605 --- /dev/null +++ b/sys/dev/ixl/ixl_pf_iov.c @@ -0,0 +1,1925 @@ +/****************************************************************************** + + Copyright (c) 2013-2015, Intel Corporation + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +******************************************************************************/ +/*$FreeBSD$*/ + +#include "ixl_pf_iov.h" + +/* Private functions */ +static void ixl_vf_map_vsi_queue(struct i40e_hw *hw, struct ixl_vf *vf, int qnum, uint32_t val); +static void ixl_vf_disable_queue_intr(struct i40e_hw *hw, uint32_t vfint_reg); +static void ixl_vf_unregister_intr(struct i40e_hw *hw, uint32_t vpint_reg); + +static bool ixl_zero_mac(const uint8_t *addr); +static bool ixl_bcast_mac(const uint8_t *addr); + +static const char * ixl_vc_opcode_str(uint16_t op); +static int ixl_vc_opcode_level(uint16_t opcode); + +static int ixl_vf_mac_valid(struct ixl_vf *vf, const uint8_t *addr); + +static int ixl_vf_alloc_vsi(struct ixl_pf *pf, struct ixl_vf *vf); +static int ixl_vf_setup_vsi(struct ixl_pf *pf, struct ixl_vf *vf); +static void ixl_vf_map_queues(struct ixl_pf *pf, struct ixl_vf *vf); +static void ixl_vf_vsi_release(struct ixl_pf *pf, struct ixl_vsi *vsi); +static void ixl_vf_release_resources(struct ixl_pf *pf, struct ixl_vf *vf); +static int ixl_flush_pcie(struct ixl_pf *pf, struct ixl_vf *vf); +static void ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf); +static void ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf); +static void ixl_send_vf_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op, enum i40e_status_code status, void *msg, uint16_t len); +static void ixl_send_vf_ack(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op); +static void ixl_send_vf_nack_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op, enum i40e_status_code status, const char *file, int line); +static void ixl_vf_version_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, uint16_t msg_size); +static void ixl_vf_reset_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, uint16_t msg_size); +static void ixl_vf_get_resources_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, uint16_t msg_size); +static int ixl_vf_config_tx_queue(struct ixl_pf *pf, struct ixl_vf *vf, struct i40e_virtchnl_txq_info *info); +static int ixl_vf_config_rx_queue(struct ixl_pf *pf, struct ixl_vf *vf, struct i40e_virtchnl_rxq_info *info); +static void ixl_vf_config_vsi_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, uint16_t msg_size); +static void ixl_vf_set_qctl(struct ixl_pf *pf, const struct i40e_virtchnl_vector_map *vector, enum i40e_queue_type cur_type, uint16_t cur_queue, + enum i40e_queue_type *last_type, uint16_t *last_queue); +static void ixl_vf_config_vector(struct ixl_pf *pf, struct ixl_vf *vf, const struct i40e_virtchnl_vector_map *vector); +static void ixl_vf_config_irq_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, uint16_t msg_size); +static void ixl_vf_enable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, uint16_t msg_size); +static void ixl_vf_disable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, uint16_t msg_size); +static void ixl_vf_add_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, uint16_t msg_size); +static void ixl_vf_del_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, uint16_t msg_size); +static enum i40e_status_code ixl_vf_enable_vlan_strip(struct ixl_pf *pf, struct ixl_vf *vf); +static void ixl_vf_add_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, uint16_t msg_size); +static void ixl_vf_del_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, uint16_t msg_size); +static void ixl_vf_config_promisc_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, uint16_t msg_size); +static void ixl_vf_get_stats_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, uint16_t msg_size); +static int ixl_vf_reserve_queues(struct ixl_pf *pf, struct ixl_vf *vf, int num_queues); + +static int ixl_adminq_err_to_errno(enum i40e_admin_queue_err err); + +void +ixl_initialize_sriov(struct ixl_pf *pf) +{ + device_t dev = pf->dev; + struct i40e_hw *hw = &pf->hw; + nvlist_t *pf_schema, *vf_schema; + int iov_error; + + /* SR-IOV is only supported when MSI-X is in use. */ + if (pf->msix <= 1) + return; + + pf_schema = pci_iov_schema_alloc_node(); + vf_schema = pci_iov_schema_alloc_node(); + pci_iov_schema_add_unicast_mac(vf_schema, "mac-addr", 0, NULL); + pci_iov_schema_add_bool(vf_schema, "mac-anti-spoof", + IOV_SCHEMA_HASDEFAULT, TRUE); + pci_iov_schema_add_bool(vf_schema, "allow-set-mac", + IOV_SCHEMA_HASDEFAULT, FALSE); + pci_iov_schema_add_bool(vf_schema, "allow-promisc", + IOV_SCHEMA_HASDEFAULT, FALSE); + pci_iov_schema_add_uint16(vf_schema, "num-queues", + IOV_SCHEMA_HASDEFAULT, + max(1, hw->func_caps.num_msix_vectors_vf - 1) % IXLV_MAX_QUEUES); + + iov_error = pci_iov_attach(dev, pf_schema, vf_schema); + if (iov_error != 0) { + device_printf(dev, + "Failed to initialize SR-IOV (error=%d)\n", + iov_error); + } else + device_printf(dev, "SR-IOV ready\n"); + + pf->vc_debug_lvl = 1; +} + +/* + * Allocate the VSI for a VF. + */ +static int +ixl_vf_alloc_vsi(struct ixl_pf *pf, struct ixl_vf *vf) +{ + device_t dev; + struct i40e_hw *hw; + struct ixl_vsi *vsi; + struct i40e_vsi_context vsi_ctx; + int i; + enum i40e_status_code code; + + hw = &pf->hw; + vsi = &pf->vsi; + dev = pf->dev; + + vsi_ctx.pf_num = hw->pf_id; + vsi_ctx.uplink_seid = pf->veb_seid; + vsi_ctx.connection_type = IXL_VSI_DATA_PORT; + vsi_ctx.vf_num = hw->func_caps.vf_base_id + vf->vf_num; + vsi_ctx.flags = I40E_AQ_VSI_TYPE_VF; + + bzero(&vsi_ctx.info, sizeof(vsi_ctx.info)); + + vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_SWITCH_VALID); + vsi_ctx.info.switch_id = htole16(0); + + vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_SECURITY_VALID); + vsi_ctx.info.sec_flags = 0; + if (vf->vf_flags & VF_FLAG_MAC_ANTI_SPOOF) + vsi_ctx.info.sec_flags |= I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK; + + vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_VLAN_VALID); + vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | + I40E_AQ_VSI_PVLAN_EMOD_NOTHING; + + vsi_ctx.info.valid_sections |= + htole16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID); + vsi_ctx.info.mapping_flags = htole16(I40E_AQ_VSI_QUE_MAP_NONCONTIG); + + /* ERJ: Only scattered allocation is supported for VFs right now */ + for (i = 0; i < vf->qtag.num_active; i++) + vsi_ctx.info.queue_mapping[i] = vf->qtag.qidx[i]; + for (; i < nitems(vsi_ctx.info.queue_mapping); i++) + vsi_ctx.info.queue_mapping[i] = htole16(I40E_AQ_VSI_QUEUE_MASK); + + vsi_ctx.info.tc_mapping[0] = htole16( + (0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | + (bsrl(vf->qtag.num_allocated) << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT)); + + code = i40e_aq_add_vsi(hw, &vsi_ctx, NULL); + if (code != I40E_SUCCESS) + return (ixl_adminq_err_to_errno(hw->aq.asq_last_status)); + vf->vsi.seid = vsi_ctx.seid; + vf->vsi.vsi_num = vsi_ctx.vsi_number; + // vf->vsi.first_queue = vf->qtag.qidx[0]; + vf->vsi.num_queues = vf->qtag.num_active; + + code = i40e_aq_get_vsi_params(hw, &vsi_ctx, NULL); + if (code != I40E_SUCCESS) + return (ixl_adminq_err_to_errno(hw->aq.asq_last_status)); + + code = i40e_aq_config_vsi_bw_limit(hw, vf->vsi.seid, 0, 0, NULL); + if (code != I40E_SUCCESS) { + device_printf(dev, "Failed to disable BW limit: %d\n", + ixl_adminq_err_to_errno(hw->aq.asq_last_status)); + return (ixl_adminq_err_to_errno(hw->aq.asq_last_status)); + } + + memcpy(&vf->vsi.info, &vsi_ctx.info, sizeof(vf->vsi.info)); + return (0); +} + +static int +ixl_vf_setup_vsi(struct ixl_pf *pf, struct ixl_vf *vf) +{ + struct i40e_hw *hw; + int error; + + hw = &pf->hw; + + error = ixl_vf_alloc_vsi(pf, vf); + if (error != 0) + return (error); + + vf->vsi.hw_filters_add = 0; + vf->vsi.hw_filters_del = 0; + ixl_add_filter(&vf->vsi, ixl_bcast_addr, IXL_VLAN_ANY); + ixl_reconfigure_filters(&vf->vsi); + + return (0); +} + +static void +ixl_vf_map_vsi_queue(struct i40e_hw *hw, struct ixl_vf *vf, int qnum, + uint32_t val) +{ + uint32_t qtable; + int index, shift; + + /* + * Two queues are mapped in a single register, so we have to do some + * gymnastics to convert the queue number into a register index and + * shift. + */ + index = qnum / 2; + shift = (qnum % 2) * I40E_VSILAN_QTABLE_QINDEX_1_SHIFT; + + qtable = i40e_read_rx_ctl(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num)); + qtable &= ~(I40E_VSILAN_QTABLE_QINDEX_0_MASK << shift); + qtable |= val << shift; + i40e_write_rx_ctl(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num), qtable); +} + +static void +ixl_vf_map_queues(struct ixl_pf *pf, struct ixl_vf *vf) +{ + struct i40e_hw *hw; + uint32_t qtable; + int i; + + hw = &pf->hw; + + /* + * Contiguous mappings aren't actually supported by the hardware, + * so we have to use non-contiguous mappings. + */ + i40e_write_rx_ctl(hw, I40E_VSILAN_QBASE(vf->vsi.vsi_num), + I40E_VSILAN_QBASE_VSIQTABLE_ENA_MASK); + + /* Enable LAN traffic on this VF */ + wr32(hw, I40E_VPLAN_MAPENA(vf->vf_num), + I40E_VPLAN_MAPENA_TXRX_ENA_MASK); + + /* Program index of each VF queue into PF queue space + * (This is only needed if QTABLE is enabled) */ + for (i = 0; i < vf->vsi.num_queues; i++) { + qtable = ixl_pf_qidx_from_vsi_qidx(&vf->qtag, i) << + I40E_VPLAN_QTABLE_QINDEX_SHIFT; + + wr32(hw, I40E_VPLAN_QTABLE(i, vf->vf_num), qtable); + } + for (; i < IXL_MAX_VSI_QUEUES; i++) + wr32(hw, I40E_VPLAN_QTABLE(i, vf->vf_num), + I40E_VPLAN_QTABLE_QINDEX_MASK); + + /* Map queues allocated to VF to its VSI; + * This mapping matches the VF-wide mapping since the VF + * is only given a single VSI */ + for (i = 0; i < vf->vsi.num_queues; i++) + ixl_vf_map_vsi_queue(hw, vf, i, + ixl_pf_qidx_from_vsi_qidx(&vf->qtag, i)); + + /* Set rest of VSI queues as unused. */ + for (; i < IXL_MAX_VSI_QUEUES; i++) + ixl_vf_map_vsi_queue(hw, vf, i, + I40E_VSILAN_QTABLE_QINDEX_0_MASK); + + ixl_flush(hw); +} + +static void +ixl_vf_vsi_release(struct ixl_pf *pf, struct ixl_vsi *vsi) +{ + struct i40e_hw *hw; + + hw = &pf->hw; + + if (vsi->seid == 0) + return; + + i40e_aq_delete_element(hw, vsi->seid, NULL); +} + +static void +ixl_vf_disable_queue_intr(struct i40e_hw *hw, uint32_t vfint_reg) +{ + + wr32(hw, vfint_reg, I40E_VFINT_DYN_CTLN_CLEARPBA_MASK); + ixl_flush(hw); +} + +static void +ixl_vf_unregister_intr(struct i40e_hw *hw, uint32_t vpint_reg) +{ + + wr32(hw, vpint_reg, I40E_VPINT_LNKLSTN_FIRSTQ_TYPE_MASK | + I40E_VPINT_LNKLSTN_FIRSTQ_INDX_MASK); + ixl_flush(hw); +} + +static void +ixl_vf_release_resources(struct ixl_pf *pf, struct ixl_vf *vf) +{ + struct i40e_hw *hw; + uint32_t vfint_reg, vpint_reg; + int i; + + hw = &pf->hw; + + ixl_vf_vsi_release(pf, &vf->vsi); + + /* Index 0 has a special register. */ + ixl_vf_disable_queue_intr(hw, I40E_VFINT_DYN_CTL0(vf->vf_num)); + + for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) { + vfint_reg = IXL_VFINT_DYN_CTLN_REG(hw, i , vf->vf_num); + ixl_vf_disable_queue_intr(hw, vfint_reg); + } + + /* Index 0 has a special register. */ + ixl_vf_unregister_intr(hw, I40E_VPINT_LNKLST0(vf->vf_num)); + + for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) { + vpint_reg = IXL_VPINT_LNKLSTN_REG(hw, i, vf->vf_num); + ixl_vf_unregister_intr(hw, vpint_reg); + } + + vf->vsi.num_queues = 0; +} + +static int +ixl_flush_pcie(struct ixl_pf *pf, struct ixl_vf *vf) +{ + struct i40e_hw *hw; + int i; + uint16_t global_vf_num; + uint32_t ciad; + + hw = &pf->hw; + global_vf_num = hw->func_caps.vf_base_id + vf->vf_num; + + wr32(hw, I40E_PF_PCI_CIAA, IXL_PF_PCI_CIAA_VF_DEVICE_STATUS | + (global_vf_num << I40E_PF_PCI_CIAA_VF_NUM_SHIFT)); + for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) { + ciad = rd32(hw, I40E_PF_PCI_CIAD); + if ((ciad & IXL_PF_PCI_CIAD_VF_TRANS_PENDING_MASK) == 0) + return (0); + DELAY(1); + } + + return (ETIMEDOUT); +} + +static void +ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf) +{ + struct i40e_hw *hw; + uint32_t vfrtrig; + + hw = &pf->hw; + + vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num)); + vfrtrig |= I40E_VPGEN_VFRTRIG_VFSWR_MASK; + wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig); + ixl_flush(hw); + + ixl_reinit_vf(pf, vf); +} + +static void +ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf) +{ + struct i40e_hw *hw; + uint32_t vfrstat, vfrtrig; + int i, error; + + hw = &pf->hw; + + error = ixl_flush_pcie(pf, vf); + if (error != 0) + device_printf(pf->dev, + "Timed out waiting for PCIe activity to stop on VF-%d\n", + vf->vf_num); + + for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) { + DELAY(10); + + vfrstat = rd32(hw, I40E_VPGEN_VFRSTAT(vf->vf_num)); + if (vfrstat & I40E_VPGEN_VFRSTAT_VFRD_MASK) + break; + } + + if (i == IXL_VF_RESET_TIMEOUT) + device_printf(pf->dev, "VF %d failed to reset\n", vf->vf_num); + + wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_COMPLETED); + + vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num)); + vfrtrig &= ~I40E_VPGEN_VFRTRIG_VFSWR_MASK; + wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig); + + if (vf->vsi.seid != 0) + ixl_disable_rings(&vf->vsi); + + ixl_vf_release_resources(pf, vf); + ixl_vf_setup_vsi(pf, vf); + ixl_vf_map_queues(pf, vf); + + wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_VFACTIVE); + ixl_flush(hw); +} + +static const char * +ixl_vc_opcode_str(uint16_t op) +{ + + switch (op) { + case I40E_VIRTCHNL_OP_VERSION: + return ("VERSION"); + case I40E_VIRTCHNL_OP_RESET_VF: + return ("RESET_VF"); + case I40E_VIRTCHNL_OP_GET_VF_RESOURCES: + return ("GET_VF_RESOURCES"); + case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE: + return ("CONFIG_TX_QUEUE"); + case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE: + return ("CONFIG_RX_QUEUE"); + case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES: + return ("CONFIG_VSI_QUEUES"); + case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP: + return ("CONFIG_IRQ_MAP"); + case I40E_VIRTCHNL_OP_ENABLE_QUEUES: + return ("ENABLE_QUEUES"); + case I40E_VIRTCHNL_OP_DISABLE_QUEUES: + return ("DISABLE_QUEUES"); + case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS: + return ("ADD_ETHER_ADDRESS"); + case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS: + return ("DEL_ETHER_ADDRESS"); + case I40E_VIRTCHNL_OP_ADD_VLAN: + return ("ADD_VLAN"); + case I40E_VIRTCHNL_OP_DEL_VLAN: + return ("DEL_VLAN"); + case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE: + return ("CONFIG_PROMISCUOUS_MODE"); + case I40E_VIRTCHNL_OP_GET_STATS: + return ("GET_STATS"); + case I40E_VIRTCHNL_OP_FCOE: + return ("FCOE"); + case I40E_VIRTCHNL_OP_EVENT: + return ("EVENT"); + case I40E_VIRTCHNL_OP_CONFIG_RSS_KEY: + return ("CONFIG_RSS_KEY"); + case I40E_VIRTCHNL_OP_CONFIG_RSS_LUT: + return ("CONFIG_RSS_LUT"); + case I40E_VIRTCHNL_OP_GET_RSS_HENA_CAPS: + return ("GET_RSS_HENA_CAPS"); + case I40E_VIRTCHNL_OP_SET_RSS_HENA: + return ("SET_RSS_HENA"); + default: + return ("UNKNOWN"); + } +} + +static int +ixl_vc_opcode_level(uint16_t opcode) +{ + switch (opcode) { + case I40E_VIRTCHNL_OP_GET_STATS: + return (10); + default: + return (5); + } +} + +static void +ixl_send_vf_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op, + enum i40e_status_code status, void *msg, uint16_t len) +{ + struct i40e_hw *hw; + int global_vf_id; + + hw = &pf->hw; + global_vf_id = hw->func_caps.vf_base_id + vf->vf_num; + + I40E_VC_DEBUG(pf, ixl_vc_opcode_level(op), + "Sending msg (op=%s[%d], status=%d) to VF-%d\n", + ixl_vc_opcode_str(op), op, status, vf->vf_num); + + i40e_aq_send_msg_to_vf(hw, global_vf_id, op, status, msg, len, NULL); +} + +static void +ixl_send_vf_ack(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op) +{ + + ixl_send_vf_msg(pf, vf, op, I40E_SUCCESS, NULL, 0); +} + +static void +ixl_send_vf_nack_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op, + enum i40e_status_code status, const char *file, int line) +{ + + I40E_VC_DEBUG(pf, 1, + "Sending NACK (op=%s[%d], err=%s[%d]) to VF-%d from %s:%d\n", + ixl_vc_opcode_str(op), op, i40e_stat_str(&pf->hw, status), + status, vf->vf_num, file, line); + ixl_send_vf_msg(pf, vf, op, status, NULL, 0); +} + +static void +ixl_vf_version_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_virtchnl_version_info reply; + + if (msg_size != sizeof(struct i40e_virtchnl_version_info)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_VERSION, + I40E_ERR_PARAM); + return; + } + + vf->version = ((struct i40e_virtchnl_version_info *)msg)->minor; + + reply.major = I40E_VIRTCHNL_VERSION_MAJOR; + reply.minor = I40E_VIRTCHNL_VERSION_MINOR; + ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_VERSION, I40E_SUCCESS, &reply, + sizeof(reply)); +} + +static void +ixl_vf_reset_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + + if (msg_size != 0) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_RESET_VF, + I40E_ERR_PARAM); + return; + } + + ixl_reset_vf(pf, vf); + + /* No response to a reset message. */ +} + +static void +ixl_vf_get_resources_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_virtchnl_vf_resource reply; + + if ((vf->version == 0 && msg_size != 0) || + (vf->version == 1 && msg_size != 4)) { + device_printf(pf->dev, "Invalid GET_VF_RESOURCES message size," + " for VF version %d.%d\n", I40E_VIRTCHNL_VERSION_MAJOR, + vf->version); + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES, + I40E_ERR_PARAM); + return; + } + + bzero(&reply, sizeof(reply)); + + if (vf->version == I40E_VIRTCHNL_VERSION_MINOR_NO_VF_CAPS) + reply.vf_offload_flags = I40E_VIRTCHNL_VF_OFFLOAD_L2 | + I40E_VIRTCHNL_VF_OFFLOAD_RSS_REG | + I40E_VIRTCHNL_VF_OFFLOAD_VLAN; + else + /* Force VF RSS setup by PF in 1.1+ VFs */ + reply.vf_offload_flags = *(u32 *)msg & ( + I40E_VIRTCHNL_VF_OFFLOAD_L2 | + I40E_VIRTCHNL_VF_OFFLOAD_RSS_PF | + I40E_VIRTCHNL_VF_OFFLOAD_VLAN); + + reply.num_vsis = 1; + reply.num_queue_pairs = vf->vsi.num_queues; + reply.max_vectors = pf->hw.func_caps.num_msix_vectors_vf; + reply.rss_key_size = 52; + reply.rss_lut_size = 64; + reply.vsi_res[0].vsi_id = vf->vsi.vsi_num; + reply.vsi_res[0].vsi_type = I40E_VSI_SRIOV; + reply.vsi_res[0].num_queue_pairs = vf->vsi.num_queues; + memcpy(reply.vsi_res[0].default_mac_addr, vf->mac, ETHER_ADDR_LEN); + + ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES, + I40E_SUCCESS, &reply, sizeof(reply)); +} + +static int +ixl_vf_config_tx_queue(struct ixl_pf *pf, struct ixl_vf *vf, + struct i40e_virtchnl_txq_info *info) +{ + struct i40e_hw *hw; + struct i40e_hmc_obj_txq txq; + uint16_t global_queue_num, global_vf_num; + enum i40e_status_code status; + uint32_t qtx_ctl; + + hw = &pf->hw; + global_queue_num = ixl_pf_qidx_from_vsi_qidx(&vf->qtag, info->queue_id); + global_vf_num = hw->func_caps.vf_base_id + vf->vf_num; + bzero(&txq, sizeof(txq)); + + DDPRINTF(pf->dev, "VF %d: PF TX queue %d / VF TX queue %d (Global VF %d)\n", + vf->vf_num, global_queue_num, info->queue_id, global_vf_num); + + status = i40e_clear_lan_tx_queue_context(hw, global_queue_num); + if (status != I40E_SUCCESS) + return (EINVAL); + + txq.base = info->dma_ring_addr / IXL_TX_CTX_BASE_UNITS; + + txq.head_wb_ena = info->headwb_enabled; + txq.head_wb_addr = info->dma_headwb_addr; + txq.qlen = info->ring_len; + txq.rdylist = le16_to_cpu(vf->vsi.info.qs_handle[0]); + txq.rdylist_act = 0; + + status = i40e_set_lan_tx_queue_context(hw, global_queue_num, &txq); + if (status != I40E_SUCCESS) + return (EINVAL); + + qtx_ctl = I40E_QTX_CTL_VF_QUEUE | + (hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) | + (global_vf_num << I40E_QTX_CTL_VFVM_INDX_SHIFT); + wr32(hw, I40E_QTX_CTL(global_queue_num), qtx_ctl); + ixl_flush(hw); + + ixl_pf_qmgr_mark_queue_configured(&vf->qtag, info->queue_id, true); + + return (0); +} + +static int +ixl_vf_config_rx_queue(struct ixl_pf *pf, struct ixl_vf *vf, + struct i40e_virtchnl_rxq_info *info) +{ + struct i40e_hw *hw; + struct i40e_hmc_obj_rxq rxq; + uint16_t global_queue_num; + enum i40e_status_code status; + + hw = &pf->hw; + global_queue_num = ixl_pf_qidx_from_vsi_qidx(&vf->qtag, info->queue_id); + bzero(&rxq, sizeof(rxq)); + + DDPRINTF(pf->dev, "VF %d: PF RX queue %d / VF RX queue %d\n", + vf->vf_num, global_queue_num, info->queue_id); + + if (info->databuffer_size > IXL_VF_MAX_BUFFER) + return (EINVAL); + + if (info->max_pkt_size > IXL_VF_MAX_FRAME || + info->max_pkt_size < ETHER_MIN_LEN) + return (EINVAL); + + if (info->splithdr_enabled) { + if (info->hdr_size > IXL_VF_MAX_HDR_BUFFER) + return (EINVAL); + + rxq.hsplit_0 = info->rx_split_pos & + (I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_L2 | + I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_IP | + I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_TCP_UDP | + I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_SCTP); + rxq.hbuff = info->hdr_size >> I40E_RXQ_CTX_HBUFF_SHIFT; + + rxq.dtype = 2; + } + + status = i40e_clear_lan_rx_queue_context(hw, global_queue_num); + if (status != I40E_SUCCESS) + return (EINVAL); + + rxq.base = info->dma_ring_addr / IXL_RX_CTX_BASE_UNITS; + rxq.qlen = info->ring_len; + + rxq.dbuff = info->databuffer_size >> I40E_RXQ_CTX_DBUFF_SHIFT; + + rxq.dsize = 1; + rxq.crcstrip = 1; + rxq.l2tsel = 1; + + rxq.rxmax = info->max_pkt_size; + rxq.tphrdesc_ena = 1; + rxq.tphwdesc_ena = 1; + rxq.tphdata_ena = 1; + rxq.tphhead_ena = 1; + rxq.lrxqthresh = 2; + rxq.prefena = 1; + + status = i40e_set_lan_rx_queue_context(hw, global_queue_num, &rxq); + if (status != I40E_SUCCESS) + return (EINVAL); + + ixl_pf_qmgr_mark_queue_configured(&vf->qtag, info->queue_id, false); + + return (0); +} + +static void +ixl_vf_config_vsi_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_virtchnl_vsi_queue_config_info *info; + struct i40e_virtchnl_queue_pair_info *pair; + uint16_t expected_msg_size; + int i; + + if (msg_size < sizeof(*info)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, + I40E_ERR_PARAM); + return; + } + + info = msg; + if (info->num_queue_pairs == 0 || info->num_queue_pairs > vf->vsi.num_queues) { + device_printf(pf->dev, "VF %d: invalid # of qpairs (msg has %d, VSI has %d)\n", + vf->vf_num, info->num_queue_pairs, vf->vsi.num_queues); + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, + I40E_ERR_PARAM); + return; + } + + expected_msg_size = sizeof(*info) + info->num_queue_pairs * sizeof(*pair); + if (msg_size != expected_msg_size) { + device_printf(pf->dev, "VF %d: size of recvd message (%d) does not match expected size (%d)\n", + vf->vf_num, msg_size, expected_msg_size); + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, + I40E_ERR_PARAM); + return; + } + + if (info->vsi_id != vf->vsi.vsi_num) { + device_printf(pf->dev, "VF %d: VSI id in recvd message (%d) does not match expected id (%d)\n", + vf->vf_num, info->vsi_id, vf->vsi.vsi_num); + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, + I40E_ERR_PARAM); + return; + } + + for (i = 0; i < info->num_queue_pairs; i++) { + pair = &info->qpair[i]; + + if (pair->txq.vsi_id != vf->vsi.vsi_num || + pair->rxq.vsi_id != vf->vsi.vsi_num || + pair->txq.queue_id != pair->rxq.queue_id || + pair->txq.queue_id >= vf->vsi.num_queues) { + + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM); + return; + } + + if (ixl_vf_config_tx_queue(pf, vf, &pair->txq) != 0) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM); + return; + } + + if (ixl_vf_config_rx_queue(pf, vf, &pair->rxq) != 0) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM); + return; + } + } + + ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES); +} + +static void +ixl_vf_set_qctl(struct ixl_pf *pf, + const struct i40e_virtchnl_vector_map *vector, + enum i40e_queue_type cur_type, uint16_t cur_queue, + enum i40e_queue_type *last_type, uint16_t *last_queue) +{ + uint32_t offset, qctl; + uint16_t itr_indx; + + if (cur_type == I40E_QUEUE_TYPE_RX) { + offset = I40E_QINT_RQCTL(cur_queue); + itr_indx = vector->rxitr_idx; + } else { + offset = I40E_QINT_TQCTL(cur_queue); + itr_indx = vector->txitr_idx; + } + + qctl = htole32((vector->vector_id << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | + (*last_type << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) | + (*last_queue << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | + I40E_QINT_RQCTL_CAUSE_ENA_MASK | + (itr_indx << I40E_QINT_RQCTL_ITR_INDX_SHIFT)); + + wr32(&pf->hw, offset, qctl); + + *last_type = cur_type; + *last_queue = cur_queue; +} + +static void +ixl_vf_config_vector(struct ixl_pf *pf, struct ixl_vf *vf, + const struct i40e_virtchnl_vector_map *vector) +{ + struct i40e_hw *hw; + u_int qindex; + enum i40e_queue_type type, last_type; + uint32_t lnklst_reg; + uint16_t rxq_map, txq_map, cur_queue, last_queue; + + hw = &pf->hw; + + rxq_map = vector->rxq_map; + txq_map = vector->txq_map; + + last_queue = IXL_END_OF_INTR_LNKLST; + last_type = I40E_QUEUE_TYPE_RX; + + /* + * The datasheet says to optimize performance, RX queues and TX queues + * should be interleaved in the interrupt linked list, so we process + * both at once here. + */ + while ((rxq_map != 0) || (txq_map != 0)) { + if (txq_map != 0) { + qindex = ffs(txq_map) - 1; + type = I40E_QUEUE_TYPE_TX; + cur_queue = ixl_pf_qidx_from_vsi_qidx(&vf->qtag, qindex); + ixl_vf_set_qctl(pf, vector, type, cur_queue, + &last_type, &last_queue); + txq_map &= ~(1 << qindex); + } + + if (rxq_map != 0) { + qindex = ffs(rxq_map) - 1; + type = I40E_QUEUE_TYPE_RX; + cur_queue = ixl_pf_qidx_from_vsi_qidx(&vf->qtag, qindex); + ixl_vf_set_qctl(pf, vector, type, cur_queue, + &last_type, &last_queue); + rxq_map &= ~(1 << qindex); + } + } + + if (vector->vector_id == 0) + lnklst_reg = I40E_VPINT_LNKLST0(vf->vf_num); + else + lnklst_reg = IXL_VPINT_LNKLSTN_REG(hw, vector->vector_id, + vf->vf_num); + wr32(hw, lnklst_reg, + (last_queue << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT) | + (last_type << I40E_VPINT_LNKLST0_FIRSTQ_TYPE_SHIFT)); + + ixl_flush(hw); +} + +static void +ixl_vf_config_irq_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_virtchnl_irq_map_info *map; + struct i40e_virtchnl_vector_map *vector; + struct i40e_hw *hw; + int i, largest_txq, largest_rxq; + + hw = &pf->hw; + + if (msg_size < sizeof(*map)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, + I40E_ERR_PARAM); + return; + } + + map = msg; + if (map->num_vectors == 0) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, + I40E_ERR_PARAM); + return; + } + + if (msg_size != sizeof(*map) + map->num_vectors * sizeof(*vector)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, + I40E_ERR_PARAM); + return; + } + + for (i = 0; i < map->num_vectors; i++) { + vector = &map->vecmap[i]; + + if ((vector->vector_id >= hw->func_caps.num_msix_vectors_vf) || + vector->vsi_id != vf->vsi.vsi_num) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, I40E_ERR_PARAM); + return; + } + + if (vector->rxq_map != 0) { + largest_rxq = fls(vector->rxq_map) - 1; + if (largest_rxq >= vf->vsi.num_queues) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, + I40E_ERR_PARAM); + return; + } + } + + if (vector->txq_map != 0) { + largest_txq = fls(vector->txq_map) - 1; + if (largest_txq >= vf->vsi.num_queues) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, + I40E_ERR_PARAM); + return; + } + } + + if (vector->rxitr_idx > IXL_MAX_ITR_IDX || + vector->txitr_idx > IXL_MAX_ITR_IDX) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, + I40E_ERR_PARAM); + return; + } + + ixl_vf_config_vector(pf, vf, vector); + } + + ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP); +} + +static void +ixl_vf_enable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_virtchnl_queue_select *select; + int error = 0; + + if (msg_size != sizeof(*select)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES, + I40E_ERR_PARAM); + return; + } + + select = msg; + if (select->vsi_id != vf->vsi.vsi_num || + select->rx_queues == 0 || select->tx_queues == 0) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES, + I40E_ERR_PARAM); + return; + } + + /* Enable TX rings selected by the VF */ + for (int i = 0; i < 32; i++) { + if ((1 << i) & select->tx_queues) { + /* Warn if queue is out of VF allocation range */ + if (i >= vf->vsi.num_queues) { + device_printf(pf->dev, "VF %d: TX ring %d is outside of VF VSI allocation!\n", + vf->vf_num, i); + break; + } + /* Skip this queue if it hasn't been configured */ + if (!ixl_pf_qmgr_is_queue_configured(&vf->qtag, i, true)) + continue; + /* Warn if this queue is already marked as enabled */ + if (ixl_pf_qmgr_is_queue_enabled(&vf->qtag, i, true)) + device_printf(pf->dev, "VF %d: TX ring %d is already enabled!\n", + vf->vf_num, i); + + error = ixl_enable_tx_ring(pf, &vf->qtag, i); + if (error) + break; + else + ixl_pf_qmgr_mark_queue_enabled(&vf->qtag, i, true); + } + } + + /* Enable RX rings selected by the VF */ + for (int i = 0; i < 32; i++) { + if ((1 << i) & select->rx_queues) { + /* Warn if queue is out of VF allocation range */ + if (i >= vf->vsi.num_queues) { + device_printf(pf->dev, "VF %d: RX ring %d is outside of VF VSI allocation!\n", + vf->vf_num, i); + break; + } + /* Skip this queue if it hasn't been configured */ + if (!ixl_pf_qmgr_is_queue_configured(&vf->qtag, i, false)) + continue; + /* Warn if this queue is already marked as enabled */ + if (ixl_pf_qmgr_is_queue_enabled(&vf->qtag, i, false)) + device_printf(pf->dev, "VF %d: RX ring %d is already enabled!\n", + vf->vf_num, i); + error = ixl_enable_rx_ring(pf, &vf->qtag, i); + if (error) + break; + else + ixl_pf_qmgr_mark_queue_enabled(&vf->qtag, i, false); + } + } + + if (error) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES, + I40E_ERR_TIMEOUT); + return; + } + + ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES); +} + +static void +ixl_vf_disable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf, + void *msg, uint16_t msg_size) +{ + struct i40e_virtchnl_queue_select *select; + int error = 0; + + if (msg_size != sizeof(*select)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES, + I40E_ERR_PARAM); + return; + } + + select = msg; + if (select->vsi_id != vf->vsi.vsi_num || + select->rx_queues == 0 || select->tx_queues == 0) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES, + I40E_ERR_PARAM); + return; + } + + /* Disable TX rings selected by the VF */ + for (int i = 0; i < 32; i++) { + if ((1 << i) & select->tx_queues) { + /* Warn if queue is out of VF allocation range */ + if (i >= vf->vsi.num_queues) { + device_printf(pf->dev, "VF %d: TX ring %d is outside of VF VSI allocation!\n", + vf->vf_num, i); + break; + } + /* Skip this queue if it hasn't been configured */ + if (!ixl_pf_qmgr_is_queue_configured(&vf->qtag, i, true)) + continue; + /* Warn if this queue is already marked as disabled */ + if (!ixl_pf_qmgr_is_queue_enabled(&vf->qtag, i, true)) { + device_printf(pf->dev, "VF %d: TX ring %d is already disabled!\n", + vf->vf_num, i); + continue; + } + error = ixl_disable_tx_ring(pf, &vf->qtag, i); + if (error) + break; + else + ixl_pf_qmgr_mark_queue_disabled(&vf->qtag, i, true); + } + } + + /* Enable RX rings selected by the VF */ + for (int i = 0; i < 32; i++) { + if ((1 << i) & select->rx_queues) { + /* Warn if queue is out of VF allocation range */ + if (i >= vf->vsi.num_queues) { + device_printf(pf->dev, "VF %d: RX ring %d is outside of VF VSI allocation!\n", + vf->vf_num, i); + break; + } + /* Skip this queue if it hasn't been configured */ + if (!ixl_pf_qmgr_is_queue_configured(&vf->qtag, i, false)) + continue; + /* Warn if this queue is already marked as disabled */ + if (!ixl_pf_qmgr_is_queue_enabled(&vf->qtag, i, false)) { + device_printf(pf->dev, "VF %d: RX ring %d is already disabled!\n", + vf->vf_num, i); + continue; + } + error = ixl_disable_rx_ring(pf, &vf->qtag, i); + if (error) + break; + else + ixl_pf_qmgr_mark_queue_disabled(&vf->qtag, i, false); + } + } + + if (error) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES, + I40E_ERR_TIMEOUT); + return; + } + + ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES); +} + +static bool +ixl_zero_mac(const uint8_t *addr) +{ + uint8_t zero[ETHER_ADDR_LEN] = {0, 0, 0, 0, 0, 0}; + + return (cmp_etheraddr(addr, zero)); +} + +static bool +ixl_bcast_mac(const uint8_t *addr) +{ + + return (cmp_etheraddr(addr, ixl_bcast_addr)); +} + +static int +ixl_vf_mac_valid(struct ixl_vf *vf, const uint8_t *addr) +{ + + if (ixl_zero_mac(addr) || ixl_bcast_mac(addr)) + return (EINVAL); + + /* + * If the VF is not allowed to change its MAC address, don't let it + * set a MAC filter for an address that is not a multicast address and + * is not its assigned MAC. + */ + if (!(vf->vf_flags & VF_FLAG_SET_MAC_CAP) && + !(ETHER_IS_MULTICAST(addr) || cmp_etheraddr(addr, vf->mac))) + return (EPERM); + + return (0); +} + +static void +ixl_vf_add_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_virtchnl_ether_addr_list *addr_list; + struct i40e_virtchnl_ether_addr *addr; + struct ixl_vsi *vsi; + int i; + size_t expected_size; + + vsi = &vf->vsi; + + if (msg_size < sizeof(*addr_list)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, + I40E_ERR_PARAM); + return; + } + + addr_list = msg; + expected_size = sizeof(*addr_list) + + addr_list->num_elements * sizeof(*addr); + + if (addr_list->num_elements == 0 || + addr_list->vsi_id != vsi->vsi_num || + msg_size != expected_size) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, + I40E_ERR_PARAM); + return; + } + + for (i = 0; i < addr_list->num_elements; i++) { + if (ixl_vf_mac_valid(vf, addr_list->list[i].addr) != 0) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM); + return; + } + } + + for (i = 0; i < addr_list->num_elements; i++) { + addr = &addr_list->list[i]; + ixl_add_filter(vsi, addr->addr, IXL_VLAN_ANY); + } + + ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS); +} + +static void +ixl_vf_del_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_virtchnl_ether_addr_list *addr_list; + struct i40e_virtchnl_ether_addr *addr; + size_t expected_size; + int i; + + if (msg_size < sizeof(*addr_list)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, + I40E_ERR_PARAM); + return; + } + + addr_list = msg; + expected_size = sizeof(*addr_list) + + addr_list->num_elements * sizeof(*addr); + + if (addr_list->num_elements == 0 || + addr_list->vsi_id != vf->vsi.vsi_num || + msg_size != expected_size) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, + I40E_ERR_PARAM); + return; + } + + for (i = 0; i < addr_list->num_elements; i++) { + addr = &addr_list->list[i]; + if (ixl_zero_mac(addr->addr) || ixl_bcast_mac(addr->addr)) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM); + return; + } + } + + for (i = 0; i < addr_list->num_elements; i++) { + addr = &addr_list->list[i]; + ixl_del_filter(&vf->vsi, addr->addr, IXL_VLAN_ANY); + } + + ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS); +} + +static enum i40e_status_code +ixl_vf_enable_vlan_strip(struct ixl_pf *pf, struct ixl_vf *vf) +{ + struct i40e_vsi_context vsi_ctx; + + vsi_ctx.seid = vf->vsi.seid; + + bzero(&vsi_ctx.info, sizeof(vsi_ctx.info)); + vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_VLAN_VALID); + vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | + I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; + return (i40e_aq_update_vsi_params(&pf->hw, &vsi_ctx, NULL)); +} + +static void +ixl_vf_add_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_virtchnl_vlan_filter_list *filter_list; + enum i40e_status_code code; + size_t expected_size; + int i; + + if (msg_size < sizeof(*filter_list)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, + I40E_ERR_PARAM); + return; + } + + filter_list = msg; + expected_size = sizeof(*filter_list) + + filter_list->num_elements * sizeof(uint16_t); + if (filter_list->num_elements == 0 || + filter_list->vsi_id != vf->vsi.vsi_num || + msg_size != expected_size) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, + I40E_ERR_PARAM); + return; + } + + if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, + I40E_ERR_PARAM); + return; + } + + for (i = 0; i < filter_list->num_elements; i++) { + if (filter_list->vlan_id[i] > EVL_VLID_MASK) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, + I40E_ERR_PARAM); + return; + } + } + + code = ixl_vf_enable_vlan_strip(pf, vf); + if (code != I40E_SUCCESS) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, + I40E_ERR_PARAM); + } + + for (i = 0; i < filter_list->num_elements; i++) + ixl_add_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]); + + ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN); +} + +static void +ixl_vf_del_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_virtchnl_vlan_filter_list *filter_list; + int i; + size_t expected_size; + + if (msg_size < sizeof(*filter_list)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN, + I40E_ERR_PARAM); + return; + } + + filter_list = msg; + expected_size = sizeof(*filter_list) + + filter_list->num_elements * sizeof(uint16_t); + if (filter_list->num_elements == 0 || + filter_list->vsi_id != vf->vsi.vsi_num || + msg_size != expected_size) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN, + I40E_ERR_PARAM); + return; + } + + for (i = 0; i < filter_list->num_elements; i++) { + if (filter_list->vlan_id[i] > EVL_VLID_MASK) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, + I40E_ERR_PARAM); + return; + } + } + + if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, + I40E_ERR_PARAM); + return; + } + + for (i = 0; i < filter_list->num_elements; i++) + ixl_del_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]); + + ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN); +} + +static void +ixl_vf_config_promisc_msg(struct ixl_pf *pf, struct ixl_vf *vf, + void *msg, uint16_t msg_size) +{ + struct i40e_virtchnl_promisc_info *info; + enum i40e_status_code code; + + if (msg_size != sizeof(*info)) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM); + return; + } + + if (!(vf->vf_flags & VF_FLAG_PROMISC_CAP)) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM); + return; + } + + info = msg; + if (info->vsi_id != vf->vsi.vsi_num) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM); + return; + } + + code = i40e_aq_set_vsi_unicast_promiscuous(&pf->hw, info->vsi_id, + info->flags & I40E_FLAG_VF_UNICAST_PROMISC, NULL, TRUE); + if (code != I40E_SUCCESS) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code); + return; + } + + code = i40e_aq_set_vsi_multicast_promiscuous(&pf->hw, info->vsi_id, + info->flags & I40E_FLAG_VF_MULTICAST_PROMISC, NULL); + if (code != I40E_SUCCESS) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code); + return; + } + + ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE); +} + +static void +ixl_vf_get_stats_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_virtchnl_queue_select *queue; + + if (msg_size != sizeof(*queue)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS, + I40E_ERR_PARAM); + return; + } + + queue = msg; + if (queue->vsi_id != vf->vsi.vsi_num) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS, + I40E_ERR_PARAM); + return; + } + + ixl_update_eth_stats(&vf->vsi); + + ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_STATS, + I40E_SUCCESS, &vf->vsi.eth_stats, sizeof(vf->vsi.eth_stats)); +} + +static void +ixl_vf_config_rss_key_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_hw *hw; + struct i40e_virtchnl_rss_key *key; + struct i40e_aqc_get_set_rss_key_data key_data; + enum i40e_status_code status; + + hw = &pf->hw; + + if (msg_size < sizeof(*key)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_RSS_KEY, + I40E_ERR_PARAM); + return; + } + + key = msg; + + if (key->key_len > 52) { + device_printf(pf->dev, "VF %d: Key size in msg (%d) is greater than max key size (%d)\n", + vf->vf_num, key->key_len, 52); + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_RSS_KEY, + I40E_ERR_PARAM); + return; + } + + if (key->vsi_id != vf->vsi.vsi_num) { + device_printf(pf->dev, "VF %d: VSI id in recvd message (%d) does not match expected id (%d)\n", + vf->vf_num, key->vsi_id, vf->vsi.vsi_num); + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_RSS_KEY, + I40E_ERR_PARAM); + return; + } + + /* Fill out hash using MAC-dependent method */ + if (hw->mac.type == I40E_MAC_X722) { + bzero(&key_data, sizeof(key_data)); + if (key->key_len <= 40) + bcopy(key->key, key_data.standard_rss_key, key->key_len); + else { + bcopy(key->key, key_data.standard_rss_key, 40); + bcopy(&key->key[40], key_data.extended_hash_key, key->key_len - 40); + } + status = i40e_aq_set_rss_key(hw, vf->vsi.vsi_num, &key_data); + if (status) { + device_printf(pf->dev, "i40e_aq_set_rss_key status %s, error %s\n", + i40e_stat_str(hw, status), i40e_aq_str(hw, hw->aq.asq_last_status)); + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_RSS_KEY, + I40E_ERR_ADMIN_QUEUE_ERROR); + return; + } + } else { + for (int i = 0; i < (key->key_len / 4); i++) + i40e_write_rx_ctl(hw, I40E_VFQF_HKEY1(i, vf->vf_num), ((u32 *)key->key)[i]); + } + + DDPRINTF(pf->dev, "VF %d: Programmed key starting with 0x%x ok!", + vf->vf_num, key->key[0]); + + ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_RSS_KEY); +} + +static void +ixl_vf_config_rss_lut_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_hw *hw; + struct i40e_virtchnl_rss_lut *lut; + enum i40e_status_code status; + + hw = &pf->hw; + + if (msg_size < sizeof(*lut)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_RSS_LUT, + I40E_ERR_PARAM); + return; + } + + lut = msg; + + if (lut->lut_entries > 64) { + device_printf(pf->dev, "VF %d: # of LUT entries in msg (%d) is greater than max (%d)\n", + vf->vf_num, lut->lut_entries, 64); + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_RSS_LUT, + I40E_ERR_PARAM); + return; + } + + if (lut->vsi_id != vf->vsi.vsi_num) { + device_printf(pf->dev, "VF %d: VSI id in recvd message (%d) does not match expected id (%d)\n", + vf->vf_num, lut->vsi_id, vf->vsi.vsi_num); + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_RSS_LUT, + I40E_ERR_PARAM); + return; + } + + /* Fill out LUT using MAC-dependent method */ + if (hw->mac.type == I40E_MAC_X722) { + status = i40e_aq_set_rss_lut(hw, vf->vsi.vsi_num, false, lut->lut, lut->lut_entries); + if (status) { + device_printf(pf->dev, "i40e_aq_set_rss_lut status %s, error %s\n", + i40e_stat_str(hw, status), i40e_aq_str(hw, hw->aq.asq_last_status)); + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_RSS_LUT, + I40E_ERR_ADMIN_QUEUE_ERROR); + return; + } + } else { + for (int i = 0; i < (lut->lut_entries / 4); i++) + i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf->vf_num), ((u32 *)lut->lut)[i]); + } + + DDPRINTF(pf->dev, "VF %d: Programmed LUT starting with 0x%x and length %d ok!", + vf->vf_num, lut->lut[0], lut->lut_entries); + + ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_RSS_LUT); +} + +static void +ixl_vf_set_rss_hena_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_hw *hw; + struct i40e_virtchnl_rss_hena *hena; + + hw = &pf->hw; + + if (msg_size < sizeof(*hena)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_SET_RSS_HENA, + I40E_ERR_PARAM); + return; + } + + hena = msg; + + /* Set HENA */ + i40e_write_rx_ctl(hw, I40E_VFQF_HENA1(0, vf->vf_num), (u32)hena->hena); + i40e_write_rx_ctl(hw, I40E_VFQF_HENA1(1, vf->vf_num), (u32)(hena->hena >> 32)); + + DDPRINTF(pf->dev, "VF %d: Programmed HENA with 0x%016lx", + vf->vf_num, hena->hena); + + ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_SET_RSS_HENA); +} + +void +ixl_handle_vf_msg(struct ixl_pf *pf, struct i40e_arq_event_info *event) +{ + struct ixl_vf *vf; + void *msg; + uint16_t vf_num, msg_size; + uint32_t opcode; + + vf_num = le16toh(event->desc.retval) - pf->hw.func_caps.vf_base_id; + opcode = le32toh(event->desc.cookie_high); + + if (vf_num >= pf->num_vfs) { + device_printf(pf->dev, "Got msg from illegal VF: %d\n", vf_num); + return; + } + + vf = &pf->vfs[vf_num]; + msg = event->msg_buf; + msg_size = event->msg_len; + + I40E_VC_DEBUG(pf, ixl_vc_opcode_level(opcode), + "Got msg %s(%d) from%sVF-%d of size %d\n", + ixl_vc_opcode_str(opcode), opcode, + (vf->vf_flags & VF_FLAG_ENABLED) ? " " : " disabled ", + vf_num, msg_size); + + /* This must be a stray msg from a previously destroyed VF. */ + if (!(vf->vf_flags & VF_FLAG_ENABLED)) + return; + + switch (opcode) { + case I40E_VIRTCHNL_OP_VERSION: + ixl_vf_version_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_RESET_VF: + ixl_vf_reset_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_GET_VF_RESOURCES: + ixl_vf_get_resources_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES: + ixl_vf_config_vsi_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP: + ixl_vf_config_irq_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_ENABLE_QUEUES: + ixl_vf_enable_queues_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_DISABLE_QUEUES: + ixl_vf_disable_queues_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS: + ixl_vf_add_mac_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS: + ixl_vf_del_mac_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_ADD_VLAN: + ixl_vf_add_vlan_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_DEL_VLAN: + ixl_vf_del_vlan_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE: + ixl_vf_config_promisc_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_GET_STATS: + ixl_vf_get_stats_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_CONFIG_RSS_KEY: + ixl_vf_config_rss_key_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_CONFIG_RSS_LUT: + ixl_vf_config_rss_lut_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_SET_RSS_HENA: + ixl_vf_set_rss_hena_msg(pf, vf, msg, msg_size); + break; + + /* These two opcodes have been superseded by CONFIG_VSI_QUEUES. */ + case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE: + case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE: + default: + i40e_send_vf_nack(pf, vf, opcode, I40E_ERR_NOT_IMPLEMENTED); + break; + } +} + +/* Handle any VFs that have reset themselves via a Function Level Reset(FLR). */ +void +ixl_handle_vflr(void *arg, int pending) +{ + struct ixl_pf *pf; + struct ixl_vf *vf; + struct i40e_hw *hw; + uint16_t global_vf_num; + uint32_t vflrstat_index, vflrstat_mask, vflrstat, icr0; + int i; + + pf = arg; + hw = &pf->hw; + + IXL_PF_LOCK(pf); + for (i = 0; i < pf->num_vfs; i++) { + global_vf_num = hw->func_caps.vf_base_id + i; + + vf = &pf->vfs[i]; + if (!(vf->vf_flags & VF_FLAG_ENABLED)) + continue; + + vflrstat_index = IXL_GLGEN_VFLRSTAT_INDEX(global_vf_num); + vflrstat_mask = IXL_GLGEN_VFLRSTAT_MASK(global_vf_num); + vflrstat = rd32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index)); + if (vflrstat & vflrstat_mask) { + wr32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index), + vflrstat_mask); + + ixl_reinit_vf(pf, vf); + } + } + + icr0 = rd32(hw, I40E_PFINT_ICR0_ENA); + icr0 |= I40E_PFINT_ICR0_ENA_VFLR_MASK; + wr32(hw, I40E_PFINT_ICR0_ENA, icr0); + ixl_flush(hw); + + IXL_PF_UNLOCK(pf); +} + +static int +ixl_adminq_err_to_errno(enum i40e_admin_queue_err err) +{ + + switch (err) { + case I40E_AQ_RC_EPERM: + return (EPERM); + case I40E_AQ_RC_ENOENT: + return (ENOENT); + case I40E_AQ_RC_ESRCH: + return (ESRCH); + case I40E_AQ_RC_EINTR: + return (EINTR); + case I40E_AQ_RC_EIO: + return (EIO); + case I40E_AQ_RC_ENXIO: + return (ENXIO); + case I40E_AQ_RC_E2BIG: + return (E2BIG); + case I40E_AQ_RC_EAGAIN: + return (EAGAIN); + case I40E_AQ_RC_ENOMEM: + return (ENOMEM); + case I40E_AQ_RC_EACCES: + return (EACCES); + case I40E_AQ_RC_EFAULT: + return (EFAULT); + case I40E_AQ_RC_EBUSY: + return (EBUSY); + case I40E_AQ_RC_EEXIST: + return (EEXIST); + case I40E_AQ_RC_EINVAL: + return (EINVAL); + case I40E_AQ_RC_ENOTTY: + return (ENOTTY); + case I40E_AQ_RC_ENOSPC: + return (ENOSPC); + case I40E_AQ_RC_ENOSYS: + return (ENOSYS); + case I40E_AQ_RC_ERANGE: + return (ERANGE); + case I40E_AQ_RC_EFLUSHED: + return (EINVAL); /* No exact equivalent in errno.h */ + case I40E_AQ_RC_BAD_ADDR: + return (EFAULT); + case I40E_AQ_RC_EMODE: + return (EPERM); + case I40E_AQ_RC_EFBIG: + return (EFBIG); + default: + return (EINVAL); + } +} + +int +ixl_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *params) +{ + struct ixl_pf *pf; + struct i40e_hw *hw; + struct ixl_vsi *pf_vsi; + enum i40e_status_code ret; + int i, error; + + pf = device_get_softc(dev); + hw = &pf->hw; + pf_vsi = &pf->vsi; + + IXL_PF_LOCK(pf); + pf->vfs = malloc(sizeof(struct ixl_vf) * num_vfs, M_IXL, M_NOWAIT | + M_ZERO); + + if (pf->vfs == NULL) { + error = ENOMEM; + goto fail; + } + + for (i = 0; i < num_vfs; i++) + sysctl_ctx_init(&pf->vfs[i].ctx); + + ret = i40e_aq_add_veb(hw, pf_vsi->uplink_seid, pf_vsi->seid, + 1, FALSE, &pf->veb_seid, FALSE, NULL); + if (ret != I40E_SUCCESS) { + error = ixl_adminq_err_to_errno(hw->aq.asq_last_status); + device_printf(dev, "add_veb failed; code=%d error=%d", ret, + error); + goto fail; + } + + ixl_enable_adminq(hw); + + pf->num_vfs = num_vfs; + IXL_PF_UNLOCK(pf); + return (0); + +fail: + free(pf->vfs, M_IXL); + pf->vfs = NULL; + IXL_PF_UNLOCK(pf); + return (error); +} + +void +ixl_iov_uninit(device_t dev) +{ + struct ixl_pf *pf; + struct i40e_hw *hw; + struct ixl_vsi *vsi; + struct ifnet *ifp; + struct ixl_vf *vfs; + int i, num_vfs; + + pf = device_get_softc(dev); + hw = &pf->hw; + vsi = &pf->vsi; + ifp = vsi->ifp; + + IXL_PF_LOCK(pf); + for (i = 0; i < pf->num_vfs; i++) { + if (pf->vfs[i].vsi.seid != 0) + i40e_aq_delete_element(hw, pf->vfs[i].vsi.seid, NULL); + ixl_pf_qmgr_release(&pf->qmgr, &pf->vfs[i].qtag); + DDPRINTF(dev, "VF %d: %d released\n", + i, pf->vfs[i].qtag.num_allocated); + DDPRINTF(dev, "Unallocated total: %d\n", ixl_pf_qmgr_get_num_free(&pf->qmgr)); + } + + if (pf->veb_seid != 0) { + i40e_aq_delete_element(hw, pf->veb_seid, NULL); + pf->veb_seid = 0; + } + + if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) { + ixl_disable_intr(vsi); + ixl_flush(hw); + } + + vfs = pf->vfs; + num_vfs = pf->num_vfs; + + pf->vfs = NULL; + pf->num_vfs = 0; + IXL_PF_UNLOCK(pf); + + /* Do this after the unlock as sysctl_ctx_free might sleep. */ + for (i = 0; i < num_vfs; i++) + sysctl_ctx_free(&vfs[i].ctx); + free(vfs, M_IXL); +} + +static int +ixl_vf_reserve_queues(struct ixl_pf *pf, struct ixl_vf *vf, int num_queues) +{ + device_t dev = pf->dev; + int error; + + /* Validate, and clamp value if invalid */ + if (num_queues < 1 || num_queues > 16) + device_printf(dev, "Invalid num-queues (%d) for VF %d\n", + num_queues, vf->vf_num); + if (num_queues < 1) { + device_printf(dev, "Setting VF %d num-queues to 1\n", vf->vf_num); + num_queues = 1; + } else if (num_queues > 16) { + device_printf(dev, "Setting VF %d num-queues to 16\n", vf->vf_num); + num_queues = 16; + } + error = ixl_pf_qmgr_alloc_scattered(&pf->qmgr, num_queues, &vf->qtag); + if (error) { + device_printf(dev, "Error allocating %d queues for VF %d's VSI\n", + num_queues, vf->vf_num); + return (ENOSPC); + } + + DDPRINTF(dev, "VF %d: %d allocated, %d active", + vf->vf_num, vf->qtag.num_allocated, vf->qtag.num_active); + DDPRINTF(dev, "Unallocated total: %d", ixl_pf_qmgr_get_num_free(&pf->qmgr)); + + return (0); +} + +int +ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *params) +{ + char sysctl_name[QUEUE_NAME_LEN]; + struct ixl_pf *pf; + struct ixl_vf *vf; + const void *mac; + size_t size; + int error; + int vf_num_queues; + + pf = device_get_softc(dev); + vf = &pf->vfs[vfnum]; + + IXL_PF_LOCK(pf); + vf->vf_num = vfnum; + + vf->vsi.back = pf; + vf->vf_flags = VF_FLAG_ENABLED; + SLIST_INIT(&vf->vsi.ftl); + + /* Reserve queue allocation from PF */ + vf_num_queues = nvlist_get_number(params, "num-queues"); + error = ixl_vf_reserve_queues(pf, vf, vf_num_queues); + if (error != 0) + goto out; + + error = ixl_vf_setup_vsi(pf, vf); + if (error != 0) + goto out; + + if (nvlist_exists_binary(params, "mac-addr")) { + mac = nvlist_get_binary(params, "mac-addr", &size); + bcopy(mac, vf->mac, ETHER_ADDR_LEN); + + if (nvlist_get_bool(params, "allow-set-mac")) + vf->vf_flags |= VF_FLAG_SET_MAC_CAP; + } else + /* + * If the administrator has not specified a MAC address then + * we must allow the VF to choose one. + */ + vf->vf_flags |= VF_FLAG_SET_MAC_CAP; + + if (nvlist_get_bool(params, "mac-anti-spoof")) + vf->vf_flags |= VF_FLAG_MAC_ANTI_SPOOF; + + if (nvlist_get_bool(params, "allow-promisc")) + vf->vf_flags |= VF_FLAG_PROMISC_CAP; + + vf->vf_flags |= VF_FLAG_VLAN_CAP; + + ixl_reset_vf(pf, vf); +out: + IXL_PF_UNLOCK(pf); + if (error == 0) { + snprintf(sysctl_name, sizeof(sysctl_name), "vf%d", vfnum); + ixl_add_vsi_sysctls(pf, &vf->vsi, &vf->ctx, sysctl_name); + } + + return (error); +} + diff --git a/sys/dev/ixl/ixl_pf_iov.h b/sys/dev/ixl/ixl_pf_iov.h new file mode 100644 index 000000000000..ae8abc208d4b --- /dev/null +++ b/sys/dev/ixl/ixl_pf_iov.h @@ -0,0 +1,62 @@ +/****************************************************************************** + + Copyright (c) 2013-2015, Intel Corporation + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +******************************************************************************/ +/*$FreeBSD$*/ + + +#ifndef _IXL_PF_IOV_H_ +#define _IXL_PF_IOV_H_ + +#include "ixl_pf.h" + +#include <sys/nv.h> +#include <sys/iov_schema.h> +#include <dev/pci/pci_iov.h> + +/* Public functions */ + +/* + * These three are DEVMETHODs required for SR-IOV PF support. + */ +int ixl_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *params); +void ixl_iov_uninit(device_t dev); +int ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *params); + +/* + * The standard PF driver needs to call these during normal execution when + * SR-IOV mode is active. + */ +void ixl_initialize_sriov(struct ixl_pf *pf); +void ixl_handle_vf_msg(struct ixl_pf *pf, struct i40e_arq_event_info *event); +void ixl_handle_vflr(void *arg, int pending); + +#endif /* _IXL_PF_IOV_H_ */ diff --git a/sys/dev/ixl/ixl_pf_main.c b/sys/dev/ixl/ixl_pf_main.c new file mode 100644 index 000000000000..d8da4cfee106 --- /dev/null +++ b/sys/dev/ixl/ixl_pf_main.c @@ -0,0 +1,5557 @@ +/****************************************************************************** + + Copyright (c) 2013-2015, Intel Corporation + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +******************************************************************************/ +/*$FreeBSD$*/ + + +#include "ixl_pf.h" + +#ifdef PCI_IOV +#include "ixl_pf_iov.h" +#endif + +#ifdef DEV_NETMAP +#include <net/netmap.h> +#include <sys/selinfo.h> +#include <dev/netmap/netmap_kern.h> +#endif /* DEV_NETMAP */ + +static int ixl_setup_queue(struct ixl_queue *, struct ixl_pf *, int); + +/* Sysctls */ +static int ixl_set_flowcntl(SYSCTL_HANDLER_ARGS); +static int ixl_set_advertise(SYSCTL_HANDLER_ARGS); +static int ixl_current_speed(SYSCTL_HANDLER_ARGS); +static int ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS); +static int ixl_sysctl_unallocated_queues(SYSCTL_HANDLER_ARGS); +static int ixl_sysctl_pf_tx_itr(SYSCTL_HANDLER_ARGS); +static int ixl_sysctl_pf_rx_itr(SYSCTL_HANDLER_ARGS); + +/* Debug Sysctls */ +static int ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS); +static int ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS); +static int ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS); +static int ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS); +static int ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS); +static int ixl_sysctl_hkey(SYSCTL_HANDLER_ARGS); +static int ixl_sysctl_hlut(SYSCTL_HANDLER_ARGS); + +void +ixl_dbg(struct ixl_pf *pf, enum ixl_dbg_mask mask, char *fmt, ...) +{ + va_list args; + + if (!(mask & pf->dbg_mask)) + return; + + va_start(args, fmt); + device_printf(pf->dev, fmt, args); + va_end(args); +} + +/* +** Put the FW, API, NVM, EEtrackID, and OEM version information into a string +*/ +void +ixl_nvm_version_str(struct i40e_hw *hw, struct sbuf *buf) +{ + u8 oem_ver = (u8)(hw->nvm.oem_ver >> 24); + u16 oem_build = (u16)((hw->nvm.oem_ver >> 16) & 0xFFFF); + u8 oem_patch = (u8)(hw->nvm.oem_ver & 0xFF); + + sbuf_printf(buf, + "fw %d.%d.%05d api %d.%d nvm %x.%02x etid %08x oem %d.%d.%d", + hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build, + hw->aq.api_maj_ver, hw->aq.api_min_ver, + (hw->nvm.version & IXL_NVM_VERSION_HI_MASK) >> + IXL_NVM_VERSION_HI_SHIFT, + (hw->nvm.version & IXL_NVM_VERSION_LO_MASK) >> + IXL_NVM_VERSION_LO_SHIFT, + hw->nvm.eetrack, + oem_ver, oem_build, oem_patch); +} + +void +ixl_print_nvm_version(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + device_t dev = pf->dev; + struct sbuf *sbuf; + + sbuf = sbuf_new_auto(); + ixl_nvm_version_str(hw, sbuf); + sbuf_finish(sbuf); + device_printf(dev, "%s\n", sbuf_data(sbuf)); + sbuf_delete(sbuf); +} + +static void +ixl_configure_tx_itr(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + struct ixl_vsi *vsi = &pf->vsi; + struct ixl_queue *que = vsi->queues; + + vsi->tx_itr_setting = pf->tx_itr; + + for (int i = 0; i < vsi->num_queues; i++, que++) { + struct tx_ring *txr = &que->txr; + + wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, i), + vsi->tx_itr_setting); + txr->itr = vsi->tx_itr_setting; + txr->latency = IXL_AVE_LATENCY; + } +} + +static void +ixl_configure_rx_itr(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + struct ixl_vsi *vsi = &pf->vsi; + struct ixl_queue *que = vsi->queues; + + vsi->rx_itr_setting = pf->rx_itr; + + for (int i = 0; i < vsi->num_queues; i++, que++) { + struct rx_ring *rxr = &que->rxr; + + wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, i), + vsi->rx_itr_setting); + rxr->itr = vsi->rx_itr_setting; + rxr->latency = IXL_AVE_LATENCY; + } +} + +/* + * Write PF ITR values to queue ITR registers. + */ +void +ixl_configure_itr(struct ixl_pf *pf) +{ + ixl_configure_tx_itr(pf); + ixl_configure_rx_itr(pf); +} + + +/********************************************************************* + * Init entry point + * + * This routine is used in two ways. It is used by the stack as + * init entry point in network interface structure. It is also used + * by the driver as a hw/sw initialization routine to get to a + * consistent state. + * + * return 0 on success, positive on failure + **********************************************************************/ +void +ixl_init_locked(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + struct ixl_vsi *vsi = &pf->vsi; + struct ifnet *ifp = vsi->ifp; + device_t dev = pf->dev; + struct i40e_filter_control_settings filter; + u8 tmpaddr[ETHER_ADDR_LEN]; + int ret; + + mtx_assert(&pf->pf_mtx, MA_OWNED); + INIT_DEBUGOUT("ixl_init_locked: begin"); + + ixl_stop_locked(pf); + + /* Get the latest mac address... User might use a LAA */ + bcopy(IF_LLADDR(vsi->ifp), tmpaddr, + I40E_ETH_LENGTH_OF_ADDRESS); + if (!cmp_etheraddr(hw->mac.addr, tmpaddr) && + (i40e_validate_mac_addr(tmpaddr) == I40E_SUCCESS)) { + ixl_del_filter(vsi, hw->mac.addr, IXL_VLAN_ANY); + bcopy(tmpaddr, hw->mac.addr, + I40E_ETH_LENGTH_OF_ADDRESS); + ret = i40e_aq_mac_address_write(hw, + I40E_AQC_WRITE_TYPE_LAA_ONLY, + hw->mac.addr, NULL); + if (ret) { + device_printf(dev, "LLA address" + "change failed!!\n"); + return; + } + } + + ixl_add_filter(vsi, hw->mac.addr, IXL_VLAN_ANY); + + /* Set the various hardware offload abilities */ + ifp->if_hwassist = 0; + if (ifp->if_capenable & IFCAP_TSO) + ifp->if_hwassist |= CSUM_TSO; + if (ifp->if_capenable & IFCAP_TXCSUM) + ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); + if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) + ifp->if_hwassist |= (CSUM_TCP_IPV6 | CSUM_UDP_IPV6); + + /* Set up the device filtering */ + bzero(&filter, sizeof(filter)); + filter.enable_ethtype = TRUE; + filter.enable_macvlan = TRUE; + filter.enable_fdir = FALSE; + filter.hash_lut_size = I40E_HASH_LUT_SIZE_512; + if (i40e_set_filter_control(hw, &filter)) + device_printf(dev, "i40e_set_filter_control() failed\n"); + + /* Prepare the VSI: rings, hmc contexts, etc... */ + if (ixl_initialize_vsi(vsi)) { + device_printf(dev, "initialize vsi failed!!\n"); + return; + } + + /* Set up RSS */ + ixl_config_rss(pf); + + /* Add protocol filters to list */ + ixl_init_filters(vsi); + + /* Setup vlan's if needed */ + ixl_setup_vlan_filters(vsi); + + /* Set up MSI/X routing and the ITR settings */ + if (pf->enable_msix) { + ixl_configure_queue_intr_msix(pf); + ixl_configure_itr(pf); + } else + ixl_configure_legacy(pf); + + ixl_enable_rings(vsi); + + i40e_aq_set_default_vsi(hw, vsi->seid, NULL); + + ixl_reconfigure_filters(vsi); + + /* And now turn on interrupts */ + ixl_enable_intr(vsi); + + /* Get link info */ + hw->phy.get_link_info = TRUE; + i40e_get_link_status(hw, &pf->link_up); + ixl_update_link_status(pf); + + /* Set initial advertised speed sysctl value */ + ixl_get_initial_advertised_speeds(pf); + + /* Start the local timer */ + callout_reset(&pf->timer, hz, ixl_local_timer, pf); + + /* Now inform the stack we're ready */ + ifp->if_drv_flags |= IFF_DRV_RUNNING; +} + + +/********************************************************************* + * + * Get the hardware capabilities + * + **********************************************************************/ + +int +ixl_get_hw_capabilities(struct ixl_pf *pf) +{ + struct i40e_aqc_list_capabilities_element_resp *buf; + struct i40e_hw *hw = &pf->hw; + device_t dev = pf->dev; + int error, len; + u16 needed; + bool again = TRUE; + + len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp); +retry: + if (!(buf = (struct i40e_aqc_list_capabilities_element_resp *) + malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO))) { + device_printf(dev, "Unable to allocate cap memory\n"); + return (ENOMEM); + } + + /* This populates the hw struct */ + error = i40e_aq_discover_capabilities(hw, buf, len, + &needed, i40e_aqc_opc_list_func_capabilities, NULL); + free(buf, M_DEVBUF); + if ((pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) && + (again == TRUE)) { + /* retry once with a larger buffer */ + again = FALSE; + len = needed; + goto retry; + } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) { + device_printf(dev, "capability discovery failed: %d\n", + pf->hw.aq.asq_last_status); + return (ENODEV); + } + + /* Capture this PF's starting queue pair */ + pf->qbase = hw->func_caps.base_queue; + +#ifdef IXL_DEBUG + device_printf(dev, "pf_id=%d, num_vfs=%d, msix_pf=%d, " + "msix_vf=%d, fd_g=%d, fd_b=%d, tx_qp=%d rx_qp=%d qbase=%d\n", + hw->pf_id, hw->func_caps.num_vfs, + hw->func_caps.num_msix_vectors, + hw->func_caps.num_msix_vectors_vf, + hw->func_caps.fd_filters_guaranteed, + hw->func_caps.fd_filters_best_effort, + hw->func_caps.num_tx_qp, + hw->func_caps.num_rx_qp, + hw->func_caps.base_queue); +#endif + /* Print a subset of the capability information. */ + device_printf(dev, "PF-ID[%d]: VFs %d, MSIX %d, VF MSIX %d, QPs %d, %s\n", + hw->pf_id, hw->func_caps.num_vfs, hw->func_caps.num_msix_vectors, + hw->func_caps.num_msix_vectors_vf, hw->func_caps.num_tx_qp, + (hw->func_caps.mdio_port_mode == 2) ? "I2C" : + (hw->func_caps.mdio_port_mode == 1) ? "MDIO dedicated" : + "MDIO shared"); + + return (error); +} + +void +ixl_cap_txcsum_tso(struct ixl_vsi *vsi, struct ifnet *ifp, int mask) +{ + device_t dev = vsi->dev; + + /* Enable/disable TXCSUM/TSO4 */ + if (!(ifp->if_capenable & IFCAP_TXCSUM) + && !(ifp->if_capenable & IFCAP_TSO4)) { + if (mask & IFCAP_TXCSUM) { + ifp->if_capenable |= IFCAP_TXCSUM; + /* enable TXCSUM, restore TSO if previously enabled */ + if (vsi->flags & IXL_FLAGS_KEEP_TSO4) { + vsi->flags &= ~IXL_FLAGS_KEEP_TSO4; + ifp->if_capenable |= IFCAP_TSO4; + } + } + else if (mask & IFCAP_TSO4) { + ifp->if_capenable |= (IFCAP_TXCSUM | IFCAP_TSO4); + vsi->flags &= ~IXL_FLAGS_KEEP_TSO4; + device_printf(dev, + "TSO4 requires txcsum, enabling both...\n"); + } + } else if((ifp->if_capenable & IFCAP_TXCSUM) + && !(ifp->if_capenable & IFCAP_TSO4)) { + if (mask & IFCAP_TXCSUM) + ifp->if_capenable &= ~IFCAP_TXCSUM; + else if (mask & IFCAP_TSO4) + ifp->if_capenable |= IFCAP_TSO4; + } else if((ifp->if_capenable & IFCAP_TXCSUM) + && (ifp->if_capenable & IFCAP_TSO4)) { + if (mask & IFCAP_TXCSUM) { + vsi->flags |= IXL_FLAGS_KEEP_TSO4; + ifp->if_capenable &= ~(IFCAP_TXCSUM | IFCAP_TSO4); + device_printf(dev, + "TSO4 requires txcsum, disabling both...\n"); + } else if (mask & IFCAP_TSO4) + ifp->if_capenable &= ~IFCAP_TSO4; + } + + /* Enable/disable TXCSUM_IPV6/TSO6 */ + if (!(ifp->if_capenable & IFCAP_TXCSUM_IPV6) + && !(ifp->if_capenable & IFCAP_TSO6)) { + if (mask & IFCAP_TXCSUM_IPV6) { + ifp->if_capenable |= IFCAP_TXCSUM_IPV6; + if (vsi->flags & IXL_FLAGS_KEEP_TSO6) { + vsi->flags &= ~IXL_FLAGS_KEEP_TSO6; + ifp->if_capenable |= IFCAP_TSO6; + } + } else if (mask & IFCAP_TSO6) { + ifp->if_capenable |= (IFCAP_TXCSUM_IPV6 | IFCAP_TSO6); + vsi->flags &= ~IXL_FLAGS_KEEP_TSO6; + device_printf(dev, + "TSO6 requires txcsum6, enabling both...\n"); + } + } else if((ifp->if_capenable & IFCAP_TXCSUM_IPV6) + && !(ifp->if_capenable & IFCAP_TSO6)) { + if (mask & IFCAP_TXCSUM_IPV6) + ifp->if_capenable &= ~IFCAP_TXCSUM_IPV6; + else if (mask & IFCAP_TSO6) + ifp->if_capenable |= IFCAP_TSO6; + } else if ((ifp->if_capenable & IFCAP_TXCSUM_IPV6) + && (ifp->if_capenable & IFCAP_TSO6)) { + if (mask & IFCAP_TXCSUM_IPV6) { + vsi->flags |= IXL_FLAGS_KEEP_TSO6; + ifp->if_capenable &= ~(IFCAP_TXCSUM_IPV6 | IFCAP_TSO6); + device_printf(dev, + "TSO6 requires txcsum6, disabling both...\n"); + } else if (mask & IFCAP_TSO6) + ifp->if_capenable &= ~IFCAP_TSO6; + } +} + +/* For the set_advertise sysctl */ +void +ixl_get_initial_advertised_speeds(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + device_t dev = pf->dev; + enum i40e_status_code status; + struct i40e_aq_get_phy_abilities_resp abilities; + + /* Set initial sysctl values */ + status = i40e_aq_get_phy_capabilities(hw, FALSE, false, &abilities, + NULL); + if (status) { + /* Non-fatal error */ + device_printf(dev, "%s: i40e_aq_get_phy_capabilities() error %d\n", + __func__, status); + return; + } + + if (abilities.link_speed & I40E_LINK_SPEED_40GB) + pf->advertised_speed |= 0x10; + if (abilities.link_speed & I40E_LINK_SPEED_20GB) + pf->advertised_speed |= 0x8; + if (abilities.link_speed & I40E_LINK_SPEED_10GB) + pf->advertised_speed |= 0x4; + if (abilities.link_speed & I40E_LINK_SPEED_1GB) + pf->advertised_speed |= 0x2; + if (abilities.link_speed & I40E_LINK_SPEED_100MB) + pf->advertised_speed |= 0x1; +} + +int +ixl_teardown_hw_structs(struct ixl_pf *pf) +{ + enum i40e_status_code status = 0; + struct i40e_hw *hw = &pf->hw; + device_t dev = pf->dev; + + /* Shutdown LAN HMC */ + if (hw->hmc.hmc_obj) { + status = i40e_shutdown_lan_hmc(hw); + if (status) { + device_printf(dev, + "init: LAN HMC shutdown failure; status %d\n", status); + goto err_out; + } + } + + // XXX: This gets called when we know the adminq is inactive; + // so we already know it's setup when we get here. + + /* Shutdown admin queue */ + status = i40e_shutdown_adminq(hw); + if (status) + device_printf(dev, + "init: Admin Queue shutdown failure; status %d\n", status); + +err_out: + return (status); +} + +int +ixl_reset(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + device_t dev = pf->dev; + u8 set_fc_err_mask; + int error = 0; + + // XXX: clear_hw() actually writes to hw registers -- maybe this isn't necessary + i40e_clear_hw(hw); + error = i40e_pf_reset(hw); + if (error) { + device_printf(dev, "init: PF reset failure"); + error = EIO; + goto err_out; + } + + error = i40e_init_adminq(hw); + if (error) { + device_printf(dev, "init: Admin queue init failure;" + " status code %d", error); + error = EIO; + goto err_out; + } + + i40e_clear_pxe_mode(hw); + + error = ixl_get_hw_capabilities(pf); + if (error) { + device_printf(dev, "init: Error retrieving HW capabilities;" + " status code %d\n", error); + goto err_out; + } + + error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, + hw->func_caps.num_rx_qp, 0, 0); + if (error) { + device_printf(dev, "init: LAN HMC init failed; status code %d\n", + error); + error = EIO; + goto err_out; + } + + error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); + if (error) { + device_printf(dev, "init: LAN HMC config failed; status code %d\n", + error); + error = EIO; + goto err_out; + } + + // XXX: possible fix for panic, but our failure recovery is still broken + error = ixl_switch_config(pf); + if (error) { + device_printf(dev, "init: ixl_switch_config() failed: %d\n", + error); + goto err_out; + } + + error = i40e_aq_set_phy_int_mask(hw, IXL_DEFAULT_PHY_INT_MASK, + NULL); + if (error) { + device_printf(dev, "init: i40e_aq_set_phy_mask() failed: err %d," + " aq_err %d\n", error, hw->aq.asq_last_status); + error = EIO; + goto err_out; + } + + error = i40e_set_fc(hw, &set_fc_err_mask, true); + if (error) { + device_printf(dev, "init: setting link flow control failed; retcode %d," + " fc_err_mask 0x%02x\n", error, set_fc_err_mask); + goto err_out; + } + + // XXX: (Rebuild VSIs?) + + /* Firmware delay workaround */ + if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) || + (hw->aq.fw_maj_ver < 4)) { + i40e_msec_delay(75); + error = i40e_aq_set_link_restart_an(hw, TRUE, NULL); + if (error) { + device_printf(dev, "init: link restart failed, aq_err %d\n", + hw->aq.asq_last_status); + goto err_out; + } + } + + +err_out: + return (error); +} + +/* +** MSIX Interrupt Handlers and Tasklets +*/ +void +ixl_handle_que(void *context, int pending) +{ + struct ixl_queue *que = context; + struct ixl_vsi *vsi = que->vsi; + struct i40e_hw *hw = vsi->hw; + struct tx_ring *txr = &que->txr; + struct ifnet *ifp = vsi->ifp; + bool more; + + if (ifp->if_drv_flags & IFF_DRV_RUNNING) { + more = ixl_rxeof(que, IXL_RX_LIMIT); + IXL_TX_LOCK(txr); + ixl_txeof(que); + if (!drbr_empty(ifp, txr->br)) + ixl_mq_start_locked(ifp, txr); + IXL_TX_UNLOCK(txr); + if (more) { + taskqueue_enqueue(que->tq, &que->task); + return; + } + } + + /* Reenable this interrupt - hmmm */ + ixl_enable_queue(hw, que->me); + return; +} + + +/********************************************************************* + * + * Legacy Interrupt Service routine + * + **********************************************************************/ +void +ixl_intr(void *arg) +{ + struct ixl_pf *pf = arg; + struct i40e_hw *hw = &pf->hw; + struct ixl_vsi *vsi = &pf->vsi; + struct ixl_queue *que = vsi->queues; + struct ifnet *ifp = vsi->ifp; + struct tx_ring *txr = &que->txr; + u32 reg, icr0, mask; + bool more_tx, more_rx; + + ++que->irqs; + + /* Protect against spurious interrupts */ + if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) + return; + + icr0 = rd32(hw, I40E_PFINT_ICR0); + + reg = rd32(hw, I40E_PFINT_DYN_CTL0); + reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK; + wr32(hw, I40E_PFINT_DYN_CTL0, reg); + + mask = rd32(hw, I40E_PFINT_ICR0_ENA); + +#ifdef PCI_IOV + if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) + taskqueue_enqueue(pf->tq, &pf->vflr_task); +#endif + + if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) { + taskqueue_enqueue(pf->tq, &pf->adminq); + return; + } + + more_rx = ixl_rxeof(que, IXL_RX_LIMIT); + + IXL_TX_LOCK(txr); + more_tx = ixl_txeof(que); + if (!drbr_empty(vsi->ifp, txr->br)) + more_tx = 1; + IXL_TX_UNLOCK(txr); + + /* re-enable other interrupt causes */ + wr32(hw, I40E_PFINT_ICR0_ENA, mask); + + /* And now the queues */ + reg = rd32(hw, I40E_QINT_RQCTL(0)); + reg |= I40E_QINT_RQCTL_CAUSE_ENA_MASK; + wr32(hw, I40E_QINT_RQCTL(0), reg); + + reg = rd32(hw, I40E_QINT_TQCTL(0)); + reg |= I40E_QINT_TQCTL_CAUSE_ENA_MASK; + reg &= ~I40E_PFINT_ICR0_INTEVENT_MASK; + wr32(hw, I40E_QINT_TQCTL(0), reg); + + ixl_enable_legacy(hw); + + return; +} + + +/********************************************************************* + * + * MSIX VSI Interrupt Service routine + * + **********************************************************************/ +void +ixl_msix_que(void *arg) +{ + struct ixl_queue *que = arg; + struct ixl_vsi *vsi = que->vsi; + struct i40e_hw *hw = vsi->hw; + struct tx_ring *txr = &que->txr; + bool more_tx, more_rx; + + /* Protect against spurious interrupts */ + if (!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING)) + return; + + ++que->irqs; + + more_rx = ixl_rxeof(que, IXL_RX_LIMIT); + + IXL_TX_LOCK(txr); + more_tx = ixl_txeof(que); + /* + ** Make certain that if the stack + ** has anything queued the task gets + ** scheduled to handle it. + */ + if (!drbr_empty(vsi->ifp, txr->br)) + more_tx = 1; + IXL_TX_UNLOCK(txr); + + ixl_set_queue_rx_itr(que); + ixl_set_queue_tx_itr(que); + + if (more_tx || more_rx) + taskqueue_enqueue(que->tq, &que->task); + else + ixl_enable_queue(hw, que->me); + + return; +} + + +/********************************************************************* + * + * MSIX Admin Queue Interrupt Service routine + * + **********************************************************************/ +void +ixl_msix_adminq(void *arg) +{ + struct ixl_pf *pf = arg; + struct i40e_hw *hw = &pf->hw; + device_t dev = pf->dev; + u32 reg, mask, rstat_reg; + bool do_task = FALSE; + + ++pf->admin_irq; + + reg = rd32(hw, I40E_PFINT_ICR0); + mask = rd32(hw, I40E_PFINT_ICR0_ENA); + + /* Check on the cause */ + if (reg & I40E_PFINT_ICR0_ADMINQ_MASK) { + mask &= ~I40E_PFINT_ICR0_ADMINQ_MASK; + do_task = TRUE; + } + + if (reg & I40E_PFINT_ICR0_MAL_DETECT_MASK) { + ixl_handle_mdd_event(pf); + mask &= ~I40E_PFINT_ICR0_MAL_DETECT_MASK; + } + + if (reg & I40E_PFINT_ICR0_GRST_MASK) { + device_printf(dev, "Reset Requested!\n"); + rstat_reg = rd32(hw, I40E_GLGEN_RSTAT); + rstat_reg = (rstat_reg & I40E_GLGEN_RSTAT_RESET_TYPE_MASK) + >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; + device_printf(dev, "Reset type: "); + switch (rstat_reg) { + /* These others might be handled similarly to an EMPR reset */ + case I40E_RESET_CORER: + printf("CORER\n"); + break; + case I40E_RESET_GLOBR: + printf("GLOBR\n"); + break; + case I40E_RESET_EMPR: + printf("EMPR\n"); + atomic_set_int(&pf->state, IXL_PF_STATE_EMPR_RESETTING); + break; + default: + printf("POR\n"); + break; + } + /* overload admin queue task to check reset progress */ + do_task = TRUE; + } + + if (reg & I40E_PFINT_ICR0_ECC_ERR_MASK) { + device_printf(dev, "ECC Error detected!\n"); + } + + if (reg & I40E_PFINT_ICR0_HMC_ERR_MASK) { + reg = rd32(hw, I40E_PFHMC_ERRORINFO); + if (reg & I40E_PFHMC_ERRORINFO_ERROR_DETECTED_MASK) { + device_printf(dev, "HMC Error detected!\n"); + device_printf(dev, "INFO 0x%08x\n", reg); + reg = rd32(hw, I40E_PFHMC_ERRORDATA); + device_printf(dev, "DATA 0x%08x\n", reg); + wr32(hw, I40E_PFHMC_ERRORINFO, 0); + } + } + + if (reg & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) { + device_printf(dev, "PCI Exception detected!\n"); + } + +#ifdef PCI_IOV + if (reg & I40E_PFINT_ICR0_VFLR_MASK) { + mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK; + taskqueue_enqueue(pf->tq, &pf->vflr_task); + } +#endif + + if (do_task) + taskqueue_enqueue(pf->tq, &pf->adminq); + else + ixl_enable_adminq(hw); +} + +void +ixl_set_promisc(struct ixl_vsi *vsi) +{ + struct ifnet *ifp = vsi->ifp; + struct i40e_hw *hw = vsi->hw; + int err, mcnt = 0; + bool uni = FALSE, multi = FALSE; + + if (ifp->if_flags & IFF_ALLMULTI) + multi = TRUE; + else { /* Need to count the multicast addresses */ + struct ifmultiaddr *ifma; + if_maddr_rlock(ifp); + TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + if (ifma->ifma_addr->sa_family != AF_LINK) + continue; + if (mcnt == MAX_MULTICAST_ADDR) + break; + mcnt++; + } + if_maddr_runlock(ifp); + } + + if (mcnt >= MAX_MULTICAST_ADDR) + multi = TRUE; + if (ifp->if_flags & IFF_PROMISC) + uni = TRUE; + + err = i40e_aq_set_vsi_unicast_promiscuous(hw, + vsi->seid, uni, NULL, TRUE); + err = i40e_aq_set_vsi_multicast_promiscuous(hw, + vsi->seid, multi, NULL); + return; +} + +/********************************************************************* + * Filter Routines + * + * Routines for multicast and vlan filter management. + * + *********************************************************************/ +void +ixl_add_multi(struct ixl_vsi *vsi) +{ + struct ifmultiaddr *ifma; + struct ifnet *ifp = vsi->ifp; + struct i40e_hw *hw = vsi->hw; + int mcnt = 0, flags; + + IOCTL_DEBUGOUT("ixl_add_multi: begin"); + + if_maddr_rlock(ifp); + /* + ** First just get a count, to decide if we + ** we simply use multicast promiscuous. + */ + TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + if (ifma->ifma_addr->sa_family != AF_LINK) + continue; + mcnt++; + } + if_maddr_runlock(ifp); + + if (__predict_false(mcnt >= MAX_MULTICAST_ADDR)) { + /* delete existing MC filters */ + ixl_del_hw_filters(vsi, mcnt); + i40e_aq_set_vsi_multicast_promiscuous(hw, + vsi->seid, TRUE, NULL); + return; + } + + mcnt = 0; + if_maddr_rlock(ifp); + TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + if (ifma->ifma_addr->sa_family != AF_LINK) + continue; + ixl_add_mc_filter(vsi, + (u8*)LLADDR((struct sockaddr_dl *) ifma->ifma_addr)); + mcnt++; + } + if_maddr_runlock(ifp); + if (mcnt > 0) { + flags = (IXL_FILTER_ADD | IXL_FILTER_USED | IXL_FILTER_MC); + ixl_add_hw_filters(vsi, flags, mcnt); + } + + IOCTL_DEBUGOUT("ixl_add_multi: end"); + return; +} + +void +ixl_del_multi(struct ixl_vsi *vsi) +{ + struct ifnet *ifp = vsi->ifp; + struct ifmultiaddr *ifma; + struct ixl_mac_filter *f; + int mcnt = 0; + bool match = FALSE; + + IOCTL_DEBUGOUT("ixl_del_multi: begin"); + + /* Search for removed multicast addresses */ + if_maddr_rlock(ifp); + SLIST_FOREACH(f, &vsi->ftl, next) { + if ((f->flags & IXL_FILTER_USED) && (f->flags & IXL_FILTER_MC)) { + match = FALSE; + TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + if (ifma->ifma_addr->sa_family != AF_LINK) + continue; + u8 *mc_addr = (u8 *)LLADDR((struct sockaddr_dl *)ifma->ifma_addr); + if (cmp_etheraddr(f->macaddr, mc_addr)) { + match = TRUE; + break; + } + } + if (match == FALSE) { + f->flags |= IXL_FILTER_DEL; + mcnt++; + } + } + } + if_maddr_runlock(ifp); + + if (mcnt > 0) + ixl_del_hw_filters(vsi, mcnt); +} + + +/********************************************************************* + * Timer routine + * + * This routine checks for link status,updates statistics, + * and runs the watchdog check. + * + * Only runs when the driver is configured UP and RUNNING. + * + **********************************************************************/ + +void +ixl_local_timer(void *arg) +{ + struct ixl_pf *pf = arg; + struct i40e_hw *hw = &pf->hw; + struct ixl_vsi *vsi = &pf->vsi; + struct ixl_queue *que = vsi->queues; + device_t dev = pf->dev; + int hung = 0; + u32 mask; + + mtx_assert(&pf->pf_mtx, MA_OWNED); + + /* Fire off the adminq task */ + taskqueue_enqueue(pf->tq, &pf->adminq); + + /* Update stats */ + ixl_update_stats_counters(pf); + + /* Check status of the queues */ + mask = (I40E_PFINT_DYN_CTLN_INTENA_MASK | + I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK); + + for (int i = 0; i < vsi->num_queues; i++, que++) { + /* Any queues with outstanding work get a sw irq */ + if (que->busy) + wr32(hw, I40E_PFINT_DYN_CTLN(que->me), mask); + /* + ** Each time txeof runs without cleaning, but there + ** are uncleaned descriptors it increments busy. If + ** we get to 5 we declare it hung. + */ + if (que->busy == IXL_QUEUE_HUNG) { + ++hung; + continue; + } + if (que->busy >= IXL_MAX_TX_BUSY) { +#ifdef IXL_DEBUG + device_printf(dev, "Warning queue %d " + "appears to be hung!\n", i); +#endif + que->busy = IXL_QUEUE_HUNG; + ++hung; + } + } + /* Only reinit if all queues show hung */ + if (hung == vsi->num_queues) + goto hung; + + callout_reset(&pf->timer, hz, ixl_local_timer, pf); + return; + +hung: + device_printf(dev, "Local Timer: HANG DETECT - Resetting!!\n"); + ixl_init_locked(pf); +} + +/* +** Note: this routine updates the OS on the link state +** the real check of the hardware only happens with +** a link interrupt. +*/ +void +ixl_update_link_status(struct ixl_pf *pf) +{ + struct ixl_vsi *vsi = &pf->vsi; + struct i40e_hw *hw = &pf->hw; + struct ifnet *ifp = vsi->ifp; + device_t dev = pf->dev; + + if (pf->link_up) { + if (vsi->link_active == FALSE) { + pf->fc = hw->fc.current_mode; + if (bootverbose) { + device_printf(dev, "Link is up %d Gbps %s," + " Flow Control: %s\n", + ((pf->link_speed == + I40E_LINK_SPEED_40GB)? 40:10), + "Full Duplex", ixl_fc_string[pf->fc]); + } + vsi->link_active = TRUE; + if_link_state_change(ifp, LINK_STATE_UP); + } + } else { /* Link down */ + if (vsi->link_active == TRUE) { + if (bootverbose) + device_printf(dev, "Link is Down\n"); + if_link_state_change(ifp, LINK_STATE_DOWN); + vsi->link_active = FALSE; + } + } + + return; +} + +/********************************************************************* + * + * This routine disables all traffic on the adapter by issuing a + * global reset on the MAC and deallocates TX/RX buffers. + * + **********************************************************************/ + +void +ixl_stop_locked(struct ixl_pf *pf) +{ + struct ixl_vsi *vsi = &pf->vsi; + struct ifnet *ifp = vsi->ifp; + + INIT_DEBUGOUT("ixl_stop: begin\n"); + + IXL_PF_LOCK_ASSERT(pf); + + /* Stop the local timer */ + callout_stop(&pf->timer); + + ixl_disable_rings_intr(vsi); + ixl_disable_rings(vsi); + + /* Tell the stack that the interface is no longer active */ + ifp->if_drv_flags &= ~(IFF_DRV_RUNNING); +} + +void +ixl_stop(struct ixl_pf *pf) +{ + IXL_PF_LOCK(pf); + ixl_stop_locked(pf); + IXL_PF_UNLOCK(pf); + + ixl_teardown_queue_msix(&pf->vsi); + ixl_free_queue_tqs(&pf->vsi); +} + +/********************************************************************* + * + * Setup MSIX Interrupt resources and handlers for the VSI + * + **********************************************************************/ +int +ixl_assign_vsi_legacy(struct ixl_pf *pf) +{ + device_t dev = pf->dev; + struct ixl_vsi *vsi = &pf->vsi; + struct ixl_queue *que = vsi->queues; + int error, rid = 0; + + if (pf->msix == 1) + rid = 1; + pf->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, + &rid, RF_SHAREABLE | RF_ACTIVE); + if (pf->res == NULL) { + device_printf(dev, "Unable to allocate" + " bus resource: vsi legacy/msi interrupt\n"); + return (ENXIO); + } + + /* Set the handler function */ + error = bus_setup_intr(dev, pf->res, + INTR_TYPE_NET | INTR_MPSAFE, NULL, + ixl_intr, pf, &pf->tag); + if (error) { + pf->res = NULL; + device_printf(dev, "Failed to register legacy/msi handler\n"); + return (error); + } + bus_describe_intr(dev, pf->res, pf->tag, "irq0"); + TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que); + TASK_INIT(&que->task, 0, ixl_handle_que, que); + que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT, + taskqueue_thread_enqueue, &que->tq); + taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que", + device_get_nameunit(dev)); + TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf); + + pf->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT, + taskqueue_thread_enqueue, &pf->tq); + taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s adminq", + device_get_nameunit(dev)); + + return (0); +} + +int +ixl_setup_adminq_tq(struct ixl_pf *pf) +{ + device_t dev = pf->dev; + int error = 0; + + /* Tasklet for Admin Queue interrupts */ + TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf); +#ifdef PCI_IOV + /* VFLR Tasklet */ + TASK_INIT(&pf->vflr_task, 0, ixl_handle_vflr, pf); +#endif + /* Create and start Admin Queue taskqueue */ + pf->tq = taskqueue_create_fast("ixl_aq", M_NOWAIT, + taskqueue_thread_enqueue, &pf->tq); + if (!pf->tq) { + device_printf(dev, "taskqueue_create_fast (for AQ) returned NULL!\n"); + return (ENOMEM); + } + error = taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s aq", + device_get_nameunit(dev)); + if (error) { + device_printf(dev, "taskqueue_start_threads (for AQ) error: %d\n", + error); + taskqueue_free(pf->tq); + return (error); + } + return (0); +} + +int +ixl_setup_queue_tqs(struct ixl_vsi *vsi) +{ + struct ixl_queue *que = vsi->queues; + device_t dev = vsi->dev; +#ifdef RSS + int cpu_id = 0; + cpuset_t cpu_mask; +#endif + + /* Create queue tasks and start queue taskqueues */ + for (int i = 0; i < vsi->num_queues; i++, que++) { + TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que); + TASK_INIT(&que->task, 0, ixl_handle_que, que); + que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT, + taskqueue_thread_enqueue, &que->tq); +#ifdef RSS + CPU_SETOF(cpu_id, &cpu_mask); + taskqueue_start_threads_cpuset(&que->tq, 1, PI_NET, + &cpu_mask, "%s (bucket %d)", + device_get_nameunit(dev), cpu_id); +#else + taskqueue_start_threads(&que->tq, 1, PI_NET, + "%s (que %d)", device_get_nameunit(dev), que->me); +#endif + } + + return (0); +} + +void +ixl_free_adminq_tq(struct ixl_pf *pf) +{ + if (pf->tq) { + taskqueue_free(pf->tq); + pf->tq = NULL; + } +} + +void +ixl_free_queue_tqs(struct ixl_vsi *vsi) +{ + struct ixl_queue *que = vsi->queues; + + for (int i = 0; i < vsi->num_queues; i++, que++) { + if (que->tq) { + taskqueue_free(que->tq); + que->tq = NULL; + } + } +} + +int +ixl_setup_adminq_msix(struct ixl_pf *pf) +{ + device_t dev = pf->dev; + int rid, error = 0; + + /* Admin IRQ rid is 1, vector is 0 */ + rid = 1; + /* Get interrupt resource from bus */ + pf->res = bus_alloc_resource_any(dev, + SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); + if (!pf->res) { + device_printf(dev, "bus_alloc_resource_any() for Admin Queue" + " interrupt failed [rid=%d]\n", rid); + return (ENXIO); + } + /* Then associate interrupt with handler */ + error = bus_setup_intr(dev, pf->res, + INTR_TYPE_NET | INTR_MPSAFE, NULL, + ixl_msix_adminq, pf, &pf->tag); + if (error) { + pf->res = NULL; + device_printf(dev, "bus_setup_intr() for Admin Queue" + " interrupt handler failed, error %d\n", error); + return (ENXIO); + } + error = bus_describe_intr(dev, pf->res, pf->tag, "aq"); + if (error) { + /* Probably non-fatal? */ + device_printf(dev, "bus_describe_intr() for Admin Queue" + " interrupt name failed, error %d\n", error); + } + pf->admvec = 0; + + return (0); +} + +/* + * Allocate interrupt resources from bus and associate an interrupt handler + * to those for the VSI's queues. + */ +int +ixl_setup_queue_msix(struct ixl_vsi *vsi) +{ + device_t dev = vsi->dev; + struct ixl_queue *que = vsi->queues; + struct tx_ring *txr; + int error, rid, vector = 1; + + /* Queue interrupt vector numbers start at 1 (adminq intr is 0) */ + for (int i = 0; i < vsi->num_queues; i++, vector++, que++) { + int cpu_id = i; + rid = vector + 1; + txr = &que->txr; + que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, + RF_SHAREABLE | RF_ACTIVE); + if (!que->res) { + device_printf(dev, "bus_alloc_resource_any() for" + " Queue %d interrupt failed [rid=%d]\n", + que->me, rid); + return (ENXIO); + } + /* Set the handler function */ + error = bus_setup_intr(dev, que->res, + INTR_TYPE_NET | INTR_MPSAFE, NULL, + ixl_msix_que, que, &que->tag); + if (error) { + device_printf(dev, "bus_setup_intr() for Queue %d" + " interrupt handler failed, error %d\n", + que->me, error); + return (error); + } + error = bus_describe_intr(dev, que->res, que->tag, "q%d", i); + if (error) { + device_printf(dev, "bus_describe_intr() for Queue %d" + " interrupt name failed, error %d\n", + que->me, error); + } + /* Bind the vector to a CPU */ +#ifdef RSS + cpu_id = rss_getcpu(i % rss_getnumbuckets()); +#endif + error = bus_bind_intr(dev, que->res, cpu_id); + if (error) { + device_printf(dev, "bus_bind_intr() for Queue %d" + " to CPU %d failed, error %d\n", + que->me, cpu_id, error); + } + que->msix = vector; + } + + return (0); +} + +/* + * When used in a virtualized environment PCI BUSMASTER capability may not be set + * so explicity set it here and rewrite the ENABLE in the MSIX control register + * at this point to cause the host to successfully initialize us. + */ +void +ixl_set_busmaster(device_t dev) +{ + u16 pci_cmd_word; + int msix_ctrl, rid; + + pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); + pci_cmd_word |= PCIM_CMD_BUSMASTEREN; + pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2); + + pci_find_cap(dev, PCIY_MSIX, &rid); + rid += PCIR_MSIX_CTRL; + msix_ctrl = pci_read_config(dev, rid, 2); + msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE; + pci_write_config(dev, rid, msix_ctrl, 2); +} + +/* + * Allocate MSI/X vectors from the OS. + * Returns 0 for legacy, 1 for MSI, >1 for MSIX. + */ +int +ixl_init_msix(struct ixl_pf *pf) +{ + device_t dev = pf->dev; + struct i40e_hw *hw = &pf->hw; + int auto_max_queues; + int rid, want, vectors, queues, available; + + /* Override by tuneable */ + if (!pf->enable_msix) + goto no_msix; + + /* Ensure proper operation in virtualized environment */ + ixl_set_busmaster(dev); + + /* First try MSI/X */ + rid = PCIR_BAR(IXL_BAR); + pf->msix_mem = bus_alloc_resource_any(dev, + SYS_RES_MEMORY, &rid, RF_ACTIVE); + if (!pf->msix_mem) { + /* May not be enabled */ + device_printf(pf->dev, + "Unable to map MSIX table\n"); + goto no_msix; + } + + available = pci_msix_count(dev); + if (available < 2) { + /* system has msix disabled (0), or only one vector (1) */ + bus_release_resource(dev, SYS_RES_MEMORY, + rid, pf->msix_mem); + pf->msix_mem = NULL; + goto no_msix; + } + + /* Clamp max number of queues based on: + * - # of MSI-X vectors available + * - # of cpus available + * - # of queues that can be assigned to the LAN VSI + */ + auto_max_queues = min(mp_ncpus, available - 1); + if (hw->mac.type == I40E_MAC_X722) + auto_max_queues = min(auto_max_queues, 128); + else + auto_max_queues = min(auto_max_queues, 64); + + /* Override with tunable value if tunable is less than autoconfig count */ + if ((pf->max_queues != 0) && (pf->max_queues <= auto_max_queues)) + queues = pf->max_queues; + /* Use autoconfig amount if that's lower */ + else if ((pf->max_queues != 0) && (pf->max_queues > auto_max_queues)) { + device_printf(dev, "ixl_max_queues (%d) is too large, using " + "autoconfig amount (%d)...\n", + pf->max_queues, auto_max_queues); + queues = auto_max_queues; + } + /* Limit maximum auto-configured queues to 8 if no user value is set */ + else + queues = min(auto_max_queues, 8); + +#ifdef RSS + /* If we're doing RSS, clamp at the number of RSS buckets */ + if (queues > rss_getnumbuckets()) + queues = rss_getnumbuckets(); +#endif + + /* + ** Want one vector (RX/TX pair) per queue + ** plus an additional for the admin queue. + */ + want = queues + 1; + if (want <= available) /* Have enough */ + vectors = want; + else { + device_printf(pf->dev, + "MSIX Configuration Problem, " + "%d vectors available but %d wanted!\n", + available, want); + return (0); /* Will go to Legacy setup */ + } + + if (pci_alloc_msix(dev, &vectors) == 0) { + device_printf(pf->dev, + "Using MSIX interrupts with %d vectors\n", vectors); + pf->msix = vectors; + pf->vsi.num_queues = queues; +#ifdef RSS + /* + * If we're doing RSS, the number of queues needs to + * match the number of RSS buckets that are configured. + * + * + If there's more queues than RSS buckets, we'll end + * up with queues that get no traffic. + * + * + If there's more RSS buckets than queues, we'll end + * up having multiple RSS buckets map to the same queue, + * so there'll be some contention. + */ + if (queues != rss_getnumbuckets()) { + device_printf(dev, + "%s: queues (%d) != RSS buckets (%d)" + "; performance will be impacted.\n", + __func__, queues, rss_getnumbuckets()); + } +#endif + return (vectors); + } +no_msix: + vectors = pci_msi_count(dev); + pf->vsi.num_queues = 1; + pf->max_queues = 1; + pf->enable_msix = 0; + if (vectors == 1 && pci_alloc_msi(dev, &vectors) == 0) + device_printf(pf->dev, "Using an MSI interrupt\n"); + else { + vectors = 0; + device_printf(pf->dev, "Using a Legacy interrupt\n"); + } + return (vectors); +} + +/* + * Configure admin queue/misc interrupt cause registers in hardware. + */ +void +ixl_configure_intr0_msix(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + u32 reg; + + /* First set up the adminq - vector 0 */ + wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */ + rd32(hw, I40E_PFINT_ICR0); /* read to clear */ + + reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | + I40E_PFINT_ICR0_ENA_GRST_MASK | + I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | + I40E_PFINT_ICR0_ENA_ADMINQ_MASK | + I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | + I40E_PFINT_ICR0_ENA_VFLR_MASK | + I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK; + wr32(hw, I40E_PFINT_ICR0_ENA, reg); + + /* + * 0x7FF is the end of the queue list. + * This means we won't use MSI-X vector 0 for a queue interrupt + * in MSIX mode. + */ + wr32(hw, I40E_PFINT_LNKLST0, 0x7FF); + /* Value is in 2 usec units, so 0x3E is 62*2 = 124 usecs. */ + wr32(hw, I40E_PFINT_ITR0(IXL_RX_ITR), 0x3E); + + wr32(hw, I40E_PFINT_DYN_CTL0, + I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK | + I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK); + + wr32(hw, I40E_PFINT_STAT_CTL0, 0); +} + +/* + * Configure queue interrupt cause registers in hardware. + */ +void +ixl_configure_queue_intr_msix(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + struct ixl_vsi *vsi = &pf->vsi; + u32 reg; + u16 vector = 1; + + for (int i = 0; i < vsi->num_queues; i++, vector++) { + wr32(hw, I40E_PFINT_DYN_CTLN(i), 0); + /* First queue type is RX / 0 */ + wr32(hw, I40E_PFINT_LNKLSTN(i), i); + + reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK | + (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | + (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | + (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | + (I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); + wr32(hw, I40E_QINT_RQCTL(i), reg); + + reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK | + (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | + (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | + (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | + (I40E_QUEUE_TYPE_RX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); + wr32(hw, I40E_QINT_TQCTL(i), reg); + } +} + +/* + * Configure for MSI single vector operation + */ +void +ixl_configure_legacy(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + u32 reg; + + wr32(hw, I40E_PFINT_ITR0(0), 0); + wr32(hw, I40E_PFINT_ITR0(1), 0); + + /* Setup "other" causes */ + reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK + | I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK + | I40E_PFINT_ICR0_ENA_GRST_MASK + | I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK + | I40E_PFINT_ICR0_ENA_GPIO_MASK + | I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK + | I40E_PFINT_ICR0_ENA_HMC_ERR_MASK + | I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK + | I40E_PFINT_ICR0_ENA_VFLR_MASK + | I40E_PFINT_ICR0_ENA_ADMINQ_MASK + ; + wr32(hw, I40E_PFINT_ICR0_ENA, reg); + + /* SW_ITR_IDX = 0, but don't change INTENA */ + wr32(hw, I40E_PFINT_DYN_CTL0, + I40E_PFINT_DYN_CTLN_SW_ITR_INDX_MASK | + I40E_PFINT_DYN_CTLN_INTENA_MSK_MASK); + /* SW_ITR_IDX = 0, OTHER_ITR_IDX = 0 */ + wr32(hw, I40E_PFINT_STAT_CTL0, 0); + + /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */ + wr32(hw, I40E_PFINT_LNKLST0, 0); + + /* Associate the queue pair to the vector and enable the q int */ + reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK + | (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) + | (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); + wr32(hw, I40E_QINT_RQCTL(0), reg); + + reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK + | (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) + | (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); + wr32(hw, I40E_QINT_TQCTL(0), reg); +} + +int +ixl_allocate_pci_resources(struct ixl_pf *pf) +{ + int rid; + struct i40e_hw *hw = &pf->hw; + device_t dev = pf->dev; + + /* Map BAR0 */ + rid = PCIR_BAR(0); + pf->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, + &rid, RF_ACTIVE); + + if (!(pf->pci_mem)) { + device_printf(dev, "Unable to allocate bus resource: PCI memory\n"); + return (ENXIO); + } + + /* Save off the PCI information */ + hw->vendor_id = pci_get_vendor(dev); + hw->device_id = pci_get_device(dev); + hw->revision_id = pci_read_config(dev, PCIR_REVID, 1); + hw->subsystem_vendor_id = + pci_read_config(dev, PCIR_SUBVEND_0, 2); + hw->subsystem_device_id = + pci_read_config(dev, PCIR_SUBDEV_0, 2); + + hw->bus.device = pci_get_slot(dev); + hw->bus.func = pci_get_function(dev); + + /* Save off register access information */ + pf->osdep.mem_bus_space_tag = + rman_get_bustag(pf->pci_mem); + pf->osdep.mem_bus_space_handle = + rman_get_bushandle(pf->pci_mem); + pf->osdep.mem_bus_space_size = rman_get_size(pf->pci_mem); + pf->osdep.flush_reg = I40E_GLGEN_STAT; + pf->hw.hw_addr = (u8 *) &pf->osdep.mem_bus_space_handle; + + pf->hw.back = &pf->osdep; + + return (0); +} + +/* + * Teardown and release the admin queue/misc vector + * interrupt. + */ +int +ixl_teardown_adminq_msix(struct ixl_pf *pf) +{ + device_t dev = pf->dev; + int rid; + + if (pf->admvec) /* we are doing MSIX */ + rid = pf->admvec + 1; + else + (pf->msix != 0) ? (rid = 1):(rid = 0); + + if (pf->tag != NULL) { + bus_teardown_intr(dev, pf->res, pf->tag); + pf->tag = NULL; + } + if (pf->res != NULL) { + bus_release_resource(dev, SYS_RES_IRQ, rid, pf->res); + pf->res = NULL; + } + + return (0); +} + +int +ixl_teardown_queue_msix(struct ixl_vsi *vsi) +{ + struct ixl_pf *pf = (struct ixl_pf *)vsi->back; + struct ixl_queue *que = vsi->queues; + device_t dev = vsi->dev; + int rid, error = 0; + + /* We may get here before stations are setup */ + if ((!pf->enable_msix) || (que == NULL)) + return (0); + + /* Release all MSIX queue resources */ + for (int i = 0; i < vsi->num_queues; i++, que++) { + rid = que->msix + 1; + if (que->tag != NULL) { + error = bus_teardown_intr(dev, que->res, que->tag); + if (error) { + device_printf(dev, "bus_teardown_intr() for" + " Queue %d interrupt failed\n", + que->me); + // return (ENXIO); + } + que->tag = NULL; + } + if (que->res != NULL) { + error = bus_release_resource(dev, SYS_RES_IRQ, rid, que->res); + if (error) { + device_printf(dev, "bus_release_resource() for" + " Queue %d interrupt failed [rid=%d]\n", + que->me, rid); + // return (ENXIO); + } + que->res = NULL; + } + } + + return (0); +} + +void +ixl_free_pci_resources(struct ixl_pf *pf) +{ + device_t dev = pf->dev; + int memrid; + + ixl_teardown_queue_msix(&pf->vsi); + ixl_teardown_adminq_msix(pf); + + if (pf->msix) + pci_release_msi(dev); + + memrid = PCIR_BAR(IXL_BAR); + + if (pf->msix_mem != NULL) + bus_release_resource(dev, SYS_RES_MEMORY, + memrid, pf->msix_mem); + + if (pf->pci_mem != NULL) + bus_release_resource(dev, SYS_RES_MEMORY, + PCIR_BAR(0), pf->pci_mem); + + return; +} + +void +ixl_add_ifmedia(struct ixl_vsi *vsi, u32 phy_type) +{ + /* Display supported media types */ + if (phy_type & (1 << I40E_PHY_TYPE_100BASE_TX)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_100_TX, 0, NULL); + + if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_T)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_T, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_SX)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_SX, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_LX)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_LX, 0, NULL); + + if (phy_type & (1 << I40E_PHY_TYPE_XAUI) || + phy_type & (1 << I40E_PHY_TYPE_XFI) || + phy_type & (1 << I40E_PHY_TYPE_10GBASE_SFPP_CU)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL); + + if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_SR)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_LR)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_LR, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_T)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_T, 0, NULL); + + if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4) || + phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4_CU) || + phy_type & (1 << I40E_PHY_TYPE_40GBASE_AOC) || + phy_type & (1 << I40E_PHY_TYPE_XLAUI) || + phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_SR4)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_LR4)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_LR4, 0, NULL); + + if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_KX)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_KX, 0, NULL); + + if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1_CU) + || phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_CR1, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_AOC)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX_LONG, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_SFI)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SFI, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KX4)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_KX4, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KR)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_KR, 0, NULL); + + if (phy_type & (1 << I40E_PHY_TYPE_20GBASE_KR2)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_20G_KR2, 0, NULL); + + if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_KR4, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_XLPPI)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_XLPPI, 0, NULL); +} + +/********************************************************************* + * + * Setup networking device structure and register an interface. + * + **********************************************************************/ +int +ixl_setup_interface(device_t dev, struct ixl_vsi *vsi) +{ + struct ifnet *ifp; + struct i40e_hw *hw = vsi->hw; + struct ixl_queue *que = vsi->queues; + struct i40e_aq_get_phy_abilities_resp abilities; + enum i40e_status_code aq_error = 0; + + INIT_DEBUGOUT("ixl_setup_interface: begin"); + + ifp = vsi->ifp = if_alloc(IFT_ETHER); + if (ifp == NULL) { + device_printf(dev, "can not allocate ifnet structure\n"); + return (-1); + } + if_initname(ifp, device_get_name(dev), device_get_unit(dev)); + ifp->if_mtu = ETHERMTU; + ifp->if_baudrate = IF_Gbps(40); + ifp->if_init = ixl_init; + ifp->if_softc = vsi; + ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; + ifp->if_ioctl = ixl_ioctl; + +#if __FreeBSD_version >= 1100036 + if_setgetcounterfn(ifp, ixl_get_counter); +#endif + + ifp->if_transmit = ixl_mq_start; + + ifp->if_qflush = ixl_qflush; + + ifp->if_snd.ifq_maxlen = que->num_desc - 2; + + vsi->max_frame_size = + ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN + + ETHER_VLAN_ENCAP_LEN; + + /* Set TSO limits */ + ifp->if_hw_tsomax = IP_MAXPACKET - (ETHER_HDR_LEN + ETHER_CRC_LEN); + ifp->if_hw_tsomaxsegcount = IXL_MAX_TSO_SEGS; + ifp->if_hw_tsomaxsegsize = PAGE_SIZE; + + /* + * Tell the upper layer(s) we support long frames. + */ + ifp->if_hdrlen = sizeof(struct ether_vlan_header); + + ifp->if_capabilities |= IFCAP_HWCSUM; + ifp->if_capabilities |= IFCAP_HWCSUM_IPV6; + ifp->if_capabilities |= IFCAP_TSO; + ifp->if_capabilities |= IFCAP_JUMBO_MTU; + ifp->if_capabilities |= IFCAP_LRO; + + /* VLAN capabilties */ + ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING + | IFCAP_VLAN_HWTSO + | IFCAP_VLAN_MTU + | IFCAP_VLAN_HWCSUM; + ifp->if_capenable = ifp->if_capabilities; + + /* + ** Don't turn this on by default, if vlans are + ** created on another pseudo device (eg. lagg) + ** then vlan events are not passed thru, breaking + ** operation, but with HW FILTER off it works. If + ** using vlans directly on the ixl driver you can + ** enable this and get full hardware tag filtering. + */ + ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; + + /* + * Specify the media types supported by this adapter and register + * callbacks to update media and link information + */ + ifmedia_init(&vsi->media, IFM_IMASK, ixl_media_change, + ixl_media_status); + + aq_error = i40e_aq_get_phy_capabilities(hw, + FALSE, TRUE, &abilities, NULL); + /* May need delay to detect fiber correctly */ + if (aq_error == I40E_ERR_UNKNOWN_PHY) { + i40e_msec_delay(200); + aq_error = i40e_aq_get_phy_capabilities(hw, FALSE, + TRUE, &abilities, NULL); + } + if (aq_error) { + if (aq_error == I40E_ERR_UNKNOWN_PHY) + device_printf(dev, "Unknown PHY type detected!\n"); + else + device_printf(dev, + "Error getting supported media types, err %d," + " AQ error %d\n", aq_error, hw->aq.asq_last_status); + return (0); + } + + ixl_add_ifmedia(vsi, abilities.phy_type); + + /* Use autoselect media by default */ + ifmedia_add(&vsi->media, IFM_ETHER | IFM_AUTO, 0, NULL); + ifmedia_set(&vsi->media, IFM_ETHER | IFM_AUTO); + + ether_ifattach(ifp, hw->mac.addr); + + return (0); +} + +/* +** Run when the Admin Queue gets a link state change interrupt. +*/ +void +ixl_link_event(struct ixl_pf *pf, struct i40e_arq_event_info *e) +{ + struct i40e_hw *hw = &pf->hw; + device_t dev = pf->dev; + struct i40e_aqc_get_link_status *status = + (struct i40e_aqc_get_link_status *)&e->desc.params.raw; + + /* Request link status from adapter */ + hw->phy.get_link_info = TRUE; + i40e_get_link_status(hw, &pf->link_up); + + /* Print out message if an unqualified module is found */ + if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) && + (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) && + (!(status->link_info & I40E_AQ_LINK_UP))) + device_printf(dev, "Link failed because " + "an unqualified module was detected!\n"); + + /* Update OS link info */ + ixl_update_link_status(pf); +} + +/********************************************************************* + * + * Get Firmware Switch configuration + * - this will need to be more robust when more complex + * switch configurations are enabled. + * + **********************************************************************/ +int +ixl_switch_config(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + struct ixl_vsi *vsi = &pf->vsi; + device_t dev = vsi->dev; + struct i40e_aqc_get_switch_config_resp *sw_config; + u8 aq_buf[I40E_AQ_LARGE_BUF]; + int ret; + u16 next = 0; + + memset(&aq_buf, 0, sizeof(aq_buf)); + sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; + ret = i40e_aq_get_switch_config(hw, sw_config, + sizeof(aq_buf), &next, NULL); + if (ret) { + device_printf(dev, "aq_get_switch_config() failed, error %d," + " aq_error %d\n", ret, pf->hw.aq.asq_last_status); + return (ret); + } + if (pf->dbg_mask & IXL_DBG_SWITCH_INFO) { + device_printf(dev, + "Switch config: header reported: %d in structure, %d total\n", + sw_config->header.num_reported, sw_config->header.num_total); + for (int i = 0; i < sw_config->header.num_reported; i++) { + device_printf(dev, + "%d: type=%d seid=%d uplink=%d downlink=%d\n", i, + sw_config->element[i].element_type, + sw_config->element[i].seid, + sw_config->element[i].uplink_seid, + sw_config->element[i].downlink_seid); + } + } + /* Simplified due to a single VSI */ + vsi->uplink_seid = sw_config->element[0].uplink_seid; + vsi->downlink_seid = sw_config->element[0].downlink_seid; + vsi->seid = sw_config->element[0].seid; + return (ret); +} + +/********************************************************************* + * + * Initialize the VSI: this handles contexts, which means things + * like the number of descriptors, buffer size, + * plus we init the rings thru this function. + * + **********************************************************************/ +int +ixl_initialize_vsi(struct ixl_vsi *vsi) +{ + struct ixl_pf *pf = vsi->back; + struct ixl_queue *que = vsi->queues; + device_t dev = vsi->dev; + struct i40e_hw *hw = vsi->hw; + struct i40e_vsi_context ctxt; + int tc_queues; + int err = 0; + + memset(&ctxt, 0, sizeof(ctxt)); + ctxt.seid = vsi->seid; + if (pf->veb_seid != 0) + ctxt.uplink_seid = pf->veb_seid; + ctxt.pf_num = hw->pf_id; + err = i40e_aq_get_vsi_params(hw, &ctxt, NULL); + if (err) { + device_printf(dev, "i40e_aq_get_vsi_params() failed, error %d" + " aq_error %d\n", err, hw->aq.asq_last_status); + return (err); + } + ixl_dbg(pf, IXL_DBG_SWITCH_INFO, + "get_vsi_params: seid: %d, uplinkseid: %d, vsi_number: %d, " + "vsis_allocated: %d, vsis_unallocated: %d, flags: 0x%x, " + "pfnum: %d, vfnum: %d, stat idx: %d, enabled: %d\n", ctxt.seid, + ctxt.uplink_seid, ctxt.vsi_number, + ctxt.vsis_allocated, ctxt.vsis_unallocated, + ctxt.flags, ctxt.pf_num, ctxt.vf_num, + ctxt.info.stat_counter_idx, ctxt.info.up_enable_bits); + /* + ** Set the queue and traffic class bits + ** - when multiple traffic classes are supported + ** this will need to be more robust. + */ + ctxt.info.valid_sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; + ctxt.info.mapping_flags |= I40E_AQ_VSI_QUE_MAP_CONTIG; + /* In contig mode, que_mapping[0] is first queue index used by this VSI */ + ctxt.info.queue_mapping[0] = 0; + /* + * This VSI will only use traffic class 0; start traffic class 0's + * queue allocation at queue 0, and assign it 2^tc_queues queues (though + * the driver may not use all of them). + */ + tc_queues = bsrl(pf->qtag.num_allocated); + ctxt.info.tc_mapping[0] = ((0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) + & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) | + ((tc_queues << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT) + & I40E_AQ_VSI_TC_QUE_NUMBER_MASK); + + /* Set VLAN receive stripping mode */ + ctxt.info.valid_sections |= I40E_AQ_VSI_PROP_VLAN_VALID; + ctxt.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL; + if (vsi->ifp->if_capenable & IFCAP_VLAN_HWTAGGING) + ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; + else + ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_NOTHING; + + /* Save VSI number and info for use later */ + vsi->vsi_num = ctxt.vsi_number; + bcopy(&ctxt.info, &vsi->info, sizeof(vsi->info)); + + /* Reset VSI statistics */ + ixl_vsi_reset_stats(vsi); + vsi->hw_filters_add = 0; + vsi->hw_filters_del = 0; + + ctxt.flags = htole16(I40E_AQ_VSI_TYPE_PF); + + err = i40e_aq_update_vsi_params(hw, &ctxt, NULL); + if (err) { + device_printf(dev, "i40e_aq_update_vsi_params() failed, error %d," + " aq_error %d\n", err, hw->aq.asq_last_status); + return (err); + } + + for (int i = 0; i < vsi->num_queues; i++, que++) { + struct tx_ring *txr = &que->txr; + struct rx_ring *rxr = &que->rxr; + struct i40e_hmc_obj_txq tctx; + struct i40e_hmc_obj_rxq rctx; + u32 txctl; + u16 size; + + /* Setup the HMC TX Context */ + size = que->num_desc * sizeof(struct i40e_tx_desc); + memset(&tctx, 0, sizeof(struct i40e_hmc_obj_txq)); + tctx.new_context = 1; + tctx.base = (txr->dma.pa/IXL_TX_CTX_BASE_UNITS); + tctx.qlen = que->num_desc; + tctx.fc_ena = 0; + tctx.rdylist = vsi->info.qs_handle[0]; /* index is TC */ + /* Enable HEAD writeback */ + tctx.head_wb_ena = 1; + tctx.head_wb_addr = txr->dma.pa + + (que->num_desc * sizeof(struct i40e_tx_desc)); + tctx.rdylist_act = 0; + err = i40e_clear_lan_tx_queue_context(hw, i); + if (err) { + device_printf(dev, "Unable to clear TX context\n"); + break; + } + err = i40e_set_lan_tx_queue_context(hw, i, &tctx); + if (err) { + device_printf(dev, "Unable to set TX context\n"); + break; + } + /* Associate the ring with this PF */ + txctl = I40E_QTX_CTL_PF_QUEUE; + txctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) & + I40E_QTX_CTL_PF_INDX_MASK); + wr32(hw, I40E_QTX_CTL(i), txctl); + ixl_flush(hw); + + /* Do ring (re)init */ + ixl_init_tx_ring(que); + + /* Next setup the HMC RX Context */ + if (vsi->max_frame_size <= MCLBYTES) + rxr->mbuf_sz = MCLBYTES; + else + rxr->mbuf_sz = MJUMPAGESIZE; + + u16 max_rxmax = rxr->mbuf_sz * hw->func_caps.rx_buf_chain_len; + + /* Set up an RX context for the HMC */ + memset(&rctx, 0, sizeof(struct i40e_hmc_obj_rxq)); + rctx.dbuff = rxr->mbuf_sz >> I40E_RXQ_CTX_DBUFF_SHIFT; + /* ignore header split for now */ + rctx.hbuff = 0 >> I40E_RXQ_CTX_HBUFF_SHIFT; + rctx.rxmax = (vsi->max_frame_size < max_rxmax) ? + vsi->max_frame_size : max_rxmax; + rctx.dtype = 0; + rctx.dsize = 1; /* do 32byte descriptors */ + rctx.hsplit_0 = 0; /* no HDR split initially */ + rctx.base = (rxr->dma.pa/IXL_RX_CTX_BASE_UNITS); + rctx.qlen = que->num_desc; + rctx.tphrdesc_ena = 1; + rctx.tphwdesc_ena = 1; + rctx.tphdata_ena = 0; + rctx.tphhead_ena = 0; + rctx.lrxqthresh = 2; + rctx.crcstrip = 1; + rctx.l2tsel = 1; + rctx.showiv = 1; + rctx.fc_ena = 0; + rctx.prefena = 1; + + err = i40e_clear_lan_rx_queue_context(hw, i); + if (err) { + device_printf(dev, + "Unable to clear RX context %d\n", i); + break; + } + err = i40e_set_lan_rx_queue_context(hw, i, &rctx); + if (err) { + device_printf(dev, "Unable to set RX context %d\n", i); + break; + } + err = ixl_init_rx_ring(que); + if (err) { + device_printf(dev, "Fail in init_rx_ring %d\n", i); + break; + } +#ifdef DEV_NETMAP + /* preserve queue */ + if (vsi->ifp->if_capenable & IFCAP_NETMAP) { + struct netmap_adapter *na = NA(vsi->ifp); + struct netmap_kring *kring = &na->rx_rings[i]; + int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring); + wr32(vsi->hw, I40E_QRX_TAIL(que->me), t); + } else +#endif /* DEV_NETMAP */ + wr32(vsi->hw, I40E_QRX_TAIL(que->me), que->num_desc - 1); + } + return (err); +} + + +/********************************************************************* + * + * Free all VSI structs. + * + **********************************************************************/ +void +ixl_free_vsi(struct ixl_vsi *vsi) +{ + struct ixl_pf *pf = (struct ixl_pf *)vsi->back; + struct ixl_queue *que = vsi->queues; + + /* Free station queues */ + if (!vsi->queues) + goto free_filters; + + for (int i = 0; i < vsi->num_queues; i++, que++) { + struct tx_ring *txr = &que->txr; + struct rx_ring *rxr = &que->rxr; + + if (!mtx_initialized(&txr->mtx)) /* uninitialized */ + continue; + IXL_TX_LOCK(txr); + ixl_free_que_tx(que); + if (txr->base) + i40e_free_dma_mem(&pf->hw, &txr->dma); + IXL_TX_UNLOCK(txr); + IXL_TX_LOCK_DESTROY(txr); + + if (!mtx_initialized(&rxr->mtx)) /* uninitialized */ + continue; + IXL_RX_LOCK(rxr); + ixl_free_que_rx(que); + if (rxr->base) + i40e_free_dma_mem(&pf->hw, &rxr->dma); + IXL_RX_UNLOCK(rxr); + IXL_RX_LOCK_DESTROY(rxr); + } + free(vsi->queues, M_DEVBUF); + +free_filters: + /* Free VSI filter list */ + ixl_free_mac_filters(vsi); +} + +void +ixl_free_mac_filters(struct ixl_vsi *vsi) +{ + struct ixl_mac_filter *f; + + while (!SLIST_EMPTY(&vsi->ftl)) { + f = SLIST_FIRST(&vsi->ftl); + SLIST_REMOVE_HEAD(&vsi->ftl, next); + free(f, M_DEVBUF); + } +} + +/* + * Fill out fields in queue struct and setup tx/rx memory and structs + */ +static int +ixl_setup_queue(struct ixl_queue *que, struct ixl_pf *pf, int index) +{ + device_t dev = pf->dev; + struct i40e_hw *hw = &pf->hw; + struct ixl_vsi *vsi = &pf->vsi; + struct tx_ring *txr = &que->txr; + struct rx_ring *rxr = &que->rxr; + int error = 0; + int rsize, tsize; + + /* ERJ: A lot of references to external objects... */ + que->num_desc = pf->ringsz; + que->me = index; + que->vsi = vsi; + + txr->que = que; + txr->tail = I40E_QTX_TAIL(que->me); + + /* Initialize the TX lock */ + snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", + device_get_nameunit(dev), que->me); + mtx_init(&txr->mtx, txr->mtx_name, NULL, MTX_DEF); + /* Create the TX descriptor ring */ + tsize = roundup2((que->num_desc * + sizeof(struct i40e_tx_desc)) + + sizeof(u32), DBA_ALIGN); + if (i40e_allocate_dma_mem(hw, + &txr->dma, i40e_mem_reserved, tsize, DBA_ALIGN)) { + device_printf(dev, + "Unable to allocate TX Descriptor memory\n"); + error = ENOMEM; + goto fail; + } + txr->base = (struct i40e_tx_desc *)txr->dma.va; + bzero((void *)txr->base, tsize); + /* Now allocate transmit soft structs for the ring */ + if (ixl_allocate_tx_data(que)) { + device_printf(dev, + "Critical Failure setting up TX structures\n"); + error = ENOMEM; + goto fail; + } + /* Allocate a buf ring */ + txr->br = buf_ring_alloc(DEFAULT_TXBRSZ, M_DEVBUF, + M_NOWAIT, &txr->mtx); + if (txr->br == NULL) { + device_printf(dev, + "Critical Failure setting up TX buf ring\n"); + error = ENOMEM; + goto fail; + } + + rsize = roundup2(que->num_desc * + sizeof(union i40e_rx_desc), DBA_ALIGN); + rxr->que = que; + rxr->tail = I40E_QRX_TAIL(que->me); + + /* Initialize the RX side lock */ + snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", + device_get_nameunit(dev), que->me); + mtx_init(&rxr->mtx, rxr->mtx_name, NULL, MTX_DEF); + + if (i40e_allocate_dma_mem(hw, + &rxr->dma, i40e_mem_reserved, rsize, 4096)) { + device_printf(dev, + "Unable to allocate RX Descriptor memory\n"); + error = ENOMEM; + goto fail; + } + rxr->base = (union i40e_rx_desc *)rxr->dma.va; + bzero((void *)rxr->base, rsize); + /* Allocate receive soft structs for the ring*/ + if (ixl_allocate_rx_data(que)) { + device_printf(dev, + "Critical Failure setting up receive structs\n"); + error = ENOMEM; + goto fail; + } + + return (0); +fail: + if (rxr->base) + i40e_free_dma_mem(&pf->hw, &rxr->dma); + if (mtx_initialized(&rxr->mtx)) + mtx_destroy(&rxr->mtx); + if (txr->br) { + buf_ring_free(txr->br, M_DEVBUF); + txr->br = NULL; + } + if (txr->base) + i40e_free_dma_mem(&pf->hw, &txr->dma); + if (mtx_initialized(&txr->mtx)) + mtx_destroy(&txr->mtx); + + return (error); +} + +/********************************************************************* + * + * Allocate memory for the VSI (virtual station interface) and their + * associated queues, rings and the descriptors associated with each, + * called only once at attach. + * + **********************************************************************/ +int +ixl_setup_stations(struct ixl_pf *pf) +{ + device_t dev = pf->dev; + struct ixl_vsi *vsi; + struct ixl_queue *que; + int error = 0; + + vsi = &pf->vsi; + vsi->back = (void *)pf; + vsi->hw = &pf->hw; + vsi->id = 0; + vsi->num_vlans = 0; + vsi->back = pf; + + /* Get memory for the station queues */ + if (!(vsi->queues = + (struct ixl_queue *) malloc(sizeof(struct ixl_queue) * + vsi->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { + device_printf(dev, "Unable to allocate queue memory\n"); + error = ENOMEM; + return (error); + } + + for (int i = 0; i < vsi->num_queues; i++) { + que = &vsi->queues[i]; + error = ixl_setup_queue(que, pf, i); + if (error) + return (error); + } + + return (0); +} + +/* +** Provide a update to the queue RX +** interrupt moderation value. +*/ +void +ixl_set_queue_rx_itr(struct ixl_queue *que) +{ + struct ixl_vsi *vsi = que->vsi; + struct ixl_pf *pf = (struct ixl_pf *)vsi->back; + struct i40e_hw *hw = vsi->hw; + struct rx_ring *rxr = &que->rxr; + u16 rx_itr; + u16 rx_latency = 0; + int rx_bytes; + + /* Idle, do nothing */ + if (rxr->bytes == 0) + return; + + if (pf->dynamic_rx_itr) { + rx_bytes = rxr->bytes/rxr->itr; + rx_itr = rxr->itr; + + /* Adjust latency range */ + switch (rxr->latency) { + case IXL_LOW_LATENCY: + if (rx_bytes > 10) { + rx_latency = IXL_AVE_LATENCY; + rx_itr = IXL_ITR_20K; + } + break; + case IXL_AVE_LATENCY: + if (rx_bytes > 20) { + rx_latency = IXL_BULK_LATENCY; + rx_itr = IXL_ITR_8K; + } else if (rx_bytes <= 10) { + rx_latency = IXL_LOW_LATENCY; + rx_itr = IXL_ITR_100K; + } + break; + case IXL_BULK_LATENCY: + if (rx_bytes <= 20) { + rx_latency = IXL_AVE_LATENCY; + rx_itr = IXL_ITR_20K; + } + break; + } + + rxr->latency = rx_latency; + + if (rx_itr != rxr->itr) { + /* do an exponential smoothing */ + rx_itr = (10 * rx_itr * rxr->itr) / + ((9 * rx_itr) + rxr->itr); + rxr->itr = rx_itr & IXL_MAX_ITR; + wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, + que->me), rxr->itr); + } + } else { /* We may have have toggled to non-dynamic */ + if (vsi->rx_itr_setting & IXL_ITR_DYNAMIC) + vsi->rx_itr_setting = pf->rx_itr; + /* Update the hardware if needed */ + if (rxr->itr != vsi->rx_itr_setting) { + rxr->itr = vsi->rx_itr_setting; + wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, + que->me), rxr->itr); + } + } + rxr->bytes = 0; + rxr->packets = 0; + return; +} + + +/* +** Provide a update to the queue TX +** interrupt moderation value. +*/ +void +ixl_set_queue_tx_itr(struct ixl_queue *que) +{ + struct ixl_vsi *vsi = que->vsi; + struct ixl_pf *pf = (struct ixl_pf *)vsi->back; + struct i40e_hw *hw = vsi->hw; + struct tx_ring *txr = &que->txr; + u16 tx_itr; + u16 tx_latency = 0; + int tx_bytes; + + + /* Idle, do nothing */ + if (txr->bytes == 0) + return; + + if (pf->dynamic_tx_itr) { + tx_bytes = txr->bytes/txr->itr; + tx_itr = txr->itr; + + switch (txr->latency) { + case IXL_LOW_LATENCY: + if (tx_bytes > 10) { + tx_latency = IXL_AVE_LATENCY; + tx_itr = IXL_ITR_20K; + } + break; + case IXL_AVE_LATENCY: + if (tx_bytes > 20) { + tx_latency = IXL_BULK_LATENCY; + tx_itr = IXL_ITR_8K; + } else if (tx_bytes <= 10) { + tx_latency = IXL_LOW_LATENCY; + tx_itr = IXL_ITR_100K; + } + break; + case IXL_BULK_LATENCY: + if (tx_bytes <= 20) { + tx_latency = IXL_AVE_LATENCY; + tx_itr = IXL_ITR_20K; + } + break; + } + + txr->latency = tx_latency; + + if (tx_itr != txr->itr) { + /* do an exponential smoothing */ + tx_itr = (10 * tx_itr * txr->itr) / + ((9 * tx_itr) + txr->itr); + txr->itr = tx_itr & IXL_MAX_ITR; + wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, + que->me), txr->itr); + } + + } else { /* We may have have toggled to non-dynamic */ + if (vsi->tx_itr_setting & IXL_ITR_DYNAMIC) + vsi->tx_itr_setting = pf->tx_itr; + /* Update the hardware if needed */ + if (txr->itr != vsi->tx_itr_setting) { + txr->itr = vsi->tx_itr_setting; + wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, + que->me), txr->itr); + } + } + txr->bytes = 0; + txr->packets = 0; + return; +} + +void +ixl_add_vsi_sysctls(struct ixl_pf *pf, struct ixl_vsi *vsi, + struct sysctl_ctx_list *ctx, const char *sysctl_name) +{ + struct sysctl_oid *tree; + struct sysctl_oid_list *child; + struct sysctl_oid_list *vsi_list; + + tree = device_get_sysctl_tree(pf->dev); + child = SYSCTL_CHILDREN(tree); + vsi->vsi_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, sysctl_name, + CTLFLAG_RD, NULL, "VSI Number"); + vsi_list = SYSCTL_CHILDREN(vsi->vsi_node); + + ixl_add_sysctls_eth_stats(ctx, vsi_list, &vsi->eth_stats); +} + +#ifdef IXL_DEBUG +/** + * ixl_sysctl_qtx_tail_handler + * Retrieves I40E_QTX_TAIL value from hardware + * for a sysctl. + */ +int +ixl_sysctl_qtx_tail_handler(SYSCTL_HANDLER_ARGS) +{ + struct ixl_queue *que; + int error; + u32 val; + + que = ((struct ixl_queue *)oidp->oid_arg1); + if (!que) return 0; + + val = rd32(que->vsi->hw, que->txr.tail); + error = sysctl_handle_int(oidp, &val, 0, req); + if (error || !req->newptr) + return error; + return (0); +} + +/** + * ixl_sysctl_qrx_tail_handler + * Retrieves I40E_QRX_TAIL value from hardware + * for a sysctl. + */ +int +ixl_sysctl_qrx_tail_handler(SYSCTL_HANDLER_ARGS) +{ + struct ixl_queue *que; + int error; + u32 val; + + que = ((struct ixl_queue *)oidp->oid_arg1); + if (!que) return 0; + + val = rd32(que->vsi->hw, que->rxr.tail); + error = sysctl_handle_int(oidp, &val, 0, req); + if (error || !req->newptr) + return error; + return (0); +} +#endif + +/* + * Used to set the Tx ITR value for all of the PF LAN VSI's queues. + * Writes to the ITR registers immediately. + */ +static int +ixl_sysctl_pf_tx_itr(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + device_t dev = pf->dev; + int error = 0; + int requested_tx_itr; + + requested_tx_itr = pf->tx_itr; + error = sysctl_handle_int(oidp, &requested_tx_itr, 0, req); + if ((error) || (req->newptr == NULL)) + return (error); + if (pf->dynamic_tx_itr) { + device_printf(dev, + "Cannot set TX itr value while dynamic TX itr is enabled\n"); + return (EINVAL); + } + if (requested_tx_itr < 0 || requested_tx_itr > IXL_MAX_ITR) { + device_printf(dev, + "Invalid TX itr value; value must be between 0 and %d\n", + IXL_MAX_ITR); + return (EINVAL); + } + + pf->tx_itr = requested_tx_itr; + ixl_configure_tx_itr(pf); + + return (error); +} + +/* + * Used to set the Rx ITR value for all of the PF LAN VSI's queues. + * Writes to the ITR registers immediately. + */ +static int +ixl_sysctl_pf_rx_itr(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + device_t dev = pf->dev; + int error = 0; + int requested_rx_itr; + + requested_rx_itr = pf->rx_itr; + error = sysctl_handle_int(oidp, &requested_rx_itr, 0, req); + if ((error) || (req->newptr == NULL)) + return (error); + if (pf->dynamic_rx_itr) { + device_printf(dev, + "Cannot set RX itr value while dynamic RX itr is enabled\n"); + return (EINVAL); + } + if (requested_rx_itr < 0 || requested_rx_itr > IXL_MAX_ITR) { + device_printf(dev, + "Invalid RX itr value; value must be between 0 and %d\n", + IXL_MAX_ITR); + return (EINVAL); + } + + pf->rx_itr = requested_rx_itr; + ixl_configure_rx_itr(pf); + + return (error); +} + +void +ixl_add_hw_stats(struct ixl_pf *pf) +{ + device_t dev = pf->dev; + struct ixl_vsi *vsi = &pf->vsi; + struct ixl_queue *queues = vsi->queues; + struct i40e_hw_port_stats *pf_stats = &pf->stats; + + struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); + struct sysctl_oid *tree = device_get_sysctl_tree(dev); + struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); + struct sysctl_oid_list *vsi_list; + + struct sysctl_oid *queue_node; + struct sysctl_oid_list *queue_list; + + struct tx_ring *txr; + struct rx_ring *rxr; + char queue_namebuf[QUEUE_NAME_LEN]; + + /* Driver statistics */ + SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events", + CTLFLAG_RD, &pf->watchdog_events, + "Watchdog timeouts"); + SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "admin_irq", + CTLFLAG_RD, &pf->admin_irq, + "Admin Queue IRQ Handled"); + + ixl_add_vsi_sysctls(pf, &pf->vsi, ctx, "pf"); + vsi_list = SYSCTL_CHILDREN(pf->vsi.vsi_node); + + /* Queue statistics */ + for (int q = 0; q < vsi->num_queues; q++) { + snprintf(queue_namebuf, QUEUE_NAME_LEN, "que%d", q); + queue_node = SYSCTL_ADD_NODE(ctx, vsi_list, + OID_AUTO, queue_namebuf, CTLFLAG_RD, NULL, "Queue #"); + queue_list = SYSCTL_CHILDREN(queue_node); + + txr = &(queues[q].txr); + rxr = &(queues[q].rxr); + + SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "mbuf_defrag_failed", + CTLFLAG_RD, &(queues[q].mbuf_defrag_failed), + "m_defrag() failed"); + SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs", + CTLFLAG_RD, &(queues[q].irqs), + "irqs on this queue"); + SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso_tx", + CTLFLAG_RD, &(queues[q].tso), + "TSO"); + SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_dmamap_failed", + CTLFLAG_RD, &(queues[q].tx_dmamap_failed), + "Driver tx dma failure in xmit"); + SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail", + CTLFLAG_RD, &(txr->no_desc), + "Queue No Descriptor Available"); + SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets", + CTLFLAG_RD, &(txr->total_packets), + "Queue Packets Transmitted"); + SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_bytes", + CTLFLAG_RD, &(txr->tx_bytes), + "Queue Bytes Transmitted"); + SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets", + CTLFLAG_RD, &(rxr->rx_packets), + "Queue Packets Received"); + SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes", + CTLFLAG_RD, &(rxr->rx_bytes), + "Queue Bytes Received"); + SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_desc_err", + CTLFLAG_RD, &(rxr->desc_errs), + "Queue Rx Descriptor Errors"); + SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rx_itr", + CTLFLAG_RD, &(rxr->itr), 0, + "Queue Rx ITR Interval"); + SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "tx_itr", + CTLFLAG_RD, &(txr->itr), 0, + "Queue Tx ITR Interval"); +#ifdef IXL_DEBUG + SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_not_done", + CTLFLAG_RD, &(rxr->not_done), + "Queue Rx Descriptors not Done"); + SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rx_next_refresh", + CTLFLAG_RD, &(rxr->next_refresh), 0, + "Queue Rx Descriptors not Done"); + SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rx_next_check", + CTLFLAG_RD, &(rxr->next_check), 0, + "Queue Rx Descriptors not Done"); + SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "qtx_tail", + CTLTYPE_UINT | CTLFLAG_RD, &queues[q], + sizeof(struct ixl_queue), + ixl_sysctl_qtx_tail_handler, "IU", + "Queue Transmit Descriptor Tail"); + SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "qrx_tail", + CTLTYPE_UINT | CTLFLAG_RD, &queues[q], + sizeof(struct ixl_queue), + ixl_sysctl_qrx_tail_handler, "IU", + "Queue Receive Descriptor Tail"); +#endif + } + + /* MAC stats */ + ixl_add_sysctls_mac_stats(ctx, child, pf_stats); +} + +void +ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *ctx, + struct sysctl_oid_list *child, + struct i40e_eth_stats *eth_stats) +{ + struct ixl_sysctl_info ctls[] = + { + {ð_stats->rx_bytes, "good_octets_rcvd", "Good Octets Received"}, + {ð_stats->rx_unicast, "ucast_pkts_rcvd", + "Unicast Packets Received"}, + {ð_stats->rx_multicast, "mcast_pkts_rcvd", + "Multicast Packets Received"}, + {ð_stats->rx_broadcast, "bcast_pkts_rcvd", + "Broadcast Packets Received"}, + {ð_stats->rx_discards, "rx_discards", "Discarded RX packets"}, + {ð_stats->tx_bytes, "good_octets_txd", "Good Octets Transmitted"}, + {ð_stats->tx_unicast, "ucast_pkts_txd", "Unicast Packets Transmitted"}, + {ð_stats->tx_multicast, "mcast_pkts_txd", + "Multicast Packets Transmitted"}, + {ð_stats->tx_broadcast, "bcast_pkts_txd", + "Broadcast Packets Transmitted"}, + // end + {0,0,0} + }; + + struct ixl_sysctl_info *entry = ctls; + while (entry->stat != 0) + { + SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, entry->name, + CTLFLAG_RD, entry->stat, + entry->description); + entry++; + } +} + +void +ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *ctx, + struct sysctl_oid_list *child, + struct i40e_hw_port_stats *stats) +{ + struct sysctl_oid *stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac", + CTLFLAG_RD, NULL, "Mac Statistics"); + struct sysctl_oid_list *stat_list = SYSCTL_CHILDREN(stat_node); + + struct i40e_eth_stats *eth_stats = &stats->eth; + ixl_add_sysctls_eth_stats(ctx, stat_list, eth_stats); + + struct ixl_sysctl_info ctls[] = + { + {&stats->crc_errors, "crc_errors", "CRC Errors"}, + {&stats->illegal_bytes, "illegal_bytes", "Illegal Byte Errors"}, + {&stats->mac_local_faults, "local_faults", "MAC Local Faults"}, + {&stats->mac_remote_faults, "remote_faults", "MAC Remote Faults"}, + {&stats->rx_length_errors, "rx_length_errors", "Receive Length Errors"}, + /* Packet Reception Stats */ + {&stats->rx_size_64, "rx_frames_64", "64 byte frames received"}, + {&stats->rx_size_127, "rx_frames_65_127", "65-127 byte frames received"}, + {&stats->rx_size_255, "rx_frames_128_255", "128-255 byte frames received"}, + {&stats->rx_size_511, "rx_frames_256_511", "256-511 byte frames received"}, + {&stats->rx_size_1023, "rx_frames_512_1023", "512-1023 byte frames received"}, + {&stats->rx_size_1522, "rx_frames_1024_1522", "1024-1522 byte frames received"}, + {&stats->rx_size_big, "rx_frames_big", "1523-9522 byte frames received"}, + {&stats->rx_undersize, "rx_undersize", "Undersized packets received"}, + {&stats->rx_fragments, "rx_fragmented", "Fragmented packets received"}, + {&stats->rx_oversize, "rx_oversized", "Oversized packets received"}, + {&stats->rx_jabber, "rx_jabber", "Received Jabber"}, + {&stats->checksum_error, "checksum_errors", "Checksum Errors"}, + /* Packet Transmission Stats */ + {&stats->tx_size_64, "tx_frames_64", "64 byte frames transmitted"}, + {&stats->tx_size_127, "tx_frames_65_127", "65-127 byte frames transmitted"}, + {&stats->tx_size_255, "tx_frames_128_255", "128-255 byte frames transmitted"}, + {&stats->tx_size_511, "tx_frames_256_511", "256-511 byte frames transmitted"}, + {&stats->tx_size_1023, "tx_frames_512_1023", "512-1023 byte frames transmitted"}, + {&stats->tx_size_1522, "tx_frames_1024_1522", "1024-1522 byte frames transmitted"}, + {&stats->tx_size_big, "tx_frames_big", "1523-9522 byte frames transmitted"}, + /* Flow control */ + {&stats->link_xon_tx, "xon_txd", "Link XON transmitted"}, + {&stats->link_xon_rx, "xon_recvd", "Link XON received"}, + {&stats->link_xoff_tx, "xoff_txd", "Link XOFF transmitted"}, + {&stats->link_xoff_rx, "xoff_recvd", "Link XOFF received"}, + /* End */ + {0,0,0} + }; + + struct ixl_sysctl_info *entry = ctls; + while (entry->stat != 0) + { + SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, entry->name, + CTLFLAG_RD, entry->stat, + entry->description); + entry++; + } +} + +void +ixl_set_rss_key(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + struct ixl_vsi *vsi = &pf->vsi; + device_t dev = pf->dev; + enum i40e_status_code status; +#ifdef RSS + u32 rss_seed[IXL_RSS_KEY_SIZE_REG]; +#else + u32 rss_seed[IXL_RSS_KEY_SIZE_REG] = {0x41b01687, + 0x183cfd8c, 0xce880440, 0x580cbc3c, + 0x35897377, 0x328b25e1, 0x4fa98922, + 0xb7d90c14, 0xd5bad70d, 0xcd15a2c1, + 0x0, 0x0, 0x0}; +#endif + +#ifdef RSS + /* Fetch the configured RSS key */ + rss_getkey((uint8_t *) &rss_seed); +#endif + /* Fill out hash function seed */ + if (hw->mac.type == I40E_MAC_X722) { + struct i40e_aqc_get_set_rss_key_data key_data; + bcopy(rss_seed, key_data.standard_rss_key, 40); + status = i40e_aq_set_rss_key(hw, vsi->vsi_num, &key_data); + if (status) + device_printf(dev, "i40e_aq_set_rss_key status %s, error %s\n", + i40e_stat_str(hw, status), i40e_aq_str(hw, hw->aq.asq_last_status)); + } else { + for (int i = 0; i < IXL_RSS_KEY_SIZE_REG; i++) + i40e_write_rx_ctl(hw, I40E_PFQF_HKEY(i), rss_seed[i]); + } +} + +/* + * Configure enabled PCTYPES for RSS. + */ +void +ixl_set_rss_pctypes(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + u64 set_hena = 0, hena; + +#ifdef RSS + u32 rss_hash_config; + + rss_hash_config = rss_gethashconfig(); + if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER); + if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP); + if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP); + if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER); + if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6); + if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP); + if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP); +#else + set_hena = IXL_DEFAULT_RSS_HENA; +#endif + hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) | + ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32); + hena |= set_hena; + i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena); + i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32)); + +} + +void +ixl_set_rss_hlut(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + device_t dev = pf->dev; + struct ixl_vsi *vsi = &pf->vsi; + int i, que_id; + int lut_entry_width; + u32 lut = 0; + enum i40e_status_code status; + + if (hw->mac.type == I40E_MAC_X722) + lut_entry_width = 7; + else + lut_entry_width = pf->hw.func_caps.rss_table_entry_width; + + /* Populate the LUT with max no. of queues in round robin fashion */ + u8 hlut_buf[512]; + for (i = 0; i < pf->hw.func_caps.rss_table_size; i++) { +#ifdef RSS + /* + * Fetch the RSS bucket id for the given indirection entry. + * Cap it at the number of configured buckets (which is + * num_queues.) + */ + que_id = rss_get_indirection_to_bucket(i); + que_id = que_id % vsi->num_queues; +#else + que_id = i % vsi->num_queues; +#endif + lut = (que_id & ((0x1 << lut_entry_width) - 1)); + hlut_buf[i] = lut; + } + + if (hw->mac.type == I40E_MAC_X722) { + status = i40e_aq_set_rss_lut(hw, vsi->vsi_num, TRUE, hlut_buf, sizeof(hlut_buf)); + if (status) + device_printf(dev, "i40e_aq_set_rss_lut status %s, error %s\n", + i40e_stat_str(hw, status), i40e_aq_str(hw, hw->aq.asq_last_status)); + } else { + for (i = 0; i < pf->hw.func_caps.rss_table_size >> 2; i++) + wr32(hw, I40E_PFQF_HLUT(i), ((u32 *)hlut_buf)[i]); + ixl_flush(hw); + } +} + +/* +** Setup the PF's RSS parameters. +*/ +void +ixl_config_rss(struct ixl_pf *pf) +{ + ixl_set_rss_key(pf); + ixl_set_rss_pctypes(pf); + ixl_set_rss_hlut(pf); +} + +/* +** This routine is run via an vlan config EVENT, +** it enables us to use the HW Filter table since +** we can get the vlan id. This just creates the +** entry in the soft version of the VFTA, init will +** repopulate the real table. +*/ +void +ixl_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) +{ + struct ixl_vsi *vsi = ifp->if_softc; + struct i40e_hw *hw = vsi->hw; + struct ixl_pf *pf = (struct ixl_pf *)vsi->back; + + if (ifp->if_softc != arg) /* Not our event */ + return; + + if ((vtag == 0) || (vtag > 4095)) /* Invalid */ + return; + + IXL_PF_LOCK(pf); + ++vsi->num_vlans; + ixl_add_filter(vsi, hw->mac.addr, vtag); + IXL_PF_UNLOCK(pf); +} + +/* +** This routine is run via an vlan +** unconfig EVENT, remove our entry +** in the soft vfta. +*/ +void +ixl_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) +{ + struct ixl_vsi *vsi = ifp->if_softc; + struct i40e_hw *hw = vsi->hw; + struct ixl_pf *pf = (struct ixl_pf *)vsi->back; + + if (ifp->if_softc != arg) + return; + + if ((vtag == 0) || (vtag > 4095)) /* Invalid */ + return; + + IXL_PF_LOCK(pf); + --vsi->num_vlans; + ixl_del_filter(vsi, hw->mac.addr, vtag); + IXL_PF_UNLOCK(pf); +} + +/* +** This routine updates vlan filters, called by init +** it scans the filter table and then updates the hw +** after a soft reset. +*/ +void +ixl_setup_vlan_filters(struct ixl_vsi *vsi) +{ + struct ixl_mac_filter *f; + int cnt = 0, flags; + + if (vsi->num_vlans == 0) + return; + /* + ** Scan the filter list for vlan entries, + ** mark them for addition and then call + ** for the AQ update. + */ + SLIST_FOREACH(f, &vsi->ftl, next) { + if (f->flags & IXL_FILTER_VLAN) { + f->flags |= + (IXL_FILTER_ADD | + IXL_FILTER_USED); + cnt++; + } + } + if (cnt == 0) { + printf("setup vlan: no filters found!\n"); + return; + } + flags = IXL_FILTER_VLAN; + flags |= (IXL_FILTER_ADD | IXL_FILTER_USED); + ixl_add_hw_filters(vsi, flags, cnt); + return; +} + +/* +** Initialize filter list and add filters that the hardware +** needs to know about. +** +** Requires VSI's filter list & seid to be set before calling. +*/ +void +ixl_init_filters(struct ixl_vsi *vsi) +{ + struct ixl_pf *pf = (struct ixl_pf *)vsi->back; + + /* Add broadcast address */ + ixl_add_filter(vsi, ixl_bcast_addr, IXL_VLAN_ANY); + + /* + * Prevent Tx flow control frames from being sent out by + * non-firmware transmitters. + * This affects every VSI in the PF. + */ + if (pf->enable_tx_fc_filter) + i40e_add_filter_to_drop_tx_flow_control_frames(vsi->hw, vsi->seid); +} + +/* +** This routine adds mulicast filters +*/ +void +ixl_add_mc_filter(struct ixl_vsi *vsi, u8 *macaddr) +{ + struct ixl_mac_filter *f; + + /* Does one already exist */ + f = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY); + if (f != NULL) + return; + + f = ixl_get_filter(vsi); + if (f == NULL) { + printf("WARNING: no filter available!!\n"); + return; + } + bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN); + f->vlan = IXL_VLAN_ANY; + f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED + | IXL_FILTER_MC); + + return; +} + +void +ixl_reconfigure_filters(struct ixl_vsi *vsi) +{ + ixl_add_hw_filters(vsi, IXL_FILTER_USED, vsi->num_macs); +} + +/* +** This routine adds macvlan filters +*/ +void +ixl_add_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan) +{ + struct ixl_mac_filter *f, *tmp; + struct ixl_pf *pf; + device_t dev; + + DEBUGOUT("ixl_add_filter: begin"); + + pf = vsi->back; + dev = pf->dev; + + /* Does one already exist */ + f = ixl_find_filter(vsi, macaddr, vlan); + if (f != NULL) + return; + /* + ** Is this the first vlan being registered, if so we + ** need to remove the ANY filter that indicates we are + ** not in a vlan, and replace that with a 0 filter. + */ + if ((vlan != IXL_VLAN_ANY) && (vsi->num_vlans == 1)) { + tmp = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY); + if (tmp != NULL) { + ixl_del_filter(vsi, macaddr, IXL_VLAN_ANY); + ixl_add_filter(vsi, macaddr, 0); + } + } + + f = ixl_get_filter(vsi); + if (f == NULL) { + device_printf(dev, "WARNING: no filter available!!\n"); + return; + } + bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN); + f->vlan = vlan; + f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED); + if (f->vlan != IXL_VLAN_ANY) + f->flags |= IXL_FILTER_VLAN; + else + vsi->num_macs++; + + ixl_add_hw_filters(vsi, f->flags, 1); + return; +} + +void +ixl_del_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan) +{ + struct ixl_mac_filter *f; + + f = ixl_find_filter(vsi, macaddr, vlan); + if (f == NULL) + return; + + f->flags |= IXL_FILTER_DEL; + ixl_del_hw_filters(vsi, 1); + vsi->num_macs--; + + /* Check if this is the last vlan removal */ + if (vlan != IXL_VLAN_ANY && vsi->num_vlans == 0) { + /* Switch back to a non-vlan filter */ + ixl_del_filter(vsi, macaddr, 0); + ixl_add_filter(vsi, macaddr, IXL_VLAN_ANY); + } + return; +} + +/* +** Find the filter with both matching mac addr and vlan id +*/ +struct ixl_mac_filter * +ixl_find_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan) +{ + struct ixl_mac_filter *f; + bool match = FALSE; + + SLIST_FOREACH(f, &vsi->ftl, next) { + if (!cmp_etheraddr(f->macaddr, macaddr)) + continue; + if (f->vlan == vlan) { + match = TRUE; + break; + } + } + + if (!match) + f = NULL; + return (f); +} + +/* +** This routine takes additions to the vsi filter +** table and creates an Admin Queue call to create +** the filters in the hardware. +*/ +void +ixl_add_hw_filters(struct ixl_vsi *vsi, int flags, int cnt) +{ + struct i40e_aqc_add_macvlan_element_data *a, *b; + struct ixl_mac_filter *f; + struct ixl_pf *pf; + struct i40e_hw *hw; + device_t dev; + int err, j = 0; + + pf = vsi->back; + dev = pf->dev; + hw = &pf->hw; + IXL_PF_LOCK_ASSERT(pf); + + a = malloc(sizeof(struct i40e_aqc_add_macvlan_element_data) * cnt, + M_DEVBUF, M_NOWAIT | M_ZERO); + if (a == NULL) { + device_printf(dev, "add_hw_filters failed to get memory\n"); + return; + } + + /* + ** Scan the filter list, each time we find one + ** we add it to the admin queue array and turn off + ** the add bit. + */ + SLIST_FOREACH(f, &vsi->ftl, next) { + if (f->flags == flags) { + b = &a[j]; // a pox on fvl long names :) + bcopy(f->macaddr, b->mac_addr, ETHER_ADDR_LEN); + if (f->vlan == IXL_VLAN_ANY) { + b->vlan_tag = 0; + b->flags = I40E_AQC_MACVLAN_ADD_IGNORE_VLAN; + } else { + b->vlan_tag = f->vlan; + b->flags = 0; + } + b->flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; + f->flags &= ~IXL_FILTER_ADD; + j++; + } + if (j == cnt) + break; + } + if (j > 0) { + err = i40e_aq_add_macvlan(hw, vsi->seid, a, j, NULL); + if (err) + device_printf(dev, "aq_add_macvlan err %d, " + "aq_error %d\n", err, hw->aq.asq_last_status); + else + vsi->hw_filters_add += j; + } + free(a, M_DEVBUF); + return; +} + +/* +** This routine takes removals in the vsi filter +** table and creates an Admin Queue call to delete +** the filters in the hardware. +*/ +void +ixl_del_hw_filters(struct ixl_vsi *vsi, int cnt) +{ + struct i40e_aqc_remove_macvlan_element_data *d, *e; + struct ixl_pf *pf; + struct i40e_hw *hw; + device_t dev; + struct ixl_mac_filter *f, *f_temp; + int err, j = 0; + + DEBUGOUT("ixl_del_hw_filters: begin\n"); + + pf = vsi->back; + hw = &pf->hw; + dev = pf->dev; + + d = malloc(sizeof(struct i40e_aqc_remove_macvlan_element_data) * cnt, + M_DEVBUF, M_NOWAIT | M_ZERO); + if (d == NULL) { + printf("del hw filter failed to get memory\n"); + return; + } + + SLIST_FOREACH_SAFE(f, &vsi->ftl, next, f_temp) { + if (f->flags & IXL_FILTER_DEL) { + e = &d[j]; // a pox on fvl long names :) + bcopy(f->macaddr, e->mac_addr, ETHER_ADDR_LEN); + e->vlan_tag = (f->vlan == IXL_VLAN_ANY ? 0 : f->vlan); + e->flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; + /* delete entry from vsi list */ + SLIST_REMOVE(&vsi->ftl, f, ixl_mac_filter, next); + free(f, M_DEVBUF); + j++; + } + if (j == cnt) + break; + } + if (j > 0) { + err = i40e_aq_remove_macvlan(hw, vsi->seid, d, j, NULL); + if (err && hw->aq.asq_last_status != I40E_AQ_RC_ENOENT) { + int sc = 0; + for (int i = 0; i < j; i++) + sc += (!d[i].error_code); + vsi->hw_filters_del += sc; + device_printf(dev, + "Failed to remove %d/%d filters, aq error %d\n", + j - sc, j, hw->aq.asq_last_status); + } else + vsi->hw_filters_del += j; + } + free(d, M_DEVBUF); + + DEBUGOUT("ixl_del_hw_filters: end\n"); + return; +} + +int +ixl_enable_tx_ring(struct ixl_pf *pf, struct ixl_pf_qtag *qtag, u16 vsi_qidx) +{ + struct i40e_hw *hw = &pf->hw; + int error = 0; + u32 reg; + u16 pf_qidx; + + pf_qidx = ixl_pf_qidx_from_vsi_qidx(qtag, vsi_qidx); + + ixl_dbg(pf, IXL_DBG_EN_DIS, + "Enabling PF TX ring %4d / VSI TX ring %4d...\n", + pf_qidx, vsi_qidx); + + i40e_pre_tx_queue_cfg(hw, pf_qidx, TRUE); + + reg = rd32(hw, I40E_QTX_ENA(pf_qidx)); + reg |= I40E_QTX_ENA_QENA_REQ_MASK | + I40E_QTX_ENA_QENA_STAT_MASK; + wr32(hw, I40E_QTX_ENA(pf_qidx), reg); + /* Verify the enable took */ + for (int j = 0; j < 10; j++) { + reg = rd32(hw, I40E_QTX_ENA(pf_qidx)); + if (reg & I40E_QTX_ENA_QENA_STAT_MASK) + break; + i40e_msec_delay(10); + } + if ((reg & I40E_QTX_ENA_QENA_STAT_MASK) == 0) { + device_printf(pf->dev, "TX queue %d still disabled!\n", + pf_qidx); + error = ETIMEDOUT; + } + + return (error); +} + +int +ixl_enable_rx_ring(struct ixl_pf *pf, struct ixl_pf_qtag *qtag, u16 vsi_qidx) +{ + struct i40e_hw *hw = &pf->hw; + int error = 0; + u32 reg; + u16 pf_qidx; + + pf_qidx = ixl_pf_qidx_from_vsi_qidx(qtag, vsi_qidx); + + ixl_dbg(pf, IXL_DBG_EN_DIS, + "Enabling PF RX ring %4d / VSI RX ring %4d...\n", + pf_qidx, vsi_qidx); + + reg = rd32(hw, I40E_QRX_ENA(pf_qidx)); + reg |= I40E_QRX_ENA_QENA_REQ_MASK | + I40E_QRX_ENA_QENA_STAT_MASK; + wr32(hw, I40E_QRX_ENA(pf_qidx), reg); + /* Verify the enable took */ + for (int j = 0; j < 10; j++) { + reg = rd32(hw, I40E_QRX_ENA(pf_qidx)); + if (reg & I40E_QRX_ENA_QENA_STAT_MASK) + break; + i40e_msec_delay(10); + } + if ((reg & I40E_QRX_ENA_QENA_STAT_MASK) == 0) { + device_printf(pf->dev, "RX queue %d still disabled!\n", + pf_qidx); + error = ETIMEDOUT; + } + + return (error); +} + +int +ixl_enable_ring(struct ixl_pf *pf, struct ixl_pf_qtag *qtag, u16 vsi_qidx) +{ + int error = 0; + + error = ixl_enable_tx_ring(pf, qtag, vsi_qidx); + /* Called function already prints error message */ + if (error) + return (error); + error = ixl_enable_rx_ring(pf, qtag, vsi_qidx); + return (error); +} + +/* For PF VSI only */ +int +ixl_enable_rings(struct ixl_vsi *vsi) +{ + struct ixl_pf *pf = vsi->back; + int error = 0; + + for (int i = 0; i < vsi->num_queues; i++) { + error = ixl_enable_ring(pf, &pf->qtag, i); + if (error) + return (error); + } + + return (error); +} + +int +ixl_disable_tx_ring(struct ixl_pf *pf, struct ixl_pf_qtag *qtag, u16 vsi_qidx) +{ + struct i40e_hw *hw = &pf->hw; + int error = 0; + u32 reg; + u16 pf_qidx; + + pf_qidx = ixl_pf_qidx_from_vsi_qidx(qtag, vsi_qidx); + + i40e_pre_tx_queue_cfg(hw, pf_qidx, FALSE); + i40e_usec_delay(500); + + reg = rd32(hw, I40E_QTX_ENA(pf_qidx)); + reg &= ~I40E_QTX_ENA_QENA_REQ_MASK; + wr32(hw, I40E_QTX_ENA(pf_qidx), reg); + /* Verify the disable took */ + for (int j = 0; j < 10; j++) { + reg = rd32(hw, I40E_QTX_ENA(pf_qidx)); + if (!(reg & I40E_QTX_ENA_QENA_STAT_MASK)) + break; + i40e_msec_delay(10); + } + if (reg & I40E_QTX_ENA_QENA_STAT_MASK) { + device_printf(pf->dev, "TX queue %d still enabled!\n", + pf_qidx); + error = ETIMEDOUT; + } + + return (error); +} + +int +ixl_disable_rx_ring(struct ixl_pf *pf, struct ixl_pf_qtag *qtag, u16 vsi_qidx) +{ + struct i40e_hw *hw = &pf->hw; + int error = 0; + u32 reg; + u16 pf_qidx; + + pf_qidx = ixl_pf_qidx_from_vsi_qidx(qtag, vsi_qidx); + + reg = rd32(hw, I40E_QRX_ENA(pf_qidx)); + reg &= ~I40E_QRX_ENA_QENA_REQ_MASK; + wr32(hw, I40E_QRX_ENA(pf_qidx), reg); + /* Verify the disable took */ + for (int j = 0; j < 10; j++) { + reg = rd32(hw, I40E_QRX_ENA(pf_qidx)); + if (!(reg & I40E_QRX_ENA_QENA_STAT_MASK)) + break; + i40e_msec_delay(10); + } + if (reg & I40E_QRX_ENA_QENA_STAT_MASK) { + device_printf(pf->dev, "RX queue %d still enabled!\n", + pf_qidx); + error = ETIMEDOUT; + } + + return (error); +} + +int +ixl_disable_ring(struct ixl_pf *pf, struct ixl_pf_qtag *qtag, u16 vsi_qidx) +{ + int error = 0; + + error = ixl_disable_tx_ring(pf, qtag, vsi_qidx); + /* Called function already prints error message */ + if (error) + return (error); + error = ixl_disable_rx_ring(pf, qtag, vsi_qidx); + return (error); +} + +/* For PF VSI only */ +int +ixl_disable_rings(struct ixl_vsi *vsi) +{ + struct ixl_pf *pf = vsi->back; + int error = 0; + + for (int i = 0; i < vsi->num_queues; i++) { + error = ixl_disable_ring(pf, &pf->qtag, i); + if (error) + return (error); + } + + return (error); +} + +/** + * ixl_handle_mdd_event + * + * Called from interrupt handler to identify possibly malicious vfs + * (But also detects events from the PF, as well) + **/ +void +ixl_handle_mdd_event(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + device_t dev = pf->dev; + bool mdd_detected = false; + bool pf_mdd_detected = false; + u32 reg; + + /* find what triggered the MDD event */ + reg = rd32(hw, I40E_GL_MDET_TX); + if (reg & I40E_GL_MDET_TX_VALID_MASK) { + u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >> + I40E_GL_MDET_TX_PF_NUM_SHIFT; + u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >> + I40E_GL_MDET_TX_EVENT_SHIFT; + u16 queue = (reg & I40E_GL_MDET_TX_QUEUE_MASK) >> + I40E_GL_MDET_TX_QUEUE_SHIFT; + device_printf(dev, + "Malicious Driver Detection event %d" + " on TX queue %d, pf number %d\n", + event, queue, pf_num); + wr32(hw, I40E_GL_MDET_TX, 0xffffffff); + mdd_detected = true; + } + reg = rd32(hw, I40E_GL_MDET_RX); + if (reg & I40E_GL_MDET_RX_VALID_MASK) { + u8 pf_num = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >> + I40E_GL_MDET_RX_FUNCTION_SHIFT; + u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >> + I40E_GL_MDET_RX_EVENT_SHIFT; + u16 queue = (reg & I40E_GL_MDET_RX_QUEUE_MASK) >> + I40E_GL_MDET_RX_QUEUE_SHIFT; + device_printf(dev, + "Malicious Driver Detection event %d" + " on RX queue %d, pf number %d\n", + event, queue, pf_num); + wr32(hw, I40E_GL_MDET_RX, 0xffffffff); + mdd_detected = true; + } + + if (mdd_detected) { + reg = rd32(hw, I40E_PF_MDET_TX); + if (reg & I40E_PF_MDET_TX_VALID_MASK) { + wr32(hw, I40E_PF_MDET_TX, 0xFFFF); + device_printf(dev, + "MDD TX event is for this function!"); + pf_mdd_detected = true; + } + reg = rd32(hw, I40E_PF_MDET_RX); + if (reg & I40E_PF_MDET_RX_VALID_MASK) { + wr32(hw, I40E_PF_MDET_RX, 0xFFFF); + device_printf(dev, + "MDD RX event is for this function!"); + pf_mdd_detected = true; + } + } + + /* re-enable mdd interrupt cause */ + reg = rd32(hw, I40E_PFINT_ICR0_ENA); + reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; + wr32(hw, I40E_PFINT_ICR0_ENA, reg); + ixl_flush(hw); +} + +void +ixl_enable_intr(struct ixl_vsi *vsi) +{ + struct ixl_pf *pf = (struct ixl_pf *)vsi->back; + struct i40e_hw *hw = vsi->hw; + struct ixl_queue *que = vsi->queues; + + if (pf->enable_msix) { + for (int i = 0; i < vsi->num_queues; i++, que++) + ixl_enable_queue(hw, que->me); + } else + ixl_enable_legacy(hw); +} + +void +ixl_disable_rings_intr(struct ixl_vsi *vsi) +{ + struct i40e_hw *hw = vsi->hw; + struct ixl_queue *que = vsi->queues; + + for (int i = 0; i < vsi->num_queues; i++, que++) + ixl_disable_queue(hw, que->me); +} + +void +ixl_disable_intr(struct ixl_vsi *vsi) +{ + struct ixl_pf *pf = (struct ixl_pf *)vsi->back; + struct i40e_hw *hw = vsi->hw; + + if (pf->enable_msix) + ixl_disable_adminq(hw); + else + ixl_disable_legacy(hw); +} + +void +ixl_enable_adminq(struct i40e_hw *hw) +{ + u32 reg; + + reg = I40E_PFINT_DYN_CTL0_INTENA_MASK | + I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | + (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); + wr32(hw, I40E_PFINT_DYN_CTL0, reg); + ixl_flush(hw); +} + +void +ixl_disable_adminq(struct i40e_hw *hw) +{ + u32 reg; + + reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT; + wr32(hw, I40E_PFINT_DYN_CTL0, reg); + ixl_flush(hw); +} + +void +ixl_enable_queue(struct i40e_hw *hw, int id) +{ + u32 reg; + + reg = I40E_PFINT_DYN_CTLN_INTENA_MASK | + I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | + (IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT); + wr32(hw, I40E_PFINT_DYN_CTLN(id), reg); +} + +void +ixl_disable_queue(struct i40e_hw *hw, int id) +{ + u32 reg; + + reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT; + wr32(hw, I40E_PFINT_DYN_CTLN(id), reg); +} + +void +ixl_enable_legacy(struct i40e_hw *hw) +{ + u32 reg; + reg = I40E_PFINT_DYN_CTL0_INTENA_MASK | + I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | + (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); + wr32(hw, I40E_PFINT_DYN_CTL0, reg); +} + +void +ixl_disable_legacy(struct i40e_hw *hw) +{ + u32 reg; + + reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT; + wr32(hw, I40E_PFINT_DYN_CTL0, reg); +} + +void +ixl_update_stats_counters(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + struct ixl_vsi *vsi = &pf->vsi; + struct ixl_vf *vf; + + struct i40e_hw_port_stats *nsd = &pf->stats; + struct i40e_hw_port_stats *osd = &pf->stats_offsets; + + /* Update hw stats */ + ixl_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port), + pf->stat_offsets_loaded, + &osd->crc_errors, &nsd->crc_errors); + ixl_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port), + pf->stat_offsets_loaded, + &osd->illegal_bytes, &nsd->illegal_bytes); + ixl_stat_update48(hw, I40E_GLPRT_GORCH(hw->port), + I40E_GLPRT_GORCL(hw->port), + pf->stat_offsets_loaded, + &osd->eth.rx_bytes, &nsd->eth.rx_bytes); + ixl_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port), + I40E_GLPRT_GOTCL(hw->port), + pf->stat_offsets_loaded, + &osd->eth.tx_bytes, &nsd->eth.tx_bytes); + ixl_stat_update32(hw, I40E_GLPRT_RDPC(hw->port), + pf->stat_offsets_loaded, + &osd->eth.rx_discards, + &nsd->eth.rx_discards); + ixl_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port), + I40E_GLPRT_UPRCL(hw->port), + pf->stat_offsets_loaded, + &osd->eth.rx_unicast, + &nsd->eth.rx_unicast); + ixl_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port), + I40E_GLPRT_UPTCL(hw->port), + pf->stat_offsets_loaded, + &osd->eth.tx_unicast, + &nsd->eth.tx_unicast); + ixl_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port), + I40E_GLPRT_MPRCL(hw->port), + pf->stat_offsets_loaded, + &osd->eth.rx_multicast, + &nsd->eth.rx_multicast); + ixl_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port), + I40E_GLPRT_MPTCL(hw->port), + pf->stat_offsets_loaded, + &osd->eth.tx_multicast, + &nsd->eth.tx_multicast); + ixl_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port), + I40E_GLPRT_BPRCL(hw->port), + pf->stat_offsets_loaded, + &osd->eth.rx_broadcast, + &nsd->eth.rx_broadcast); + ixl_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port), + I40E_GLPRT_BPTCL(hw->port), + pf->stat_offsets_loaded, + &osd->eth.tx_broadcast, + &nsd->eth.tx_broadcast); + + ixl_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port), + pf->stat_offsets_loaded, + &osd->tx_dropped_link_down, + &nsd->tx_dropped_link_down); + ixl_stat_update32(hw, I40E_GLPRT_MLFC(hw->port), + pf->stat_offsets_loaded, + &osd->mac_local_faults, + &nsd->mac_local_faults); + ixl_stat_update32(hw, I40E_GLPRT_MRFC(hw->port), + pf->stat_offsets_loaded, + &osd->mac_remote_faults, + &nsd->mac_remote_faults); + ixl_stat_update32(hw, I40E_GLPRT_RLEC(hw->port), + pf->stat_offsets_loaded, + &osd->rx_length_errors, + &nsd->rx_length_errors); + + /* Flow control (LFC) stats */ + ixl_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port), + pf->stat_offsets_loaded, + &osd->link_xon_rx, &nsd->link_xon_rx); + ixl_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port), + pf->stat_offsets_loaded, + &osd->link_xon_tx, &nsd->link_xon_tx); + ixl_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port), + pf->stat_offsets_loaded, + &osd->link_xoff_rx, &nsd->link_xoff_rx); + ixl_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port), + pf->stat_offsets_loaded, + &osd->link_xoff_tx, &nsd->link_xoff_tx); + + /* Packet size stats rx */ + ixl_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port), + I40E_GLPRT_PRC64L(hw->port), + pf->stat_offsets_loaded, + &osd->rx_size_64, &nsd->rx_size_64); + ixl_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port), + I40E_GLPRT_PRC127L(hw->port), + pf->stat_offsets_loaded, + &osd->rx_size_127, &nsd->rx_size_127); + ixl_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port), + I40E_GLPRT_PRC255L(hw->port), + pf->stat_offsets_loaded, + &osd->rx_size_255, &nsd->rx_size_255); + ixl_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port), + I40E_GLPRT_PRC511L(hw->port), + pf->stat_offsets_loaded, + &osd->rx_size_511, &nsd->rx_size_511); + ixl_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port), + I40E_GLPRT_PRC1023L(hw->port), + pf->stat_offsets_loaded, + &osd->rx_size_1023, &nsd->rx_size_1023); + ixl_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port), + I40E_GLPRT_PRC1522L(hw->port), + pf->stat_offsets_loaded, + &osd->rx_size_1522, &nsd->rx_size_1522); + ixl_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port), + I40E_GLPRT_PRC9522L(hw->port), + pf->stat_offsets_loaded, + &osd->rx_size_big, &nsd->rx_size_big); + + /* Packet size stats tx */ + ixl_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port), + I40E_GLPRT_PTC64L(hw->port), + pf->stat_offsets_loaded, + &osd->tx_size_64, &nsd->tx_size_64); + ixl_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port), + I40E_GLPRT_PTC127L(hw->port), + pf->stat_offsets_loaded, + &osd->tx_size_127, &nsd->tx_size_127); + ixl_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port), + I40E_GLPRT_PTC255L(hw->port), + pf->stat_offsets_loaded, + &osd->tx_size_255, &nsd->tx_size_255); + ixl_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port), + I40E_GLPRT_PTC511L(hw->port), + pf->stat_offsets_loaded, + &osd->tx_size_511, &nsd->tx_size_511); + ixl_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port), + I40E_GLPRT_PTC1023L(hw->port), + pf->stat_offsets_loaded, + &osd->tx_size_1023, &nsd->tx_size_1023); + ixl_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port), + I40E_GLPRT_PTC1522L(hw->port), + pf->stat_offsets_loaded, + &osd->tx_size_1522, &nsd->tx_size_1522); + ixl_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port), + I40E_GLPRT_PTC9522L(hw->port), + pf->stat_offsets_loaded, + &osd->tx_size_big, &nsd->tx_size_big); + + ixl_stat_update32(hw, I40E_GLPRT_RUC(hw->port), + pf->stat_offsets_loaded, + &osd->rx_undersize, &nsd->rx_undersize); + ixl_stat_update32(hw, I40E_GLPRT_RFC(hw->port), + pf->stat_offsets_loaded, + &osd->rx_fragments, &nsd->rx_fragments); + ixl_stat_update32(hw, I40E_GLPRT_ROC(hw->port), + pf->stat_offsets_loaded, + &osd->rx_oversize, &nsd->rx_oversize); + ixl_stat_update32(hw, I40E_GLPRT_RJC(hw->port), + pf->stat_offsets_loaded, + &osd->rx_jabber, &nsd->rx_jabber); + pf->stat_offsets_loaded = true; + /* End hw stats */ + + /* Update vsi stats */ + ixl_update_vsi_stats(vsi); + + for (int i = 0; i < pf->num_vfs; i++) { + vf = &pf->vfs[i]; + if (vf->vf_flags & VF_FLAG_ENABLED) + ixl_update_eth_stats(&pf->vfs[i].vsi); + } +} + +int +ixl_rebuild_hw_structs_after_reset(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + struct ixl_vsi *vsi = &pf->vsi; + device_t dev = pf->dev; + bool is_up = false; + int error = 0; + + is_up = !!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING); + + /* Teardown */ + if (is_up) + ixl_stop(pf); + error = i40e_shutdown_lan_hmc(hw); + if (error) + device_printf(dev, + "Shutdown LAN HMC failed with code %d\n", error); + ixl_disable_adminq(hw); + ixl_teardown_adminq_msix(pf); + error = i40e_shutdown_adminq(hw); + if (error) + device_printf(dev, + "Shutdown Admin queue failed with code %d\n", error); + + /* Setup */ + error = i40e_init_adminq(hw); + if (error != 0 && error != I40E_ERR_FIRMWARE_API_VERSION) { + device_printf(dev, "Unable to initialize Admin Queue, error %d\n", + error); + } + error = ixl_setup_adminq_msix(pf); + if (error) { + device_printf(dev, "ixl_setup_adminq_msix error: %d\n", + error); + } + ixl_configure_intr0_msix(pf); + ixl_enable_adminq(hw); + error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, + hw->func_caps.num_rx_qp, 0, 0); + if (error) { + device_printf(dev, "init_lan_hmc failed: %d\n", error); + } + error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); + if (error) { + device_printf(dev, "configure_lan_hmc failed: %d\n", error); + } + if (is_up) + ixl_init(pf); + + return (0); +} + +void +ixl_handle_empr_reset(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + device_t dev = pf->dev; + int count = 0; + u32 reg; + + /* Typically finishes within 3-4 seconds */ + while (count++ < 100) { + reg = rd32(hw, I40E_GLGEN_RSTAT) + & I40E_GLGEN_RSTAT_DEVSTATE_MASK; + if (reg) + i40e_msec_delay(100); + else + break; + } + ixl_dbg(pf, IXL_DBG_INFO, + "EMPR reset wait count: %d\n", count); + + device_printf(dev, "Rebuilding driver state...\n"); + ixl_rebuild_hw_structs_after_reset(pf); + device_printf(dev, "Rebuilding driver state done.\n"); + + atomic_clear_int(&pf->state, IXL_PF_STATE_EMPR_RESETTING); +} + +/* +** Tasklet handler for MSIX Adminq interrupts +** - do outside interrupt since it might sleep +*/ +void +ixl_do_adminq(void *context, int pending) +{ + struct ixl_pf *pf = context; + struct i40e_hw *hw = &pf->hw; + struct i40e_arq_event_info event; + i40e_status ret; + device_t dev = pf->dev; + u32 loop = 0; + u16 opcode, result; + + if (pf->state & IXL_PF_STATE_EMPR_RESETTING) { + /* Flag cleared at end of this function */ + ixl_handle_empr_reset(pf); + return; + } + + /* Admin Queue handling */ + event.buf_len = IXL_AQ_BUF_SZ; + event.msg_buf = malloc(event.buf_len, + M_DEVBUF, M_NOWAIT | M_ZERO); + if (!event.msg_buf) { + device_printf(dev, "%s: Unable to allocate memory for Admin" + " Queue event!\n", __func__); + return; + } + + IXL_PF_LOCK(pf); + /* clean and process any events */ + do { + ret = i40e_clean_arq_element(hw, &event, &result); + if (ret) + break; + opcode = LE16_TO_CPU(event.desc.opcode); + ixl_dbg(pf, IXL_DBG_AQ, + "%s: Admin Queue event: %#06x\n", __func__, opcode); + switch (opcode) { + case i40e_aqc_opc_get_link_status: + ixl_link_event(pf, &event); + break; + case i40e_aqc_opc_send_msg_to_pf: +#ifdef PCI_IOV + ixl_handle_vf_msg(pf, &event); +#endif + break; + case i40e_aqc_opc_event_lan_overflow: + default: + break; + } + + } while (result && (loop++ < IXL_ADM_LIMIT)); + + free(event.msg_buf, M_DEVBUF); + + /* + * If there are still messages to process, reschedule ourselves. + * Otherwise, re-enable our interrupt. + */ + if (result > 0) + taskqueue_enqueue(pf->tq, &pf->adminq); + else + ixl_enable_adminq(hw); + + IXL_PF_UNLOCK(pf); +} + +/** + * Update VSI-specific ethernet statistics counters. + **/ +void +ixl_update_eth_stats(struct ixl_vsi *vsi) +{ + struct ixl_pf *pf = (struct ixl_pf *)vsi->back; + struct i40e_hw *hw = &pf->hw; + struct i40e_eth_stats *es; + struct i40e_eth_stats *oes; + struct i40e_hw_port_stats *nsd; + u16 stat_idx = vsi->info.stat_counter_idx; + + es = &vsi->eth_stats; + oes = &vsi->eth_stats_offsets; + nsd = &pf->stats; + + /* Gather up the stats that the hw collects */ + ixl_stat_update32(hw, I40E_GLV_TEPC(stat_idx), + vsi->stat_offsets_loaded, + &oes->tx_errors, &es->tx_errors); + ixl_stat_update32(hw, I40E_GLV_RDPC(stat_idx), + vsi->stat_offsets_loaded, + &oes->rx_discards, &es->rx_discards); + + ixl_stat_update48(hw, I40E_GLV_GORCH(stat_idx), + I40E_GLV_GORCL(stat_idx), + vsi->stat_offsets_loaded, + &oes->rx_bytes, &es->rx_bytes); + ixl_stat_update48(hw, I40E_GLV_UPRCH(stat_idx), + I40E_GLV_UPRCL(stat_idx), + vsi->stat_offsets_loaded, + &oes->rx_unicast, &es->rx_unicast); + ixl_stat_update48(hw, I40E_GLV_MPRCH(stat_idx), + I40E_GLV_MPRCL(stat_idx), + vsi->stat_offsets_loaded, + &oes->rx_multicast, &es->rx_multicast); + ixl_stat_update48(hw, I40E_GLV_BPRCH(stat_idx), + I40E_GLV_BPRCL(stat_idx), + vsi->stat_offsets_loaded, + &oes->rx_broadcast, &es->rx_broadcast); + + ixl_stat_update48(hw, I40E_GLV_GOTCH(stat_idx), + I40E_GLV_GOTCL(stat_idx), + vsi->stat_offsets_loaded, + &oes->tx_bytes, &es->tx_bytes); + ixl_stat_update48(hw, I40E_GLV_UPTCH(stat_idx), + I40E_GLV_UPTCL(stat_idx), + vsi->stat_offsets_loaded, + &oes->tx_unicast, &es->tx_unicast); + ixl_stat_update48(hw, I40E_GLV_MPTCH(stat_idx), + I40E_GLV_MPTCL(stat_idx), + vsi->stat_offsets_loaded, + &oes->tx_multicast, &es->tx_multicast); + ixl_stat_update48(hw, I40E_GLV_BPTCH(stat_idx), + I40E_GLV_BPTCL(stat_idx), + vsi->stat_offsets_loaded, + &oes->tx_broadcast, &es->tx_broadcast); + vsi->stat_offsets_loaded = true; +} + +void +ixl_update_vsi_stats(struct ixl_vsi *vsi) +{ + struct ixl_pf *pf; + struct ifnet *ifp; + struct i40e_eth_stats *es; + u64 tx_discards; + + struct i40e_hw_port_stats *nsd; + + pf = vsi->back; + ifp = vsi->ifp; + es = &vsi->eth_stats; + nsd = &pf->stats; + + ixl_update_eth_stats(vsi); + + tx_discards = es->tx_discards + nsd->tx_dropped_link_down; + for (int i = 0; i < vsi->num_queues; i++) + tx_discards += vsi->queues[i].txr.br->br_drops; + + /* Update ifnet stats */ + IXL_SET_IPACKETS(vsi, es->rx_unicast + + es->rx_multicast + + es->rx_broadcast); + IXL_SET_OPACKETS(vsi, es->tx_unicast + + es->tx_multicast + + es->tx_broadcast); + IXL_SET_IBYTES(vsi, es->rx_bytes); + IXL_SET_OBYTES(vsi, es->tx_bytes); + IXL_SET_IMCASTS(vsi, es->rx_multicast); + IXL_SET_OMCASTS(vsi, es->tx_multicast); + + IXL_SET_IERRORS(vsi, nsd->crc_errors + nsd->illegal_bytes + + nsd->rx_undersize + nsd->rx_oversize + nsd->rx_fragments + + nsd->rx_jabber); + IXL_SET_OERRORS(vsi, es->tx_errors); + IXL_SET_IQDROPS(vsi, es->rx_discards + nsd->eth.rx_discards); + IXL_SET_OQDROPS(vsi, tx_discards); + IXL_SET_NOPROTO(vsi, es->rx_unknown_protocol); + IXL_SET_COLLISIONS(vsi, 0); +} + +/** + * Reset all of the stats for the given pf + **/ +void +ixl_pf_reset_stats(struct ixl_pf *pf) +{ + bzero(&pf->stats, sizeof(struct i40e_hw_port_stats)); + bzero(&pf->stats_offsets, sizeof(struct i40e_hw_port_stats)); + pf->stat_offsets_loaded = false; +} + +/** + * Resets all stats of the given vsi + **/ +void +ixl_vsi_reset_stats(struct ixl_vsi *vsi) +{ + bzero(&vsi->eth_stats, sizeof(struct i40e_eth_stats)); + bzero(&vsi->eth_stats_offsets, sizeof(struct i40e_eth_stats)); + vsi->stat_offsets_loaded = false; +} + +/** + * Read and update a 48 bit stat from the hw + * + * Since the device stats are not reset at PFReset, they likely will not + * be zeroed when the driver starts. We'll save the first values read + * and use them as offsets to be subtracted from the raw values in order + * to report stats that count from zero. + **/ +void +ixl_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg, + bool offset_loaded, u64 *offset, u64 *stat) +{ + u64 new_data; + +#if defined(__FreeBSD__) && (__FreeBSD_version >= 1000000) && defined(__amd64__) + new_data = rd64(hw, loreg); +#else + /* + * Use two rd32's instead of one rd64; FreeBSD versions before + * 10 don't support 64-bit bus reads/writes. + */ + new_data = rd32(hw, loreg); + new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32; +#endif + + if (!offset_loaded) + *offset = new_data; + if (new_data >= *offset) + *stat = new_data - *offset; + else + *stat = (new_data + ((u64)1 << 48)) - *offset; + *stat &= 0xFFFFFFFFFFFFULL; +} + +/** + * Read and update a 32 bit stat from the hw + **/ +void +ixl_stat_update32(struct i40e_hw *hw, u32 reg, + bool offset_loaded, u64 *offset, u64 *stat) +{ + u32 new_data; + + new_data = rd32(hw, reg); + if (!offset_loaded) + *offset = new_data; + if (new_data >= *offset) + *stat = (u32)(new_data - *offset); + else + *stat = (u32)((new_data + ((u64)1 << 32)) - *offset); +} + +void +ixl_add_device_sysctls(struct ixl_pf *pf) +{ + device_t dev = pf->dev; + + struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); + struct sysctl_oid_list *ctx_list = + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)); + + struct sysctl_oid *debug_node; + struct sysctl_oid_list *debug_list; + + /* Set up sysctls */ + SYSCTL_ADD_PROC(ctx, ctx_list, + OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW, + pf, 0, ixl_set_flowcntl, "I", IXL_SYSCTL_HELP_FC); + + SYSCTL_ADD_PROC(ctx, ctx_list, + OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW, + pf, 0, ixl_set_advertise, "I", IXL_SYSCTL_HELP_SET_ADVERTISE); + + SYSCTL_ADD_PROC(ctx, ctx_list, + OID_AUTO, "current_speed", CTLTYPE_STRING | CTLFLAG_RD, + pf, 0, ixl_current_speed, "A", "Current Port Speed"); + + SYSCTL_ADD_PROC(ctx, ctx_list, + OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD, + pf, 0, ixl_sysctl_show_fw, "A", "Firmware version"); + + SYSCTL_ADD_PROC(ctx, ctx_list, + OID_AUTO, "unallocated_queues", CTLTYPE_INT | CTLFLAG_RD, + pf, 0, ixl_sysctl_unallocated_queues, "I", + "Queues not allocated to a PF or VF"); + + SYSCTL_ADD_PROC(ctx, ctx_list, + OID_AUTO, "tx_itr", CTLTYPE_INT | CTLFLAG_RW, + pf, 0, ixl_sysctl_pf_tx_itr, "I", + "Immediately set TX ITR value for all queues"); + + SYSCTL_ADD_PROC(ctx, ctx_list, + OID_AUTO, "rx_itr", CTLTYPE_INT | CTLFLAG_RW, + pf, 0, ixl_sysctl_pf_rx_itr, "I", + "Immediately set RX ITR value for all queues"); + + SYSCTL_ADD_INT(ctx, ctx_list, + OID_AUTO, "dynamic_rx_itr", CTLFLAG_RW, + &pf->dynamic_rx_itr, 0, "Enable dynamic RX ITR"); + + SYSCTL_ADD_INT(ctx, ctx_list, + OID_AUTO, "dynamic_tx_itr", CTLFLAG_RW, + &pf->dynamic_tx_itr, 0, "Enable dynamic TX ITR"); + + /* Add sysctls meant to print debug information, but don't list them + * in "sysctl -a" output. */ + debug_node = SYSCTL_ADD_NODE(ctx, ctx_list, + OID_AUTO, "debug", CTLFLAG_RD | CTLFLAG_SKIP, NULL, "Debug Sysctls"); + debug_list = SYSCTL_CHILDREN(debug_node); + + SYSCTL_ADD_UINT(ctx, debug_list, + OID_AUTO, "shared_debug_mask", CTLFLAG_RW, + &pf->hw.debug_mask, 0, "Shared code debug message level"); + + SYSCTL_ADD_UINT(ctx, debug_list, + OID_AUTO, "core_debug_mask", CTLFLAG_RW, + &pf->dbg_mask, 0, "Non-hared code debug message level"); + + SYSCTL_ADD_PROC(ctx, debug_list, + OID_AUTO, "link_status", CTLTYPE_STRING | CTLFLAG_RD, + pf, 0, ixl_sysctl_link_status, "A", IXL_SYSCTL_HELP_LINK_STATUS); + + SYSCTL_ADD_PROC(ctx, debug_list, + OID_AUTO, "phy_abilities", CTLTYPE_STRING | CTLFLAG_RD, + pf, 0, ixl_sysctl_phy_abilities, "A", "PHY Abilities"); + + SYSCTL_ADD_PROC(ctx, debug_list, + OID_AUTO, "filter_list", CTLTYPE_STRING | CTLFLAG_RD, + pf, 0, ixl_sysctl_sw_filter_list, "A", "SW Filter List"); + + SYSCTL_ADD_PROC(ctx, debug_list, + OID_AUTO, "hw_res_alloc", CTLTYPE_STRING | CTLFLAG_RD, + pf, 0, ixl_sysctl_hw_res_alloc, "A", "HW Resource Allocation"); + + SYSCTL_ADD_PROC(ctx, debug_list, + OID_AUTO, "switch_config", CTLTYPE_STRING | CTLFLAG_RD, + pf, 0, ixl_sysctl_switch_config, "A", "HW Switch Configuration"); + + SYSCTL_ADD_PROC(ctx, debug_list, + OID_AUTO, "rss_key", CTLTYPE_STRING | CTLFLAG_RD, + pf, 0, ixl_sysctl_hkey, "A", "View RSS key"); + + SYSCTL_ADD_PROC(ctx, debug_list, + OID_AUTO, "rss_lut", CTLTYPE_STRING | CTLFLAG_RD, + pf, 0, ixl_sysctl_hlut, "A", "View RSS lookup table"); +#ifdef PCI_IOV + SYSCTL_ADD_UINT(ctx, debug_list, + OID_AUTO, "vc_debug_level", CTLFLAG_RW, &pf->vc_debug_lvl, + 0, "PF/VF Virtual Channel debug level"); +#endif +} + +/* + * Primarily for finding out how many queues can be assigned to VFs, + * at runtime. + */ +static int +ixl_sysctl_unallocated_queues(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + int queues; + + IXL_PF_LOCK(pf); + queues = (int)ixl_pf_qmgr_get_num_free(&pf->qmgr); + IXL_PF_UNLOCK(pf); + + return sysctl_handle_int(oidp, NULL, queues, req); +} + +/* +** Set flow control using sysctl: +** 0 - off +** 1 - rx pause +** 2 - tx pause +** 3 - full +*/ +int +ixl_set_flowcntl(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + struct i40e_hw *hw = &pf->hw; + device_t dev = pf->dev; + int requested_fc, error = 0; + enum i40e_status_code aq_error = 0; + u8 fc_aq_err = 0; + + /* Get request */ + requested_fc = pf->fc; + error = sysctl_handle_int(oidp, &requested_fc, 0, req); + if ((error) || (req->newptr == NULL)) + return (error); + if (requested_fc < 0 || requested_fc > 3) { + device_printf(dev, + "Invalid fc mode; valid modes are 0 through 3\n"); + return (EINVAL); + } + + /* Set fc ability for port */ + hw->fc.requested_mode = requested_fc; + aq_error = i40e_set_fc(hw, &fc_aq_err, TRUE); + if (aq_error) { + device_printf(dev, + "%s: Error setting new fc mode %d; fc_err %#x\n", + __func__, aq_error, fc_aq_err); + return (EIO); + } + pf->fc = requested_fc; + + /* Get new link state */ + i40e_msec_delay(250); + hw->phy.get_link_info = TRUE; + i40e_get_link_status(hw, &pf->link_up); + + return (0); +} + +int +ixl_current_speed(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + struct i40e_hw *hw = &pf->hw; + int error = 0, index = 0; + + char *speeds[] = { + "Unknown", + "100M", + "1G", + "10G", + "40G", + "20G" + }; + + ixl_update_link_status(pf); + + switch (hw->phy.link_info.link_speed) { + case I40E_LINK_SPEED_100MB: + index = 1; + break; + case I40E_LINK_SPEED_1GB: + index = 2; + break; + case I40E_LINK_SPEED_10GB: + index = 3; + break; + case I40E_LINK_SPEED_40GB: + index = 4; + break; + case I40E_LINK_SPEED_20GB: + index = 5; + break; + case I40E_LINK_SPEED_UNKNOWN: + default: + index = 0; + break; + } + + error = sysctl_handle_string(oidp, speeds[index], + strlen(speeds[index]), req); + return (error); +} + +int +ixl_set_advertised_speeds(struct ixl_pf *pf, int speeds) +{ + struct i40e_hw *hw = &pf->hw; + device_t dev = pf->dev; + struct i40e_aq_get_phy_abilities_resp abilities; + struct i40e_aq_set_phy_config config; + enum i40e_status_code aq_error = 0; + + /* Get current capability information */ + aq_error = i40e_aq_get_phy_capabilities(hw, + FALSE, FALSE, &abilities, NULL); + if (aq_error) { + device_printf(dev, + "%s: Error getting phy capabilities %d," + " aq error: %d\n", __func__, aq_error, + hw->aq.asq_last_status); + return (EIO); + } + + /* Prepare new config */ + bzero(&config, sizeof(config)); + config.phy_type = abilities.phy_type; + config.abilities = abilities.abilities + | I40E_AQ_PHY_ENABLE_ATOMIC_LINK; + config.eee_capability = abilities.eee_capability; + config.eeer = abilities.eeer_val; + config.low_power_ctrl = abilities.d3_lpan; + /* Translate into aq cmd link_speed */ + if (speeds & 0x10) + config.link_speed |= I40E_LINK_SPEED_40GB; + if (speeds & 0x8) + config.link_speed |= I40E_LINK_SPEED_20GB; + if (speeds & 0x4) + config.link_speed |= I40E_LINK_SPEED_10GB; + if (speeds & 0x2) + config.link_speed |= I40E_LINK_SPEED_1GB; + if (speeds & 0x1) + config.link_speed |= I40E_LINK_SPEED_100MB; + + /* Do aq command & restart link */ + aq_error = i40e_aq_set_phy_config(hw, &config, NULL); + if (aq_error) { + device_printf(dev, + "%s: Error setting new phy config %d," + " aq error: %d\n", __func__, aq_error, + hw->aq.asq_last_status); + return (EAGAIN); + } + + /* + ** This seems a bit heavy handed, but we + ** need to get a reinit on some devices + */ + IXL_PF_LOCK(pf); + ixl_stop_locked(pf); + ixl_init_locked(pf); + IXL_PF_UNLOCK(pf); + + return (0); +} + +/* +** Control link advertise speed: +** Flags: +** 0x1 - advertise 100 Mb +** 0x2 - advertise 1G +** 0x4 - advertise 10G +** 0x8 - advertise 20G +** 0x10 - advertise 40G +** +** Set to 0 to disable link +*/ +int +ixl_set_advertise(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + struct i40e_hw *hw = &pf->hw; + device_t dev = pf->dev; + int requested_ls = 0; + int error = 0; + + /* Read in new mode */ + requested_ls = pf->advertised_speed; + error = sysctl_handle_int(oidp, &requested_ls, 0, req); + if ((error) || (req->newptr == NULL)) + return (error); + /* Check for sane value */ + if (requested_ls > 0x10) { + device_printf(dev, "Invalid advertised speed; " + "valid modes are 0x1 through 0x10\n"); + return (EINVAL); + } + /* Then check for validity based on adapter type */ + switch (hw->device_id) { + case I40E_DEV_ID_1G_BASE_T_X722: + /* 1G BaseT */ + if (requested_ls & ~(0x2)) { + device_printf(dev, + "Only 1G speeds supported on this device.\n"); + return (EINVAL); + } + break; + case I40E_DEV_ID_10G_BASE_T: + case I40E_DEV_ID_10G_BASE_T4: + /* 10G BaseT */ + if (requested_ls & ~(0x7)) { + device_printf(dev, + "Only 100M/1G/10G speeds supported on this device.\n"); + return (EINVAL); + } + break; + case I40E_DEV_ID_20G_KR2: + case I40E_DEV_ID_20G_KR2_A: + /* 20G */ + if (requested_ls & ~(0xE)) { + device_printf(dev, + "Only 1G/10G/20G speeds supported on this device.\n"); + return (EINVAL); + } + break; + case I40E_DEV_ID_KX_B: + case I40E_DEV_ID_QSFP_A: + case I40E_DEV_ID_QSFP_B: + /* 40G */ + if (requested_ls & ~(0x10)) { + device_printf(dev, + "Only 40G speeds supported on this device.\n"); + return (EINVAL); + } + break; + default: + /* 10G (1G) */ + if (requested_ls & ~(0x6)) { + device_printf(dev, + "Only 1/10G speeds supported on this device.\n"); + return (EINVAL); + } + break; + } + + /* Exit if no change */ + if (pf->advertised_speed == requested_ls) + return (0); + + error = ixl_set_advertised_speeds(pf, requested_ls); + if (error) + return (error); + + pf->advertised_speed = requested_ls; + ixl_update_link_status(pf); + return (0); +} + +/* +** Get the width and transaction speed of +** the bus this adapter is plugged into. +*/ +void +ixl_get_bus_info(struct i40e_hw *hw, device_t dev) +{ + u16 link; + u32 offset; + + /* Some devices don't use PCIE */ + if (hw->mac.type == I40E_MAC_X722) + return; + + /* Read PCI Express Capabilities Link Status Register */ + pci_find_cap(dev, PCIY_EXPRESS, &offset); + link = pci_read_config(dev, offset + PCIER_LINK_STA, 2); + + /* Fill out hw struct with PCIE info */ + i40e_set_pci_config_data(hw, link); + + /* Use info to print out bandwidth messages */ + device_printf(dev,"PCI Express Bus: Speed %s %s\n", + ((hw->bus.speed == i40e_bus_speed_8000) ? "8.0GT/s": + (hw->bus.speed == i40e_bus_speed_5000) ? "5.0GT/s": + (hw->bus.speed == i40e_bus_speed_2500) ? "2.5GT/s":"Unknown"), + (hw->bus.width == i40e_bus_width_pcie_x8) ? "Width x8" : + (hw->bus.width == i40e_bus_width_pcie_x4) ? "Width x4" : + (hw->bus.width == i40e_bus_width_pcie_x1) ? "Width x1" : + ("Unknown")); + + if ((hw->bus.width <= i40e_bus_width_pcie_x8) && + (hw->bus.speed < i40e_bus_speed_8000)) { + device_printf(dev, "PCI-Express bandwidth available" + " for this device may be insufficient for" + " optimal performance.\n"); + device_printf(dev, "For optimal performance, a x8 " + "PCIE Gen3 slot is required.\n"); + } +} + +static int +ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + struct i40e_hw *hw = &pf->hw; + struct sbuf *sbuf; + + sbuf = sbuf_new_for_sysctl(NULL, NULL, 128, req); + ixl_nvm_version_str(hw, sbuf); + sbuf_finish(sbuf); + sbuf_delete(sbuf); + + return 0; +} + +void +ixl_print_nvm_cmd(device_t dev, struct i40e_nvm_access *nvma) +{ + if ((nvma->command == I40E_NVM_READ) && + ((nvma->config & 0xFF) == 0xF) && + (((nvma->config & 0xF00) >> 8) == 0xF) && + (nvma->offset == 0) && + (nvma->data_size == 1)) { + // device_printf(dev, "- Get Driver Status Command\n"); + } + else if (nvma->command == I40E_NVM_READ) { + + } + else { + switch (nvma->command) { + case 0xB: + device_printf(dev, "- command: I40E_NVM_READ\n"); + break; + case 0xC: + device_printf(dev, "- command: I40E_NVM_WRITE\n"); + break; + default: + device_printf(dev, "- command: unknown 0x%08x\n", nvma->command); + break; + } + + device_printf(dev, "- config (ptr) : 0x%02x\n", nvma->config & 0xFF); + device_printf(dev, "- config (flags): 0x%01x\n", (nvma->config & 0xF00) >> 8); + device_printf(dev, "- offset : 0x%08x\n", nvma->offset); + device_printf(dev, "- data_s : 0x%08x\n", nvma->data_size); + } +} + +int +ixl_handle_nvmupd_cmd(struct ixl_pf *pf, struct ifdrv *ifd) +{ + struct i40e_hw *hw = &pf->hw; + struct i40e_nvm_access *nvma; + device_t dev = pf->dev; + enum i40e_status_code status = 0; + int perrno; + + DEBUGFUNC("ixl_handle_nvmupd_cmd"); + + /* Sanity checks */ + if (ifd->ifd_len < sizeof(struct i40e_nvm_access) || + ifd->ifd_data == NULL) { + device_printf(dev, "%s: incorrect ifdrv length or data pointer\n", + __func__); + device_printf(dev, "%s: ifdrv length: %lu, sizeof(struct i40e_nvm_access): %lu\n", + __func__, ifd->ifd_len, sizeof(struct i40e_nvm_access)); + device_printf(dev, "%s: data pointer: %p\n", __func__, + ifd->ifd_data); + return (EINVAL); + } + + nvma = (struct i40e_nvm_access *)ifd->ifd_data; + + if (pf->dbg_mask & IXL_DBG_NVMUPD) + ixl_print_nvm_cmd(dev, nvma); + + if (pf->state & IXL_PF_STATE_EMPR_RESETTING) { + int count = 0; + while (count++ < 100) { + i40e_msec_delay(100); + if (!(pf->state & IXL_PF_STATE_EMPR_RESETTING)) + break; + } + } + + if (!(pf->state & IXL_PF_STATE_EMPR_RESETTING)) { + IXL_PF_LOCK(pf); + status = i40e_nvmupd_command(hw, nvma, nvma->data, &perrno); + IXL_PF_UNLOCK(pf); + } else { + perrno = -EBUSY; + } + + if (status) + device_printf(dev, "i40e_nvmupd_command status %d, perrno %d\n", + status, perrno); + + /* + * -EPERM is actually ERESTART, which the kernel interprets as it needing + * to run this ioctl again. So use -EACCES for -EPERM instead. + */ + if (perrno == -EPERM) + return (-EACCES); + else + return (perrno); +} + +/********************************************************************* + * + * Media Ioctl callback + * + * This routine is called whenever the user queries the status of + * the interface using ifconfig. + * + **********************************************************************/ +void +ixl_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) +{ + struct ixl_vsi *vsi = ifp->if_softc; + struct ixl_pf *pf = vsi->back; + struct i40e_hw *hw = &pf->hw; + + INIT_DEBUGOUT("ixl_media_status: begin"); + IXL_PF_LOCK(pf); + + hw->phy.get_link_info = TRUE; + i40e_get_link_status(hw, &pf->link_up); + ixl_update_link_status(pf); + + ifmr->ifm_status = IFM_AVALID; + ifmr->ifm_active = IFM_ETHER; + + if (!pf->link_up) { + IXL_PF_UNLOCK(pf); + return; + } + + ifmr->ifm_status |= IFM_ACTIVE; + + /* Hardware always does full-duplex */ + ifmr->ifm_active |= IFM_FDX; + + switch (hw->phy.link_info.phy_type) { + /* 100 M */ + case I40E_PHY_TYPE_100BASE_TX: + ifmr->ifm_active |= IFM_100_TX; + break; + /* 1 G */ + case I40E_PHY_TYPE_1000BASE_T: + ifmr->ifm_active |= IFM_1000_T; + break; + case I40E_PHY_TYPE_1000BASE_SX: + ifmr->ifm_active |= IFM_1000_SX; + break; + case I40E_PHY_TYPE_1000BASE_LX: + ifmr->ifm_active |= IFM_1000_LX; + break; + case I40E_PHY_TYPE_1000BASE_T_OPTICAL: + ifmr->ifm_active |= IFM_OTHER; + break; + /* 10 G */ + case I40E_PHY_TYPE_10GBASE_SFPP_CU: + ifmr->ifm_active |= IFM_10G_TWINAX; + break; + case I40E_PHY_TYPE_10GBASE_SR: + ifmr->ifm_active |= IFM_10G_SR; + break; + case I40E_PHY_TYPE_10GBASE_LR: + ifmr->ifm_active |= IFM_10G_LR; + break; + case I40E_PHY_TYPE_10GBASE_T: + ifmr->ifm_active |= IFM_10G_T; + break; + case I40E_PHY_TYPE_XAUI: + case I40E_PHY_TYPE_XFI: + case I40E_PHY_TYPE_10GBASE_AOC: + ifmr->ifm_active |= IFM_OTHER; + break; + /* 40 G */ + case I40E_PHY_TYPE_40GBASE_CR4: + case I40E_PHY_TYPE_40GBASE_CR4_CU: + ifmr->ifm_active |= IFM_40G_CR4; + break; + case I40E_PHY_TYPE_40GBASE_SR4: + ifmr->ifm_active |= IFM_40G_SR4; + break; + case I40E_PHY_TYPE_40GBASE_LR4: + ifmr->ifm_active |= IFM_40G_LR4; + break; + case I40E_PHY_TYPE_XLAUI: + ifmr->ifm_active |= IFM_OTHER; + break; + case I40E_PHY_TYPE_1000BASE_KX: + ifmr->ifm_active |= IFM_1000_KX; + break; + case I40E_PHY_TYPE_SGMII: + ifmr->ifm_active |= IFM_1000_SGMII; + break; + /* ERJ: What's the difference between these? */ + case I40E_PHY_TYPE_10GBASE_CR1_CU: + case I40E_PHY_TYPE_10GBASE_CR1: + ifmr->ifm_active |= IFM_10G_CR1; + break; + case I40E_PHY_TYPE_10GBASE_KX4: + ifmr->ifm_active |= IFM_10G_KX4; + break; + case I40E_PHY_TYPE_10GBASE_KR: + ifmr->ifm_active |= IFM_10G_KR; + break; + case I40E_PHY_TYPE_SFI: + ifmr->ifm_active |= IFM_10G_SFI; + break; + /* Our single 20G media type */ + case I40E_PHY_TYPE_20GBASE_KR2: + ifmr->ifm_active |= IFM_20G_KR2; + break; + case I40E_PHY_TYPE_40GBASE_KR4: + ifmr->ifm_active |= IFM_40G_KR4; + break; + case I40E_PHY_TYPE_XLPPI: + case I40E_PHY_TYPE_40GBASE_AOC: + ifmr->ifm_active |= IFM_40G_XLPPI; + break; + /* Unknown to driver */ + default: + ifmr->ifm_active |= IFM_UNKNOWN; + break; + } + /* Report flow control status as well */ + if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX) + ifmr->ifm_active |= IFM_ETH_TXPAUSE; + if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX) + ifmr->ifm_active |= IFM_ETH_RXPAUSE; + + IXL_PF_UNLOCK(pf); +} + +void +ixl_init(void *arg) +{ + struct ixl_pf *pf = arg; + struct ixl_vsi *vsi = &pf->vsi; + device_t dev = pf->dev; + int error = 0; + + /* + * If the aq is dead here, it probably means something outside of the driver + * did something to the adapter, like a PF reset. + * So rebuild the driver's state here if that occurs. + */ + if (!i40e_check_asq_alive(&pf->hw)) { + device_printf(dev, "Admin Queue is down; resetting...\n"); + IXL_PF_LOCK(pf); + ixl_teardown_hw_structs(pf); + ixl_reset(pf); + IXL_PF_UNLOCK(pf); + } + + /* + * Set up LAN queue interrupts here. + * Kernel interrupt setup functions cannot be called while holding a lock, + * so this is done outside of init_locked(). + */ + if (pf->msix > 1) { + /* Teardown existing interrupts, if they exist */ + ixl_teardown_queue_msix(vsi); + ixl_free_queue_tqs(vsi); + /* Then set them up again */ + error = ixl_setup_queue_msix(vsi); + if (error) + device_printf(dev, "ixl_setup_queue_msix() error: %d\n", + error); + error = ixl_setup_queue_tqs(vsi); + if (error) + device_printf(dev, "ixl_setup_queue_tqs() error: %d\n", + error); + } else + // possibly broken + error = ixl_assign_vsi_legacy(pf); + if (error) { + device_printf(pf->dev, "assign_vsi_msix/legacy error: %d\n", error); + return; + } + + IXL_PF_LOCK(pf); + ixl_init_locked(pf); + IXL_PF_UNLOCK(pf); +} + +/* + * NOTE: Fortville does not support forcing media speeds. Instead, + * use the set_advertise sysctl to set the speeds Fortville + * will advertise or be allowed to operate at. + */ +int +ixl_media_change(struct ifnet * ifp) +{ + struct ixl_vsi *vsi = ifp->if_softc; + struct ifmedia *ifm = &vsi->media; + + INIT_DEBUGOUT("ixl_media_change: begin"); + + if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) + return (EINVAL); + + if_printf(ifp, "Use 'advertise_speed' sysctl to change advertised speeds\n"); + + return (ENODEV); +} + +/********************************************************************* + * Ioctl entry point + * + * ixl_ioctl is called when the user wants to configure the + * interface. + * + * return 0 on success, positive on failure + **********************************************************************/ + +int +ixl_ioctl(struct ifnet * ifp, u_long command, caddr_t data) +{ + struct ixl_vsi *vsi = ifp->if_softc; + struct ixl_pf *pf = vsi->back; + struct ifreq *ifr = (struct ifreq *)data; + struct ifdrv *ifd = (struct ifdrv *)data; +#if defined(INET) || defined(INET6) + struct ifaddr *ifa = (struct ifaddr *)data; + bool avoid_reset = FALSE; +#endif + int error = 0; + + switch (command) { + + case SIOCSIFADDR: +#ifdef INET + if (ifa->ifa_addr->sa_family == AF_INET) + avoid_reset = TRUE; +#endif +#ifdef INET6 + if (ifa->ifa_addr->sa_family == AF_INET6) + avoid_reset = TRUE; +#endif +#if defined(INET) || defined(INET6) + /* + ** Calling init results in link renegotiation, + ** so we avoid doing it when possible. + */ + if (avoid_reset) { + ifp->if_flags |= IFF_UP; + if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) + ixl_init(pf); +#ifdef INET + if (!(ifp->if_flags & IFF_NOARP)) + arp_ifinit(ifp, ifa); +#endif + } else + error = ether_ioctl(ifp, command, data); + break; +#endif + case SIOCSIFMTU: + IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); + if (ifr->ifr_mtu > IXL_MAX_FRAME - + ETHER_HDR_LEN - ETHER_CRC_LEN - ETHER_VLAN_ENCAP_LEN) { + error = EINVAL; + } else { + IXL_PF_LOCK(pf); + ifp->if_mtu = ifr->ifr_mtu; + vsi->max_frame_size = + ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN + + ETHER_VLAN_ENCAP_LEN; + ixl_init_locked(pf); + IXL_PF_UNLOCK(pf); + } + break; + case SIOCSIFFLAGS: + IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); + IXL_PF_LOCK(pf); + if (ifp->if_flags & IFF_UP) { + if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { + if ((ifp->if_flags ^ pf->if_flags) & + (IFF_PROMISC | IFF_ALLMULTI)) { + ixl_set_promisc(vsi); + } + } else { + IXL_PF_UNLOCK(pf); + ixl_init(pf); + IXL_PF_LOCK(pf); + } + } else { + if (ifp->if_drv_flags & IFF_DRV_RUNNING) { + IXL_PF_UNLOCK(pf); + ixl_stop(pf); + IXL_PF_LOCK(pf); + } + } + pf->if_flags = ifp->if_flags; + IXL_PF_UNLOCK(pf); + break; + case SIOCSDRVSPEC: + case SIOCGDRVSPEC: + IOCTL_DEBUGOUT("ioctl: SIOCxDRVSPEC (Get/Set Driver-specific " + "Info)\n"); + + /* NVM update command */ + if (ifd->ifd_cmd == I40E_NVM_ACCESS) + error = ixl_handle_nvmupd_cmd(pf, ifd); + else + error = EINVAL; + break; + case SIOCADDMULTI: + IOCTL_DEBUGOUT("ioctl: SIOCADDMULTI"); + if (ifp->if_drv_flags & IFF_DRV_RUNNING) { + IXL_PF_LOCK(pf); + ixl_disable_intr(vsi); + ixl_add_multi(vsi); + ixl_enable_intr(vsi); + IXL_PF_UNLOCK(pf); + } + break; + case SIOCDELMULTI: + IOCTL_DEBUGOUT("ioctl: SIOCDELMULTI"); + if (ifp->if_drv_flags & IFF_DRV_RUNNING) { + IXL_PF_LOCK(pf); + ixl_disable_intr(vsi); + ixl_del_multi(vsi); + ixl_enable_intr(vsi); + IXL_PF_UNLOCK(pf); + } + break; + case SIOCSIFMEDIA: + case SIOCGIFMEDIA: + case SIOCGIFXMEDIA: + IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); + error = ifmedia_ioctl(ifp, ifr, &vsi->media, command); + break; + case SIOCSIFCAP: + { + int mask = ifr->ifr_reqcap ^ ifp->if_capenable; + IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); + + ixl_cap_txcsum_tso(vsi, ifp, mask); + + if (mask & IFCAP_RXCSUM) + ifp->if_capenable ^= IFCAP_RXCSUM; + if (mask & IFCAP_RXCSUM_IPV6) + ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; + if (mask & IFCAP_LRO) + ifp->if_capenable ^= IFCAP_LRO; + if (mask & IFCAP_VLAN_HWTAGGING) + ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; + if (mask & IFCAP_VLAN_HWFILTER) + ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; + if (mask & IFCAP_VLAN_HWTSO) + ifp->if_capenable ^= IFCAP_VLAN_HWTSO; + if (ifp->if_drv_flags & IFF_DRV_RUNNING) { + IXL_PF_LOCK(pf); + ixl_init_locked(pf); + IXL_PF_UNLOCK(pf); + } + VLAN_CAPABILITIES(ifp); + + break; + } + + default: + IOCTL_DEBUGOUT("ioctl: UNKNOWN (0x%X)\n", (int)command); + error = ether_ioctl(ifp, command, data); + break; + } + + return (error); +} + +static char * +ixl_phy_type_string(u32 bit_pos) +{ + static char * phy_types_str[32] = { + "SGMII", + "1000BASE-KX", + "10GBASE-KX4", + "10GBASE-KR", + "40GBASE-KR4", + "XAUI", + "XFI", + "SFI", + "XLAUI", + "XLPPI", + "40GBASE-CR4", + "10GBASE-CR1", + "Reserved (12)", + "Reserved (13)", + "Reserved (14)", + "Reserved (15)", + "Reserved (16)", + "100BASE-TX", + "1000BASE-T", + "10GBASE-T", + "10GBASE-SR", + "10GBASE-LR", + "10GBASE-SFP+Cu", + "10GBASE-CR1", + "40GBASE-CR4", + "40GBASE-SR4", + "40GBASE-LR4", + "1000BASE-SX", + "1000BASE-LX", + "1000BASE-T Optical", + "20GBASE-KR2", + "Reserved (31)" + }; + + if (bit_pos > 31) return "Invalid"; + return phy_types_str[bit_pos]; +} + + +static int +ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + struct i40e_hw *hw = &pf->hw; + device_t dev = pf->dev; + struct i40e_link_status link_status; + enum i40e_status_code status; + struct sbuf *buf; + int error = 0; + + buf = sbuf_new_for_sysctl(NULL, NULL, 128, req); + if (!buf) { + device_printf(dev, "Could not allocate sbuf for sysctl output.\n"); + return (ENOMEM); + } + + status = i40e_aq_get_link_info(hw, true, &link_status, NULL); + if (status) { + device_printf(dev, + "%s: i40e_aq_get_link_info() status %s, aq error %s\n", + __func__, i40e_stat_str(hw, status), + i40e_aq_str(hw, hw->aq.asq_last_status)); + sbuf_delete(buf); + return (EIO); + } + + sbuf_printf(buf, "\n" + "PHY Type : 0x%02x<%s>\n" + "Speed : 0x%02x\n" + "Link info: 0x%02x\n" + "AN info : 0x%02x\n" + "Ext info : 0x%02x\n" + "Max Frame: %d\n" + "Pacing : 0x%02x\n" + "CRC En? : %s\n", + link_status.phy_type, ixl_phy_type_string(link_status.phy_type), + link_status.link_speed, + link_status.link_info, link_status.an_info, + link_status.ext_info, link_status.max_frame_size, + link_status.pacing, + (link_status.crc_enable) ? "Yes" : "No"); + + error = sbuf_finish(buf); + if (error) + device_printf(dev, "Error finishing sbuf: %d\n", error); + + sbuf_delete(buf); + return (error); +} + +static int +ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + struct i40e_hw *hw = &pf->hw; + device_t dev = pf->dev; + enum i40e_status_code status; + struct i40e_aq_get_phy_abilities_resp abilities; + struct sbuf *buf; + int error = 0; + + buf = sbuf_new_for_sysctl(NULL, NULL, 128, req); + if (!buf) { + device_printf(dev, "Could not allocate sbuf for sysctl output.\n"); + return (ENOMEM); + } + + status = i40e_aq_get_phy_capabilities(hw, + TRUE, FALSE, &abilities, NULL); + if (status) { + device_printf(dev, + "%s: i40e_aq_get_phy_capabilities() status %s, aq error %s\n", + __func__, i40e_stat_str(hw, status), + i40e_aq_str(hw, hw->aq.asq_last_status)); + sbuf_delete(buf); + return (EIO); + } + + sbuf_printf(buf, "\n" + "PHY Type : %08x", + abilities.phy_type); + + if (abilities.phy_type != 0) { + sbuf_printf(buf, "<"); + for (int i = 0; i < 32; i++) + if ((1 << i) & abilities.phy_type) + sbuf_printf(buf, "%s,", ixl_phy_type_string(i)); + sbuf_printf(buf, ">\n"); + } + + sbuf_printf(buf, + "Speed : %02x\n" + "Abilities: %02x\n" + "EEE cap : %04x\n" + "EEER reg : %08x\n" + "D3 Lpan : %02x\n" + "ID : %02x %02x %02x %02x\n" + "ModType : %02x %02x %02x", + abilities.link_speed, + abilities.abilities, abilities.eee_capability, + abilities.eeer_val, abilities.d3_lpan, + abilities.phy_id[0], abilities.phy_id[1], + abilities.phy_id[2], abilities.phy_id[3], + abilities.module_type[0], abilities.module_type[1], + abilities.module_type[2]); + + error = sbuf_finish(buf); + if (error) + device_printf(dev, "Error finishing sbuf: %d\n", error); + + sbuf_delete(buf); + return (error); +} + +static int +ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + struct ixl_vsi *vsi = &pf->vsi; + struct ixl_mac_filter *f; + char *buf, *buf_i; + + int error = 0; + int ftl_len = 0; + int ftl_counter = 0; + int buf_len = 0; + int entry_len = 42; + + SLIST_FOREACH(f, &vsi->ftl, next) { + ftl_len++; + } + + if (ftl_len < 1) { + sysctl_handle_string(oidp, "(none)", 6, req); + return (0); + } + + buf_len = sizeof(char) * (entry_len + 1) * ftl_len + 2; + buf = buf_i = malloc(buf_len, M_DEVBUF, M_NOWAIT); + + sprintf(buf_i++, "\n"); + SLIST_FOREACH(f, &vsi->ftl, next) { + sprintf(buf_i, + MAC_FORMAT ", vlan %4d, flags %#06x", + MAC_FORMAT_ARGS(f->macaddr), f->vlan, f->flags); + buf_i += entry_len; + /* don't print '\n' for last entry */ + if (++ftl_counter != ftl_len) { + sprintf(buf_i, "\n"); + buf_i++; + } + } + + error = sysctl_handle_string(oidp, buf, strlen(buf), req); + if (error) + printf("sysctl error: %d\n", error); + free(buf, M_DEVBUF); + return error; +} + +#define IXL_SW_RES_SIZE 0x14 +int +ixl_res_alloc_cmp(const void *a, const void *b) +{ + const struct i40e_aqc_switch_resource_alloc_element_resp *one, *two; + one = (const struct i40e_aqc_switch_resource_alloc_element_resp *)a; + two = (const struct i40e_aqc_switch_resource_alloc_element_resp *)b; + + return ((int)one->resource_type - (int)two->resource_type); +} + +/* + * Longest string length: 25 + */ +char * +ixl_switch_res_type_string(u8 type) +{ + char * ixl_switch_res_type_strings[0x14] = { + "VEB", + "VSI", + "Perfect Match MAC address", + "S-tag", + "(Reserved)", + "Multicast hash entry", + "Unicast hash entry", + "VLAN", + "VSI List entry", + "(Reserved)", + "VLAN Statistic Pool", + "Mirror Rule", + "Queue Set", + "Inner VLAN Forward filter", + "(Reserved)", + "Inner MAC", + "IP", + "GRE/VN1 Key", + "VN2 Key", + "Tunneling Port" + }; + + if (type < 0x14) + return ixl_switch_res_type_strings[type]; + else + return "(Reserved)"; +} + +static int +ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + struct i40e_hw *hw = &pf->hw; + device_t dev = pf->dev; + struct sbuf *buf; + enum i40e_status_code status; + int error = 0; + + u8 num_entries; + struct i40e_aqc_switch_resource_alloc_element_resp resp[IXL_SW_RES_SIZE]; + + buf = sbuf_new_for_sysctl(NULL, NULL, 128, req); + if (!buf) { + device_printf(dev, "Could not allocate sbuf for output.\n"); + return (ENOMEM); + } + + bzero(resp, sizeof(resp)); + status = i40e_aq_get_switch_resource_alloc(hw, &num_entries, + resp, + IXL_SW_RES_SIZE, + NULL); + if (status) { + device_printf(dev, + "%s: get_switch_resource_alloc() error %s, aq error %s\n", + __func__, i40e_stat_str(hw, status), + i40e_aq_str(hw, hw->aq.asq_last_status)); + sbuf_delete(buf); + return (error); + } + + /* Sort entries by type for display */ + qsort(resp, num_entries, + sizeof(struct i40e_aqc_switch_resource_alloc_element_resp), + &ixl_res_alloc_cmp); + + sbuf_cat(buf, "\n"); + sbuf_printf(buf, "# of entries: %d\n", num_entries); + sbuf_printf(buf, + " Type | Guaranteed | Total | Used | Un-allocated\n" + " | (this) | (all) | (this) | (all) \n"); + for (int i = 0; i < num_entries; i++) { + sbuf_printf(buf, + "%25s | %10d %5d %6d %12d", + ixl_switch_res_type_string(resp[i].resource_type), + resp[i].guaranteed, + resp[i].total, + resp[i].used, + resp[i].total_unalloced); + if (i < num_entries - 1) + sbuf_cat(buf, "\n"); + } + + error = sbuf_finish(buf); + if (error) + device_printf(dev, "Error finishing sbuf: %d\n", error); + + sbuf_delete(buf); + return (error); +} + +/* +** Caller must init and delete sbuf; this function will clear and +** finish it for caller. +** +** XXX: Cannot use the SEID for this, since there is no longer a +** fixed mapping between SEID and element type. +*/ +char * +ixl_switch_element_string(struct sbuf *s, + struct i40e_aqc_switch_config_element_resp *element) +{ + sbuf_clear(s); + + switch (element->element_type) { + case I40E_AQ_SW_ELEM_TYPE_MAC: + sbuf_printf(s, "MAC %3d", element->element_info); + break; + case I40E_AQ_SW_ELEM_TYPE_PF: + sbuf_printf(s, "PF %3d", element->element_info); + break; + case I40E_AQ_SW_ELEM_TYPE_VF: + sbuf_printf(s, "VF %3d", element->element_info); + break; + case I40E_AQ_SW_ELEM_TYPE_EMP: + sbuf_cat(s, "EMP"); + break; + case I40E_AQ_SW_ELEM_TYPE_BMC: + sbuf_cat(s, "BMC"); + break; + case I40E_AQ_SW_ELEM_TYPE_PV: + sbuf_cat(s, "PV"); + break; + case I40E_AQ_SW_ELEM_TYPE_VEB: + sbuf_cat(s, "VEB"); + break; + case I40E_AQ_SW_ELEM_TYPE_PA: + sbuf_cat(s, "PA"); + break; + case I40E_AQ_SW_ELEM_TYPE_VSI: + sbuf_printf(s, "VSI %3d", element->element_info); + break; + default: + sbuf_cat(s, "?"); + break; + } + + sbuf_finish(s); + return sbuf_data(s); +} + +static int +ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + struct i40e_hw *hw = &pf->hw; + device_t dev = pf->dev; + struct sbuf *buf; + struct sbuf *nmbuf; + enum i40e_status_code status; + int error = 0; + u16 next = 0; + u8 aq_buf[I40E_AQ_LARGE_BUF]; + + struct i40e_aqc_get_switch_config_resp *sw_config; + sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; + + buf = sbuf_new_for_sysctl(NULL, NULL, 128, req); + if (!buf) { + device_printf(dev, "Could not allocate sbuf for sysctl output.\n"); + return (ENOMEM); + } + + status = i40e_aq_get_switch_config(hw, sw_config, + sizeof(aq_buf), &next, NULL); + if (status) { + device_printf(dev, + "%s: aq_get_switch_config() error %s, aq error %s\n", + __func__, i40e_stat_str(hw, status), + i40e_aq_str(hw, hw->aq.asq_last_status)); + sbuf_delete(buf); + return error; + } + if (next) + device_printf(dev, "%s: TODO: get more config with SEID %d\n", + __func__, next); + + nmbuf = sbuf_new_auto(); + if (!nmbuf) { + device_printf(dev, "Could not allocate sbuf for name output.\n"); + sbuf_delete(buf); + return (ENOMEM); + } + + sbuf_cat(buf, "\n"); + /* Assuming <= 255 elements in switch */ + sbuf_printf(buf, "# of reported elements: %d\n", sw_config->header.num_reported); + sbuf_printf(buf, "total # of elements: %d\n", sw_config->header.num_total); + /* Exclude: + ** Revision -- all elements are revision 1 for now + */ + sbuf_printf(buf, + "SEID ( Name ) | Uplink | Downlink | Conn Type\n" + " | | | (uplink)\n"); + for (int i = 0; i < sw_config->header.num_reported; i++) { + // "%4d (%8s) | %8s %8s %#8x", + sbuf_printf(buf, "%4d", sw_config->element[i].seid); + sbuf_cat(buf, " "); + sbuf_printf(buf, "(%8s)", ixl_switch_element_string(nmbuf, + &sw_config->element[i])); + sbuf_cat(buf, " | "); + sbuf_printf(buf, "%8d", sw_config->element[i].uplink_seid); + sbuf_cat(buf, " "); + sbuf_printf(buf, "%8d", sw_config->element[i].downlink_seid); + sbuf_cat(buf, " "); + sbuf_printf(buf, "%#8x", sw_config->element[i].connection_type); + if (i < sw_config->header.num_reported - 1) + sbuf_cat(buf, "\n"); + } + sbuf_delete(nmbuf); + + error = sbuf_finish(buf); + if (error) + device_printf(dev, "Error finishing sbuf: %d\n", error); + + sbuf_delete(buf); + + return (error); +} + +static int +ixl_sysctl_hkey(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + struct i40e_hw *hw = &pf->hw; + device_t dev = pf->dev; + struct sbuf *buf; + int error = 0; + enum i40e_status_code status; + u32 reg; + + struct i40e_aqc_get_set_rss_key_data key_data; + + buf = sbuf_new_for_sysctl(NULL, NULL, 128, req); + if (!buf) { + device_printf(dev, "Could not allocate sbuf for output.\n"); + return (ENOMEM); + } + + sbuf_cat(buf, "\n"); + if (hw->mac.type == I40E_MAC_X722) { + bzero(key_data.standard_rss_key, sizeof(key_data.standard_rss_key)); + status = i40e_aq_get_rss_key(hw, pf->vsi.vsi_num, &key_data); + if (status) + device_printf(dev, "i40e_aq_get_rss_key status %s, error %s\n", + i40e_stat_str(hw, status), i40e_aq_str(hw, hw->aq.asq_last_status)); + sbuf_printf(buf, "%40D", (u_char *)key_data.standard_rss_key, ""); + } else { + for (int i = 0; i < IXL_RSS_KEY_SIZE_REG; i++) { + reg = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i)); + sbuf_printf(buf, "%4D", (u_char *)®, ""); + } + } + + error = sbuf_finish(buf); + if (error) + device_printf(dev, "Error finishing sbuf: %d\n", error); + sbuf_delete(buf); + + return (error); +} + +static int +ixl_sysctl_hlut(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + struct i40e_hw *hw = &pf->hw; + device_t dev = pf->dev; + struct sbuf *buf; + int error = 0; + enum i40e_status_code status; + u8 hlut[512]; + u32 reg; + + buf = sbuf_new_for_sysctl(NULL, NULL, 128, req); + if (!buf) { + device_printf(dev, "Could not allocate sbuf for output.\n"); + return (ENOMEM); + } + + sbuf_cat(buf, "\n"); + if (hw->mac.type == I40E_MAC_X722) { + bzero(hlut, sizeof(hlut)); + status = i40e_aq_get_rss_lut(hw, pf->vsi.vsi_num, TRUE, hlut, sizeof(hlut)); + if (status) + device_printf(dev, "i40e_aq_get_rss_lut status %s, error %s\n", + i40e_stat_str(hw, status), i40e_aq_str(hw, hw->aq.asq_last_status)); + sbuf_printf(buf, "%512D", (u_char *)hlut, ""); + } else { + for (int i = 0; i < hw->func_caps.rss_table_size >> 2; i++) { + reg = rd32(hw, I40E_PFQF_HLUT(i)); + sbuf_printf(buf, "%4D", (u_char *)®, ""); + } + } + + error = sbuf_finish(buf); + if (error) + device_printf(dev, "Error finishing sbuf: %d\n", error); + sbuf_delete(buf); + + return (error); +} + diff --git a/sys/dev/ixl/ixl_pf_qmgr.c b/sys/dev/ixl/ixl_pf_qmgr.c new file mode 100644 index 000000000000..f2842e584dc4 --- /dev/null +++ b/sys/dev/ixl/ixl_pf_qmgr.c @@ -0,0 +1,308 @@ +/****************************************************************************** + + Copyright (c) 2013-2015, Intel Corporation + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +******************************************************************************/ +/*$FreeBSD$*/ + + +#include "ixl_pf_qmgr.h" + +static int ixl_pf_qmgr_find_free_contiguous_block(struct ixl_pf_qmgr *qmgr, int num); + +int +ixl_pf_qmgr_init(struct ixl_pf_qmgr *qmgr, u16 num_queues) +{ + if (num_queues < 1) + return (EINVAL); + + qmgr->num_queues = num_queues; + qmgr->qinfo = malloc(num_queues * sizeof(struct ixl_pf_qmgr_qinfo), + M_IXL, M_ZERO | M_WAITOK); + if (qmgr->qinfo == NULL) + return ENOMEM; + + return (0); +} + +int +ixl_pf_qmgr_alloc_contiguous(struct ixl_pf_qmgr *qmgr, u16 num, struct ixl_pf_qtag *qtag) +{ + int i; + int avail; + int block_start; + u16 alloc_size; + + if (qtag == NULL || num < 1) + return (EINVAL); + + /* We have to allocate in power-of-two chunks, so get next power of two */ + alloc_size = (u16)next_power_of_two(num); + + /* Don't try if there aren't enough queues */ + avail = ixl_pf_qmgr_get_num_free(qmgr); + if (avail < alloc_size) + return (ENOSPC); + + block_start = ixl_pf_qmgr_find_free_contiguous_block(qmgr, alloc_size); + if (block_start < 0) + return (ENOSPC); + + /* Mark queues as allocated */ + for (i = block_start; i < block_start + alloc_size; i++) + qmgr->qinfo[i].allocated = true; + + bzero(qtag, sizeof(*qtag)); + qtag->qmgr = qmgr; + qtag->type = IXL_PF_QALLOC_CONTIGUOUS; + qtag->qidx[0] = block_start; + qtag->num_allocated = num; + qtag->num_active = alloc_size; + + return (0); +} + +/* + * NB: indices is u16 because this is the queue index width used in the Add VSI AQ command + */ +int +ixl_pf_qmgr_alloc_scattered(struct ixl_pf_qmgr *qmgr, u16 num, struct ixl_pf_qtag *qtag) +{ + int i; + int avail, count = 0; + u16 alloc_size; + + if (qtag == NULL || num < 1 || num > 16) + return (EINVAL); + + /* We have to allocate in power-of-two chunks, so get next power of two */ + alloc_size = (u16)next_power_of_two(num); + + avail = ixl_pf_qmgr_get_num_free(qmgr); + if (avail < alloc_size) + return (ENOSPC); + + bzero(qtag, sizeof(*qtag)); + qtag->qmgr = qmgr; + qtag->type = IXL_PF_QALLOC_SCATTERED; + qtag->num_active = num; + qtag->num_allocated = alloc_size; + + for (i = 0; i < qmgr->num_queues; i++) { + if (!qmgr->qinfo[i].allocated) { + qtag->qidx[count] = i; + count++; + qmgr->qinfo[i].allocated = true; + if (count == alloc_size) + return (0); + } + } + + // Shouldn't get here + return (EDOOFUS); +} + +int +ixl_pf_qmgr_release(struct ixl_pf_qmgr *qmgr, struct ixl_pf_qtag *qtag) +{ + u16 i, qidx; + + if (qtag == NULL) + return (EINVAL); + + if (qtag->type == IXL_PF_QALLOC_SCATTERED) { + for (i = 0; i < qtag->num_allocated; i++) { + qidx = qtag->qidx[i]; + bzero(&qmgr->qinfo[qidx], sizeof(qmgr->qinfo[qidx])); + } + } else { + u16 first_index = qtag->qidx[0]; + for (i = first_index; i < first_index + qtag->num_allocated; i++) + bzero(&qmgr->qinfo[i], sizeof(qmgr->qinfo[qidx])); + } + + qtag->qmgr = NULL; + return (0); +} + +int +ixl_pf_qmgr_get_num_queues(struct ixl_pf_qmgr *qmgr) +{ + return (qmgr->num_queues); +} + +/* + * ERJ: This assumes the info array isn't longer than INT_MAX. + * This assumption might cause a y3k bug or something, I'm sure. + */ +int +ixl_pf_qmgr_get_num_free(struct ixl_pf_qmgr *qmgr) +{ + int count = 0; + + for (int i = 0; i < qmgr->num_queues; i++) { + if (!qmgr->qinfo[i].allocated) + count++; + } + + return (count); +} + +int +ixl_pf_qmgr_get_first_free(struct ixl_pf_qmgr *qmgr, u16 start) +{ + int i; + + if (start > qmgr->num_queues - 1) + return (-EINVAL); + + for (i = start; i < qmgr->num_queues; i++) { + if (qmgr->qinfo[i].allocated) + continue; + else + return (i); + } + + // No free queues + return (-ENOSPC); +} + +void +ixl_pf_qmgr_destroy(struct ixl_pf_qmgr *qmgr) +{ + free(qmgr->qinfo, M_IXL); + qmgr->qinfo = NULL; +} + +void +ixl_pf_qmgr_mark_queue_enabled(struct ixl_pf_qtag *qtag, u16 vsi_qidx, bool tx) +{ + MPASS(qtag != NULL); + + struct ixl_pf_qmgr *qmgr = qtag->qmgr; + u16 pf_qidx = ixl_pf_qidx_from_vsi_qidx(qtag, vsi_qidx); + if (tx) + qmgr->qinfo[pf_qidx].tx_enabled = true; + else + qmgr->qinfo[pf_qidx].rx_enabled = true; +} + +void +ixl_pf_qmgr_mark_queue_disabled(struct ixl_pf_qtag *qtag, u16 vsi_qidx, bool tx) +{ + MPASS(qtag != NULL); + + struct ixl_pf_qmgr *qmgr = qtag->qmgr; + u16 pf_qidx = ixl_pf_qidx_from_vsi_qidx(qtag, vsi_qidx); + if (tx) + qmgr->qinfo[pf_qidx].tx_enabled = false; + else + qmgr->qinfo[pf_qidx].rx_enabled = false; +} + +void +ixl_pf_qmgr_mark_queue_configured(struct ixl_pf_qtag *qtag, u16 vsi_qidx, bool tx) +{ + MPASS(qtag != NULL); + + struct ixl_pf_qmgr *qmgr = qtag->qmgr; + u16 pf_qidx = ixl_pf_qidx_from_vsi_qidx(qtag, vsi_qidx); + if (tx) + qmgr->qinfo[pf_qidx].tx_configured = true; + else + qmgr->qinfo[pf_qidx].rx_configured = true; +} + +bool +ixl_pf_qmgr_is_queue_enabled(struct ixl_pf_qtag *qtag, u16 vsi_qidx, bool tx) +{ + MPASS(qtag != NULL); + + struct ixl_pf_qmgr *qmgr = qtag->qmgr; + u16 pf_qidx = ixl_pf_qidx_from_vsi_qidx(qtag, vsi_qidx); + if (tx) + return (qmgr->qinfo[pf_qidx].tx_enabled); + else + return (qmgr->qinfo[pf_qidx].rx_enabled); +} + +bool +ixl_pf_qmgr_is_queue_configured(struct ixl_pf_qtag *qtag, u16 vsi_qidx, bool tx) +{ + MPASS(qtag != NULL); + + struct ixl_pf_qmgr *qmgr = qtag->qmgr; + u16 pf_qidx = ixl_pf_qidx_from_vsi_qidx(qtag, vsi_qidx); + if (tx) + return (qmgr->qinfo[pf_qidx].tx_configured); + else + return (qmgr->qinfo[pf_qidx].rx_configured); +} + +u16 +ixl_pf_qidx_from_vsi_qidx(struct ixl_pf_qtag *qtag, u16 index) +{ + MPASS(index < qtag->num_allocated); + + if (qtag->type == IXL_PF_QALLOC_CONTIGUOUS) + return qtag->qidx[0] + index; + else + return qtag->qidx[index]; +} + +/* Static Functions */ + +static int +ixl_pf_qmgr_find_free_contiguous_block(struct ixl_pf_qmgr *qmgr, int num) +{ + int i; + int count = 0; + bool block_started = false; + int possible_start; + + for (i = 0; i < qmgr->num_queues; i++) { + if (!qmgr->qinfo[i].allocated) { + if (!block_started) { + block_started = true; + possible_start = i; + } + count++; + if (count == num) + return (possible_start); + } else { /* this queue is already allocated */ + block_started = false; + count = 0; + } + } + + /* Can't find a contiguous block of the requested size */ + return (-1); +} + diff --git a/sys/dev/ixl/ixl_pf_qmgr.h b/sys/dev/ixl/ixl_pf_qmgr.h new file mode 100644 index 000000000000..d6ad431bd605 --- /dev/null +++ b/sys/dev/ixl/ixl_pf_qmgr.h @@ -0,0 +1,109 @@ +/****************************************************************************** + + Copyright (c) 2013-2015, Intel Corporation + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +******************************************************************************/ +/*$FreeBSD$*/ + + +#include "ixl_pf.h" + +#ifndef _IXL_PF_QMGR_H_ +#define _IXL_PF_QMGR_H_ + +/* + * Primarily manages the queues that need to be allocated to VSIs. + * + * Cardinality: There should only be one of these in a PF. + * Lifetime: Created and initialized in attach(); destroyed in detach(). + */ + +#define IXL_MAX_SCATTERED_QUEUES 16 +#define IXL_MAX_CONTIGUOUS_QUEUES_XL710 64 +#define IXL_MAX_CONTIGUOUS_QUEUES_X722 128 + +/* Structures */ + +/* Manager */ +struct ixl_pf_qmgr_qinfo { + bool allocated; + bool tx_enabled; + bool rx_enabled; + bool tx_configured; + bool rx_configured; +}; + +struct ixl_pf_qmgr { + u16 num_queues; + struct ixl_pf_qmgr_qinfo *qinfo; +}; + +/* Tag */ +enum ixl_pf_qmgr_qalloc_type { + IXL_PF_QALLOC_CONTIGUOUS, + IXL_PF_QALLOC_SCATTERED +}; + +struct ixl_pf_qtag { + struct ixl_pf_qmgr *qmgr; + enum ixl_pf_qmgr_qalloc_type type; + u16 qidx[IXL_MAX_SCATTERED_QUEUES]; + u16 num_allocated; + u16 num_active; +}; + +/* Public manager functions */ +int ixl_pf_qmgr_init(struct ixl_pf_qmgr *qmgr, u16 num_queues); +void ixl_pf_qmgr_destroy(struct ixl_pf_qmgr *qmgr); + +int ixl_pf_qmgr_get_num_queues(struct ixl_pf_qmgr *qmgr); +int ixl_pf_qmgr_get_first_free(struct ixl_pf_qmgr *qmgr, u16 start); +int ixl_pf_qmgr_get_num_free(struct ixl_pf_qmgr *qmgr); + +/* Allocate queues for a VF VSI */ +int ixl_pf_qmgr_alloc_scattered(struct ixl_pf_qmgr *qmgr, u16 num, struct ixl_pf_qtag *qtag); +/* Allocate queues for the LAN VSIs, or X722 VF VSIs */ +int ixl_pf_qmgr_alloc_contiguous(struct ixl_pf_qmgr *qmgr, u16 num, struct ixl_pf_qtag *qtag); +/* Release a queue allocation */ +int ixl_pf_qmgr_release(struct ixl_pf_qmgr *qmgr, struct ixl_pf_qtag *qtag); + +/* Help manage queues used in VFs */ +/* Typically hardware refers to RX as 0 and TX as 1, so continue that convention here */ +void ixl_pf_qmgr_mark_queue_enabled(struct ixl_pf_qtag *qtag, u16 vsi_qidx, bool tx); +void ixl_pf_qmgr_mark_queue_disabled(struct ixl_pf_qtag *qtag, u16 vsi_qidx, bool tx); +void ixl_pf_qmgr_mark_queue_configured(struct ixl_pf_qtag *qtag, u16 vsi_qidx, bool tx); +bool ixl_pf_qmgr_is_queue_enabled(struct ixl_pf_qtag *qtag, u16 vsi_qidx, bool tx); +bool ixl_pf_qmgr_is_queue_configured(struct ixl_pf_qtag *qtag, u16 vsi_qidx, bool tx); + +/* Public tag functions */ +u16 ixl_pf_qidx_from_vsi_qidx(struct ixl_pf_qtag *qtag, u16 index); + +#endif /* _IXL_PF_QMGR_H_ */ + diff --git a/sys/dev/ixl/ixl_txrx.c b/sys/dev/ixl/ixl_txrx.c index d3aa7bf7b259..5cf54fa9410a 100644 --- a/sys/dev/ixl/ixl_txrx.c +++ b/sys/dev/ixl/ixl_txrx.c @@ -35,7 +35,7 @@ /* ** IXL driver TX/RX Routines: ** This was seperated to allow usage by -** both the BASE and the VF drivers. +** both the PF and VF drivers. */ #ifndef IXL_STANDALONE_BUILD @@ -58,15 +58,37 @@ static int ixl_tx_setup_offload(struct ixl_queue *, struct mbuf *, u32 *, u32 *); static bool ixl_tso_setup(struct ixl_queue *, struct mbuf *); -static __inline void ixl_rx_discard(struct rx_ring *, int); -static __inline void ixl_rx_input(struct rx_ring *, struct ifnet *, +static inline void ixl_rx_discard(struct rx_ring *, int); +static inline void ixl_rx_input(struct rx_ring *, struct ifnet *, struct mbuf *, u8); +static inline bool ixl_tso_detect_sparse(struct mbuf *mp); +static int ixl_tx_setup_offload(struct ixl_queue *que, + struct mbuf *mp, u32 *cmd, u32 *off); +static inline u32 ixl_get_tx_head(struct ixl_queue *que); + #ifdef DEV_NETMAP #include <dev/netmap/if_ixl_netmap.h> #endif /* DEV_NETMAP */ /* + * @key key is saved into this parameter + */ +void +ixl_get_default_rss_key(u32 *key) +{ + MPASS(key != NULL); + + u32 rss_seed[IXL_RSS_KEY_SIZE_REG] = {0x41b01687, + 0x183cfd8c, 0xce880440, 0x580cbc3c, + 0x35897377, 0x328b25e1, 0x4fa98922, + 0xb7d90c14, 0xd5bad70d, 0xcd15a2c1, + 0x0, 0x0, 0x0}; + + bcopy(rss_seed, key, IXL_RSS_KEY_SIZE); +} + +/* ** Multiqueue Transmit driver */ int @@ -98,13 +120,6 @@ ixl_mq_start(struct ifnet *ifp, struct mbuf *m) i = m->m_pkthdr.flowid % vsi->num_queues; } else i = curcpu % vsi->num_queues; - /* - ** This may not be perfect, but until something - ** better comes along it will keep from scheduling - ** on stalled queues. - */ - if (((1 << i) & vsi->active_queues) == 0) - i = ffsl(vsi->active_queues); que = &vsi->queues[i]; txr = &que->txr; @@ -239,7 +254,7 @@ ixl_xmit(struct ixl_queue *que, struct mbuf **m_headp) struct ixl_tx_buf *buf; struct i40e_tx_desc *txd = NULL; struct mbuf *m_head, *m; - int i, j, error, nsegs, maxsegs; + int i, j, error, nsegs; int first, last = 0; u16 vtag = 0; u32 cmd, off; @@ -259,12 +274,10 @@ ixl_xmit(struct ixl_queue *que, struct mbuf **m_headp) buf = &txr->buffers[first]; map = buf->map; tag = txr->tx_tag; - maxsegs = IXL_MAX_TX_SEGS; if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { /* Use larger mapping for TSO */ tag = txr->tso_tag; - maxsegs = IXL_MAX_TSO_SEGS; if (ixl_tso_detect_sparse(m_head)) { m = m_defrag(m_head, M_NOWAIT); if (m == NULL) { @@ -299,19 +312,19 @@ ixl_xmit(struct ixl_queue *que, struct mbuf **m_headp) *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); if (error == ENOMEM) { - que->tx_dma_setup++; + que->tx_dmamap_failed++; return (error); } else if (error != 0) { - que->tx_dma_setup++; + que->tx_dmamap_failed++; m_freem(*m_headp); *m_headp = NULL; return (error); } } else if (error == ENOMEM) { - que->tx_dma_setup++; + que->tx_dmamap_failed++; return (error); } else if (error != 0) { - que->tx_dma_setup++; + que->tx_dmamap_failed++; m_freem(*m_headp); *m_headp = NULL; return (error); @@ -804,6 +817,7 @@ ixl_tso_setup(struct ixl_queue *que, struct mbuf *mp) type = I40E_TX_DESC_DTYPE_CONTEXT; cmd = I40E_TX_CTX_DESC_TSO; + /* ERJ: this must not be less than 64 */ mss = mp->m_pkthdr.tso_segsz; type_cmd_tso_mss = ((u64)type << I40E_TXD_CTX_QW1_DTYPE_SHIFT) | @@ -1374,7 +1388,7 @@ ixl_free_que_rx(struct ixl_queue *que) return; } -static __inline void +static inline void ixl_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u8 ptype) { @@ -1405,7 +1419,7 @@ ixl_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u8 ptype) } -static __inline void +static inline void ixl_rx_discard(struct rx_ring *rxr, int i) { struct ixl_rx_buf *rbuf; @@ -1532,7 +1546,7 @@ ixl_rxeof(struct ixl_queue *que, int count) for (i = rxr->next_check; count != 0;) { struct mbuf *sendmp, *mh, *mp; - u32 rsc, status, error; + u32 status, error; u16 hlen, plen, vtag; u64 qword; u8 ptype; @@ -1565,7 +1579,6 @@ ixl_rxeof(struct ixl_queue *que, int count) count--; sendmp = NULL; nbuf = NULL; - rsc = 0; cur->wb.qword1.status_error_len = 0; rbuf = &rxr->buffers[i]; mh = rbuf->m_head; @@ -1673,10 +1686,6 @@ ixl_rxeof(struct ixl_queue *que, int count) sendmp = mp; sendmp->m_flags |= M_PKTHDR; sendmp->m_pkthdr.len = mp->m_len; - if (vtag) { - sendmp->m_pkthdr.ether_vtag = vtag; - sendmp->m_flags |= M_VLANTAG; - } } /* Pass the head pointer on */ if (eop == 0) { @@ -1695,6 +1704,11 @@ ixl_rxeof(struct ixl_queue *que, int count) /* capture data for dynamic ITR adjustment */ rxr->packets++; rxr->bytes += sendmp->m_pkthdr.len; + /* Set VLAN tag (field only valid in eop desc) */ + if (vtag) { + sendmp->m_pkthdr.ether_vtag = vtag; + sendmp->m_flags |= M_VLANTAG; + } if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) ixl_rx_checksum(sendmp, status, error, ptype); #ifdef RSS diff --git a/sys/dev/ixl/ixlv.h b/sys/dev/ixl/ixlv.h index a131ca0a4b22..888b4e0a2bc7 100644 --- a/sys/dev/ixl/ixlv.h +++ b/sys/dev/ixl/ixlv.h @@ -44,7 +44,7 @@ #define IXLV_AQ_TIMEOUT (1 * hz) #define IXLV_CALLOUT_TIMO (hz / 50) /* 20 msec */ -#define IXLV_FLAG_AQ_ENABLE_QUEUES (u32)(1) +#define IXLV_FLAG_AQ_ENABLE_QUEUES (u32)(1 << 0) #define IXLV_FLAG_AQ_DISABLE_QUEUES (u32)(1 << 1) #define IXLV_FLAG_AQ_ADD_MAC_FILTER (u32)(1 << 2) #define IXLV_FLAG_AQ_ADD_VLAN_FILTER (u32)(1 << 3) @@ -55,6 +55,10 @@ #define IXLV_FLAG_AQ_HANDLE_RESET (u32)(1 << 8) #define IXLV_FLAG_AQ_CONFIGURE_PROMISC (u32)(1 << 9) #define IXLV_FLAG_AQ_GET_STATS (u32)(1 << 10) +#define IXLV_FLAG_AQ_CONFIG_RSS_KEY (u32)(1 << 11) +#define IXLV_FLAG_AQ_SET_RSS_HENA (u32)(1 << 12) +#define IXLV_FLAG_AQ_GET_RSS_HENA_CAPS (u32)(1 << 13) +#define IXLV_FLAG_AQ_CONFIG_RSS_LUT (u32)(1 << 14) /* printf %b arg */ #define IXLV_FLAGS \ @@ -62,9 +66,17 @@ "\4ADD_VLAN_FILTER\5DEL_MAC_FILTER\6DEL_VLAN_FILTER" \ "\7CONFIGURE_QUEUES\10MAP_VECTORS\11HANDLE_RESET" \ "\12CONFIGURE_PROMISC\13GET_STATS" - -/* Hack for compatibility with 1.0.x linux pf driver */ -#define I40E_VIRTCHNL_OP_EVENT 17 +#define IXLV_PRINTF_VF_OFFLOAD_FLAGS \ + "\20\1I40E_VIRTCHNL_VF_OFFLOAD_L2" \ + "\2I40E_VIRTCHNL_VF_OFFLOAD_IWARP" \ + "\3I40E_VIRTCHNL_VF_OFFLOAD_FCOE" \ + "\4I40E_VIRTCHNL_VF_OFFLOAD_RSS_AQ" \ + "\5I40E_VIRTCHNL_VF_OFFLOAD_RSS_REG" \ + "\6I40E_VIRTCHNL_VF_OFFLOAD_WB_ON_ITR" \ + "\21I40E_VIRTCHNL_VF_OFFLOAD_VLAN" \ + "\22I40E_VIRTCHNL_VF_OFFLOAD_RX_POLLING" \ + "\23I40E_VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2" \ + "\24I40E_VIRTCHNL_VF_OFFLOAD_RSS_PF" /* Driver state */ enum ixlv_state_t { @@ -80,9 +92,11 @@ enum ixlv_state_t { IXLV_INIT_MAPPING, IXLV_INIT_ENABLE, IXLV_INIT_COMPLETE, - IXLV_RUNNING, + IXLV_RUNNING, }; +/* Structs */ + struct ixlv_mac_filter { SLIST_ENTRY(ixlv_mac_filter) next; u8 macaddr[ETHER_ADDR_LEN]; @@ -101,12 +115,13 @@ SLIST_HEAD(vlan_list, ixlv_vlan_filter); struct ixlv_sc { struct i40e_hw hw; struct i40e_osdep osdep; - struct device *dev; + device_t dev; struct resource *pci_mem; struct resource *msix_mem; enum ixlv_state_t init_state; + int init_in_progress; /* * Interrupt resources @@ -154,6 +169,10 @@ struct ixlv_sc { struct ixl_vc_cmd del_vlan_cmd; struct ixl_vc_cmd add_multi_cmd; struct ixl_vc_cmd del_multi_cmd; + struct ixl_vc_cmd config_rss_key_cmd; + struct ixl_vc_cmd get_rss_hena_caps_cmd; + struct ixl_vc_cmd set_rss_hena_cmd; + struct ixl_vc_cmd config_rss_lut_cmd; /* Virtual comm channel */ struct i40e_virtchnl_vf_resource *vf_res; @@ -209,5 +228,9 @@ void ixlv_del_vlans(struct ixlv_sc *); void ixlv_update_stats_counters(struct ixlv_sc *, struct i40e_eth_stats *); void ixlv_update_link_status(struct ixlv_sc *); +void ixlv_get_default_rss_key(u32 *, bool); +void ixlv_config_rss_key(struct ixlv_sc *); +void ixlv_set_rss_hena(struct ixlv_sc *); +void ixlv_config_rss_lut(struct ixlv_sc *); #endif /* _IXLV_H_ */ diff --git a/sys/dev/ixl/ixlvc.c b/sys/dev/ixl/ixlvc.c index 6a367eab3a6d..7b041c18a8d7 100644 --- a/sys/dev/ixl/ixlvc.c +++ b/sys/dev/ixl/ixlvc.c @@ -69,8 +69,10 @@ static int ixl_vc_validate_vf_msg(struct ixlv_sc *sc, u32 v_opcode, valid_len = sizeof(struct i40e_virtchnl_version_info); break; case I40E_VIRTCHNL_OP_RESET_VF: + valid_len = 0; + break; case I40E_VIRTCHNL_OP_GET_VF_RESOURCES: - // TODO: valid length in api v1.0 is 0, v1.1 is 4 + /* Valid length in api v1.0 is 0, v1.1 is 4 */ valid_len = 4; break; case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE: @@ -218,7 +220,7 @@ ixlv_verify_api_ver(struct ixlv_sc *sc) i40e_status err; int retries = 0; - event.buf_len = IXL_AQ_BUFSZ; + event.buf_len = IXL_AQ_BUF_SZ; event.msg_buf = malloc(event.buf_len, M_DEVBUF, M_NOWAIT); if (!event.msg_buf) { err = ENOMEM; @@ -230,7 +232,7 @@ ixlv_verify_api_ver(struct ixlv_sc *sc) goto out_alloc; /* Initial delay here is necessary */ - i40e_msec_delay(100); + i40e_msec_pause(100); err = i40e_clean_arq_element(hw, &event, NULL); if (err == I40E_ERR_ADMIN_QUEUE_NO_WORK) continue; @@ -288,7 +290,7 @@ ixlv_send_vf_config_msg(struct ixlv_sc *sc) u32 caps; caps = I40E_VIRTCHNL_VF_OFFLOAD_L2 | - I40E_VIRTCHNL_VF_OFFLOAD_RSS_REG | + I40E_VIRTCHNL_VF_OFFLOAD_RSS_PF | I40E_VIRTCHNL_VF_OFFLOAD_VLAN; if (sc->pf_version == I40E_VIRTCHNL_VERSION_MINOR_NO_VF_CAPS) @@ -331,7 +333,7 @@ ixlv_get_vf_config(struct ixlv_sc *sc) err = i40e_clean_arq_element(hw, &event, NULL); if (err == I40E_ERR_ADMIN_QUEUE_NO_WORK) { if (++retries <= IXLV_AQ_MAX_ERR) - i40e_msec_delay(10); + i40e_msec_pause(10); } else if ((enum i40e_virtchnl_ops)le32toh(event.desc.cookie_high) != I40E_VIRTCHNL_OP_GET_VF_RESOURCES) { DDPRINTF(dev, "Received a response from PF," @@ -498,7 +500,7 @@ ixlv_map_queues(struct ixlv_sc *sc) vm->vecmap[i].txq_map = (1 << que->me); vm->vecmap[i].rxq_map = (1 << que->me); vm->vecmap[i].rxitr_idx = 0; - vm->vecmap[i].txitr_idx = 0; + vm->vecmap[i].txitr_idx = 1; } /* Misc vector last - this is only for AdminQ messages */ @@ -570,13 +572,6 @@ ixlv_add_vlans(struct ixlv_sc *sc) if (i == cnt) break; } - // ERJ: Should this be taken out? - if (i == 0) { /* Should not happen... */ - device_printf(dev, "%s: i == 0?\n", __func__); - ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_ADD_VLAN_FILTER, - I40E_SUCCESS); - return; - } ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_ADD_VLAN, (u8 *)v, len); free(v, M_DEVBUF); @@ -640,13 +635,6 @@ ixlv_del_vlans(struct ixlv_sc *sc) if (i == cnt) break; } - // ERJ: Take this out? - if (i == 0) { /* Should not happen... */ - device_printf(dev, "%s: i == 0?\n", __func__); - ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_DEL_VLAN_FILTER, - I40E_SUCCESS); - return; - } ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_DEL_VLAN, (u8 *)v, len); free(v, M_DEVBUF); @@ -842,6 +830,97 @@ ixlv_update_stats_counters(struct ixlv_sc *sc, struct i40e_eth_stats *es) vsi->eth_stats = *es; } +void +ixlv_config_rss_key(struct ixlv_sc *sc) +{ + struct i40e_virtchnl_rss_key *rss_key_msg; + int msg_len, key_length; + u8 rss_seed[IXL_RSS_KEY_SIZE]; + +#ifdef RSS + /* Fetch the configured RSS key */ + rss_getkey((uint8_t *) &rss_seed); +#else + ixl_get_default_rss_key((u32 *)rss_seed); +#endif + + /* Send the fetched key */ + key_length = IXL_RSS_KEY_SIZE; + msg_len = sizeof(struct i40e_virtchnl_rss_key) + (sizeof(u8) * key_length) - 1; + rss_key_msg = malloc(msg_len, M_DEVBUF, M_NOWAIT | M_ZERO); + if (rss_key_msg == NULL) { + device_printf(sc->dev, "Unable to allocate msg memory for RSS key msg.\n"); + return; + } + + rss_key_msg->vsi_id = sc->vsi_res->vsi_id; + rss_key_msg->key_len = key_length; + bcopy(rss_seed, &rss_key_msg->key[0], key_length); + + DDPRINTF(sc->dev, "config_rss: vsi_id %d, key_len %d", + rss_key_msg->vsi_id, rss_key_msg->key_len); + + ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_CONFIG_RSS_KEY, + (u8 *)rss_key_msg, msg_len); + + free(rss_key_msg, M_DEVBUF); +} + +void +ixlv_set_rss_hena(struct ixlv_sc *sc) +{ + struct i40e_virtchnl_rss_hena hena; + + hena.hena = IXL_DEFAULT_RSS_HENA; + + ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_SET_RSS_HENA, + (u8 *)&hena, sizeof(hena)); +} + +void +ixlv_config_rss_lut(struct ixlv_sc *sc) +{ + struct i40e_virtchnl_rss_lut *rss_lut_msg; + int msg_len; + u16 lut_length; + u32 lut; + int i, que_id; + + lut_length = IXL_RSS_VSI_LUT_SIZE; + msg_len = sizeof(struct i40e_virtchnl_rss_lut) + (lut_length * sizeof(u8)) - 1; + rss_lut_msg = malloc(msg_len, M_DEVBUF, M_NOWAIT | M_ZERO); + if (rss_lut_msg == NULL) { + device_printf(sc->dev, "Unable to allocate msg memory for RSS lut msg.\n"); + return; + } + + rss_lut_msg->vsi_id = sc->vsi_res->vsi_id; + /* Each LUT entry is a max of 1 byte, so this is easy */ + rss_lut_msg->lut_entries = lut_length; + + /* Populate the LUT with max no. of queues in round robin fashion */ + for (i = 0; i < lut_length; i++) { +#ifdef RSS + /* + * Fetch the RSS bucket id for the given indirection entry. + * Cap it at the number of configured buckets (which is + * num_queues.) + */ + que_id = rss_get_indirection_to_bucket(i); + que_id = que_id % sc->vsi.num_queues; +#else + que_id = i % sc->vsi.num_queues; +#endif + lut = que_id & IXL_RSS_VSI_LUT_ENTRY_MASK; + rss_lut_msg->lut[i] = lut; + } + + ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_CONFIG_RSS_LUT, + (u8 *)rss_lut_msg, msg_len); + + free(rss_lut_msg, M_DEVBUF); +} + /* ** ixlv_vc_completion ** @@ -940,7 +1019,7 @@ ixlv_vc_completion(struct ixlv_sc *sc, ixlv_enable_intr(vsi); /* And inform the stack we're ready */ vsi->ifp->if_drv_flags |= IFF_DRV_RUNNING; - vsi->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; + /* TODO: Clear a state flag, so we know we're ready to run init again */ } break; case I40E_VIRTCHNL_OP_DISABLE_QUEUES: @@ -950,7 +1029,7 @@ ixlv_vc_completion(struct ixlv_sc *sc, /* Turn off all interrupts */ ixlv_disable_intr(vsi); /* Tell the stack that the interface is no longer active */ - vsi->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); + vsi->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING); } break; case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES: @@ -961,6 +1040,18 @@ ixlv_vc_completion(struct ixlv_sc *sc, ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_MAP_VECTORS, v_retval); break; + case I40E_VIRTCHNL_OP_CONFIG_RSS_KEY: + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_CONFIG_RSS_KEY, + v_retval); + break; + case I40E_VIRTCHNL_OP_SET_RSS_HENA: + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_SET_RSS_HENA, + v_retval); + break; + case I40E_VIRTCHNL_OP_CONFIG_RSS_LUT: + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_CONFIG_RSS_LUT, + v_retval); + break; default: #ifdef IXL_DEBUG device_printf(dev, @@ -1008,6 +1099,18 @@ ixl_vc_send_cmd(struct ixlv_sc *sc, uint32_t request) case IXLV_FLAG_AQ_ENABLE_QUEUES: ixlv_enable_queues(sc); break; + + case IXLV_FLAG_AQ_CONFIG_RSS_KEY: + ixlv_config_rss_key(sc); + break; + + case IXLV_FLAG_AQ_SET_RSS_HENA: + ixlv_set_rss_hena(sc); + break; + + case IXLV_FLAG_AQ_CONFIG_RSS_LUT: + ixlv_config_rss_lut(sc); + break; } } diff --git a/sys/dev/kbd/kbd.c b/sys/dev/kbd/kbd.c index 6a386088ff1b..ff49ee61b2c8 100644 --- a/sys/dev/kbd/kbd.c +++ b/sys/dev/kbd/kbd.c @@ -284,8 +284,8 @@ kbd_unregister(keyboard_t *kbd) } /* find a function table by the driver name */ -keyboard_switch_t -*kbd_get_switch(char *driver) +keyboard_switch_t * +kbd_get_switch(char *driver) { const keyboard_driver_t **list; const keyboard_driver_t *p; @@ -419,8 +419,8 @@ kbd_change_callback(keyboard_t *kbd, void *id, kbd_callback_func_t *func, } /* get a keyboard structure */ -keyboard_t -*kbd_get_keyboard(int index) +keyboard_t * +kbd_get_keyboard(int index) { if ((index < 0) || (index >= keyboards)) return (NULL); @@ -1118,8 +1118,8 @@ fkey_change_ok(fkeytab_t *oldkey, fkeyarg_t *newkey, struct thread *td) #endif /* get a pointer to the string associated with the given function key */ -u_char -*genkbd_get_fkeystr(keyboard_t *kbd, int fkey, size_t *len) +u_char * +genkbd_get_fkeystr(keyboard_t *kbd, int fkey, size_t *len) { if (kbd == NULL) return (NULL); @@ -1131,8 +1131,8 @@ u_char } /* diagnostic dump */ -static char -*get_kbd_type_name(int type) +static char * +get_kbd_type_name(int type) { static struct { int type; diff --git a/sys/dev/mcd/mcd.c b/sys/dev/mcd/mcd.c deleted file mode 100644 index dacbb3d3e2f0..000000000000 --- a/sys/dev/mcd/mcd.c +++ /dev/null @@ -1,1652 +0,0 @@ -/*- - * Copyright 1993 by Holger Veit (data part) - * Copyright 1993 by Brian Moore (audio part) - * Changes Copyright 1993 by Gary Clark II - * Changes Copyright (C) 1994-1995 by Andrey A. Chernov, Moscow, Russia - * - * Rewrote probe routine to work on newer Mitsumi drives. - * Additional changes (C) 1994 by Jordan K. Hubbard - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This software was developed by Holger Veit and Brian Moore - * for use with "386BSD" and similar operating systems. - * "Similar operating systems" includes mainly non-profit oriented - * systems for research and education, including but not restricted to - * "NetBSD", "FreeBSD", "Mach" (by CMU). - * 4. Neither the name of the developer(s) nor the name "386BSD" - * may be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE DEVELOPER(S) ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE DEVELOPER(S) BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, - * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT - * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#include <sys/cdefs.h> -__FBSDID("$FreeBSD$"); -static const char __used COPYRIGHT[] = "mcd-driver (C)1993 by H.Veit & B.Moore"; - -#include <sys/param.h> -#include <sys/systm.h> -#include <sys/kernel.h> -#include <sys/conf.h> -#include <sys/fcntl.h> -#include <sys/bio.h> -#include <sys/cdio.h> -#include <sys/disk.h> -#include <sys/bus.h> - -#include <machine/bus.h> -#include <machine/resource.h> -#include <sys/rman.h> - -#include <isa/isavar.h> - -#include <dev/mcd/mcdreg.h> -#include <dev/mcd/mcdvar.h> - -#define MCD_TRACE(format, args...) \ -{ \ - if (sc->debug) { \ - device_printf(sc->dev, "status=0x%02x: ", \ - sc->data.status); \ - printf(format, ## args); \ - } \ -} - -#define RAW_PART 2 - -/* flags */ -#define MCDVALID 0x0001 /* parameters loaded */ -#define MCDINIT 0x0002 /* device is init'd */ -#define MCDNEWMODEL 0x0004 /* device is new model */ -#define MCDLABEL 0x0008 /* label is read */ -#define MCDPROBING 0x0010 /* probing */ -#define MCDREADRAW 0x0020 /* read raw mode (2352 bytes) */ -#define MCDVOLINFO 0x0040 /* already read volinfo */ -#define MCDTOC 0x0080 /* already read toc */ -#define MCDMBXBSY 0x0100 /* local mbx is busy */ - -/* status */ -#define MCDAUDIOBSY MCD_ST_AUDIOBSY /* playing audio */ -#define MCDDSKCHNG MCD_ST_DSKCHNG /* sensed change of disk */ -#define MCDDSKIN MCD_ST_DSKIN /* sensed disk in drive */ -#define MCDDOOROPEN MCD_ST_DOOROPEN /* sensed door open */ - -/* These are apparently the different states a mitsumi can get up to */ -#define MCDCDABSENT 0x0030 -#define MCDCDPRESENT 0x0020 -#define MCDSCLOSED 0x0080 -#define MCDSOPEN 0x00a0 - -#define MCD_MD_UNKNOWN (-1) - -#define MCD_TYPE_UNKNOWN 0 -#define MCD_TYPE_LU002S 1 -#define MCD_TYPE_LU005S 2 -#define MCD_TYPE_LU006S 3 -#define MCD_TYPE_FX001 4 -#define MCD_TYPE_FX001D 5 - -/* reader state machine */ -#define MCD_S_BEGIN 0 -#define MCD_S_BEGIN1 1 -#define MCD_S_WAITSTAT 2 -#define MCD_S_WAITMODE 3 -#define MCD_S_WAITREAD 4 - -/* prototypes */ -static void mcd_start(struct mcd_softc *); -#ifdef NOTYET -static void mcd_configure(struct mcd_softc *sc); -#endif -static int mcd_get(struct mcd_softc *, char *buf, int nmax); -static int mcd_setflags(struct mcd_softc *); -static int mcd_getstat(struct mcd_softc *,int sflg); -static int mcd_send(struct mcd_softc *, int cmd,int nretrys); -static void hsg2msf(int hsg, bcd_t *msf); -static int msf2hsg(bcd_t *msf, int relative); -static int mcd_volinfo(struct mcd_softc *); -static int mcd_waitrdy(struct mcd_softc *,int dly); -static void mcd_timeout(void *arg); -static void mcd_doread(struct mcd_softc *, int state, struct mcd_mbx *mbxin); -static void mcd_soft_reset(struct mcd_softc *); -static int mcd_hard_reset(struct mcd_softc *); -static int mcd_setmode(struct mcd_softc *, int mode); -static int mcd_getqchan(struct mcd_softc *, struct mcd_qchninfo *q); -static int mcd_subchan(struct mcd_softc *, struct ioc_read_subchannel *sc, - int nocopyout); -static int mcd_toc_header(struct mcd_softc *, struct ioc_toc_header *th); -static int mcd_read_toc(struct mcd_softc *); -static int mcd_toc_entrys(struct mcd_softc *, struct ioc_read_toc_entry *te); -#if 0 -static int mcd_toc_entry(struct mcd_softc *, struct ioc_read_toc_single_entry *te); -#endif -static int mcd_stop(struct mcd_softc *); -static int mcd_eject(struct mcd_softc *); -static int mcd_inject(struct mcd_softc *); -static int mcd_playtracks(struct mcd_softc *, struct ioc_play_track *pt); -static int mcd_play(struct mcd_softc *, struct mcd_read2 *pb); -static int mcd_playmsf(struct mcd_softc *, struct ioc_play_msf *pt); -static int mcd_playblocks(struct mcd_softc *, struct ioc_play_blocks *); -static int mcd_pause(struct mcd_softc *); -static int mcd_resume(struct mcd_softc *); -static int mcd_lock_door(struct mcd_softc *, int lock); -static int mcd_close_tray(struct mcd_softc *); -static int mcd_size(struct cdev *dev); - -static d_open_t mcdopen; -static d_close_t mcdclose; -static d_ioctl_t mcdioctl; -static d_strategy_t mcdstrategy; - -static struct cdevsw mcd_cdevsw = { - .d_version = D_VERSION, - .d_open = mcdopen, - .d_close = mcdclose, - .d_read = physread, - .d_ioctl = mcdioctl, - .d_strategy = mcdstrategy, - .d_name = "mcd", - .d_flags = D_DISK, -}; - -#define MCD_RETRYS 5 -#define MCD_RDRETRYS 8 - -#define CLOSE_TRAY_SECS 8 -#define DISK_SENSE_SECS 3 -#define WAIT_FRAC 4 - -/* several delays */ -#define RDELAY_WAITSTAT 300 -#define RDELAY_WAITMODE 300 -#define RDELAY_WAITREAD 800 - -#define MIN_DELAY 15 -#define DELAY_GETREPLY 5000000 - -int -mcd_attach(struct mcd_softc *sc) -{ - int unit; - - unit = device_get_unit(sc->dev); - - MCD_LOCK(sc); - sc->data.flags |= MCDINIT; - mcd_soft_reset(sc); - bioq_init(&sc->data.head); - -#ifdef NOTYET - /* wire controller for interrupts and dma */ - mcd_configure(sc); -#endif - MCD_UNLOCK(sc); - /* name filled in probe */ - sc->mcd_dev_t = make_dev(&mcd_cdevsw, 8 * unit, - UID_ROOT, GID_OPERATOR, 0640, "mcd%d", unit); - - sc->mcd_dev_t->si_drv1 = (void *)sc; - callout_init_mtx(&sc->timer, &sc->mtx, 0); - - return (0); -} - -static int -mcdopen(struct cdev *dev, int flags, int fmt, struct thread *td) -{ - struct mcd_softc *sc; - int r,retry; - - sc = (struct mcd_softc *)dev->si_drv1; - - /* invalidated in the meantime? mark all open part's invalid */ - MCD_LOCK(sc); - if (!(sc->data.flags & MCDVALID) && sc->data.openflags) { - MCD_UNLOCK(sc); - return (ENXIO); - } - - if (mcd_getstat(sc, 1) == -1) { - MCD_UNLOCK(sc); - return (EIO); - } - - if ( (sc->data.status & (MCDDSKCHNG|MCDDOOROPEN)) - || !(sc->data.status & MCDDSKIN)) - for (retry = 0; retry < DISK_SENSE_SECS * WAIT_FRAC; retry++) { - (void) mtx_sleep(sc, &sc->mtx, PSOCK | PCATCH, - "mcdsn1", hz/WAIT_FRAC); - if ((r = mcd_getstat(sc, 1)) == -1) { - MCD_UNLOCK(sc); - return (EIO); - } - if (r != -2) - break; - } - - if (sc->data.status & MCDDOOROPEN) { - MCD_UNLOCK(sc); - device_printf(sc->dev, "door is open\n"); - return (ENXIO); - } - if (!(sc->data.status & MCDDSKIN)) { - MCD_UNLOCK(sc); - device_printf(sc->dev, "no CD inside\n"); - return (ENXIO); - } - if (sc->data.status & MCDDSKCHNG) { - MCD_UNLOCK(sc); - device_printf(sc->dev, "CD not sensed\n"); - return (ENXIO); - } - - if (mcd_size(dev) < 0) { - MCD_UNLOCK(sc); - device_printf(sc->dev, "failed to get disk size\n"); - return (ENXIO); - } - - sc->data.openflags = 1; - sc->data.partflags |= MCDREADRAW; - sc->data.flags |= MCDVALID; - - (void) mcd_lock_door(sc, MCD_LK_LOCK); - if (!(sc->data.flags & MCDVALID)) { - MCD_UNLOCK(sc); - return (ENXIO); - } - - r = mcd_read_toc(sc); - MCD_UNLOCK(sc); - return (r); -} - -static int -mcdclose(struct cdev *dev, int flags, int fmt, struct thread *td) -{ - struct mcd_softc *sc; - - sc = (struct mcd_softc *)dev->si_drv1; - - MCD_LOCK(sc); - KASSERT(sc->data.openflags, ("device not open")); - - (void) mcd_lock_door(sc, MCD_LK_UNLOCK); - sc->data.openflags = 0; - sc->data.partflags &= ~MCDREADRAW; - MCD_UNLOCK(sc); - - return (0); -} - -static void -mcdstrategy(struct bio *bp) -{ - struct mcd_softc *sc; - - sc = (struct mcd_softc *)bp->bio_dev->si_drv1; - - /* if device invalidated (e.g. media change, door open), error */ - MCD_LOCK(sc); - if (!(sc->data.flags & MCDVALID)) { - device_printf(sc->dev, "media changed\n"); - bp->bio_error = EIO; - goto bad; - } - - /* read only */ - if (!(bp->bio_cmd == BIO_READ)) { - bp->bio_error = EROFS; - goto bad; - } - - /* no data to read */ - if (bp->bio_bcount == 0) - goto done; - - if (!(sc->data.flags & MCDTOC)) { - bp->bio_error = EIO; - goto bad; - } - - bp->bio_resid = 0; - - /* queue it */ - bioq_disksort(&sc->data.head, bp); - - /* now check whether we can perform processing */ - mcd_start(sc); - MCD_UNLOCK(sc); - return; - -bad: - bp->bio_flags |= BIO_ERROR; -done: - MCD_UNLOCK(sc); - bp->bio_resid = bp->bio_bcount; - biodone(bp); - return; -} - -static void -mcd_start(struct mcd_softc *sc) -{ - struct bio *bp; - - MCD_ASSERT_LOCKED(sc); - if (sc->data.flags & MCDMBXBSY) { - return; - } - - bp = bioq_takefirst(&sc->data.head); - if (bp != 0) { - /* block found to process, dequeue */ - /*MCD_TRACE("mcd_start: found block bp=0x%x\n",bp,0,0,0);*/ - sc->data.flags |= MCDMBXBSY; - } else { - /* nothing to do */ - return; - } - - sc->data.mbx.retry = MCD_RETRYS; - sc->data.mbx.bp = bp; - - mcd_doread(sc, MCD_S_BEGIN,&(sc->data.mbx)); - return; -} - -static int -mcdioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flags, struct thread *td) -{ - struct mcd_softc *sc; - int retry,r; - - sc = (struct mcd_softc *)dev->si_drv1; - - MCD_LOCK(sc); - if (mcd_getstat(sc, 1) == -1) { /* detect disk change too */ - MCD_UNLOCK(sc); - return (EIO); - } -MCD_TRACE("ioctl called 0x%lx\n", cmd); - - switch (cmd) { - case CDIOCSETPATCH: - case CDIOCGETVOL: - case CDIOCSETVOL: - case CDIOCSETMONO: - case CDIOCSETSTERIO: - case CDIOCSETMUTE: - case CDIOCSETLEFT: - case CDIOCSETRIGHT: - MCD_UNLOCK(sc); - return (EINVAL); - case CDIOCEJECT: - r = mcd_eject(sc); - MCD_UNLOCK(sc); - return (r); - case CDIOCSETDEBUG: - sc->data.debug = 1; - MCD_UNLOCK(sc); - return (0); - case CDIOCCLRDEBUG: - sc->data.debug = 0; - MCD_UNLOCK(sc); - return (0); - case CDIOCRESET: - r = mcd_hard_reset(sc); - MCD_UNLOCK(sc); - return (r); - case CDIOCALLOW: - r = mcd_lock_door(sc, MCD_LK_UNLOCK); - MCD_UNLOCK(sc); - return (r); - case CDIOCPREVENT: - r = mcd_lock_door(sc, MCD_LK_LOCK); - MCD_UNLOCK(sc); - return (r); - case CDIOCCLOSE: - r = mcd_inject(sc); - MCD_UNLOCK(sc); - return (r); - } - - if (!(sc->data.flags & MCDVALID)) { - if ( (sc->data.status & (MCDDSKCHNG|MCDDOOROPEN)) - || !(sc->data.status & MCDDSKIN)) - for (retry = 0; retry < DISK_SENSE_SECS * WAIT_FRAC; retry++) { - (void) mtx_sleep(sc, &sc->mtx, PSOCK | PCATCH, - "mcdsn2", hz/WAIT_FRAC); - if ((r = mcd_getstat(sc, 1)) == -1) { - MCD_UNLOCK(sc); - return (EIO); - } - if (r != -2) - break; - } - if ( (sc->data.status & (MCDDOOROPEN|MCDDSKCHNG)) - || !(sc->data.status & MCDDSKIN) - || mcd_size(dev) < 0 - ) { - MCD_UNLOCK(sc); - return (ENXIO); - } - sc->data.flags |= MCDVALID; - sc->data.partflags |= MCDREADRAW; - (void) mcd_lock_door(sc, MCD_LK_LOCK); - if (!(sc->data.flags & MCDVALID)) { - MCD_UNLOCK(sc); - return (ENXIO); - } - } - - switch (cmd) { - case DIOCGMEDIASIZE: - *(off_t *)addr = (off_t)sc->data.disksize * sc->data.blksize; - r = 0; - break; - case DIOCGSECTORSIZE: - *(u_int *)addr = sc->data.blksize; - r = 0; - break; - case CDIOCPLAYTRACKS: - r = mcd_playtracks(sc, (struct ioc_play_track *) addr); - break; - case CDIOCPLAYBLOCKS: - r = mcd_playblocks(sc, (struct ioc_play_blocks *) addr); - break; - case CDIOCPLAYMSF: - r = mcd_playmsf(sc, (struct ioc_play_msf *) addr); - break; - case CDIOCREADSUBCHANNEL_SYSSPACE: - return mcd_subchan(sc, (struct ioc_read_subchannel *) addr, 1); - case CDIOCREADSUBCHANNEL: - return mcd_subchan(sc, (struct ioc_read_subchannel *) addr, 0); - case CDIOREADTOCHEADER: - r = mcd_toc_header(sc, (struct ioc_toc_header *) addr); - break; - case CDIOREADTOCENTRYS: - return mcd_toc_entrys(sc, (struct ioc_read_toc_entry *) addr); - case CDIOCRESUME: - r = mcd_resume(sc); - break; - case CDIOCPAUSE: - r = mcd_pause(sc); - break; - case CDIOCSTART: - if (mcd_setmode(sc, MCD_MD_COOKED) != 0) - r = EIO; - else - r = 0; - break; - case CDIOCSTOP: - r = mcd_stop(sc); - break; - default: - r = ENOTTY; - } - MCD_UNLOCK(sc); - return (r); -} - -static int -mcd_size(struct cdev *dev) -{ - struct mcd_softc *sc; - int size; - - sc = (struct mcd_softc *)dev->si_drv1; - - if (mcd_volinfo(sc) == 0) { - sc->data.blksize = MCDBLK; - size = msf2hsg(sc->data.volinfo.vol_msf, 0); - sc->data.disksize = size * (MCDBLK/DEV_BSIZE); - return (0); - } - return (-1); -} - -/*************************************************************** - * lower level of driver starts here - **************************************************************/ - -#ifdef NOTDEF -static char -irqs[] = { - 0x00,0x00,0x10,0x20,0x00,0x30,0x00,0x00, - 0x00,0x10,0x40,0x50,0x00,0x00,0x00,0x00 -}; - -static char -drqs[] = { - 0x00,0x01,0x00,0x03,0x00,0x05,0x06,0x07, -}; -#endif - -#ifdef NOT_YET -static void -mcd_configure(struct mcd_softc *sc) -{ - MCD_WRITE(sc, MCD_REG_CONFIG, sc->data.config); -} -#endif - -/* Wait for non-busy - return 0 on timeout */ -static int -twiddle_thumbs(struct mcd_softc *sc, int count, char *whine) -{ - int i; - - for (i = 0; i < count; i++) { - if (!(MCD_READ(sc, MCD_FLAGS) & MFL_STATUS_NOT_AVAIL)) - return (1); - } - if (bootverbose) - device_printf(sc->dev, "timeout %s\n", whine); - return (0); -} - -/* check to see if a Mitsumi CD-ROM is attached to the ISA bus */ - -int -mcd_probe(struct mcd_softc *sc) -{ - int i, j; - unsigned char stbytes[3]; - - sc->data.flags = MCDPROBING; - -#ifdef NOTDEF - /* get irq/drq configuration word */ - sc->data.config = irqs[dev->id_irq]; /* | drqs[dev->id_drq];*/ -#else - sc->data.config = 0; -#endif - - /* send a reset */ - MCD_WRITE(sc, MCD_FLAGS, M_RESET); - - /* - * delay awhile by getting any pending garbage (old data) and - * throwing it away. - */ - for (i = 1000000; i != 0; i--) - (void)MCD_READ(sc, MCD_FLAGS); - - /* Get status */ - MCD_WRITE(sc, MCD_DATA, MCD_CMDGETSTAT); - if (!twiddle_thumbs(sc, 1000000, "getting status")) - return (ENXIO); /* Timeout */ - /* Get version information */ - MCD_WRITE(sc, MCD_DATA, MCD_CMDCONTINFO); - for (j = 0; j < 3; j++) { - if (!twiddle_thumbs(sc, 3000, "getting version info")) - return (ENXIO); - stbytes[j] = (MCD_READ(sc, MCD_DATA) & 0xFF); - } - if (stbytes[1] == stbytes[2]) - return (ENXIO); - if (stbytes[2] >= 4 || stbytes[1] != 'M') { - MCD_WRITE(sc, MCD_CTRL, M_PICKLE); - sc->data.flags |= MCDNEWMODEL; - } - sc->data.read_command = MCD_CMDSINGLESPEEDREAD; - switch (stbytes[1]) { - case 'M': - if (stbytes[2] <= 2) { - sc->data.type = MCD_TYPE_LU002S; - sc->data.name = "Mitsumi LU002S"; - } else if (stbytes[2] <= 5) { - sc->data.type = MCD_TYPE_LU005S; - sc->data.name = "Mitsumi LU005S"; - } else { - sc->data.type = MCD_TYPE_LU006S; - sc->data.name = "Mitsumi LU006S"; - } - break; - case 'F': - sc->data.type = MCD_TYPE_FX001; - sc->data.name = "Mitsumi FX001"; - break; - case 'D': - sc->data.type = MCD_TYPE_FX001D; - sc->data.name = "Mitsumi FX001D"; - sc->data.read_command = MCD_CMDDOUBLESPEEDREAD; - break; - default: - sc->data.type = MCD_TYPE_UNKNOWN; - sc->data.name = "Mitsumi ???"; - break; - } - - if (bootverbose) - device_printf(sc->dev, "type %s, version info: %c %x\n", - sc->data.name, stbytes[1], stbytes[2]); - - return (0); -} - - -static int -mcd_waitrdy(struct mcd_softc *sc, int dly) -{ - int i; - - /* wait until flag port senses status ready */ - for (i=0; i<dly; i+=MIN_DELAY) { - if (!(MCD_READ(sc, MCD_FLAGS) & MFL_STATUS_NOT_AVAIL)) - return (0); - DELAY(MIN_DELAY); - } - return (-1); -} - -static int -mcd_getreply(struct mcd_softc *sc, int dly) -{ - - /* wait data to become ready */ - if (mcd_waitrdy(sc, dly)<0) { - device_printf(sc->dev, "timeout getreply\n"); - return (-1); - } - - /* get the data */ - return (MCD_READ(sc, MCD_REG_STATUS) & 0xFF); -} - -static int -mcd_getstat(struct mcd_softc *sc, int sflg) -{ - int i; - - /* get the status */ - if (sflg) - MCD_WRITE(sc, MCD_REG_COMMAND, MCD_CMDGETSTAT); - i = mcd_getreply(sc, DELAY_GETREPLY); - if (i<0 || (i & MCD_ST_CMDCHECK)) { - sc->data.curr_mode = MCD_MD_UNKNOWN; - return (-1); - } - - sc->data.status = i; - - if (mcd_setflags(sc) < 0) - return (-2); - return (sc->data.status); -} - -static int -mcd_setflags(struct mcd_softc *sc) -{ - - /* check flags */ - if ( (sc->data.status & (MCDDSKCHNG|MCDDOOROPEN)) - || !(sc->data.status & MCDDSKIN)) { - MCD_TRACE("setflags: sensed DSKCHNG or DOOROPEN or !DSKIN\n"); - mcd_soft_reset(sc); - return (-1); - } - - if (sc->data.status & MCDAUDIOBSY) - sc->data.audio_status = CD_AS_PLAY_IN_PROGRESS; - else if (sc->data.audio_status == CD_AS_PLAY_IN_PROGRESS) - sc->data.audio_status = CD_AS_PLAY_COMPLETED; - return (0); -} - -static int -mcd_get(struct mcd_softc *sc, char *buf, int nmax) -{ - int i,k; - - for (i=0; i<nmax; i++) { - /* wait for data */ - if ((k = mcd_getreply(sc, DELAY_GETREPLY)) < 0) { - device_printf(sc->dev, "timeout mcd_get\n"); - return (-1); - } - buf[i] = k; - } - return (i); -} - -static int -mcd_send(struct mcd_softc *sc, int cmd,int nretrys) -{ - int i,k=0; - -/*MCD_TRACE("mcd_send: command = 0x%02x\n",cmd,0,0,0);*/ - for (i=0; i<nretrys; i++) { - MCD_WRITE(sc, MCD_REG_COMMAND, cmd); - if ((k=mcd_getstat(sc, 0)) != -1) - break; - } - if (k == -2) { - device_printf(sc->dev, "media changed\n"); - return (-1); - } - if (i == nretrys) { - device_printf(sc->dev, "mcd_send retry cnt exceeded\n"); - return (-1); - } -/*MCD_TRACE("mcd_send: done\n",0,0,0,0);*/ - return (0); -} - -static void -hsg2msf(int hsg, bcd_t *msf) -{ - hsg += 150; - F_msf(msf) = bin2bcd(hsg % 75); - hsg /= 75; - S_msf(msf) = bin2bcd(hsg % 60); - hsg /= 60; - M_msf(msf) = bin2bcd(hsg); -} - -static int -msf2hsg(bcd_t *msf, int relative) -{ - return (bcd2bin(M_msf(msf)) * 60 + bcd2bin(S_msf(msf))) * 75 + - bcd2bin(F_msf(msf)) - (!relative) * 150; -} - -static int -mcd_volinfo(struct mcd_softc *sc) -{ - - /* Just return if we already have it */ - if (sc->data.flags & MCDVOLINFO) return (0); - -/*MCD_TRACE("mcd_volinfo: enter\n",0,0,0,0);*/ - - /* send volume info command */ - if (mcd_send(sc, MCD_CMDGETVOLINFO,MCD_RETRYS) < 0) - return (EIO); - - /* get data */ - if (mcd_get(sc, (char*) &sc->data.volinfo,sizeof(struct mcd_volinfo)) < 0) { - device_printf(sc->dev, "mcd_volinfo: error read data\n"); - return (EIO); - } - - if (sc->data.volinfo.trk_low > 0 && - sc->data.volinfo.trk_high >= sc->data.volinfo.trk_low - ) { - sc->data.flags |= MCDVOLINFO; /* volinfo is OK */ - return (0); - } - - return (EINVAL); -} - -/* state machine to process read requests - * initialize with MCD_S_BEGIN: calculate sizes, and read status - * MCD_S_WAITSTAT: wait for status reply, set mode - * MCD_S_WAITMODE: waits for status reply from set mode, set read command - * MCD_S_WAITREAD: wait for read ready, read data - */ -static void -mcd_timeout(void *arg) -{ - struct mcd_softc *sc; - - sc = (struct mcd_softc *)arg; - - MCD_ASSERT_LOCKED(sc); - mcd_doread(sc, sc->ch_state, sc->ch_mbxsave); -} - -static void -mcd_doread(struct mcd_softc *sc, int state, struct mcd_mbx *mbxin) -{ - struct mcd_mbx *mbx; - struct bio *bp; - int rm, i, k; - struct mcd_read2 rbuf; - int blknum; - caddr_t addr; - - MCD_ASSERT_LOCKED(sc); - mbx = (state!=MCD_S_BEGIN) ? sc->ch_mbxsave : mbxin; - bp = mbx->bp; - -loop: - switch (state) { - case MCD_S_BEGIN: - mbx = sc->ch_mbxsave = mbxin; - - case MCD_S_BEGIN1: -retry_status: - /* get status */ - MCD_WRITE(sc, MCD_REG_COMMAND, MCD_CMDGETSTAT); - mbx->count = RDELAY_WAITSTAT; - sc->ch_state = MCD_S_WAITSTAT; - callout_reset(&sc->timer, hz/100, mcd_timeout, sc); /* XXX */ - return; - case MCD_S_WAITSTAT: - sc->ch_state = MCD_S_WAITSTAT; - callout_stop(&sc->timer); - if (mbx->count-- >= 0) { - if (MCD_READ(sc, MCD_FLAGS) & MFL_STATUS_NOT_AVAIL) { - sc->ch_state = MCD_S_WAITSTAT; - callout_reset(&sc->timer, hz/100, - mcd_timeout, sc); /* XXX */ - return; - } - sc->data.status = MCD_READ(sc, MCD_REG_STATUS) & 0xFF; - if (sc->data.status & MCD_ST_CMDCHECK) - goto retry_status; - if (mcd_setflags(sc) < 0) - goto changed; - MCD_TRACE("got WAITSTAT delay=%d\n", - RDELAY_WAITSTAT-mbx->count); - /* reject, if audio active */ - if (sc->data.status & MCDAUDIOBSY) { - device_printf(sc->dev, "audio is active\n"); - goto readerr; - } - -retry_mode: - /* to check for raw/cooked mode */ - if (sc->data.flags & MCDREADRAW) { - rm = MCD_MD_RAW; - mbx->sz = MCDRBLK; - } else { - rm = MCD_MD_COOKED; - mbx->sz = sc->data.blksize; - } - - if (rm == sc->data.curr_mode) - goto modedone; - - mbx->count = RDELAY_WAITMODE; - - sc->data.curr_mode = MCD_MD_UNKNOWN; - mbx->mode = rm; - MCD_WRITE(sc, MCD_REG_COMMAND, MCD_CMDSETMODE); - MCD_WRITE(sc, MCD_REG_COMMAND, rm); - - sc->ch_state = MCD_S_WAITMODE; - callout_reset(&sc->timer, hz / 100, mcd_timeout, sc); /* XXX */ - return; - } else { - device_printf(sc->dev, "timeout getstatus\n"); - goto readerr; - } - - case MCD_S_WAITMODE: - sc->ch_state = MCD_S_WAITMODE; - callout_stop(&sc->timer); - if (mbx->count-- < 0) { - device_printf(sc->dev, "timeout set mode\n"); - goto readerr; - } - if (MCD_READ(sc, MCD_FLAGS) & MFL_STATUS_NOT_AVAIL) { - sc->ch_state = MCD_S_WAITMODE; - callout_reset(&sc->timer, hz / 100, mcd_timeout, sc); - return; - } - sc->data.status = MCD_READ(sc, MCD_REG_STATUS) & 0xFF; - if (sc->data.status & MCD_ST_CMDCHECK) { - sc->data.curr_mode = MCD_MD_UNKNOWN; - goto retry_mode; - } - if (mcd_setflags(sc) < 0) - goto changed; - sc->data.curr_mode = mbx->mode; - MCD_TRACE("got WAITMODE delay=%d\n", - RDELAY_WAITMODE-mbx->count); -modedone: - /* for first block */ - mbx->nblk = howmany(bp->bio_bcount, mbx->sz); - mbx->skip = 0; - -nextblock: - blknum = bp->bio_offset / mbx->sz + mbx->skip/mbx->sz; - - MCD_TRACE("mcd_doread: read blknum=%d for bp=%p\n", - blknum, bp); - - /* build parameter block */ - hsg2msf(blknum,rbuf.start_msf); -retry_read: - /* send the read command */ - MCD_WRITE(sc, MCD_REG_COMMAND, sc->data.read_command); - MCD_WRITE(sc, MCD_REG_COMMAND, rbuf.start_msf[0]); - MCD_WRITE(sc, MCD_REG_COMMAND, rbuf.start_msf[1]); - MCD_WRITE(sc, MCD_REG_COMMAND, rbuf.start_msf[2]); - MCD_WRITE(sc, MCD_REG_COMMAND, 0); - MCD_WRITE(sc, MCD_REG_COMMAND, 0); - MCD_WRITE(sc, MCD_REG_COMMAND, 1); - - /* Spin briefly (<= 2ms) to avoid missing next block */ - for (i = 0; i < 20; i++) { - k = MCD_READ(sc, MCD_FLAGS); - if (!(k & MFL_DATA_NOT_AVAIL)) - goto got_it; - DELAY(100); - } - - mbx->count = RDELAY_WAITREAD; - sc->ch_state = MCD_S_WAITREAD; - callout_reset(&sc->timer, hz / 100, mcd_timeout, sc); /* XXX */ - return; - case MCD_S_WAITREAD: - sc->ch_state = MCD_S_WAITREAD; - callout_stop(&sc->timer); - if (mbx->count-- > 0) { - k = MCD_READ(sc, MCD_FLAGS); - if (!(k & MFL_DATA_NOT_AVAIL)) { /* XXX */ - MCD_TRACE("got data delay=%d\n", - RDELAY_WAITREAD-mbx->count); - got_it: - /* data is ready */ - addr = bp->bio_data + mbx->skip; - - MCD_WRITE(sc, MCD_REG_CTL2,0x04); /* XXX */ - for (i=0; i<mbx->sz; i++) - *addr++ = MCD_READ(sc, MCD_REG_RDATA); - MCD_WRITE(sc, MCD_REG_CTL2,0x0c); /* XXX */ - - k = MCD_READ(sc, MCD_FLAGS); - /* If we still have some junk, read it too */ - if (!(k & MFL_DATA_NOT_AVAIL)) { - MCD_WRITE(sc, MCD_REG_CTL2, 0x04); /* XXX */ - (void)MCD_READ(sc, MCD_REG_RDATA); - (void)MCD_READ(sc, MCD_REG_RDATA); - MCD_WRITE(sc, MCD_REG_CTL2, 0x0c); /* XXX */ - } - - if (--mbx->nblk > 0) { - mbx->skip += mbx->sz; - goto nextblock; - } - - /* return buffer */ - bp->bio_resid = 0; - biodone(bp); - - sc->data.flags &= ~(MCDMBXBSY|MCDREADRAW); - mcd_start(sc); - return; - } - if (!(k & MFL_STATUS_NOT_AVAIL)) { - sc->data.status = MCD_READ(sc, MCD_REG_STATUS) & 0xFF; - if (sc->data.status & MCD_ST_CMDCHECK) - goto retry_read; - if (mcd_setflags(sc) < 0) - goto changed; - } - sc->ch_state = MCD_S_WAITREAD; - callout_reset(&sc->timer, hz / 100, mcd_timeout, sc); /* XXX */ - return; - } else { - device_printf(sc->dev, "timeout read data\n"); - goto readerr; - } - } - -readerr: - if (mbx->retry-- > 0) { - device_printf(sc->dev, "retrying\n"); - state = MCD_S_BEGIN1; - goto loop; - } -harderr: - /* invalidate the buffer */ - bp->bio_flags |= BIO_ERROR; - bp->bio_resid = bp->bio_bcount; - biodone(bp); - - sc->data.flags &= ~(MCDMBXBSY|MCDREADRAW); - mcd_start(sc); - return; - -changed: - device_printf(sc->dev, "media changed\n"); - goto harderr; - -#ifdef NOTDEF - device_printf(sc->dev, "unit timeout, resetting\n"); - MCD_WRITE(sc, MCD_REG_RESET, MCD_CMDRESET); - DELAY(300000); - (void)mcd_getstat(sc, 1); - (void)mcd_getstat(sc, 1); - /*sc->data.status &= ~MCDDSKCHNG; */ - sc->data.debug = 1; /* preventive set debug mode */ - -#endif - -} - -static int -mcd_lock_door(struct mcd_softc *sc, int lock) -{ - - MCD_WRITE(sc, MCD_REG_COMMAND, MCD_CMDLOCKDRV); - MCD_WRITE(sc, MCD_REG_COMMAND, lock); - if (mcd_getstat(sc, 0) == -1) - return (EIO); - return (0); -} - -static int -mcd_close_tray(struct mcd_softc *sc) -{ - int retry, r; - - if (mcd_getstat(sc, 1) == -1) - return (EIO); - if (sc->data.status & MCDDOOROPEN) { - MCD_WRITE(sc, MCD_REG_COMMAND, MCD_CMDCLOSETRAY); - for (retry = 0; retry < CLOSE_TRAY_SECS * WAIT_FRAC; retry++) { - if (MCD_READ(sc, MCD_FLAGS) & MFL_STATUS_NOT_AVAIL) - (void) mtx_sleep(sc, &sc->mtx, PSOCK | PCATCH, - "mcdcls", hz/WAIT_FRAC); - else { - if ((r = mcd_getstat(sc, 0)) == -1) - return (EIO); - return (0); - } - } - return (ENXIO); - } - return (0); -} - -static int -mcd_eject(struct mcd_softc *sc) -{ - int r; - - if (mcd_getstat(sc, 1) == -1) /* detect disk change too */ - return (EIO); - if (sc->data.status & MCDDOOROPEN) - return (0); - if ((r = mcd_stop(sc)) == EIO) - return (r); - MCD_WRITE(sc, MCD_REG_COMMAND, MCD_CMDEJECTDISK); - if (mcd_getstat(sc, 0) == -1) - return (EIO); - return (0); -} - -static int -mcd_inject(struct mcd_softc *sc) -{ - - if (mcd_getstat(sc, 1) == -1) /* detect disk change too */ - return (EIO); - if (sc->data.status & MCDDOOROPEN) - return mcd_close_tray(sc); - return (0); -} - -static int -mcd_hard_reset(struct mcd_softc *sc) -{ - - MCD_WRITE(sc, MCD_REG_RESET, MCD_CMDRESET); - sc->data.curr_mode = MCD_MD_UNKNOWN; - sc->data.audio_status = CD_AS_AUDIO_INVALID; - return (0); -} - -static void -mcd_soft_reset(struct mcd_softc *sc) -{ - - sc->data.flags &= (MCDINIT|MCDPROBING|MCDNEWMODEL); - sc->data.curr_mode = MCD_MD_UNKNOWN; - sc->data.partflags = 0; - sc->data.audio_status = CD_AS_AUDIO_INVALID; -} - -static int -mcd_setmode(struct mcd_softc *sc, int mode) -{ - int retry, st; - - if (sc->data.curr_mode == mode) - return (0); - if (sc->data.debug) - device_printf(sc->dev, "setting mode to %d\n", mode); - for(retry=0; retry<MCD_RETRYS; retry++) - { - sc->data.curr_mode = MCD_MD_UNKNOWN; - MCD_WRITE(sc, MCD_REG_COMMAND, MCD_CMDSETMODE); - MCD_WRITE(sc, MCD_REG_COMMAND, mode); - if ((st = mcd_getstat(sc, 0)) >= 0) { - sc->data.curr_mode = mode; - return (0); - } - if (st == -2) { - device_printf(sc->dev, "media changed\n"); - break; - } - } - - return (-1); -} - -static int -mcd_toc_header(struct mcd_softc *sc, struct ioc_toc_header *th) -{ - int r; - - if ((r = mcd_volinfo(sc)) != 0) - return (r); - - th->starting_track = bcd2bin(sc->data.volinfo.trk_low); - th->ending_track = bcd2bin(sc->data.volinfo.trk_high); - th->len = 2 * sizeof(u_char) /* start & end tracks */ + - (th->ending_track + 1 - th->starting_track + 1) * - sizeof(struct cd_toc_entry); - - return (0); -} - -static int -mcd_read_toc(struct mcd_softc *sc) -{ - struct ioc_toc_header th; - struct mcd_qchninfo q; - int rc, trk, idx, retry; - - /* Only read TOC if needed */ - if (sc->data.flags & MCDTOC) - return (0); - - if (sc->data.debug) - device_printf(sc->dev, "reading toc header\n"); - - if ((rc = mcd_toc_header(sc, &th)) != 0) - return (rc); - - if (mcd_send(sc, MCD_CMDSTOPAUDIO, MCD_RETRYS) < 0) - return (EIO); - - if (mcd_setmode(sc, MCD_MD_TOC) != 0) - return (EIO); - - if (sc->data.debug) - device_printf(sc->dev, "get_toc reading qchannel info\n"); - - for(trk=th.starting_track; trk<=th.ending_track; trk++) - sc->data.toc[trk].idx_no = 0; - trk = th.ending_track - th.starting_track + 1; - for(retry=0; retry<600 && trk>0; retry++) - { - if (mcd_getqchan(sc, &q) < 0) break; - idx = bcd2bin(q.idx_no); - if (idx>=th.starting_track && idx<=th.ending_track && q.trk_no==0) { - if (sc->data.toc[idx].idx_no == 0) { - sc->data.toc[idx] = q; - trk--; - } - } - } - - if (mcd_setmode(sc, MCD_MD_COOKED) != 0) - return (EIO); - - if (trk != 0) - return (ENXIO); - - /* add a fake last+1 */ - idx = th.ending_track + 1; - sc->data.toc[idx].control = sc->data.toc[idx-1].control; - sc->data.toc[idx].addr_type = sc->data.toc[idx-1].addr_type; - sc->data.toc[idx].trk_no = 0; - sc->data.toc[idx].idx_no = MCD_LASTPLUS1; - sc->data.toc[idx].hd_pos_msf[0] = sc->data.volinfo.vol_msf[0]; - sc->data.toc[idx].hd_pos_msf[1] = sc->data.volinfo.vol_msf[1]; - sc->data.toc[idx].hd_pos_msf[2] = sc->data.volinfo.vol_msf[2]; - - if (sc->data.debug) - { int i; - for (i = th.starting_track; i <= idx; i++) - device_printf(sc->dev, "trk %d idx %d pos %d %d %d\n", - i, - sc->data.toc[i].idx_no > 0x99 ? sc->data.toc[i].idx_no : - bcd2bin(sc->data.toc[i].idx_no), - bcd2bin(sc->data.toc[i].hd_pos_msf[0]), - bcd2bin(sc->data.toc[i].hd_pos_msf[1]), - bcd2bin(sc->data.toc[i].hd_pos_msf[2])); - } - - sc->data.flags |= MCDTOC; - - return (0); -} - -#if 0 -static int -mcd_toc_entry(struct mcd_softc *sc, struct ioc_read_toc_single_entry *te) -{ - struct ioc_toc_header th; - int rc, trk; - - if (te->address_format != CD_MSF_FORMAT - && te->address_format != CD_LBA_FORMAT) - return (EINVAL); - - /* Copy the toc header */ - if ((rc = mcd_toc_header(sc, &th)) != 0) - return (rc); - - /* verify starting track */ - trk = te->track; - if (trk == 0) - trk = th.starting_track; - else if (trk == MCD_LASTPLUS1) - trk = th.ending_track + 1; - else if (trk < th.starting_track || trk > th.ending_track + 1) - return (EINVAL); - - /* Make sure we have a valid toc */ - if ((rc=mcd_read_toc(sc)) != 0) - return (rc); - - /* Copy the TOC data. */ - if (sc->data.toc[trk].idx_no == 0) - return (EIO); - - te->entry.control = sc->data.toc[trk].control; - te->entry.addr_type = sc->data.toc[trk].addr_type; - te->entry.track = - sc->data.toc[trk].idx_no > 0x99 ? sc->data.toc[trk].idx_no : - bcd2bin(sc->data.toc[trk].idx_no); - switch (te->address_format) { - case CD_MSF_FORMAT: - te->entry.addr.msf.unused = 0; - te->entry.addr.msf.minute = bcd2bin(sc->data.toc[trk].hd_pos_msf[0]); - te->entry.addr.msf.second = bcd2bin(sc->data.toc[trk].hd_pos_msf[1]); - te->entry.addr.msf.frame = bcd2bin(sc->data.toc[trk].hd_pos_msf[2]); - break; - case CD_LBA_FORMAT: - te->entry.addr.lba = htonl(msf2hsg(sc->data.toc[trk].hd_pos_msf, 0)); - break; - } - return (0); -} -#endif - -static int -mcd_toc_entrys(struct mcd_softc *sc, struct ioc_read_toc_entry *te) -{ - struct cd_toc_entry entries[MCD_MAXTOCS]; - struct ioc_toc_header th; - int rc, n, trk, len; - - if ( te->data_len < sizeof(entries[0]) - || (te->data_len % sizeof(entries[0])) != 0 - || (te->address_format != CD_MSF_FORMAT - && te->address_format != CD_LBA_FORMAT) - ) - return (EINVAL); - - /* Copy the toc header */ - if ((rc = mcd_toc_header(sc, &th)) != 0) - return (rc); - - /* verify starting track */ - trk = te->starting_track; - if (trk == 0) - trk = th.starting_track; - else if (trk == MCD_LASTPLUS1) - trk = th.ending_track + 1; - else if (trk < th.starting_track || trk > th.ending_track + 1) - return (EINVAL); - - len = ((th.ending_track + 1 - trk) + 1) * - sizeof(entries[0]); - if (te->data_len < len) - len = te->data_len; - if (len > sizeof(entries)) - return (EINVAL); - - /* Make sure we have a valid toc */ - if ((rc=mcd_read_toc(sc)) != 0) - return (rc); - - /* Copy the TOC data. */ - for (n = 0; len > 0 && trk <= th.ending_track + 1; trk++) { - if (sc->data.toc[trk].idx_no == 0) - continue; - entries[n].control = sc->data.toc[trk].control; - entries[n].addr_type = sc->data.toc[trk].addr_type; - entries[n].track = - sc->data.toc[trk].idx_no > 0x99 ? sc->data.toc[trk].idx_no : - bcd2bin(sc->data.toc[trk].idx_no); - switch (te->address_format) { - case CD_MSF_FORMAT: - entries[n].addr.msf.unused = 0; - entries[n].addr.msf.minute = bcd2bin(sc->data.toc[trk].hd_pos_msf[0]); - entries[n].addr.msf.second = bcd2bin(sc->data.toc[trk].hd_pos_msf[1]); - entries[n].addr.msf.frame = bcd2bin(sc->data.toc[trk].hd_pos_msf[2]); - break; - case CD_LBA_FORMAT: - entries[n].addr.lba = htonl(msf2hsg(sc->data.toc[trk].hd_pos_msf, 0)); - break; - } - len -= sizeof(struct cd_toc_entry); - n++; - } - - /* copy the data back */ - MCD_UNLOCK(sc); - return copyout(entries, te->data, n * sizeof(struct cd_toc_entry)); -} - -static int -mcd_stop(struct mcd_softc *sc) -{ - - /* Verify current status */ - if (sc->data.audio_status != CD_AS_PLAY_IN_PROGRESS && - sc->data.audio_status != CD_AS_PLAY_PAUSED && - sc->data.audio_status != CD_AS_PLAY_COMPLETED) { - if (sc->data.debug) - device_printf(sc->dev, - "stop attempted when not playing, audio status %d\n", - sc->data.audio_status); - return (EINVAL); - } - if (sc->data.audio_status == CD_AS_PLAY_IN_PROGRESS) - if (mcd_send(sc, MCD_CMDSTOPAUDIO, MCD_RETRYS) < 0) - return (EIO); - sc->data.audio_status = CD_AS_PLAY_COMPLETED; - return (0); -} - -static int -mcd_getqchan(struct mcd_softc *sc, struct mcd_qchninfo *q) -{ - - if (mcd_send(sc, MCD_CMDGETQCHN, MCD_RETRYS) < 0) - return (-1); - if (mcd_get(sc, (char *) q, sizeof(struct mcd_qchninfo)) < 0) - return (-1); - if (sc->data.debug) { - device_printf(sc->dev, - "getqchan control=0x%x addr_type=0x%x trk=%d ind=%d ttm=%d:%d.%d dtm=%d:%d.%d\n", - q->control, q->addr_type, - bcd2bin(q->trk_no), - bcd2bin(q->idx_no), - bcd2bin(q->trk_size_msf[0]), - bcd2bin(q->trk_size_msf[1]), - bcd2bin(q->trk_size_msf[2]), - bcd2bin(q->hd_pos_msf[0]), - bcd2bin(q->hd_pos_msf[1]), - bcd2bin(q->hd_pos_msf[2])); - } - return (0); -} - -static int -mcd_subchan(struct mcd_softc *sc, struct ioc_read_subchannel *sch, int nocopyout) -{ - struct mcd_qchninfo q; - struct cd_sub_channel_info data; - int lba; - - if (sc->data.debug) - device_printf(sc->dev, "subchan af=%d, df=%d\n", - sch->address_format, - sch->data_format); - - if (sch->address_format != CD_MSF_FORMAT && - sch->address_format != CD_LBA_FORMAT) - return (EINVAL); - - if (sch->data_format != CD_CURRENT_POSITION && - sch->data_format != CD_MEDIA_CATALOG) - return (EINVAL); - - if (mcd_setmode(sc, MCD_MD_COOKED) != 0) - return (EIO); - - if (mcd_getqchan(sc, &q) < 0) - return (EIO); - - data.header.audio_status = sc->data.audio_status; - data.what.position.data_format = sch->data_format; - - switch (sch->data_format) { - case CD_MEDIA_CATALOG: - data.what.media_catalog.mc_valid = 1; - data.what.media_catalog.mc_number[0] = '\0'; - break; - - case CD_CURRENT_POSITION: - data.what.position.control = q.control; - data.what.position.addr_type = q.addr_type; - data.what.position.track_number = bcd2bin(q.trk_no); - data.what.position.index_number = bcd2bin(q.idx_no); - switch (sch->address_format) { - case CD_MSF_FORMAT: - data.what.position.reladdr.msf.unused = 0; - data.what.position.reladdr.msf.minute = bcd2bin(q.trk_size_msf[0]); - data.what.position.reladdr.msf.second = bcd2bin(q.trk_size_msf[1]); - data.what.position.reladdr.msf.frame = bcd2bin(q.trk_size_msf[2]); - data.what.position.absaddr.msf.unused = 0; - data.what.position.absaddr.msf.minute = bcd2bin(q.hd_pos_msf[0]); - data.what.position.absaddr.msf.second = bcd2bin(q.hd_pos_msf[1]); - data.what.position.absaddr.msf.frame = bcd2bin(q.hd_pos_msf[2]); - break; - case CD_LBA_FORMAT: - lba = msf2hsg(q.trk_size_msf, 1); - /* - * Pre-gap has index number of 0, and decreasing MSF - * address. Must be converted to negative LBA, per - * SCSI spec. - */ - if (data.what.position.index_number == 0) - lba = -lba; - data.what.position.reladdr.lba = htonl(lba); - data.what.position.absaddr.lba = htonl(msf2hsg(q.hd_pos_msf, 0)); - break; - } - break; - } - - MCD_UNLOCK(sc); - if (nocopyout == 0) - return copyout(&data, sch->data, min(sizeof(struct cd_sub_channel_info), sch->data_len)); - bcopy(&data, sch->data, min(sizeof(struct cd_sub_channel_info), sch->data_len)); - return (0); -} - -static int -mcd_playmsf(struct mcd_softc *sc, struct ioc_play_msf *p) -{ - struct mcd_read2 pb; - - if (sc->data.debug) - device_printf(sc->dev, "playmsf: from %d:%d.%d to %d:%d.%d\n", - p->start_m, p->start_s, p->start_f, - p->end_m, p->end_s, p->end_f); - - if ((p->start_m * 60 * 75 + p->start_s * 75 + p->start_f) >= - (p->end_m * 60 * 75 + p->end_s * 75 + p->end_f) || - (p->end_m * 60 * 75 + p->end_s * 75 + p->end_f) > - M_msf(sc->data.volinfo.vol_msf) * 60 * 75 + - S_msf(sc->data.volinfo.vol_msf) * 75 + - F_msf(sc->data.volinfo.vol_msf)) - return (EINVAL); - - pb.start_msf[0] = bin2bcd(p->start_m); - pb.start_msf[1] = bin2bcd(p->start_s); - pb.start_msf[2] = bin2bcd(p->start_f); - pb.end_msf[0] = bin2bcd(p->end_m); - pb.end_msf[1] = bin2bcd(p->end_s); - pb.end_msf[2] = bin2bcd(p->end_f); - - if (mcd_setmode(sc, MCD_MD_COOKED) != 0) - return (EIO); - - return mcd_play(sc, &pb); -} - -static int -mcd_playtracks(struct mcd_softc *sc, struct ioc_play_track *pt) -{ - struct mcd_read2 pb; - int a = pt->start_track; - int z = pt->end_track; - int rc, i; - - if ((rc = mcd_read_toc(sc)) != 0) - return (rc); - - if (sc->data.debug) - device_printf(sc->dev, "playtracks from %d:%d to %d:%d\n", - a, pt->start_index, z, pt->end_index); - - if ( a < bcd2bin(sc->data.volinfo.trk_low) - || a > bcd2bin(sc->data.volinfo.trk_high) - || a > z - || z < bcd2bin(sc->data.volinfo.trk_low) - || z > bcd2bin(sc->data.volinfo.trk_high)) - return (EINVAL); - - for (i = 0; i < 3; i++) { - pb.start_msf[i] = sc->data.toc[a].hd_pos_msf[i]; - pb.end_msf[i] = sc->data.toc[z+1].hd_pos_msf[i]; - } - - if (mcd_setmode(sc, MCD_MD_COOKED) != 0) - return (EIO); - - return mcd_play(sc, &pb); -} - -static int -mcd_playblocks(struct mcd_softc *sc, struct ioc_play_blocks *p) -{ - struct mcd_read2 pb; - - if (sc->data.debug) - device_printf(sc->dev, "playblocks: blkno %d length %d\n", - p->blk, p->len); - - if (p->blk > sc->data.disksize || p->len > sc->data.disksize || - p->blk < 0 || p->len < 0 || - (p->blk + p->len) > sc->data.disksize) - return (EINVAL); - - hsg2msf(p->blk, pb.start_msf); - hsg2msf(p->blk + p->len, pb.end_msf); - - if (mcd_setmode(sc, MCD_MD_COOKED) != 0) - return (EIO); - - return mcd_play(sc, &pb); -} - -static int -mcd_play(struct mcd_softc *sc, struct mcd_read2 *pb) -{ - int retry, st = -1, status; - - sc->data.lastpb = *pb; - for(retry=0; retry<MCD_RETRYS; retry++) { - - critical_enter(); - MCD_WRITE(sc, MCD_REG_COMMAND, MCD_CMDSINGLESPEEDREAD); - MCD_WRITE(sc, MCD_REG_COMMAND, pb->start_msf[0]); - MCD_WRITE(sc, MCD_REG_COMMAND, pb->start_msf[1]); - MCD_WRITE(sc, MCD_REG_COMMAND, pb->start_msf[2]); - MCD_WRITE(sc, MCD_REG_COMMAND, pb->end_msf[0]); - MCD_WRITE(sc, MCD_REG_COMMAND, pb->end_msf[1]); - MCD_WRITE(sc, MCD_REG_COMMAND, pb->end_msf[2]); - critical_exit(); - - status=mcd_getstat(sc, 0); - if (status == -1) - continue; - else if (status != -2) - st = 0; - break; - } - - if (status == -2) { - device_printf(sc->dev, "media changed\n"); - return (ENXIO); - } - if (sc->data.debug) - device_printf(sc->dev, - "mcd_play retry=%d, status=0x%02x\n", retry, status); - if (st < 0) - return (ENXIO); - sc->data.audio_status = CD_AS_PLAY_IN_PROGRESS; - return (0); -} - -static int -mcd_pause(struct mcd_softc *sc) -{ - struct mcd_qchninfo q; - int rc; - - /* Verify current status */ - if (sc->data.audio_status != CD_AS_PLAY_IN_PROGRESS && - sc->data.audio_status != CD_AS_PLAY_PAUSED) { - if (sc->data.debug) - device_printf(sc->dev, - "pause attempted when not playing, audio status %d\n", - sc->data.audio_status); - return (EINVAL); - } - - /* Get the current position */ - if (mcd_getqchan(sc, &q) < 0) - return (EIO); - - /* Copy it into lastpb */ - sc->data.lastpb.start_msf[0] = q.hd_pos_msf[0]; - sc->data.lastpb.start_msf[1] = q.hd_pos_msf[1]; - sc->data.lastpb.start_msf[2] = q.hd_pos_msf[2]; - - /* Stop playing */ - if ((rc=mcd_stop(sc)) != 0) - return (rc); - - /* Set the proper status and exit */ - sc->data.audio_status = CD_AS_PLAY_PAUSED; - return (0); -} - -static int -mcd_resume(struct mcd_softc *sc) -{ - - if (sc->data.audio_status != CD_AS_PLAY_PAUSED) - return (EINVAL); - return mcd_play(sc, &sc->data.lastpb); -} diff --git a/sys/dev/mcd/mcd_isa.c b/sys/dev/mcd/mcd_isa.c deleted file mode 100644 index 92e8adb57192..000000000000 --- a/sys/dev/mcd/mcd_isa.c +++ /dev/null @@ -1,202 +0,0 @@ -/* - */ - -#include <sys/cdefs.h> -__FBSDID("$FreeBSD$"); - -#include <sys/param.h> -#include <sys/systm.h> -#include <sys/kernel.h> -#include <sys/module.h> -#include <sys/conf.h> -#include <sys/fcntl.h> -#include <sys/bio.h> -#include <sys/cdio.h> -#include <sys/bus.h> - -#include <sys/lock.h> -#include <sys/mutex.h> - -#include <machine/bus.h> -#include <machine/resource.h> -#include <sys/rman.h> - -#include <isa/isavar.h> - -#include <dev/mcd/mcdreg.h> -#include <dev/mcd/mcdvar.h> - -static int mcd_isa_probe (device_t); -static int mcd_isa_attach (device_t); -static int mcd_isa_detach (device_t); - -static int mcd_alloc_resources (device_t); -static void mcd_release_resources (device_t); - -static int -mcd_isa_probe (device_t dev) -{ - struct mcd_softc * sc; - int error; - - /* No pnp support */ - if (isa_get_vendorid(dev)) - return (ENXIO); - - /* IO port must be configured. */ - if (bus_get_resource_start(dev, SYS_RES_IOPORT, 0) == 0) - return (ENXIO); - - sc = device_get_softc(dev); - sc->dev = dev; - sc->port_rid = 0; - sc->port_type = SYS_RES_IOPORT; - error = mcd_alloc_resources(dev); - if (error) - goto fail; - - error = mcd_probe(sc); - if (error) { - device_printf(dev, "Probe failed.\n"); - goto fail; - } - - device_set_desc(dev, sc->data.name); - -fail: - mcd_release_resources(dev); - return (error); -} - -static int -mcd_isa_attach (device_t dev) -{ - struct mcd_softc * sc; - int error; - - sc = device_get_softc(dev); - error = 0; - - sc->dev = dev; - sc->port_rid = 0; - sc->port_type = SYS_RES_IOPORT; - error = mcd_alloc_resources(dev); - if (error) - goto fail; - - error = mcd_probe(sc); - if (error) { - device_printf(dev, "Re-Probe failed.\n"); - goto fail; - } - - error = mcd_attach(sc); - if (error) { - device_printf(dev, "Attach failed.\n"); - goto fail; - } - - return (0); -fail: - mcd_release_resources(dev); - return (error); -} - -static int -mcd_isa_detach (device_t dev) -{ - struct mcd_softc * sc; - int error; - - sc = device_get_softc(dev); - error = 0; - - destroy_dev(sc->mcd_dev_t); - - mcd_release_resources(dev); - - return (error); -} - -static int -mcd_alloc_resources (device_t dev) -{ - struct mcd_softc * sc; - int error; - - sc = device_get_softc(dev); - error = 0; - mtx_init(&sc->mtx, "mcd", NULL, MTX_DEF); - - if (sc->port_type) { - sc->port = bus_alloc_resource_any(dev, sc->port_type, - &sc->port_rid, RF_ACTIVE); - if (sc->port == NULL) { - device_printf(dev, "Unable to allocate PORT resource.\n"); - error = ENOMEM; - goto bad; - } - } - - if (sc->irq_type) { - sc->irq = bus_alloc_resource_any(dev, sc->irq_type, - &sc->irq_rid, RF_ACTIVE); - if (sc->irq == NULL) { - device_printf(dev, "Unable to allocate IRQ resource.\n"); - error = ENOMEM; - goto bad; - } - } - - if (sc->drq_type) { - sc->drq = bus_alloc_resource_any(dev, sc->drq_type, - &sc->drq_rid, RF_ACTIVE); - if (sc->drq == NULL) { - device_printf(dev, "Unable to allocate DRQ resource.\n"); - error = ENOMEM; - goto bad; - } - } - -bad: - return (error); -} - -static void -mcd_release_resources (device_t dev) -{ - struct mcd_softc * sc; - - sc = device_get_softc(dev); - - if (sc->irq_ih) - bus_teardown_intr(dev, sc->irq, sc->irq_ih); - if (sc->port) - bus_release_resource(dev, sc->port_type, sc->port_rid, sc->port); - if (sc->irq) - bus_release_resource(dev, sc->irq_type, sc->irq_rid, sc->irq); - if (sc->drq) - bus_release_resource(dev, sc->drq_type, sc->drq_rid, sc->drq); - - mtx_destroy(&sc->mtx); - - return; -} - -static device_method_t mcd_isa_methods[] = { - DEVMETHOD(device_probe, mcd_isa_probe), - DEVMETHOD(device_attach, mcd_isa_attach), - DEVMETHOD(device_detach, mcd_isa_detach), - - { 0, 0 } -}; - -static driver_t mcd_isa_driver = { - "mcd", - mcd_isa_methods, - sizeof(struct mcd_softc) -}; - -static devclass_t mcd_devclass; - -DRIVER_MODULE(mcd, isa, mcd_isa_driver, mcd_devclass, NULL, 0); diff --git a/sys/dev/mcd/mcdreg.h b/sys/dev/mcd/mcdreg.h deleted file mode 100644 index 451fc5de76c3..000000000000 --- a/sys/dev/mcd/mcdreg.h +++ /dev/null @@ -1,219 +0,0 @@ -/*- - * Copyright 1993 by Holger Veit (data part) - * Copyright 1993 by Brian Moore (audio part) - * Changes Copyright 1993 by Gary Clark II - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This software was developed by Holger Veit and Brian Moore - * for use with "386BSD" and similar operating systems. - * "Similar operating systems" includes mainly non-profit oriented - * systems for research and education, including but not restricted to - * "NetBSD", "FreeBSD", "Mach" (by CMU). - * 4. Neither the name of the developer(s) nor the name "386BSD" - * may be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE DEVELOPER(S) ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE DEVELOPER(S) BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, - * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT - * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * This file contains definitions for some cdrom control commands - * and status codes. This info was "inherited" from the DOS MTMCDE.SYS - * driver, and is thus not complete (and may even be wrong). Some day - * the manufacturer or anyone else might provide better documentation, - * so this file (and the driver) will then have a better quality. - * - * $FreeBSD$ - */ - -#ifndef MCD_H -#define MCD_H - -/* toc */ -#define MCD_MAXTOCS 104 /* from the Linux driver */ -#define MCD_LASTPLUS1 170 /* special toc entry */ - -typedef unsigned char bcd_t; -#define M_msf(msf) msf[0] -#define S_msf(msf) msf[1] -#define F_msf(msf) msf[2] - -/* io lines used */ -#define MCD_IO_BASE 0x300 - -#define MCD_REG_COMMAND 0 -#define MCD_REG_STATUS 0 -#define MCD_REG_RDATA 0 - -#define MCD_REG_RESET 1 -#define MCD_REG_CTL2 2 /* XXX Is this right? */ -#define MCD_REG_CONFIG 3 - -#define MCD_MASK_DMA 0x07 /* bits 2-0 = DMA channel */ -#define MCD_MASK_IRQ 0x70 /* bits 6-4 = INT number */ - /* 001 = int 2,9 */ - /* 010 = int 3 */ - /* 011 = int 5 */ - /* 100 = int 10 */ - /* 101 = int 11 */ -/* flags */ -#define MFL_DATA_NOT_AVAIL 0x02 -#define MFL_STATUS_NOT_AVAIL 0x04 - -/* New Commands */ -#define M_RESET 0x00 -#define M_PICKLE 0x04 - -/* ports */ -#define MCD_DATA 0 -#define MCD_FLAGS 1 -#define MCD_CTRL 2 -#define CHANNEL 3 /* XXX ??? */ - -/* Status bits */ -#define MCD_ST_DOOROPEN 0x80 -#define MCD_ST_DSKIN 0x40 -#define MCD_ST_DSKCHNG 0x20 -#define MCD_ST_SPINNING 0x10 -#define MCD_ST_AUDIODISK 0x08 /* Audio Disk is in */ -#define MCD_ST_BUSY 0x04 -#define MCD_ST_AUDIOBSY 0x02 /* Audio Disk is Playing */ -#define MCD_ST_CMDCHECK 0x01 /* Command error */ - -/* commands known by the controller */ -#define MCD_CMDRESET 0x00 -#define MCD_CMDGETVOLINFO 0x10 /* gets mcd_volinfo */ -#define MCD_CMDGETDISKINFO 0x11 /* gets mcd_disk information */ -#define MCD_CMDGETQCHN 0x20 /* gets mcd_qchninfo */ -#define MCD_CMDGETSENSE 0x30 /* gets sense info */ -#define MCD_CMDGETSTAT 0x40 /* gets a byte of status */ - -#define MCD_CMDSETMODE 0x50 /* set transmission mode, needs byte */ - -#define MCD_MDBIT_TESTMODE 0x80 /* 0 = DATALENGTH setting is valid */ -#define MCD_MDBIT_DATALENGTH 0x40 /* 0 = Read User Data Only */ - /* 1 = Read Raw sectors (2352 bytes) */ - -#define MCDBLK 2048 /* for cooked mode */ -#define MCDRBLK sizeof(struct mcd_rawsector) /* for raw mode */ - -#define MCD_MDBIT_ECCMODE 0x20 /* 0 = Use secondary correction */ - /* 1 = Don't use secondary ECC */ -#define MCD_MDBIT_SPINDOWN 0x08 /* 0 = Spin Up, 1 = Spin Down */ -#define MCD_MDBIT_GET_TOC 0x04 /* 0 = Get UPC on next GETQCHAN */ - /* 1 = Get TOC on GETQCHAN */ -#define MCD_MDBIT_MUTEDATA 0x01 /* 1 = Don't play back Data as audio */ - -#define MCD_MD_RAW (MCD_MDBIT_DATALENGTH|MCD_MDBIT_ECCMODE|MCD_MDBIT_MUTEDATA) -#define MCD_MD_COOKED (MCD_MDBIT_MUTEDATA) -#define MCD_MD_TOC (MCD_MDBIT_GET_TOC|MCD_MDBIT_MUTEDATA) - -#define MCD_CMDSTOPAUDIO 0x70 -#define MCD_CMDSTOPAUDIOTIME 0x80 -#define MCD_CMDGETVOLUME 0x8E /* gets mcd_volume */ -#define MCD_CMDSETDRIVEMODE 0xA0 /* Set drive mode */ -#define MCD_READUPC 0xA2 /* Get UPC info */ -#define MCD_CMDSETVOLUME 0xAE /* sets mcd_volume */ -#define MCD_CMDREAD1 0xB0 /* read n sectors */ -#define MCD_CMDSINGLESPEEDREAD 0xC0 /* read from-to */ -#define MCD_CMDSTARTAUDIOMSF 0xC1 /* read audio data */ -#define MCD_CMDDOUBLESPEEDREAD 0xC1 /* Read lots of data from the drive */ -#define MCD_CMDGETDRIVEMODE 0xC2 /* Get the drive mode */ -#define MCD_CMDREAD 0xC3 /* Read data from the drive */ -#define MCD_CMDSETINTERLEAVE 0xC8 /* Adjust the interleave */ -#define MCD_CMDCONTINFO 0xDC /* Get controller info */ -#define MCD_CMDSTOP 0xF0 /* Stop everything */ -#define MCD_CMDEJECTDISK 0xF6 -#define MCD_CMDCLOSETRAY 0xF8 - -#define MCD_CMDLOCKDRV 0xFE /* needs byte */ -#define MCD_LK_UNLOCK 0x00 -#define MCD_LK_LOCK 0x01 -#define MCD_LK_TEST 0x02 - -/* DMA Enable Stuff */ -#define MCD_DMA_IRQFLAGS 0x10 /* Set data0 for IRQ click */ - -#define MCD_DMA_PREIRQ 0x01 /* All of these are for */ -#define MCD_DMA_POSTIRQ 0x02 /* MCD_DMA_IRQFLAG... */ -#define MCD_DMA_ERRIRQ 0x04 /* */ - -#define MCD_DMA_TIMEOUT 0x08 /* Set data0 for DMA timeout */ -#define MCD_DMA_UPCFLAG 0x04 /* 1 = Next command will be READUPC */ - -#define MCD_DMA_DMAMODE 0x02 /* 1 = Data uses DMA */ -#define MCD_DMA_TRANSFERLENGTH 0x01 /* data0 = MSB, data1 = LSB of block length */ - -struct mcd_dma_mode { - u_char dma_mode; - u_char data0; /* If dma_mode & 0x10: Use IRQ settings */ - u_char data1; /* Used if dma_mode & 0x01 */ -} __packed; - -struct mcd_volinfo { - bcd_t trk_low; - bcd_t trk_high; - bcd_t vol_msf[3]; - bcd_t trk1_msf[3]; -} __packed; - -struct mcd_qchninfo { - u_char addr_type:4; - u_char control:4; - u_char trk_no; - u_char idx_no; - bcd_t trk_size_msf[3]; - u_char :8; - bcd_t hd_pos_msf[3]; -} __packed; - -struct mcd_volume { - u_char v0l; - u_char v0rs; - u_char v0r; - u_char v0ls; -} __packed; - -struct mcd_holdtime { - u_char units_of_ten_seconds; - /* If this is 0, the default (12) is used */ -} __packed; - -struct mcd_read1 { - bcd_t start_msf[3]; - u_char nsec[3]; -} __packed; - -struct mcd_read2 { - bcd_t start_msf[3]; - bcd_t end_msf[3]; -} __packed; - -struct mcd_rawsector { - u_char sync1[12]; - u_char header[4]; - u_char subheader1[4]; - u_char subheader2[4]; - u_char data[MCDBLK]; - u_char ecc_bits[280]; -} __packed; - -#endif /* MCD_H */ diff --git a/sys/dev/mcd/mcdvar.h b/sys/dev/mcd/mcdvar.h deleted file mode 100644 index 71e45d0b0bd2..000000000000 --- a/sys/dev/mcd/mcdvar.h +++ /dev/null @@ -1,71 +0,0 @@ -/* - * $FreeBSD$ - */ - -struct mcd_mbx { - short retry; - short nblk; - int sz; - u_long skip; - struct bio * bp; - short count; - short mode; -}; - -struct mcd_data { - short type; - char * name; - short config; - short flags; - u_char read_command; - short status; - int blksize; - u_long disksize; - int partflags; - int openflags; - struct mcd_volinfo volinfo; - struct mcd_qchninfo toc[MCD_MAXTOCS]; - short audio_status; - short curr_mode; - struct mcd_read2 lastpb; - short debug; - struct bio_queue_head head; /* head of bio queue */ - struct mcd_mbx mbx; -}; - -struct mcd_softc { - device_t dev; - struct cdev *mcd_dev_t; - int debug; - - struct resource * port; - int port_rid; - int port_type; - - struct resource * irq; - int irq_rid; - int irq_type; - void * irq_ih; - - struct resource * drq; - int drq_rid; - int drq_type; - - struct mtx mtx; - - struct callout timer; - int ch_state; - struct mcd_mbx * ch_mbxsave; - - struct mcd_data data; -}; - -#define MCD_LOCK(_sc) mtx_lock(&_sc->mtx) -#define MCD_UNLOCK(_sc) mtx_unlock(&_sc->mtx) -#define MCD_ASSERT_LOCKED(_sc) mtx_assert(&_sc->mtx, MA_OWNED) - -#define MCD_READ(_sc, _reg) bus_read_1(_sc->port, _reg) -#define MCD_WRITE(_sc, _reg, _val) bus_write_1(_sc->port, _reg, _val) - -int mcd_probe (struct mcd_softc *); -int mcd_attach (struct mcd_softc *); diff --git a/sys/dev/mlx5/mlx5_en/en.h b/sys/dev/mlx5/mlx5_en/en.h index e894a3fa6251..0ed0a0e12380 100644 --- a/sys/dev/mlx5/mlx5_en/en.h +++ b/sys/dev/mlx5/mlx5_en/en.h @@ -59,10 +59,6 @@ #include <machine/bus.h> -#ifdef HAVE_TURBO_LRO -#include "tcp_tlro.h" -#endif - #include <dev/mlx5/driver.h> #include <dev/mlx5/qp.h> #include <dev/mlx5/cq.h> @@ -460,11 +456,7 @@ struct mlx5e_rq { struct ifnet *ifp; struct mlx5e_rq_stats stats; struct mlx5e_cq cq; -#ifdef HAVE_TURBO_LRO - struct tlro_ctrl lro; -#else struct lro_ctrl lro; -#endif volatile int enabled; int ix; diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c index 4a8029bd95d9..2136567d2e15 100644 --- a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c +++ b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c @@ -666,10 +666,15 @@ mlx5e_create_rq(struct mlx5e_channel *c, } wq_sz = mlx5_wq_ll_get_size(&rq->wq); + + err = -tcp_lro_init_args(&rq->lro, c->ifp, TCP_LRO_ENTRIES, wq_sz); + if (err) + goto err_rq_wq_destroy; + rq->mbuf = malloc(wq_sz * sizeof(rq->mbuf[0]), M_MLX5EN, M_WAITOK | M_ZERO); if (rq->mbuf == NULL) { err = -ENOMEM; - goto err_rq_wq_destroy; + goto err_lro_init; } for (i = 0; i != wq_sz; i++) { struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, i); @@ -694,20 +699,12 @@ mlx5e_create_rq(struct mlx5e_channel *c, mlx5e_create_stats(&rq->stats.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet), buffer, mlx5e_rq_stats_desc, MLX5E_RQ_STATS_NUM, rq->stats.arg); - -#ifdef HAVE_TURBO_LRO - if (tcp_tlro_init(&rq->lro, c->ifp, MLX5E_BUDGET_MAX) != 0) - rq->lro.mbuf = NULL; -#else - if (tcp_lro_init(&rq->lro)) - rq->lro.lro_cnt = 0; - else - rq->lro.ifp = c->ifp; -#endif return (0); err_rq_mbuf_free: free(rq->mbuf, M_MLX5EN); +err_lro_init: + tcp_lro_free(&rq->lro); err_rq_wq_destroy: mlx5_wq_destroy(&rq->wq_ctrl); err_free_dma_tag: @@ -726,11 +723,8 @@ mlx5e_destroy_rq(struct mlx5e_rq *rq) sysctl_ctx_free(&rq->stats.ctx); /* free leftover LRO packets, if any */ -#ifdef HAVE_TURBO_LRO - tcp_tlro_free(&rq->lro); -#else tcp_lro_free(&rq->lro); -#endif + wq_sz = mlx5_wq_ll_get_size(&rq->wq); for (i = 0; i != wq_sz; i++) { if (rq->mbuf[i].mbuf != NULL) { @@ -3114,6 +3108,13 @@ mlx5e_destroy_ifp(struct mlx5_core_dev *mdev, void *vpriv) /* don't allow more IOCTLs */ priv->gone = 1; + /* + * Clear the device description to avoid use after free, + * because the bsddev is not destroyed when this module is + * unloaded: + */ + device_set_desc(mdev->pdev->dev.bsddev, NULL); + /* XXX wait a bit to allow IOCTL handlers to complete */ pause("W", hz); diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c index 6bc2c17370a6..5c793473e63c 100644 --- a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c +++ b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c @@ -369,15 +369,9 @@ mlx5e_poll_rx_cq(struct mlx5e_rq *rq, int budget) mlx5e_build_rx_mbuf(cqe, rq, mb, byte_cnt); rq->stats.packets++; -#ifdef HAVE_TURBO_LRO - if (mb->m_pkthdr.csum_flags == 0 || - (rq->ifp->if_capenable & IFCAP_LRO) == 0 || - rq->lro.mbuf == NULL) { - /* normal input */ - rq->ifp->if_input(rq->ifp, mb); - } else { - tcp_tlro_rx(&rq->lro, mb); - } + +#if !defined(HAVE_TCP_LRO_RX) + tcp_lro_queue_mbuf(&rq->lro, mb); #else if (mb->m_pkthdr.csum_flags == 0 || (rq->ifp->if_capenable & IFCAP_LRO) == 0 || @@ -395,9 +389,6 @@ wq_ll_pop: /* ensure cq space is freed before enabling more cqes */ wmb(); -#ifndef HAVE_TURBO_LRO - tcp_lro_flush_all(&rq->lro); -#endif return (i); } @@ -437,8 +428,6 @@ mlx5e_rx_cq_comp(struct mlx5_core_cq *mcq) } mlx5e_post_rx_wqes(rq); mlx5e_cq_arm(&rq->cq); -#ifdef HAVE_TURBO_LRO - tcp_tlro_flush(&rq->lro, 1); -#endif + tcp_lro_flush_all(&rq->lro); mtx_unlock(&rq->mtx); } diff --git a/sys/dev/mlx5/mlx5_en/tcp_tlro.c b/sys/dev/mlx5/mlx5_en/tcp_tlro.c deleted file mode 100644 index 4b8190455636..000000000000 --- a/sys/dev/mlx5/mlx5_en/tcp_tlro.c +++ /dev/null @@ -1,697 +0,0 @@ -/*- - * Copyright (c) 2015 Mellanox Technologies. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include <sys/cdefs.h> -__FBSDID("$FreeBSD$"); - -#include "opt_inet.h" -#include "opt_inet6.h" - -#include <sys/param.h> -#include <sys/libkern.h> -#include <sys/mbuf.h> -#include <sys/lock.h> -#include <sys/mutex.h> -#include <sys/sysctl.h> -#include <sys/malloc.h> -#include <sys/kernel.h> -#include <sys/endian.h> -#include <sys/socket.h> -#include <sys/sockopt.h> -#include <sys/smp.h> - -#include <net/if.h> -#include <net/if_var.h> -#include <net/ethernet.h> - -#if defined(INET) || defined(INET6) -#include <netinet/in.h> -#endif - -#ifdef INET -#include <netinet/ip.h> -#endif - -#ifdef INET6 -#include <netinet/ip6.h> -#endif - -#include <netinet/tcp_var.h> - -#include "tcp_tlro.h" - -#ifndef M_HASHTYPE_LRO_TCP -#ifndef KLD_MODULE -#warning "M_HASHTYPE_LRO_TCP is not defined" -#endif -#define M_HASHTYPE_LRO_TCP 254 -#endif - -static SYSCTL_NODE(_net_inet_tcp, OID_AUTO, tlro, - CTLFLAG_RW, 0, "TCP turbo LRO parameters"); - -static MALLOC_DEFINE(M_TLRO, "TLRO", "Turbo LRO"); - -static int tlro_min_rate = 20; /* Hz */ - -SYSCTL_INT(_net_inet_tcp_tlro, OID_AUTO, min_rate, CTLFLAG_RWTUN, - &tlro_min_rate, 0, "Minimum serving rate in Hz"); - -static int tlro_max_packet = IP_MAXPACKET; - -SYSCTL_INT(_net_inet_tcp_tlro, OID_AUTO, max_packet, CTLFLAG_RWTUN, - &tlro_max_packet, 0, "Maximum packet size in bytes"); - -typedef struct { - uint32_t value; -} __packed uint32_p_t; - -static uint16_t -tcp_tlro_csum(const uint32_p_t *p, size_t l) -{ - const uint32_p_t *pend = p + (l / 4); - uint64_t cs; - - for (cs = 0; p != pend; p++) - cs += le32toh(p->value); - while (cs > 0xffff) - cs = (cs >> 16) + (cs & 0xffff); - return (cs); -} - -static void * -tcp_tlro_get_header(const struct mbuf *m, const u_int off, - const u_int len) -{ - if (m->m_len < (off + len)) - return (NULL); - return (mtod(m, char *) + off); -} - -static uint8_t -tcp_tlro_info_save_timestamp(struct tlro_mbuf_data *pinfo) -{ - struct tcphdr *tcp = pinfo->tcp; - uint32_t *ts_ptr; - - if (tcp->th_off < ((TCPOLEN_TSTAMP_APPA + sizeof(*tcp)) >> 2)) - return (0); - - ts_ptr = (uint32_t *)(tcp + 1); - if (*ts_ptr != ntohl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) | - (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP)) - return (0); - - /* Save timestamps */ - pinfo->tcp_ts = ts_ptr[1]; - pinfo->tcp_ts_reply = ts_ptr[2]; - return (1); -} - -static void -tcp_tlro_info_restore_timestamp(struct tlro_mbuf_data *pinfoa, - struct tlro_mbuf_data *pinfob) -{ - struct tcphdr *tcp = pinfoa->tcp; - uint32_t *ts_ptr; - - if (tcp->th_off < ((TCPOLEN_TSTAMP_APPA + sizeof(*tcp)) >> 2)) - return; - - ts_ptr = (uint32_t *)(tcp + 1); - if (*ts_ptr != ntohl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) | - (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP)) - return; - - /* Restore timestamps */ - ts_ptr[1] = pinfob->tcp_ts; - ts_ptr[2] = pinfob->tcp_ts_reply; -} - -static void -tcp_tlro_extract_header(struct tlro_mbuf_data *pinfo, struct mbuf *m, int seq) -{ - uint8_t *phdr = (uint8_t *)pinfo->buf; - struct ether_header *eh; - struct ether_vlan_header *vlan; -#ifdef INET - struct ip *ip; -#endif -#ifdef INET6 - struct ip6_hdr *ip6; -#endif - struct tcphdr *tcp; - uint16_t etype; - int diff; - int off; - - /* Fill in information */ - pinfo->head = m; - pinfo->last_tick = ticks; - pinfo->sequence = seq; - pinfo->pprev = &m_last(m)->m_next; - - off = sizeof(*eh); - if (m->m_len < off) - goto error; - eh = tcp_tlro_get_header(m, 0, sizeof(*eh)); - if (eh == NULL) - goto error; - memcpy(phdr, &eh->ether_dhost, ETHER_ADDR_LEN); - phdr += ETHER_ADDR_LEN; - memcpy(phdr, &eh->ether_type, sizeof(eh->ether_type)); - phdr += sizeof(eh->ether_type); - etype = ntohs(eh->ether_type); - - if (etype == ETHERTYPE_VLAN) { - vlan = tcp_tlro_get_header(m, off, sizeof(*vlan)); - if (vlan == NULL) - goto error; - memcpy(phdr, &vlan->evl_tag, sizeof(vlan->evl_tag) + - sizeof(vlan->evl_proto)); - phdr += sizeof(vlan->evl_tag) + sizeof(vlan->evl_proto); - etype = ntohs(vlan->evl_proto); - off += sizeof(*vlan) - sizeof(*eh); - } - switch (etype) { -#ifdef INET - case ETHERTYPE_IP: - /* - * Cannot LRO: - * - Non-IP packets - * - Fragmented packets - * - Packets with IPv4 options - * - Non-TCP packets - */ - ip = tcp_tlro_get_header(m, off, sizeof(*ip)); - if (ip == NULL || - (ip->ip_off & htons(IP_MF | IP_OFFMASK)) != 0 || - (ip->ip_p != IPPROTO_TCP) || - (ip->ip_hl << 2) != sizeof(*ip)) - goto error; - - /* Legacy IP has a header checksum that needs to be correct */ - if (!(m->m_pkthdr.csum_flags & CSUM_IP_CHECKED)) { - /* Verify IP header */ - if (tcp_tlro_csum((uint32_p_t *)ip, sizeof(*ip)) != 0xFFFF) - m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED; - else - m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED | - CSUM_IP_VALID; - } - /* Only accept valid checksums */ - if (!(m->m_pkthdr.csum_flags & CSUM_IP_VALID) || - !(m->m_pkthdr.csum_flags & CSUM_DATA_VALID)) - goto error; - memcpy(phdr, &ip->ip_src, sizeof(ip->ip_src) + - sizeof(ip->ip_dst)); - phdr += sizeof(ip->ip_src) + sizeof(ip->ip_dst); - if (M_HASHTYPE_GET(m) == M_HASHTYPE_LRO_TCP) - pinfo->ip_len = m->m_pkthdr.len - off; - else - pinfo->ip_len = ntohs(ip->ip_len); - pinfo->ip_hdrlen = sizeof(*ip); - pinfo->ip.v4 = ip; - pinfo->ip_version = 4; - off += sizeof(*ip); - break; -#endif -#ifdef INET6 - case ETHERTYPE_IPV6: - /* - * Cannot LRO: - * - Non-IP packets - * - Packets with IPv6 options - * - Non-TCP packets - */ - ip6 = tcp_tlro_get_header(m, off, sizeof(*ip6)); - if (ip6 == NULL || ip6->ip6_nxt != IPPROTO_TCP) - goto error; - if (!(m->m_pkthdr.csum_flags & CSUM_DATA_VALID)) - goto error; - memcpy(phdr, &ip6->ip6_src, sizeof(struct in6_addr) + - sizeof(struct in6_addr)); - phdr += sizeof(struct in6_addr) + sizeof(struct in6_addr); - if (M_HASHTYPE_GET(m) == M_HASHTYPE_LRO_TCP) - pinfo->ip_len = m->m_pkthdr.len - off; - else - pinfo->ip_len = ntohs(ip6->ip6_plen) + sizeof(*ip6); - pinfo->ip_hdrlen = sizeof(*ip6); - pinfo->ip.v6 = ip6; - pinfo->ip_version = 6; - off += sizeof(*ip6); - break; -#endif - default: - goto error; - } - tcp = tcp_tlro_get_header(m, off, sizeof(*tcp)); - if (tcp == NULL) - goto error; - memcpy(phdr, &tcp->th_sport, sizeof(tcp->th_sport) + - sizeof(tcp->th_dport)); - phdr += sizeof(tcp->th_sport) + - sizeof(tcp->th_dport); - /* Store TCP header length */ - *phdr++ = tcp->th_off; - if (tcp->th_off < (sizeof(*tcp) >> 2)) - goto error; - - /* Compute offset to data payload */ - pinfo->tcp_len = (tcp->th_off << 2); - off += pinfo->tcp_len; - - /* Store more info */ - pinfo->data_off = off; - pinfo->tcp = tcp; - - /* Try to save timestamp, if any */ - *phdr++ = tcp_tlro_info_save_timestamp(pinfo); - - /* Verify offset and IP/TCP length */ - if (off > m->m_pkthdr.len || - pinfo->ip_len < pinfo->tcp_len) - goto error; - - /* Compute data payload length */ - pinfo->data_len = (pinfo->ip_len - pinfo->tcp_len - pinfo->ip_hdrlen); - - /* Trim any padded data */ - diff = (m->m_pkthdr.len - off) - pinfo->data_len; - if (diff != 0) { - if (diff < 0) - goto error; - else - m_adj(m, -diff); - } - /* Compute header length */ - pinfo->buf_length = phdr - (uint8_t *)pinfo->buf; - /* Zero-pad rest of buffer */ - memset(phdr, 0, TLRO_MAX_HEADER - pinfo->buf_length); - return; -error: - pinfo->buf_length = 0; -} - -static int -tcp_tlro_cmp64(const uint64_t *pa, const uint64_t *pb) -{ - int64_t diff = 0; - unsigned x; - - for (x = 0; x != TLRO_MAX_HEADER / 8; x++) { - /* - * NOTE: Endianness does not matter in this - * comparisation: - */ - diff = pa[x] - pb[x]; - if (diff != 0) - goto done; - } -done: - if (diff < 0) - return (-1); - else if (diff > 0) - return (1); - return (0); -} - -static int -tcp_tlro_compare_header(const void *_ppa, const void *_ppb) -{ - const struct tlro_mbuf_ptr *ppa = _ppa; - const struct tlro_mbuf_ptr *ppb = _ppb; - struct tlro_mbuf_data *pinfoa = ppa->data; - struct tlro_mbuf_data *pinfob = ppb->data; - int ret; - - ret = (pinfoa->head == NULL) - (pinfob->head == NULL); - if (ret != 0) - goto done; - - ret = pinfoa->buf_length - pinfob->buf_length; - if (ret != 0) - goto done; - if (pinfoa->buf_length != 0) { - ret = tcp_tlro_cmp64(pinfoa->buf, pinfob->buf); - if (ret != 0) - goto done; - ret = ntohl(pinfoa->tcp->th_seq) - ntohl(pinfob->tcp->th_seq); - if (ret != 0) - goto done; - ret = ntohl(pinfoa->tcp->th_ack) - ntohl(pinfob->tcp->th_ack); - if (ret != 0) - goto done; - ret = pinfoa->sequence - pinfob->sequence; - if (ret != 0) - goto done; - } -done: - return (ret); -} - -static void -tcp_tlro_sort(struct tlro_ctrl *tlro) -{ - if (tlro->curr == 0) - return; - - qsort(tlro->mbuf, tlro->curr, sizeof(struct tlro_mbuf_ptr), - &tcp_tlro_compare_header); -} - -static int -tcp_tlro_get_ticks(void) -{ - int to = tlro_min_rate; - - if (to < 1) - to = 1; - to = hz / to; - if (to < 1) - to = 1; - return (to); -} - -static void -tcp_tlro_combine(struct tlro_ctrl *tlro, int force) -{ - struct tlro_mbuf_data *pinfoa; - struct tlro_mbuf_data *pinfob; - uint32_t cs; - int curr_ticks = ticks; - int ticks_limit = tcp_tlro_get_ticks(); - unsigned x; - unsigned y; - unsigned z; - int temp; - - if (tlro->curr == 0) - return; - - for (y = 0; y != tlro->curr;) { - struct mbuf *m; - - pinfoa = tlro->mbuf[y].data; - for (x = y + 1; x != tlro->curr; x++) { - pinfob = tlro->mbuf[x].data; - if (pinfoa->buf_length != pinfob->buf_length || - tcp_tlro_cmp64(pinfoa->buf, pinfob->buf) != 0) - break; - } - if (pinfoa->buf_length == 0) { - /* Forward traffic which cannot be combined */ - for (z = y; z != x; z++) { - /* Just forward packets */ - pinfob = tlro->mbuf[z].data; - - m = pinfob->head; - - /* Reset info structure */ - pinfob->head = NULL; - pinfob->buf_length = 0; - - /* Do stats */ - tlro->lro_flushed++; - - /* Input packet to network layer */ - (*tlro->ifp->if_input) (tlro->ifp, m); - } - y = z; - continue; - } - - /* Compute current checksum subtracted some header parts */ - temp = (pinfoa->ip_len - pinfoa->ip_hdrlen); - cs = ((temp & 0xFF) << 8) + ((temp & 0xFF00) >> 8) + - tcp_tlro_csum((uint32_p_t *)pinfoa->tcp, pinfoa->tcp_len); - - /* Append all fragments into one block */ - for (z = y + 1; z != x; z++) { - - pinfob = tlro->mbuf[z].data; - - /* Check for command packets */ - if ((pinfoa->tcp->th_flags & ~(TH_ACK | TH_PUSH)) || - (pinfob->tcp->th_flags & ~(TH_ACK | TH_PUSH))) - break; - - /* Check if there is enough space */ - if ((pinfoa->ip_len + pinfob->data_len) > tlro_max_packet) - break; - - /* Try to append the new segment */ - temp = ntohl(pinfoa->tcp->th_seq) + pinfoa->data_len; - if (temp != (int)ntohl(pinfob->tcp->th_seq)) - break; - - temp = pinfob->ip_len - pinfob->ip_hdrlen; - cs += ((temp & 0xFF) << 8) + ((temp & 0xFF00) >> 8) + - tcp_tlro_csum((uint32_p_t *)pinfob->tcp, pinfob->tcp_len); - /* Remove fields which appear twice */ - cs += (IPPROTO_TCP << 8); - if (pinfob->ip_version == 4) { - cs += tcp_tlro_csum((uint32_p_t *)&pinfob->ip.v4->ip_src, 4); - cs += tcp_tlro_csum((uint32_p_t *)&pinfob->ip.v4->ip_dst, 4); - } else { - cs += tcp_tlro_csum((uint32_p_t *)&pinfob->ip.v6->ip6_src, 16); - cs += tcp_tlro_csum((uint32_p_t *)&pinfob->ip.v6->ip6_dst, 16); - } - /* Remainder computation */ - while (cs > 0xffff) - cs = (cs >> 16) + (cs & 0xffff); - - /* Update window and ack sequence number */ - pinfoa->tcp->th_ack = pinfob->tcp->th_ack; - pinfoa->tcp->th_win = pinfob->tcp->th_win; - - /* Check if we should restore the timestamp */ - tcp_tlro_info_restore_timestamp(pinfoa, pinfob); - - /* Accumulate TCP flags */ - pinfoa->tcp->th_flags |= pinfob->tcp->th_flags; - - /* update lengths */ - pinfoa->ip_len += pinfob->data_len; - pinfoa->data_len += pinfob->data_len; - - /* Clear mbuf pointer - packet is accumulated */ - m = pinfob->head; - - /* Reset info structure */ - pinfob->head = NULL; - pinfob->buf_length = 0; - - /* Append data to mbuf [y] */ - m_adj(m, pinfob->data_off); - /* Delete mbuf tags, if any */ - m_tag_delete_chain(m, NULL); - /* Clear packet header flag */ - m->m_flags &= ~M_PKTHDR; - - /* Concat mbuf(s) to end of list */ - pinfoa->pprev[0] = m; - m = m_last(m); - pinfoa->pprev = &m->m_next; - pinfoa->head->m_pkthdr.len += pinfob->data_len; - } - /* Compute new TCP header checksum */ - pinfoa->tcp->th_sum = 0; - - temp = pinfoa->ip_len - pinfoa->ip_hdrlen; - cs = (cs ^ 0xFFFF) + - tcp_tlro_csum((uint32_p_t *)pinfoa->tcp, pinfoa->tcp_len) + - ((temp & 0xFF) << 8) + ((temp & 0xFF00) >> 8); - - /* Remainder computation */ - while (cs > 0xffff) - cs = (cs >> 16) + (cs & 0xffff); - - /* Update new checksum */ - pinfoa->tcp->th_sum = ~htole16(cs); - - /* Update IP length, if any */ - if (pinfoa->ip_version == 4) { - if (pinfoa->ip_len > IP_MAXPACKET) { - M_HASHTYPE_SET(pinfoa->head, M_HASHTYPE_LRO_TCP); - pinfoa->ip.v4->ip_len = htons(IP_MAXPACKET); - } else { - pinfoa->ip.v4->ip_len = htons(pinfoa->ip_len); - } - } else { - if (pinfoa->ip_len > (IP_MAXPACKET + sizeof(*pinfoa->ip.v6))) { - M_HASHTYPE_SET(pinfoa->head, M_HASHTYPE_LRO_TCP); - pinfoa->ip.v6->ip6_plen = htons(IP_MAXPACKET); - } else { - temp = pinfoa->ip_len - sizeof(*pinfoa->ip.v6); - pinfoa->ip.v6->ip6_plen = htons(temp); - } - } - - temp = curr_ticks - pinfoa->last_tick; - /* Check if packet should be forwarded */ - if (force != 0 || z != x || temp >= ticks_limit || - pinfoa->data_len == 0) { - - /* Compute new IPv4 header checksum */ - if (pinfoa->ip_version == 4) { - pinfoa->ip.v4->ip_sum = 0; - cs = tcp_tlro_csum((uint32_p_t *)pinfoa->ip.v4, - sizeof(*pinfoa->ip.v4)); - pinfoa->ip.v4->ip_sum = ~htole16(cs); - } - /* Forward packet */ - m = pinfoa->head; - - /* Reset info structure */ - pinfoa->head = NULL; - pinfoa->buf_length = 0; - - /* Do stats */ - tlro->lro_flushed++; - - /* Input packet to network layer */ - (*tlro->ifp->if_input) (tlro->ifp, m); - } - y = z; - } - - /* Cleanup all NULL heads */ - for (y = 0; y != tlro->curr; y++) { - if (tlro->mbuf[y].data->head == NULL) { - for (z = y + 1; z != tlro->curr; z++) { - struct tlro_mbuf_ptr ptemp; - if (tlro->mbuf[z].data->head == NULL) - continue; - ptemp = tlro->mbuf[y]; - tlro->mbuf[y] = tlro->mbuf[z]; - tlro->mbuf[z] = ptemp; - y++; - } - break; - } - } - tlro->curr = y; -} - -static void -tcp_tlro_cleanup(struct tlro_ctrl *tlro) -{ - while (tlro->curr != 0 && - tlro->mbuf[tlro->curr - 1].data->head == NULL) - tlro->curr--; -} - -void -tcp_tlro_flush(struct tlro_ctrl *tlro, int force) -{ - if (tlro->curr == 0) - return; - - tcp_tlro_sort(tlro); - tcp_tlro_cleanup(tlro); - tcp_tlro_combine(tlro, force); -} - -int -tcp_tlro_init(struct tlro_ctrl *tlro, struct ifnet *ifp, - int max_mbufs) -{ - ssize_t size; - uint32_t x; - - /* Set zero defaults */ - memset(tlro, 0, sizeof(*tlro)); - - /* Compute size needed for data */ - size = (sizeof(struct tlro_mbuf_ptr) * max_mbufs) + - (sizeof(struct tlro_mbuf_data) * max_mbufs); - - /* Range check */ - if (max_mbufs <= 0 || size <= 0 || ifp == NULL) - return (EINVAL); - - /* Setup tlro control structure */ - tlro->mbuf = malloc(size, M_TLRO, M_WAITOK | M_ZERO); - tlro->max = max_mbufs; - tlro->ifp = ifp; - - /* Setup pointer array */ - for (x = 0; x != tlro->max; x++) { - tlro->mbuf[x].data = ((struct tlro_mbuf_data *) - &tlro->mbuf[max_mbufs]) + x; - } - return (0); -} - -void -tcp_tlro_free(struct tlro_ctrl *tlro) -{ - struct tlro_mbuf_data *pinfo; - struct mbuf *m; - uint32_t y; - - /* Check if not setup */ - if (tlro->mbuf == NULL) - return; - /* Free MBUF array and any leftover MBUFs */ - for (y = 0; y != tlro->max; y++) { - - pinfo = tlro->mbuf[y].data; - - m = pinfo->head; - - /* Reset info structure */ - pinfo->head = NULL; - pinfo->buf_length = 0; - - m_freem(m); - } - free(tlro->mbuf, M_TLRO); - /* Reset buffer */ - memset(tlro, 0, sizeof(*tlro)); -} - -void -tcp_tlro_rx(struct tlro_ctrl *tlro, struct mbuf *m) -{ - if (m->m_len > 0 && tlro->curr < tlro->max) { - /* do stats */ - tlro->lro_queued++; - - /* extract header */ - tcp_tlro_extract_header(tlro->mbuf[tlro->curr++].data, - m, tlro->sequence++); - } else if (tlro->ifp != NULL) { - /* do stats */ - tlro->lro_flushed++; - - /* input packet to network layer */ - (*tlro->ifp->if_input) (tlro->ifp, m); - } else { - /* packet drop */ - m_freem(m); - } -} diff --git a/sys/dev/mlx5/mlx5_en/tcp_tlro.h b/sys/dev/mlx5/mlx5_en/tcp_tlro.h deleted file mode 100644 index 40ebdf681902..000000000000 --- a/sys/dev/mlx5/mlx5_en/tcp_tlro.h +++ /dev/null @@ -1,83 +0,0 @@ -/*- - * Copyright (c) 2015 Mellanox Technologies. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * $FreeBSD$ - */ - -#ifndef _TCP_TLRO_H_ -#define _TCP_TLRO_H_ - -#define TLRO_MAX_HEADER 64 /* bytes */ - -struct ip; -struct ip6_hdr; -struct tcphdr; - -struct tlro_mbuf_data { - union { -#ifdef INET - struct ip *v4; -#endif -#ifdef INET6 - struct ip6_hdr *v6; -#endif - } ip; - struct tcphdr *tcp; - struct mbuf *head; - struct mbuf **pprev; - int last_tick; - int sequence; - int data_len; - int data_off; - int ip_hdrlen; - int ip_len; - uint32_t tcp_ts; - uint32_t tcp_ts_reply; - uint16_t tcp_len; - uint8_t ip_version; - uint8_t buf_length; /* in 32-bit words */ - uint64_t buf[TLRO_MAX_HEADER / 8]; -} __aligned(256); - -struct tlro_mbuf_ptr { - struct tlro_mbuf_data *data; -}; - -/* NB: This is part of driver structs */ -struct tlro_ctrl { - struct ifnet *ifp; - struct tlro_mbuf_ptr *mbuf; - uint64_t lro_queued; - uint64_t lro_flushed; - uint32_t max; - uint32_t curr; - int sequence; -}; - -int tcp_tlro_init(struct tlro_ctrl *, struct ifnet *, int); -void tcp_tlro_free(struct tlro_ctrl *); -void tcp_tlro_flush(struct tlro_ctrl *, int); -void tcp_tlro_rx(struct tlro_ctrl *, struct mbuf *); - -#endif /* _TCP_TLRO_H_ */ diff --git a/sys/dev/mpt/mpt_pci.c b/sys/dev/mpt/mpt_pci.c index b014874e5553..7caf6d4f568b 100644 --- a/sys/dev/mpt/mpt_pci.c +++ b/sys/dev/mpt/mpt_pci.c @@ -654,10 +654,6 @@ mpt_dma_mem_alloc(struct mpt_softc *mpt) len = sizeof (request_t) * MPT_MAX_REQUESTS(mpt); mpt->request_pool = (request_t *)malloc(len, M_DEVBUF, M_WAITOK|M_ZERO); - if (mpt->request_pool == NULL) { - mpt_prt(mpt, "cannot allocate request pool\n"); - return (1); - } /* * Create a parent dma tag for this device. diff --git a/sys/dev/msk/if_msk.c b/sys/dev/msk/if_msk.c index 458d7fdeb43c..cc1351db0c46 100644 --- a/sys/dev/msk/if_msk.c +++ b/sys/dev/msk/if_msk.c @@ -1953,12 +1953,6 @@ mskc_attach(device_t dev) goto fail; } mmd = malloc(sizeof(struct msk_mii_data), M_DEVBUF, M_WAITOK | M_ZERO); - if (mmd == NULL) { - device_printf(dev, "failed to allocate memory for " - "ivars of PORT_A\n"); - error = ENXIO; - goto fail; - } mmd->port = MSK_PORT_A; mmd->pmd = sc->msk_pmd; mmd->mii_flags |= MIIF_DOPAUSE; @@ -1977,12 +1971,6 @@ mskc_attach(device_t dev) } mmd = malloc(sizeof(struct msk_mii_data), M_DEVBUF, M_WAITOK | M_ZERO); - if (mmd == NULL) { - device_printf(dev, "failed to allocate memory for " - "ivars of PORT_B\n"); - error = ENXIO; - goto fail; - } mmd->port = MSK_PORT_B; mmd->pmd = sc->msk_pmd; if (sc->msk_pmd == 'L' || sc->msk_pmd == 'S') diff --git a/sys/dev/nand/nand_generic.c b/sys/dev/nand/nand_generic.c index 59a6bf2001c1..272b9590b1f8 100644 --- a/sys/dev/nand/nand_generic.c +++ b/sys/dev/nand/nand_generic.c @@ -228,8 +228,6 @@ generic_nand_attach(device_t dev) if (ivar->is_onfi) { onfi_chip_params = malloc(sizeof(struct onfi_chip_params), M_NAND, M_WAITOK | M_ZERO); - if (onfi_chip_params == NULL) - return (ENOMEM); if (onfi_read_parameter(chip, onfi_chip_params)) { nand_debug(NDBG_GEN,"Could not read parameter page!\n"); @@ -741,10 +739,6 @@ onfi_is_blk_bad(device_t device, uint32_t block_number, uint8_t *bad) chip = device_get_softc(device); oob = malloc(chip->chip_geom.oob_size, M_NAND, M_WAITOK); - if (!oob) { - device_printf(device, "%s: cannot allocate oob\n", __func__); - return (ENOMEM); - } page_number = block_number * chip->chip_geom.pgs_per_blk; *bad = 0; @@ -1001,10 +995,6 @@ generic_is_blk_bad(device_t dev, uint32_t block, uint8_t *bad) chip = device_get_softc(dev); oob = malloc(chip->chip_geom.oob_size, M_NAND, M_WAITOK); - if (!oob) { - device_printf(dev, "%s: cannot allocate OOB\n", __func__); - return (ENOMEM); - } page_number = block * chip->chip_geom.pgs_per_blk; *bad = 0; diff --git a/sys/dev/nand/nandsim_chip.c b/sys/dev/nand/nandsim_chip.c index 5b568abf58b2..70ccbc9a32b8 100644 --- a/sys/dev/nand/nandsim_chip.c +++ b/sys/dev/nand/nandsim_chip.c @@ -90,8 +90,6 @@ nandsim_chip_init(struct nandsim_softc* sc, uint8_t chip_num, int error; chip = malloc(sizeof(*chip), M_NANDSIM, M_WAITOK | M_ZERO); - if (!chip) - return (NULL); mtx_init(&chip->ns_lock, "nandsim lock", NULL, MTX_DEF); callout_init(&chip->ns_callout, 1); @@ -206,9 +204,6 @@ nandsim_blk_state_init(struct nandsim_chip *chip, uint32_t size, chip->blk_state = malloc(size * sizeof(struct nandsim_block_state), M_NANDSIM, M_WAITOK | M_ZERO); - if (!chip->blk_state) { - return (-1); - } for (i = 0; i < size; i++) { if (wear_lev) diff --git a/sys/dev/netmap/netmap_mem2.c b/sys/dev/netmap/netmap_mem2.c index 023604d49535..fd0c06bb8b57 100644 --- a/sys/dev/netmap/netmap_mem2.c +++ b/sys/dev/netmap/netmap_mem2.c @@ -201,7 +201,7 @@ NMD_DEFNACB(void, rings_delete); static int netmap_mem_map(struct netmap_obj_pool *, struct netmap_adapter *); static int netmap_mem_unmap(struct netmap_obj_pool *, struct netmap_adapter *); -static int nm_mem_assign_group(struct netmap_mem_d *, struct device *); +static int nm_mem_assign_group(struct netmap_mem_d *, device_t); #define NMA_LOCK_INIT(n) NM_MTX_INIT((n)->nm_mtx) #define NMA_LOCK_DESTROY(n) NM_MTX_DESTROY((n)->nm_mtx) @@ -456,7 +456,7 @@ nm_mem_release_id(struct netmap_mem_d *nmd) } static int -nm_mem_assign_group(struct netmap_mem_d *nmd, struct device *dev) +nm_mem_assign_group(struct netmap_mem_d *nmd, device_t dev) { int err = 0, id; id = nm_iommu_group_id(dev); diff --git a/sys/dev/ntb/if_ntb/if_ntb.c b/sys/dev/ntb/if_ntb/if_ntb.c index 5ff1b9ae1764..c67ae0dc682f 100644 --- a/sys/dev/ntb/if_ntb/if_ntb.c +++ b/sys/dev/ntb/if_ntb/if_ntb.c @@ -76,7 +76,7 @@ __FBSDID("$FreeBSD$"); static SYSCTL_NODE(_hw, OID_AUTO, if_ntb, CTLFLAG_RW, 0, "if_ntb"); -static unsigned g_if_ntb_num_queues = 1; +static unsigned g_if_ntb_num_queues = UINT_MAX; SYSCTL_UINT(_hw_if_ntb, OID_AUTO, num_queues, CTLFLAG_RWTUN, &g_if_ntb_num_queues, 0, "Number of queues per interface"); @@ -144,7 +144,8 @@ ntb_net_attach(device_t dev) if_initname(ifp, device_get_name(dev), device_get_unit(dev)); if_setdev(ifp, dev); - sc->num_queues = g_if_ntb_num_queues; + sc->num_queues = min(g_if_ntb_num_queues, + ntb_transport_queue_count(dev)); sc->queues = malloc(sc->num_queues * sizeof(struct ntb_net_queue), M_DEVBUF, M_WAITOK | M_ZERO); sc->mtu = INT_MAX; @@ -152,8 +153,7 @@ ntb_net_attach(device_t dev) q = &sc->queues[i]; q->sc = sc; q->ifp = ifp; - q->qp = ntb_transport_create_queue(q, - device_get_parent(dev), &handlers); + q->qp = ntb_transport_create_queue(dev, i, &handlers, q); if (q->qp == NULL) break; sc->mtu = imin(sc->mtu, ntb_transport_max_size(q->qp)); @@ -167,6 +167,7 @@ ntb_net_attach(device_t dev) callout_init(&q->queue_full, 1); } sc->num_queues = i; + device_printf(dev, "%d queue(s)\n", sc->num_queues); if_setinitfn(ifp, ntb_net_init); if_setsoftc(ifp, sc); @@ -492,10 +493,9 @@ static void create_random_local_eui48(u_char *eaddr) { static uint8_t counter = 0; - uint32_t seed = ticks; eaddr[0] = EUI48_LOCALLY_ADMINISTERED; - memcpy(&eaddr[1], &seed, sizeof(uint32_t)); + arc4rand(&eaddr[1], 4, 0); eaddr[5] = counter++; } diff --git a/sys/dev/ntb/ntb.c b/sys/dev/ntb/ntb.c index 1973cbf0fa28..44c0c618646c 100644 --- a/sys/dev/ntb/ntb.c +++ b/sys/dev/ntb/ntb.c @@ -31,6 +31,8 @@ __FBSDID("$FreeBSD$"); #include <sys/kernel.h> #include <sys/systm.h> #include <sys/bus.h> +#include <sys/rmlock.h> +#include <sys/malloc.h> #include <sys/module.h> #include <sys/sysctl.h> @@ -39,4 +41,422 @@ __FBSDID("$FreeBSD$"); devclass_t ntb_hw_devclass; SYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls"); +struct ntb_child { + device_t dev; + int enabled; + int mwoff; + int mwcnt; + int spadoff; + int spadcnt; + int dboff; + int dbmask; + void *ctx; + const struct ntb_ctx_ops *ctx_ops; + struct rmlock ctx_lock; + struct ntb_child *next; +}; + +int +ntb_register_device(device_t dev) +{ + struct ntb_child **cpp = device_get_softc(dev); + struct ntb_child *nc; + int i, mw, mwu, mwt, spad, spadu, spadt, db, dbu, dbt; + char cfg[128] = ""; + char buf[32]; + char *n, *np, *c, *p, *name; + + mwu = 0; + mwt = NTB_MW_COUNT(dev); + spadu = 0; + spadt = NTB_SPAD_COUNT(dev); + dbu = 0; + dbt = flsll(NTB_DB_VALID_MASK(dev)); + + device_printf(dev, "%d memory windows, %d scratchpads, " + "%d doorbells\n", mwt, spadt, dbt); + + snprintf(buf, sizeof(buf), "hint.%s.%d.config", device_get_name(dev), + device_get_unit(dev)); + TUNABLE_STR_FETCH(buf, cfg, sizeof(cfg)); + n = cfg; + i = 0; + while ((c = strsep(&n, ",")) != NULL) { + np = c; + name = strsep(&np, ":"); + if (name != NULL && name[0] == 0) + name = NULL; + p = strsep(&np, ":"); + mw = (p && p[0] != 0) ? strtol(p, NULL, 10) : mwt - mwu; + p = strsep(&np, ":"); + spad = (p && p[0] != 0) ? strtol(p, NULL, 10) : spadt - spadu; + db = (np && np[0] != 0) ? strtol(np, NULL, 10) : dbt - dbu; + + if (mw > mwt - mwu || spad > spadt - spadu || db > dbt - dbu) { + device_printf(dev, "Not enough resources for config\n"); + break; + } + + nc = malloc(sizeof(*nc), M_DEVBUF, M_WAITOK | M_ZERO); + nc->mwoff = mwu; + nc->mwcnt = mw; + nc->spadoff = spadu; + nc->spadcnt = spad; + nc->dboff = dbu; + nc->dbmask = (db == 0) ? 0 : (0xffffffffffffffff >> (64 - db)); + rm_init(&nc->ctx_lock, "ntb ctx"); + nc->dev = device_add_child(dev, name, -1); + if (nc->dev == NULL) { + ntb_unregister_device(dev); + return (ENOMEM); + } + device_set_ivars(nc->dev, nc); + *cpp = nc; + cpp = &nc->next; + + if (bootverbose) { + device_printf(dev, "%d \"%s\":", i, name); + if (mw > 0) { + printf(" memory windows %d", mwu); + if (mw > 1) + printf("-%d", mwu + mw - 1); + } + if (spad > 0) { + printf(" scratchpads %d", spadu); + if (spad > 1) + printf("-%d", spadu + spad - 1); + } + if (db > 0) { + printf(" doorbells %d", dbu); + if (db > 1) + printf("-%d", dbu + db - 1); + } + printf("\n"); + } + + mwu += mw; + spadu += spad; + dbu += db; + i++; + } + + bus_generic_attach(dev); + return (0); +} + +int +ntb_unregister_device(device_t dev) +{ + struct ntb_child **cpp = device_get_softc(dev); + struct ntb_child *nc; + int error = 0; + + while ((nc = *cpp) != NULL) { + *cpp = (*cpp)->next; + error = device_delete_child(dev, nc->dev); + if (error) + break; + rm_destroy(&nc->ctx_lock); + free(nc, M_DEVBUF); + } + return (error); +} + +void +ntb_link_event(device_t dev) +{ + struct ntb_child **cpp = device_get_softc(dev); + struct ntb_child *nc; + struct rm_priotracker ctx_tracker; + + for (nc = *cpp; nc != NULL; nc = nc->next) { + rm_rlock(&nc->ctx_lock, &ctx_tracker); + if (nc->ctx_ops != NULL && nc->ctx_ops->link_event != NULL) + nc->ctx_ops->link_event(nc->ctx); + rm_runlock(&nc->ctx_lock, &ctx_tracker); + } +} + +void +ntb_db_event(device_t dev, uint32_t vec) +{ + struct ntb_child **cpp = device_get_softc(dev); + struct ntb_child *nc; + struct rm_priotracker ctx_tracker; + + for (nc = *cpp; nc != NULL; nc = nc->next) { + rm_rlock(&nc->ctx_lock, &ctx_tracker); + if (nc->ctx_ops != NULL && nc->ctx_ops->db_event != NULL) + nc->ctx_ops->db_event(nc->ctx, vec); + rm_runlock(&nc->ctx_lock, &ctx_tracker); + } +} + +bool +ntb_link_is_up(device_t ntb, enum ntb_speed *speed, enum ntb_width *width) +{ + + return (NTB_LINK_IS_UP(device_get_parent(ntb), speed, width)); +} + +int +ntb_link_enable(device_t ntb, enum ntb_speed speed, enum ntb_width width) +{ + struct ntb_child *nc = device_get_ivars(ntb); + struct ntb_child **cpp = device_get_softc(device_get_parent(nc->dev)); + struct ntb_child *nc1; + + for (nc1 = *cpp; nc1 != NULL; nc1 = nc1->next) { + if (nc1->enabled) { + nc->enabled = 1; + return (0); + } + } + nc->enabled = 1; + return (NTB_LINK_ENABLE(device_get_parent(ntb), speed, width)); +} + +int +ntb_link_disable(device_t ntb) +{ + struct ntb_child *nc = device_get_ivars(ntb); + struct ntb_child **cpp = device_get_softc(device_get_parent(nc->dev)); + struct ntb_child *nc1; + + if (!nc->enabled) + return (0); + nc->enabled = 0; + for (nc1 = *cpp; nc1 != NULL; nc1 = nc1->next) { + if (nc1->enabled) + return (0); + } + return (NTB_LINK_DISABLE(device_get_parent(ntb))); +} + +bool +ntb_link_enabled(device_t ntb) +{ + struct ntb_child *nc = device_get_ivars(ntb); + + return (nc->enabled && NTB_LINK_ENABLED(device_get_parent(ntb))); +} + +int +ntb_set_ctx(device_t ntb, void *ctx, const struct ntb_ctx_ops *ctx_ops) +{ + struct ntb_child *nc = device_get_ivars(ntb); + + if (ctx == NULL || ctx_ops == NULL) + return (EINVAL); + + rm_wlock(&nc->ctx_lock); + if (nc->ctx_ops != NULL) { + rm_wunlock(&nc->ctx_lock); + return (EINVAL); + } + nc->ctx = ctx; + nc->ctx_ops = ctx_ops; + rm_wunlock(&nc->ctx_lock); + + return (0); +} + +void * +ntb_get_ctx(device_t ntb, const struct ntb_ctx_ops **ctx_ops) +{ + struct ntb_child *nc = device_get_ivars(ntb); + + KASSERT(nc->ctx != NULL && nc->ctx_ops != NULL, ("bogus")); + if (ctx_ops != NULL) + *ctx_ops = nc->ctx_ops; + return (nc->ctx); +} + +void +ntb_clear_ctx(device_t ntb) +{ + struct ntb_child *nc = device_get_ivars(ntb); + + rm_wlock(&nc->ctx_lock); + nc->ctx = NULL; + nc->ctx_ops = NULL; + rm_wunlock(&nc->ctx_lock); +} + +uint8_t +ntb_mw_count(device_t ntb) +{ + struct ntb_child *nc = device_get_ivars(ntb); + + return (nc->mwcnt); +} + +int +ntb_mw_get_range(device_t ntb, unsigned mw_idx, vm_paddr_t *base, + caddr_t *vbase, size_t *size, size_t *align, size_t *align_size, + bus_addr_t *plimit) +{ + struct ntb_child *nc = device_get_ivars(ntb); + + return (NTB_MW_GET_RANGE(device_get_parent(ntb), mw_idx + nc->mwoff, + base, vbase, size, align, align_size, plimit)); +} + +int +ntb_mw_set_trans(device_t ntb, unsigned mw_idx, bus_addr_t addr, size_t size) +{ + struct ntb_child *nc = device_get_ivars(ntb); + + return (NTB_MW_SET_TRANS(device_get_parent(ntb), mw_idx + nc->mwoff, + addr, size)); +} + +int +ntb_mw_clear_trans(device_t ntb, unsigned mw_idx) +{ + struct ntb_child *nc = device_get_ivars(ntb); + + return (NTB_MW_CLEAR_TRANS(device_get_parent(ntb), mw_idx + nc->mwoff)); +} + +int +ntb_mw_get_wc(device_t ntb, unsigned mw_idx, vm_memattr_t *mode) +{ + struct ntb_child *nc = device_get_ivars(ntb); + + return (NTB_MW_GET_WC(device_get_parent(ntb), mw_idx + nc->mwoff, mode)); +} + +int +ntb_mw_set_wc(device_t ntb, unsigned mw_idx, vm_memattr_t mode) +{ + struct ntb_child *nc = device_get_ivars(ntb); + + return (NTB_MW_SET_WC(device_get_parent(ntb), mw_idx + nc->mwoff, mode)); +} + +uint8_t +ntb_spad_count(device_t ntb) +{ + struct ntb_child *nc = device_get_ivars(ntb); + + return (nc->spadcnt); +} + +void +ntb_spad_clear(device_t ntb) +{ + struct ntb_child *nc = device_get_ivars(ntb); + unsigned i; + + for (i = 0; i < nc->spadcnt; i++) + NTB_SPAD_WRITE(device_get_parent(ntb), i + nc->spadoff, 0); +} + +int +ntb_spad_write(device_t ntb, unsigned int idx, uint32_t val) +{ + struct ntb_child *nc = device_get_ivars(ntb); + + return (NTB_SPAD_WRITE(device_get_parent(ntb), idx + nc->spadoff, val)); +} + +int +ntb_spad_read(device_t ntb, unsigned int idx, uint32_t *val) +{ + struct ntb_child *nc = device_get_ivars(ntb); + + return (NTB_SPAD_READ(device_get_parent(ntb), idx + nc->spadoff, val)); +} + +int +ntb_peer_spad_write(device_t ntb, unsigned int idx, uint32_t val) +{ + struct ntb_child *nc = device_get_ivars(ntb); + + return (NTB_PEER_SPAD_WRITE(device_get_parent(ntb), idx + nc->spadoff, + val)); +} + +int +ntb_peer_spad_read(device_t ntb, unsigned int idx, uint32_t *val) +{ + struct ntb_child *nc = device_get_ivars(ntb); + + return (NTB_PEER_SPAD_READ(device_get_parent(ntb), idx + nc->spadoff, + val)); +} + +uint64_t +ntb_db_valid_mask(device_t ntb) +{ + struct ntb_child *nc = device_get_ivars(ntb); + + return (nc->dbmask); +} + +int +ntb_db_vector_count(device_t ntb) +{ + + return (NTB_DB_VECTOR_COUNT(device_get_parent(ntb))); +} + +uint64_t +ntb_db_vector_mask(device_t ntb, uint32_t vector) +{ + struct ntb_child *nc = device_get_ivars(ntb); + + return ((NTB_DB_VECTOR_MASK(device_get_parent(ntb), vector) + >> nc->dboff) & nc->dbmask); +} + +int +ntb_peer_db_addr(device_t ntb, bus_addr_t *db_addr, vm_size_t *db_size) +{ + + return (NTB_PEER_DB_ADDR(device_get_parent(ntb), db_addr, db_size)); +} + +void +ntb_db_clear(device_t ntb, uint64_t bits) +{ + struct ntb_child *nc = device_get_ivars(ntb); + + return (NTB_DB_CLEAR(device_get_parent(ntb), bits << nc->dboff)); +} + +void +ntb_db_clear_mask(device_t ntb, uint64_t bits) +{ + struct ntb_child *nc = device_get_ivars(ntb); + + return (NTB_DB_CLEAR_MASK(device_get_parent(ntb), bits << nc->dboff)); +} + +uint64_t +ntb_db_read(device_t ntb) +{ + struct ntb_child *nc = device_get_ivars(ntb); + + return ((NTB_DB_READ(device_get_parent(ntb)) >> nc->dboff) + & nc->dbmask); +} + +void +ntb_db_set_mask(device_t ntb, uint64_t bits) +{ + struct ntb_child *nc = device_get_ivars(ntb); + + return (NTB_DB_SET_MASK(device_get_parent(ntb), bits << nc->dboff)); +} + +void +ntb_peer_db_set(device_t ntb, uint64_t bits) +{ + struct ntb_child *nc = device_get_ivars(ntb); + + return (NTB_PEER_DB_SET(device_get_parent(ntb), bits << nc->dboff)); +} + MODULE_VERSION(ntb, 1); diff --git a/sys/dev/ntb/ntb.h b/sys/dev/ntb/ntb.h index 5218cbd7be39..8593c65b4cf0 100644 --- a/sys/dev/ntb/ntb.h +++ b/sys/dev/ntb/ntb.h @@ -34,4 +34,376 @@ extern devclass_t ntb_hw_devclass; SYSCTL_DECL(_hw_ntb); +int ntb_register_device(device_t ntb); +int ntb_unregister_device(device_t ntb); + +/* + * ntb_link_event() - notify driver context of a change in link status + * @ntb: NTB device context + * + * Notify the driver context that the link status may have changed. The driver + * should call intb_link_is_up() to get the current status. + */ +void ntb_link_event(device_t ntb); + +/* + * ntb_db_event() - notify driver context of a doorbell event + * @ntb: NTB device context + * @vector: Interrupt vector number + * + * Notify the driver context of a doorbell event. If hardware supports + * multiple interrupt vectors for doorbells, the vector number indicates which + * vector received the interrupt. The vector number is relative to the first + * vector used for doorbells, starting at zero, and must be less than + * ntb_db_vector_count(). The driver may call ntb_db_read() to check which + * doorbell bits need service, and ntb_db_vector_mask() to determine which of + * those bits are associated with the vector number. + */ +void ntb_db_event(device_t ntb, uint32_t vec); + +/* + * ntb_link_is_up() - get the current ntb link state + * @ntb: NTB device context + * @speed: OUT - The link speed expressed as PCIe generation number + * @width: OUT - The link width expressed as the number of PCIe lanes + * + * RETURNS: true or false based on the hardware link state + */ +bool ntb_link_is_up(device_t ntb, enum ntb_speed *speed, enum ntb_width *width); + +/* + * ntb_link_enable() - enable the link on the secondary side of the ntb + * @ntb: NTB device context + * @max_speed: The maximum link speed expressed as PCIe generation number[0] + * @max_width: The maximum link width expressed as the number of PCIe lanes[0] + * + * Enable the link on the secondary side of the ntb. This can only be done + * from the primary side of the ntb in primary or b2b topology. The ntb device + * should train the link to its maximum speed and width, or the requested speed + * and width, whichever is smaller, if supported. + * + * Return: Zero on success, otherwise an error number. + * + * [0]: Only NTB_SPEED_AUTO and NTB_WIDTH_AUTO are valid inputs; other speed + * and width input will be ignored. + */ +int ntb_link_enable(device_t ntb, enum ntb_speed speed, enum ntb_width width); + +/* + * ntb_link_disable() - disable the link on the secondary side of the ntb + * @ntb: NTB device context + * + * Disable the link on the secondary side of the ntb. This can only be done + * from the primary side of the ntb in primary or b2b topology. The ntb device + * should disable the link. Returning from this call must indicate that a + * barrier has passed, though with no more writes may pass in either direction + * across the link, except if this call returns an error number. + * + * Return: Zero on success, otherwise an error number. + */ +int ntb_link_disable(device_t ntb); + +/* + * get enable status of the link on the secondary side of the ntb + */ +bool ntb_link_enabled(device_t ntb); + +/* + * ntb_set_ctx() - associate a driver context with an ntb device + * @ntb: NTB device context + * @ctx: Driver context + * @ctx_ops: Driver context operations + * + * Associate a driver context and operations with a ntb device. The context is + * provided by the client driver, and the driver may associate a different + * context with each ntb device. + * + * Return: Zero if the context is associated, otherwise an error number. + */ +int ntb_set_ctx(device_t ntb, void *ctx, const struct ntb_ctx_ops *ctx_ops); + +/* + * ntb_set_ctx() - get a driver context associated with an ntb device + * @ntb: NTB device context + * @ctx_ops: Driver context operations + * + * Get a driver context and operations associated with a ntb device. + */ +void * ntb_get_ctx(device_t ntb, const struct ntb_ctx_ops **ctx_ops); + +/* + * ntb_clear_ctx() - disassociate any driver context from an ntb device + * @ntb: NTB device context + * + * Clear any association that may exist between a driver context and the ntb + * device. + */ +void ntb_clear_ctx(device_t ntb); + +/* + * ntb_mw_count() - Get the number of memory windows available for KPI + * consumers. + * + * (Excludes any MW wholly reserved for register access.) + */ +uint8_t ntb_mw_count(device_t ntb); + +/* + * ntb_mw_get_range() - get the range of a memory window + * @ntb: NTB device context + * @idx: Memory window number + * @base: OUT - the base address for mapping the memory window + * @size: OUT - the size for mapping the memory window + * @align: OUT - the base alignment for translating the memory window + * @align_size: OUT - the size alignment for translating the memory window + * + * Get the range of a memory window. NULL may be given for any output + * parameter if the value is not needed. The base and size may be used for + * mapping the memory window, to access the peer memory. The alignment and + * size may be used for translating the memory window, for the peer to access + * memory on the local system. + * + * Return: Zero on success, otherwise an error number. + */ +int ntb_mw_get_range(device_t ntb, unsigned mw_idx, vm_paddr_t *base, + caddr_t *vbase, size_t *size, size_t *align, size_t *align_size, + bus_addr_t *plimit); + +/* + * ntb_mw_set_trans() - set the translation of a memory window + * @ntb: NTB device context + * @idx: Memory window number + * @addr: The dma address local memory to expose to the peer + * @size: The size of the local memory to expose to the peer + * + * Set the translation of a memory window. The peer may access local memory + * through the window starting at the address, up to the size. The address + * must be aligned to the alignment specified by ntb_mw_get_range(). The size + * must be aligned to the size alignment specified by ntb_mw_get_range(). The + * address must be below the plimit specified by ntb_mw_get_range() (i.e. for + * 32-bit BARs). + * + * Return: Zero on success, otherwise an error number. + */ +int ntb_mw_set_trans(device_t ntb, unsigned mw_idx, bus_addr_t addr, + size_t size); + +/* + * ntb_mw_clear_trans() - clear the translation of a memory window + * @ntb: NTB device context + * @idx: Memory window number + * + * Clear the translation of a memory window. The peer may no longer access + * local memory through the window. + * + * Return: Zero on success, otherwise an error number. + */ +int ntb_mw_clear_trans(device_t ntb, unsigned mw_idx); + +/* + * ntb_mw_get_wc - Get the write-combine status of a memory window + * + * Returns: Zero on success, setting *wc; otherwise an error number (e.g. if + * idx is an invalid memory window). + * + * Mode is a VM_MEMATTR_* type. + */ +int ntb_mw_get_wc(device_t ntb, unsigned mw_idx, vm_memattr_t *mode); + +/* + * ntb_mw_set_wc - Set the write-combine status of a memory window + * + * If 'mode' matches the current status, this does nothing and succeeds. Mode + * is a VM_MEMATTR_* type. + * + * Returns: Zero on success, setting the caching attribute on the virtual + * mapping of the BAR; otherwise an error number (e.g. if idx is an invalid + * memory window, or if changing the caching attribute fails). + */ +int ntb_mw_set_wc(device_t ntb, unsigned mw_idx, vm_memattr_t mode); + +/* + * ntb_spad_count() - get the total scratch regs usable + * @ntb: pointer to ntb_softc instance + * + * This function returns the max 32bit scratchpad registers usable by the + * upper layer. + * + * RETURNS: total number of scratch pad registers available + */ +uint8_t ntb_spad_count(device_t ntb); + +/* + * ntb_get_max_spads() - zero local scratch registers + * @ntb: pointer to ntb_softc instance + * + * This functions overwrites all local scratchpad registers with zeroes. + */ +void ntb_spad_clear(device_t ntb); + +/* + * ntb_spad_write() - write to the secondary scratchpad register + * @ntb: pointer to ntb_softc instance + * @idx: index to the scratchpad register, 0 based + * @val: the data value to put into the register + * + * This function allows writing of a 32bit value to the indexed scratchpad + * register. The register resides on the secondary (external) side. + * + * RETURNS: An appropriate ERRNO error value on error, or zero for success. + */ +int ntb_spad_write(device_t ntb, unsigned int idx, uint32_t val); + +/* + * ntb_spad_read() - read from the primary scratchpad register + * @ntb: pointer to ntb_softc instance + * @idx: index to scratchpad register, 0 based + * @val: pointer to 32bit integer for storing the register value + * + * This function allows reading of the 32bit scratchpad register on + * the primary (internal) side. + * + * RETURNS: An appropriate ERRNO error value on error, or zero for success. + */ +int ntb_spad_read(device_t ntb, unsigned int idx, uint32_t *val); + +/* + * ntb_peer_spad_write() - write to the secondary scratchpad register + * @ntb: pointer to ntb_softc instance + * @idx: index to the scratchpad register, 0 based + * @val: the data value to put into the register + * + * This function allows writing of a 32bit value to the indexed scratchpad + * register. The register resides on the secondary (external) side. + * + * RETURNS: An appropriate ERRNO error value on error, or zero for success. + */ +int ntb_peer_spad_write(device_t ntb, unsigned int idx, uint32_t val); + +/* + * ntb_peer_spad_read() - read from the primary scratchpad register + * @ntb: pointer to ntb_softc instance + * @idx: index to scratchpad register, 0 based + * @val: pointer to 32bit integer for storing the register value + * + * This function allows reading of the 32bit scratchpad register on + * the primary (internal) side. + * + * RETURNS: An appropriate ERRNO error value on error, or zero for success. + */ +int ntb_peer_spad_read(device_t ntb, unsigned int idx, uint32_t *val); + +/* + * ntb_db_valid_mask() - get a mask of doorbell bits supported by the ntb + * @ntb: NTB device context + * + * Hardware may support different number or arrangement of doorbell bits. + * + * Return: A mask of doorbell bits supported by the ntb. + */ +uint64_t ntb_db_valid_mask(device_t ntb); + +/* + * ntb_db_vector_count() - get the number of doorbell interrupt vectors + * @ntb: NTB device context. + * + * Hardware may support different number of interrupt vectors. + * + * Return: The number of doorbell interrupt vectors. + */ +int ntb_db_vector_count(device_t ntb); + +/* + * ntb_db_vector_mask() - get a mask of doorbell bits serviced by a vector + * @ntb: NTB device context + * @vector: Doorbell vector number + * + * Each interrupt vector may have a different number or arrangement of bits. + * + * Return: A mask of doorbell bits serviced by a vector. + */ +uint64_t ntb_db_vector_mask(device_t ntb, uint32_t vector); + +/* + * ntb_peer_db_addr() - address and size of the peer doorbell register + * @ntb: NTB device context. + * @db_addr: OUT - The address of the peer doorbell register. + * @db_size: OUT - The number of bytes to write the peer doorbell register. + * + * Return the address of the peer doorbell register. This may be used, for + * example, by drivers that offload memory copy operations to a dma engine. + * The drivers may wish to ring the peer doorbell at the completion of memory + * copy operations. For efficiency, and to simplify ordering of operations + * between the dma memory copies and the ringing doorbell, the driver may + * append one additional dma memory copy with the doorbell register as the + * destination, after the memory copy operations. + * + * Return: Zero on success, otherwise an error number. + * + * Note that writing the peer doorbell via a memory window will *not* generate + * an interrupt on the remote host; that must be done separately. + */ +int ntb_peer_db_addr(device_t ntb, bus_addr_t *db_addr, vm_size_t *db_size); + +/* + * ntb_db_clear() - clear bits in the local doorbell register + * @ntb: NTB device context. + * @db_bits: Doorbell bits to clear. + * + * Clear bits in the local doorbell register, arming the bits for the next + * doorbell. + * + * Return: Zero on success, otherwise an error number. + */ +void ntb_db_clear(device_t ntb, uint64_t bits); + +/* + * ntb_db_clear_mask() - clear bits in the local doorbell mask + * @ntb: NTB device context. + * @db_bits: Doorbell bits to clear. + * + * Clear bits in the local doorbell mask register, allowing doorbell interrupts + * from being generated for those doorbell bits. If a doorbell bit is already + * set at the time the mask is cleared, and the corresponding mask bit is + * changed from set to clear, then the ntb driver must ensure that + * ntb_db_event() is called. If the hardware does not generate the interrupt + * on clearing the mask bit, then the driver must call ntb_db_event() anyway. + * + * Return: Zero on success, otherwise an error number. + */ +void ntb_db_clear_mask(device_t ntb, uint64_t bits); + +/* + * ntb_db_read() - read the local doorbell register + * @ntb: NTB device context. + * + * Read the local doorbell register, and return the bits that are set. + * + * Return: The bits currently set in the local doorbell register. + */ +uint64_t ntb_db_read(device_t ntb); + +/* + * ntb_db_set_mask() - set bits in the local doorbell mask + * @ntb: NTB device context. + * @db_bits: Doorbell mask bits to set. + * + * Set bits in the local doorbell mask register, preventing doorbell interrupts + * from being generated for those doorbell bits. Bits that were already set + * must remain set. + * + * Return: Zero on success, otherwise an error number. + */ +void ntb_db_set_mask(device_t ntb, uint64_t bits); + +/* + * ntb_peer_db_set() - Set the doorbell on the secondary/external side + * @ntb: pointer to ntb_softc instance + * @bit: doorbell bits to ring + * + * This function allows triggering of a doorbell on the secondary/external + * side that will initiate an interrupt on the remote host + */ +void ntb_peer_db_set(device_t ntb, uint64_t bits); + #endif /* _NTB_H_ */ diff --git a/sys/dev/ntb/ntb_hw/ntb_hw.c b/sys/dev/ntb/ntb_hw/ntb_hw.c index 071ef42b55e4..dac36997cdb1 100644 --- a/sys/dev/ntb/ntb_hw/ntb_hw.c +++ b/sys/dev/ntb/ntb_hw/ntb_hw.c @@ -51,7 +51,6 @@ __FBSDID("$FreeBSD$"); #include <sys/pciio.h> #include <sys/queue.h> #include <sys/rman.h> -#include <sys/rmlock.h> #include <sys/sbuf.h> #include <sys/sysctl.h> #include <vm/vm.h> @@ -201,6 +200,9 @@ struct ntb_msix_data { }; struct ntb_softc { + /* ntb.c context. Do not move! Must go first! */ + void *ntb_store; + device_t device; enum ntb_device_type type; uint32_t features; @@ -219,10 +221,7 @@ struct ntb_softc { struct callout heartbeat_timer; struct callout lr_timer; - void *ntb_ctx; - const struct ntb_ctx_ops *ctx_ops; struct ntb_vec *msix_vec; - struct rmlock ctx_lock; uint32_t ppd; enum ntb_conn_type conn_type; @@ -284,72 +283,74 @@ bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle, } #endif -#define ntb_bar_read(SIZE, bar, offset) \ +#define intel_ntb_bar_read(SIZE, bar, offset) \ bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ ntb->bar_info[(bar)].pci_bus_handle, (offset)) -#define ntb_bar_write(SIZE, bar, offset, val) \ +#define intel_ntb_bar_write(SIZE, bar, offset, val) \ bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) -#define ntb_reg_read(SIZE, offset) ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) -#define ntb_reg_write(SIZE, offset, val) \ - ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) -#define ntb_mw_read(SIZE, offset) \ - ntb_bar_read(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), offset) -#define ntb_mw_write(SIZE, offset, val) \ - ntb_bar_write(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ +#define intel_ntb_reg_read(SIZE, offset) \ + intel_ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) +#define intel_ntb_reg_write(SIZE, offset, val) \ + intel_ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) +#define intel_ntb_mw_read(SIZE, offset) \ + intel_ntb_bar_read(SIZE, intel_ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ + offset) +#define intel_ntb_mw_write(SIZE, offset, val) \ + intel_ntb_bar_write(SIZE, intel_ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ offset, val) -static int ntb_probe(device_t device); -static int ntb_attach(device_t device); -static int ntb_detach(device_t device); -static uint64_t ntb_db_valid_mask(device_t dev); -static void ntb_spad_clear(device_t dev); -static uint64_t ntb_db_vector_mask(device_t dev, uint32_t vector); -static bool ntb_link_is_up(device_t dev, enum ntb_speed *speed, +static int intel_ntb_probe(device_t device); +static int intel_ntb_attach(device_t device); +static int intel_ntb_detach(device_t device); +static uint64_t intel_ntb_db_valid_mask(device_t dev); +static void intel_ntb_spad_clear(device_t dev); +static uint64_t intel_ntb_db_vector_mask(device_t dev, uint32_t vector); +static bool intel_ntb_link_is_up(device_t dev, enum ntb_speed *speed, enum ntb_width *width); -static int ntb_link_enable(device_t dev, enum ntb_speed speed, +static int intel_ntb_link_enable(device_t dev, enum ntb_speed speed, enum ntb_width width); -static int ntb_link_disable(device_t dev); -static int ntb_spad_read(device_t dev, unsigned int idx, uint32_t *val); -static int ntb_peer_spad_write(device_t dev, unsigned int idx, uint32_t val); +static int intel_ntb_link_disable(device_t dev); +static int intel_ntb_spad_read(device_t dev, unsigned int idx, uint32_t *val); +static int intel_ntb_peer_spad_write(device_t dev, unsigned int idx, uint32_t val); -static unsigned ntb_user_mw_to_idx(struct ntb_softc *, unsigned uidx); -static inline enum ntb_bar ntb_mw_to_bar(struct ntb_softc *, unsigned mw); +static unsigned intel_ntb_user_mw_to_idx(struct ntb_softc *, unsigned uidx); +static inline enum ntb_bar intel_ntb_mw_to_bar(struct ntb_softc *, unsigned mw); static inline bool bar_is_64bit(struct ntb_softc *, enum ntb_bar); static inline void bar_get_xlat_params(struct ntb_softc *, enum ntb_bar, uint32_t *base, uint32_t *xlat, uint32_t *lmt); -static int ntb_map_pci_bars(struct ntb_softc *ntb); -static int ntb_mw_set_wc_internal(struct ntb_softc *, unsigned idx, +static int intel_ntb_map_pci_bars(struct ntb_softc *ntb); +static int intel_ntb_mw_set_wc_internal(struct ntb_softc *, unsigned idx, vm_memattr_t); static void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *, const char *); static int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); static int map_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); -static void ntb_unmap_pci_bar(struct ntb_softc *ntb); -static int ntb_remap_msix(device_t, uint32_t desired, uint32_t avail); -static int ntb_init_isr(struct ntb_softc *ntb); -static int ntb_setup_legacy_interrupt(struct ntb_softc *ntb); -static int ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors); -static void ntb_teardown_interrupts(struct ntb_softc *ntb); -static inline uint64_t ntb_vec_mask(struct ntb_softc *, uint64_t db_vector); -static void ntb_interrupt(struct ntb_softc *, uint32_t vec); +static void intel_ntb_unmap_pci_bar(struct ntb_softc *ntb); +static int intel_ntb_remap_msix(device_t, uint32_t desired, uint32_t avail); +static int intel_ntb_init_isr(struct ntb_softc *ntb); +static int intel_ntb_setup_legacy_interrupt(struct ntb_softc *ntb); +static int intel_ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors); +static void intel_ntb_teardown_interrupts(struct ntb_softc *ntb); +static inline uint64_t intel_ntb_vec_mask(struct ntb_softc *, uint64_t db_vector); +static void intel_ntb_interrupt(struct ntb_softc *, uint32_t vec); static void ndev_vec_isr(void *arg); static void ndev_irq_isr(void *arg); static inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff); static inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t); static inline void db_iowrite_raw(struct ntb_softc *, uint64_t regoff, uint64_t); -static int ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors); -static void ntb_free_msix_vec(struct ntb_softc *ntb); -static void ntb_get_msix_info(struct ntb_softc *ntb); -static void ntb_exchange_msix(void *); -static struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); -static void ntb_detect_max_mw(struct ntb_softc *ntb); -static int ntb_detect_xeon(struct ntb_softc *ntb); -static int ntb_detect_atom(struct ntb_softc *ntb); -static int ntb_xeon_init_dev(struct ntb_softc *ntb); -static int ntb_atom_init_dev(struct ntb_softc *ntb); -static void ntb_teardown_xeon(struct ntb_softc *ntb); +static int intel_ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors); +static void intel_ntb_free_msix_vec(struct ntb_softc *ntb); +static void intel_ntb_get_msix_info(struct ntb_softc *ntb); +static void intel_ntb_exchange_msix(void *); +static struct ntb_hw_info *intel_ntb_get_device_info(uint32_t device_id); +static void intel_ntb_detect_max_mw(struct ntb_softc *ntb); +static int intel_ntb_detect_xeon(struct ntb_softc *ntb); +static int intel_ntb_detect_atom(struct ntb_softc *ntb); +static int intel_ntb_xeon_init_dev(struct ntb_softc *ntb); +static int intel_ntb_atom_init_dev(struct ntb_softc *ntb); +static void intel_ntb_teardown_xeon(struct ntb_softc *ntb); static void configure_atom_secondary_side_bars(struct ntb_softc *ntb); static void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx, enum ntb_bar regbar); @@ -362,15 +363,13 @@ static int xeon_setup_b2b_mw(struct ntb_softc *, static inline bool link_is_up(struct ntb_softc *ntb); static inline bool _xeon_link_is_up(struct ntb_softc *ntb); static inline bool atom_link_is_err(struct ntb_softc *ntb); -static inline enum ntb_speed ntb_link_sta_speed(struct ntb_softc *); -static inline enum ntb_width ntb_link_sta_width(struct ntb_softc *); +static inline enum ntb_speed intel_ntb_link_sta_speed(struct ntb_softc *); +static inline enum ntb_width intel_ntb_link_sta_width(struct ntb_softc *); static void atom_link_hb(void *arg); -static void ntb_link_event(device_t dev); -static void ntb_db_event(device_t dev, uint32_t vec); static void recover_atom_link(void *arg); -static bool ntb_poll_link(struct ntb_softc *ntb); +static bool intel_ntb_poll_link(struct ntb_softc *ntb); static void save_bar_parameters(struct ntb_pci_bar_info *bar); -static void ntb_sysctl_init(struct ntb_softc *); +static void intel_ntb_sysctl_init(struct ntb_softc *); static int sysctl_handle_features(SYSCTL_HANDLER_ARGS); static int sysctl_handle_link_admin(SYSCTL_HANDLER_ARGS); static int sysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS); @@ -380,7 +379,7 @@ static int sysctl_handle_register(SYSCTL_HANDLER_ARGS); static unsigned g_ntb_hw_debug_level; SYSCTL_UINT(_hw_ntb, OID_AUTO, debug_level, CTLFLAG_RWTUN, &g_ntb_hw_debug_level, 0, "ntb_hw log level -- higher is more verbose"); -#define ntb_printf(lvl, ...) do { \ +#define intel_ntb_printf(lvl, ...) do { \ if ((lvl) <= g_ntb_hw_debug_level) { \ device_printf(ntb->device, __VA_ARGS__); \ } \ @@ -403,7 +402,7 @@ SYSCTL_UINT(_hw_ntb, OID_AUTO, default_mw_pat, CTLFLAG_RDTUN, "UC-: " __XSTRING(_NTB_PAT_UCM)); static inline vm_memattr_t -ntb_pat_flags(void) +intel_ntb_pat_flags(void) { switch (g_ntb_mw_pat) { @@ -429,7 +428,7 @@ ntb_pat_flags(void) * anywhere better yet. */ static inline const char * -ntb_vm_memattr_to_str(vm_memattr_t pat) +intel_ntb_vm_memattr_to_str(vm_memattr_t pat) { switch (pat) { @@ -621,11 +620,11 @@ MALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); * OS <-> Driver linkage functions */ static int -ntb_probe(device_t device) +intel_ntb_probe(device_t device) { struct ntb_hw_info *p; - p = ntb_get_device_info(pci_get_devid(device)); + p = intel_ntb_get_device_info(pci_get_devid(device)); if (p == NULL) return (ENXIO); @@ -634,14 +633,14 @@ ntb_probe(device_t device) } static int -ntb_attach(device_t device) +intel_ntb_attach(device_t device) { struct ntb_softc *ntb; struct ntb_hw_info *p; int error; ntb = device_get_softc(device); - p = ntb_get_device_info(pci_get_devid(device)); + p = intel_ntb_get_device_info(pci_get_devid(device)); ntb->device = device; ntb->type = p->type; @@ -654,54 +653,52 @@ ntb_attach(device_t device) callout_init(&ntb->lr_timer, 1); callout_init(&ntb->peer_msix_work, 1); mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN); - rm_init(&ntb->ctx_lock, "ntb ctx"); if (ntb->type == NTB_ATOM) - error = ntb_detect_atom(ntb); + error = intel_ntb_detect_atom(ntb); else - error = ntb_detect_xeon(ntb); + error = intel_ntb_detect_xeon(ntb); if (error != 0) goto out; - ntb_detect_max_mw(ntb); + intel_ntb_detect_max_mw(ntb); pci_enable_busmaster(ntb->device); - error = ntb_map_pci_bars(ntb); + error = intel_ntb_map_pci_bars(ntb); if (error != 0) goto out; if (ntb->type == NTB_ATOM) - error = ntb_atom_init_dev(ntb); + error = intel_ntb_atom_init_dev(ntb); else - error = ntb_xeon_init_dev(ntb); + error = intel_ntb_xeon_init_dev(ntb); if (error != 0) goto out; - ntb_spad_clear(device); + intel_ntb_spad_clear(device); - ntb_poll_link(ntb); + intel_ntb_poll_link(ntb); - ntb_sysctl_init(ntb); + intel_ntb_sysctl_init(ntb); /* Attach children to this controller */ - device_add_child(device, NULL, -1); - bus_generic_attach(device); + error = ntb_register_device(device); out: if (error != 0) - ntb_detach(device); + intel_ntb_detach(device); return (error); } static int -ntb_detach(device_t device) +intel_ntb_detach(device_t device) { struct ntb_softc *ntb; ntb = device_get_softc(device); /* Detach & delete all children */ - device_delete_children(device); + ntb_unregister_device(device); if (ntb->self_reg != NULL) { DB_MASK_LOCK(ntb); @@ -713,13 +710,12 @@ ntb_detach(device_t device) callout_drain(&ntb->peer_msix_work); pci_disable_busmaster(ntb->device); if (ntb->type == NTB_XEON) - ntb_teardown_xeon(ntb); - ntb_teardown_interrupts(ntb); + intel_ntb_teardown_xeon(ntb); + intel_ntb_teardown_interrupts(ntb); mtx_destroy(&ntb->db_mask_lock); - rm_destroy(&ntb->ctx_lock); - ntb_unmap_pci_bar(ntb); + intel_ntb_unmap_pci_bar(ntb); return (0); } @@ -728,7 +724,7 @@ ntb_detach(device_t device) * Driver internal routines */ static inline enum ntb_bar -ntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw) +intel_ntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw) { KASSERT(mw < ntb->mw_count, @@ -784,7 +780,7 @@ bar_get_xlat_params(struct ntb_softc *ntb, enum ntb_bar bar, uint32_t *base, } static int -ntb_map_pci_bars(struct ntb_softc *ntb) +intel_ntb_map_pci_bars(struct ntb_softc *ntb) { int rc; @@ -910,7 +906,7 @@ map_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) * Optionally, mark MW BARs as anything other than UC to improve * performance. */ - mapmode = ntb_pat_flags(); + mapmode = intel_ntb_pat_flags(); if (mapmode == bar->map_mode) return (0); @@ -923,7 +919,7 @@ map_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) PCI_RID2BAR(bar->pci_resource_id), bar->vbase, (char *)bar->vbase + bar->size - 1, (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), - ntb_vm_memattr_to_str(mapmode)); + intel_ntb_vm_memattr_to_str(mapmode)); } else device_printf(ntb->device, "Unable to mark BAR%d v:[%p-%p] p:[%p-%p] as " @@ -931,13 +927,13 @@ map_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) PCI_RID2BAR(bar->pci_resource_id), bar->vbase, (char *)bar->vbase + bar->size - 1, (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), - ntb_vm_memattr_to_str(mapmode), rc); + intel_ntb_vm_memattr_to_str(mapmode), rc); /* Proceed anyway */ return (0); } static void -ntb_unmap_pci_bar(struct ntb_softc *ntb) +intel_ntb_unmap_pci_bar(struct ntb_softc *ntb) { struct ntb_pci_bar_info *current_bar; int i; @@ -952,7 +948,7 @@ ntb_unmap_pci_bar(struct ntb_softc *ntb) } static int -ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors) +intel_ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors) { uint32_t i; int rc; @@ -1005,7 +1001,7 @@ SYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN, * round-robin fashion. */ static int -ntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail) +intel_ntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail) { u_int *vectors; uint32_t i; @@ -1025,7 +1021,7 @@ ntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail) } static int -ntb_init_isr(struct ntb_softc *ntb) +intel_ntb_init_isr(struct ntb_softc *ntb) { uint32_t desired_vectors, num_vectors; int rc; @@ -1051,7 +1047,7 @@ ntb_init_isr(struct ntb_softc *ntb) num_vectors--; if (rc == 0 && num_vectors < desired_vectors) { - rc = ntb_remap_msix(ntb->device, desired_vectors, + rc = intel_ntb_remap_msix(ntb->device, desired_vectors, num_vectors); if (rc == 0) num_vectors = desired_vectors; @@ -1072,7 +1068,7 @@ ntb_init_isr(struct ntb_softc *ntb) ntb->db_vec_count = 1; ntb->db_vec_shift = XEON_DB_TOTAL_SHIFT; - rc = ntb_setup_legacy_interrupt(ntb); + rc = intel_ntb_setup_legacy_interrupt(ntb); } else { if (num_vectors - 1 != XEON_NONLINK_DB_MSIX_BITS && HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { @@ -1082,22 +1078,20 @@ ntb_init_isr(struct ntb_softc *ntb) return (EINVAL); } - ntb_create_msix_vec(ntb, num_vectors); - rc = ntb_setup_msix(ntb, num_vectors); - if (rc == 0 && HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) - ntb_get_msix_info(ntb); + intel_ntb_create_msix_vec(ntb, num_vectors); + rc = intel_ntb_setup_msix(ntb, num_vectors); } if (rc != 0) { device_printf(ntb->device, "Error allocating interrupts: %d\n", rc); - ntb_free_msix_vec(ntb); + intel_ntb_free_msix_vec(ntb); } return (rc); } static int -ntb_setup_legacy_interrupt(struct ntb_softc *ntb) +intel_ntb_setup_legacy_interrupt(struct ntb_softc *ntb) { int rc; @@ -1124,7 +1118,7 @@ ntb_setup_legacy_interrupt(struct ntb_softc *ntb) } static void -ntb_teardown_interrupts(struct ntb_softc *ntb) +intel_ntb_teardown_interrupts(struct ntb_softc *ntb) { struct ntb_int_info *current_int; int i; @@ -1140,7 +1134,7 @@ ntb_teardown_interrupts(struct ntb_softc *ntb) rman_get_rid(current_int->res), current_int->res); } - ntb_free_msix_vec(ntb); + intel_ntb_free_msix_vec(ntb); pci_release_msi(ntb->device); } @@ -1153,11 +1147,11 @@ db_ioread(struct ntb_softc *ntb, uint64_t regoff) { if (ntb->type == NTB_ATOM) - return (ntb_reg_read(8, regoff)); + return (intel_ntb_reg_read(8, regoff)); KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); - return (ntb_reg_read(2, regoff)); + return (intel_ntb_reg_read(2, regoff)); } static inline void @@ -1179,16 +1173,16 @@ db_iowrite_raw(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) { if (ntb->type == NTB_ATOM) { - ntb_reg_write(8, regoff, val); + intel_ntb_reg_write(8, regoff, val); return; } KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); - ntb_reg_write(2, regoff, (uint16_t)val); + intel_ntb_reg_write(2, regoff, (uint16_t)val); } static void -ntb_db_set_mask(device_t dev, uint64_t bits) +intel_ntb_db_set_mask(device_t dev, uint64_t bits) { struct ntb_softc *ntb = device_get_softc(dev); @@ -1200,7 +1194,7 @@ ntb_db_set_mask(device_t dev, uint64_t bits) } static void -ntb_db_clear_mask(device_t dev, uint64_t bits) +intel_ntb_db_clear_mask(device_t dev, uint64_t bits) { struct ntb_softc *ntb = device_get_softc(dev); uint64_t ibits; @@ -1217,7 +1211,7 @@ ntb_db_clear_mask(device_t dev, uint64_t bits) if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { /* Simulate fake interrupts if unmasked DB bits are set. */ for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { - if ((ibits & ntb_db_vector_mask(dev, i)) != 0) + if ((ibits & intel_ntb_db_vector_mask(dev, i)) != 0) swi_sched(ntb->int_info[i].tag, 0); } } else { @@ -1227,7 +1221,7 @@ ntb_db_clear_mask(device_t dev, uint64_t bits) } static uint64_t -ntb_db_read(device_t dev) +intel_ntb_db_read(device_t dev) { struct ntb_softc *ntb = device_get_softc(dev); @@ -1238,7 +1232,7 @@ ntb_db_read(device_t dev) } static void -ntb_db_clear(device_t dev, uint64_t bits) +intel_ntb_db_clear(device_t dev, uint64_t bits) { struct ntb_softc *ntb = device_get_softc(dev); @@ -1258,7 +1252,7 @@ ntb_db_clear(device_t dev, uint64_t bits) } static inline uint64_t -ntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector) +intel_ntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector) { uint64_t shift, mask; @@ -1281,15 +1275,15 @@ ntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector) } static void -ntb_interrupt(struct ntb_softc *ntb, uint32_t vec) +intel_ntb_interrupt(struct ntb_softc *ntb, uint32_t vec) { uint64_t vec_mask; ntb->last_ts = ticks; - vec_mask = ntb_vec_mask(ntb, vec); + vec_mask = intel_ntb_vec_mask(ntb, vec); if ((vec_mask & ntb->db_link_mask) != 0) { - if (ntb_poll_link(ntb)) + if (intel_ntb_poll_link(ntb)) ntb_link_event(ntb->device); } @@ -1318,18 +1312,18 @@ ndev_vec_isr(void *arg) { struct ntb_vec *nvec = arg; - ntb_interrupt(nvec->ntb, nvec->num); + intel_ntb_interrupt(nvec->ntb, nvec->num); } static void ndev_irq_isr(void *arg) { /* If we couldn't set up MSI-X, we only have the one vector. */ - ntb_interrupt(arg, 0); + intel_ntb_interrupt(arg, 0); } static int -ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors) +intel_ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors) { uint32_t i; @@ -1344,7 +1338,7 @@ ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors) } static void -ntb_free_msix_vec(struct ntb_softc *ntb) +intel_ntb_free_msix_vec(struct ntb_softc *ntb) { if (ntb->msix_vec == NULL) @@ -1355,7 +1349,7 @@ ntb_free_msix_vec(struct ntb_softc *ntb) } static void -ntb_get_msix_info(struct ntb_softc *ntb) +intel_ntb_get_msix_info(struct ntb_softc *ntb) { struct pci_devinfo *dinfo; struct pcicfg_msix *msix; @@ -1371,7 +1365,7 @@ ntb_get_msix_info(struct ntb_softc *ntb) laddr = bus_read_4(msix->msix_table_res, offset + PCI_MSIX_ENTRY_LOWER_ADDR); - ntb_printf(2, "local MSIX addr(%u): 0x%x\n", i, laddr); + intel_ntb_printf(2, "local MSIX addr(%u): 0x%x\n", i, laddr); KASSERT((laddr & MSI_INTEL_ADDR_BASE) == MSI_INTEL_ADDR_BASE, ("local MSIX addr 0x%x not in MSI base 0x%x", laddr, @@ -1380,14 +1374,14 @@ ntb_get_msix_info(struct ntb_softc *ntb) data = bus_read_4(msix->msix_table_res, offset + PCI_MSIX_ENTRY_DATA); - ntb_printf(2, "local MSIX data(%u): 0x%x\n", i, data); + intel_ntb_printf(2, "local MSIX data(%u): 0x%x\n", i, data); ntb->msix_data[i].nmd_data = data; } } static struct ntb_hw_info * -ntb_get_device_info(uint32_t device_id) +intel_ntb_get_device_info(uint32_t device_id) { struct ntb_hw_info *ep = pci_ids; @@ -1400,15 +1394,15 @@ ntb_get_device_info(uint32_t device_id) } static void -ntb_teardown_xeon(struct ntb_softc *ntb) +intel_ntb_teardown_xeon(struct ntb_softc *ntb) { if (ntb->reg != NULL) - ntb_link_disable(ntb->device); + intel_ntb_link_disable(ntb->device); } static void -ntb_detect_max_mw(struct ntb_softc *ntb) +intel_ntb_detect_max_mw(struct ntb_softc *ntb) { if (ntb->type == NTB_ATOM) { @@ -1423,7 +1417,7 @@ ntb_detect_max_mw(struct ntb_softc *ntb) } static int -ntb_detect_xeon(struct ntb_softc *ntb) +intel_ntb_detect_xeon(struct ntb_softc *ntb) { uint8_t ppd, conn_type; @@ -1471,7 +1465,7 @@ ntb_detect_xeon(struct ntb_softc *ntb) } static int -ntb_detect_atom(struct ntb_softc *ntb) +intel_ntb_detect_atom(struct ntb_softc *ntb) { uint32_t ppd, conn_type; @@ -1496,7 +1490,7 @@ ntb_detect_atom(struct ntb_softc *ntb) } static int -ntb_xeon_init_dev(struct ntb_softc *ntb) +intel_ntb_xeon_init_dev(struct ntb_softc *ntb) { int rc; @@ -1521,9 +1515,9 @@ ntb_xeon_init_dev(struct ntb_softc *ntb) ntb->fake_db_bell = 0; ntb->msix_mw_idx = (ntb->mw_count + g_ntb_msix_idx) % ntb->mw_count; - ntb_printf(2, "Setting up MSIX mw idx %d means %u\n", + intel_ntb_printf(2, "Setting up MSIX mw idx %d means %u\n", g_ntb_msix_idx, ntb->msix_mw_idx); - rc = ntb_mw_set_wc_internal(ntb, ntb->msix_mw_idx, + rc = intel_ntb_mw_set_wc_internal(ntb, ntb->msix_mw_idx, VM_MEMATTR_UNCACHEABLE); KASSERT(rc == 0, ("shouldn't fail")); } else if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) { @@ -1536,9 +1530,9 @@ ntb_xeon_init_dev(struct ntb_softc *ntb) */ ntb->b2b_mw_idx = (ntb->mw_count + g_ntb_mw_idx) % ntb->mw_count; - ntb_printf(2, "Setting up b2b mw idx %d means %u\n", + intel_ntb_printf(2, "Setting up b2b mw idx %d means %u\n", g_ntb_mw_idx, ntb->b2b_mw_idx); - rc = ntb_mw_set_wc_internal(ntb, ntb->b2b_mw_idx, + rc = intel_ntb_mw_set_wc_internal(ntb, ntb->b2b_mw_idx, VM_MEMATTR_UNCACHEABLE); KASSERT(rc == 0, ("shouldn't fail")); } else if (HAS_FEATURE(ntb, NTB_B2BDOORBELL_BIT14)) @@ -1564,7 +1558,7 @@ ntb_xeon_init_dev(struct ntb_softc *ntb) return (rc); /* Enable Bus Master and Memory Space on the secondary side */ - ntb_reg_write(2, XEON_SPCICMD_OFFSET, + intel_ntb_reg_write(2, XEON_SPCICMD_OFFSET, PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); /* @@ -1575,12 +1569,12 @@ ntb_xeon_init_dev(struct ntb_softc *ntb) db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); DB_MASK_UNLOCK(ntb); - rc = ntb_init_isr(ntb); + rc = intel_ntb_init_isr(ntb); return (rc); } static int -ntb_atom_init_dev(struct ntb_softc *ntb) +intel_ntb_atom_init_dev(struct ntb_softc *ntb) { int error; @@ -1607,15 +1601,15 @@ ntb_atom_init_dev(struct ntb_softc *ntb) configure_atom_secondary_side_bars(ntb); /* Enable Bus Master and Memory Space on the secondary side */ - ntb_reg_write(2, ATOM_SPCICMD_OFFSET, + intel_ntb_reg_write(2, ATOM_SPCICMD_OFFSET, PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); - error = ntb_init_isr(ntb); + error = intel_ntb_init_isr(ntb); if (error != 0) return (error); /* Initiate PCI-E link training */ - ntb_link_enable(ntb->device, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); + intel_ntb_link_enable(ntb->device, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); callout_reset(&ntb->heartbeat_timer, 0, atom_link_hb, ntb); @@ -1628,19 +1622,19 @@ configure_atom_secondary_side_bars(struct ntb_softc *ntb) { if (ntb->dev_type == NTB_DEV_USD) { - ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, + intel_ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, XEON_B2B_BAR2_ADDR64); - ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, + intel_ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, XEON_B2B_BAR4_ADDR64); - ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); - ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); + intel_ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); + intel_ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); } else { - ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, + intel_ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, XEON_B2B_BAR2_ADDR64); - ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, + intel_ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, XEON_B2B_BAR4_ADDR64); - ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); - ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); + intel_ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); + intel_ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); } } @@ -1698,20 +1692,20 @@ xeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t bar_addr, } if (!bar_is_64bit(ntb, idx)) { - ntb_reg_write(4, base_reg, bar_addr); - reg_val = ntb_reg_read(4, base_reg); + intel_ntb_reg_write(4, base_reg, bar_addr); + reg_val = intel_ntb_reg_read(4, base_reg); (void)reg_val; - ntb_reg_write(4, lmt_reg, bar_addr); - reg_val = ntb_reg_read(4, lmt_reg); + intel_ntb_reg_write(4, lmt_reg, bar_addr); + reg_val = intel_ntb_reg_read(4, lmt_reg); (void)reg_val; } else { - ntb_reg_write(8, base_reg, bar_addr); - reg_val = ntb_reg_read(8, base_reg); + intel_ntb_reg_write(8, base_reg, bar_addr); + reg_val = intel_ntb_reg_read(8, base_reg); (void)reg_val; - ntb_reg_write(8, lmt_reg, bar_addr); - reg_val = ntb_reg_read(8, lmt_reg); + intel_ntb_reg_write(8, lmt_reg, bar_addr); + reg_val = intel_ntb_reg_read(8, lmt_reg); (void)reg_val; } } @@ -1723,11 +1717,11 @@ xeon_set_pbar_xlat(struct ntb_softc *ntb, uint64_t base_addr, enum ntb_bar idx) bar = &ntb->bar_info[idx]; if (HAS_FEATURE(ntb, NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) { - ntb_reg_write(4, bar->pbarxlat_off, base_addr); - base_addr = ntb_reg_read(4, bar->pbarxlat_off); + intel_ntb_reg_write(4, bar->pbarxlat_off, base_addr); + base_addr = intel_ntb_reg_read(4, bar->pbarxlat_off); } else { - ntb_reg_write(8, bar->pbarxlat_off, base_addr); - base_addr = ntb_reg_read(8, bar->pbarxlat_off); + intel_ntb_reg_write(8, bar->pbarxlat_off, base_addr); + base_addr = intel_ntb_reg_read(8, bar->pbarxlat_off); } (void)base_addr; } @@ -1746,7 +1740,7 @@ xeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr, b2b_bar_num = NTB_CONFIG_BAR; ntb->b2b_off = 0; } else { - b2b_bar_num = ntb_mw_to_bar(ntb, ntb->b2b_mw_idx); + b2b_bar_num = intel_ntb_mw_to_bar(ntb, ntb->b2b_mw_idx); KASSERT(b2b_bar_num > 0 && b2b_bar_num < NTB_MAX_BARS, ("invalid b2b mw bar")); @@ -1786,7 +1780,7 @@ xeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr, else KASSERT(false, ("invalid bar")); - ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, bar_addr); + intel_ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, bar_addr); /* * Other SBARs are normally hit by the PBAR xlat, except for the b2b @@ -1807,8 +1801,8 @@ xeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr, NTB_B2B_BAR_2, b2b_bar_num); /* Zero incoming translation addrs */ - ntb_reg_write(8, XEON_SBAR2XLAT_OFFSET, 0); - ntb_reg_write(8, XEON_SBAR4XLAT_OFFSET, 0); + intel_ntb_reg_write(8, XEON_SBAR2XLAT_OFFSET, 0); + intel_ntb_reg_write(8, XEON_SBAR4XLAT_OFFSET, 0); if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { uint32_t xlat_reg, lmt_reg; @@ -1818,26 +1812,26 @@ xeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr, * We point the chosen MSIX MW BAR xlat to remote LAPIC for * workaround */ - bar_num = ntb_mw_to_bar(ntb, ntb->msix_mw_idx); + bar_num = intel_ntb_mw_to_bar(ntb, ntb->msix_mw_idx); bar_get_xlat_params(ntb, bar_num, NULL, &xlat_reg, &lmt_reg); if (bar_is_64bit(ntb, bar_num)) { - ntb_reg_write(8, xlat_reg, MSI_INTEL_ADDR_BASE); - ntb->msix_xlat = ntb_reg_read(8, xlat_reg); - ntb_reg_write(8, lmt_reg, 0); + intel_ntb_reg_write(8, xlat_reg, MSI_INTEL_ADDR_BASE); + ntb->msix_xlat = intel_ntb_reg_read(8, xlat_reg); + intel_ntb_reg_write(8, lmt_reg, 0); } else { - ntb_reg_write(4, xlat_reg, MSI_INTEL_ADDR_BASE); - ntb->msix_xlat = ntb_reg_read(4, xlat_reg); - ntb_reg_write(4, lmt_reg, 0); + intel_ntb_reg_write(4, xlat_reg, MSI_INTEL_ADDR_BASE); + ntb->msix_xlat = intel_ntb_reg_read(4, xlat_reg); + intel_ntb_reg_write(4, lmt_reg, 0); } ntb->peer_lapic_bar = &ntb->bar_info[bar_num]; } - (void)ntb_reg_read(8, XEON_SBAR2XLAT_OFFSET); - (void)ntb_reg_read(8, XEON_SBAR4XLAT_OFFSET); + (void)intel_ntb_reg_read(8, XEON_SBAR2XLAT_OFFSET); + (void)intel_ntb_reg_read(8, XEON_SBAR4XLAT_OFFSET); /* Zero outgoing translation limits (whole bar size windows) */ - ntb_reg_write(8, XEON_PBAR2LMT_OFFSET, 0); - ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); + intel_ntb_reg_write(8, XEON_PBAR2LMT_OFFSET, 0); + intel_ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); /* Set outgoing translation offsets */ xeon_set_pbar_xlat(ntb, peer_addr->bar2_addr64, NTB_B2B_BAR_1); @@ -1866,8 +1860,8 @@ xeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr, * B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits * at a time. */ - ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff); - ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32); + intel_ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff); + intel_ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32); return (0); } @@ -1899,11 +1893,11 @@ atom_link_is_err(struct ntb_softc *ntb) KASSERT(ntb->type == NTB_ATOM, ("ntb type")); - status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); + status = intel_ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); if ((status & ATOM_LTSSMSTATEJMP_FORCEDETECT) != 0) return (true); - status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); + status = intel_ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); return ((status & ATOM_IBIST_ERR_OFLOW) != 0); } @@ -1926,7 +1920,7 @@ atom_link_hb(void *arg) goto out; } - if (ntb_poll_link(ntb)) + if (intel_ntb_poll_link(ntb)) ntb_link_event(ntb->device); if (!link_is_up(ntb) && atom_link_is_err(ntb)) { @@ -1945,137 +1939,47 @@ atom_perform_link_restart(struct ntb_softc *ntb) uint32_t status; /* Driver resets the NTB ModPhy lanes - magic! */ - ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0xe0); - ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x40); - ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x60); - ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0x60); + intel_ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0xe0); + intel_ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x40); + intel_ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x60); + intel_ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0x60); /* Driver waits 100ms to allow the NTB ModPhy to settle */ pause("ModPhy", hz / 10); /* Clear AER Errors, write to clear */ - status = ntb_reg_read(4, ATOM_ERRCORSTS_OFFSET); + status = intel_ntb_reg_read(4, ATOM_ERRCORSTS_OFFSET); status &= PCIM_AER_COR_REPLAY_ROLLOVER; - ntb_reg_write(4, ATOM_ERRCORSTS_OFFSET, status); + intel_ntb_reg_write(4, ATOM_ERRCORSTS_OFFSET, status); /* Clear unexpected electrical idle event in LTSSM, write to clear */ - status = ntb_reg_read(4, ATOM_LTSSMERRSTS0_OFFSET); + status = intel_ntb_reg_read(4, ATOM_LTSSMERRSTS0_OFFSET); status |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI; - ntb_reg_write(4, ATOM_LTSSMERRSTS0_OFFSET, status); + intel_ntb_reg_write(4, ATOM_LTSSMERRSTS0_OFFSET, status); /* Clear DeSkew Buffer error, write to clear */ - status = ntb_reg_read(4, ATOM_DESKEWSTS_OFFSET); + status = intel_ntb_reg_read(4, ATOM_DESKEWSTS_OFFSET); status |= ATOM_DESKEWSTS_DBERR; - ntb_reg_write(4, ATOM_DESKEWSTS_OFFSET, status); + intel_ntb_reg_write(4, ATOM_DESKEWSTS_OFFSET, status); - status = ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); + status = intel_ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); status &= ATOM_IBIST_ERR_OFLOW; - ntb_reg_write(4, ATOM_IBSTERRRCRVSTS0_OFFSET, status); + intel_ntb_reg_write(4, ATOM_IBSTERRRCRVSTS0_OFFSET, status); /* Releases the NTB state machine to allow the link to retrain */ - status = ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); + status = intel_ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); status &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT; - ntb_reg_write(4, ATOM_LTSSMSTATEJMP_OFFSET, status); -} - -static int -ntb_set_ctx(device_t dev, void *ctx, const struct ntb_ctx_ops *ops) -{ - struct ntb_softc *ntb = device_get_softc(dev); - - if (ctx == NULL || ops == NULL) - return (EINVAL); - - rm_wlock(&ntb->ctx_lock); - if (ntb->ctx_ops != NULL) { - rm_wunlock(&ntb->ctx_lock); - return (EINVAL); - } - ntb->ntb_ctx = ctx; - ntb->ctx_ops = ops; - rm_wunlock(&ntb->ctx_lock); - - return (0); -} - -/* - * It is expected that this will only be used from contexts where the ctx_lock - * is not needed to protect ntb_ctx lifetime. - */ -static void * -ntb_get_ctx(device_t dev, const struct ntb_ctx_ops **ops) -{ - struct ntb_softc *ntb = device_get_softc(dev); - - KASSERT(ntb->ntb_ctx != NULL && ntb->ctx_ops != NULL, ("bogus")); - if (ops != NULL) - *ops = ntb->ctx_ops; - return (ntb->ntb_ctx); -} - -static void -ntb_clear_ctx(device_t dev) -{ - struct ntb_softc *ntb = device_get_softc(dev); - - rm_wlock(&ntb->ctx_lock); - ntb->ntb_ctx = NULL; - ntb->ctx_ops = NULL; - rm_wunlock(&ntb->ctx_lock); -} - -/* - * ntb_link_event() - notify driver context of a change in link status - * @ntb: NTB device context - * - * Notify the driver context that the link status may have changed. The driver - * should call ntb_link_is_up() to get the current status. - */ -static void -ntb_link_event(device_t dev) -{ - struct ntb_softc *ntb = device_get_softc(dev); - struct rm_priotracker ctx_tracker; - - rm_rlock(&ntb->ctx_lock, &ctx_tracker); - if (ntb->ctx_ops != NULL && ntb->ctx_ops->link_event != NULL) - ntb->ctx_ops->link_event(ntb->ntb_ctx); - rm_runlock(&ntb->ctx_lock, &ctx_tracker); -} - -/* - * ntb_db_event() - notify driver context of a doorbell event - * @ntb: NTB device context - * @vector: Interrupt vector number - * - * Notify the driver context of a doorbell event. If hardware supports - * multiple interrupt vectors for doorbells, the vector number indicates which - * vector received the interrupt. The vector number is relative to the first - * vector used for doorbells, starting at zero, and must be less than - * ntb_db_vector_count(). The driver may call ntb_db_read() to check which - * doorbell bits need service, and ntb_db_vector_mask() to determine which of - * those bits are associated with the vector number. - */ -static void -ntb_db_event(device_t dev, uint32_t vec) -{ - struct ntb_softc *ntb = device_get_softc(dev); - struct rm_priotracker ctx_tracker; - - rm_rlock(&ntb->ctx_lock, &ctx_tracker); - if (ntb->ctx_ops != NULL && ntb->ctx_ops->db_event != NULL) - ntb->ctx_ops->db_event(ntb->ntb_ctx, vec); - rm_runlock(&ntb->ctx_lock, &ctx_tracker); + intel_ntb_reg_write(4, ATOM_LTSSMSTATEJMP_OFFSET, status); } static int -ntb_link_enable(device_t dev, enum ntb_speed speed __unused, +intel_ntb_link_enable(device_t dev, enum ntb_speed speed __unused, enum ntb_width width __unused) { struct ntb_softc *ntb = device_get_softc(dev); uint32_t cntl; - ntb_printf(2, "%s\n", __func__); + intel_ntb_printf(2, "%s\n", __func__); if (ntb->type == NTB_ATOM) { pci_write_config(ntb->device, NTB_PPD_OFFSET, @@ -2088,41 +1992,41 @@ ntb_link_enable(device_t dev, enum ntb_speed speed __unused, return (0); } - cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); + cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP; if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP; - ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); + intel_ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); return (0); } static int -ntb_link_disable(device_t dev) +intel_ntb_link_disable(device_t dev) { struct ntb_softc *ntb = device_get_softc(dev); uint32_t cntl; - ntb_printf(2, "%s\n", __func__); + intel_ntb_printf(2, "%s\n", __func__); if (ntb->conn_type == NTB_CONN_TRANSPARENT) { ntb_link_event(dev); return (0); } - cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); + cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP); if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP); cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; - ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); + intel_ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); return (0); } static bool -ntb_link_enabled(device_t dev) +intel_ntb_link_enabled(device_t dev) { struct ntb_softc *ntb = device_get_softc(dev); uint32_t cntl; @@ -2135,7 +2039,7 @@ ntb_link_enabled(device_t dev) if (ntb->conn_type == NTB_CONN_TRANSPARENT) return (true); - cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); + cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); return ((cntl & NTB_CNTL_LINK_DISABLE) == 0); } @@ -2160,11 +2064,11 @@ recover_atom_link(void *arg) if (atom_link_is_err(ntb)) goto retry; - status32 = ntb_reg_read(4, ntb->reg->ntb_ctl); + status32 = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); if ((status32 & ATOM_CNTL_LINK_DOWN) != 0) goto out; - status32 = ntb_reg_read(4, ntb->reg->lnk_sta); + status32 = intel_ntb_reg_read(4, ntb->reg->lnk_sta); width = NTB_LNK_STA_WIDTH(status32); speed = status32 & NTB_LINK_SPEED_MASK; @@ -2187,18 +2091,18 @@ retry: * Polls the HW link status register(s); returns true if something has changed. */ static bool -ntb_poll_link(struct ntb_softc *ntb) +intel_ntb_poll_link(struct ntb_softc *ntb) { uint32_t ntb_cntl; uint16_t reg_val; if (ntb->type == NTB_ATOM) { - ntb_cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); + ntb_cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); if (ntb_cntl == ntb->ntb_ctl) return (false); ntb->ntb_ctl = ntb_cntl; - ntb->lnk_sta = ntb_reg_read(4, ntb->reg->lnk_sta); + ntb->lnk_sta = intel_ntb_reg_read(4, ntb->reg->lnk_sta); } else { db_iowrite_raw(ntb, ntb->self_reg->db_bell, ntb->db_link_mask); @@ -2212,7 +2116,7 @@ ntb_poll_link(struct ntb_softc *ntb) if (_xeon_link_is_up(ntb)) { if (!ntb->peer_msix_good) { callout_reset(&ntb->peer_msix_work, 0, - ntb_exchange_msix, ntb); + intel_ntb_exchange_msix, ntb); return (false); } } else { @@ -2225,7 +2129,7 @@ ntb_poll_link(struct ntb_softc *ntb) } static inline enum ntb_speed -ntb_link_sta_speed(struct ntb_softc *ntb) +intel_ntb_link_sta_speed(struct ntb_softc *ntb) { if (!link_is_up(ntb)) @@ -2234,7 +2138,7 @@ ntb_link_sta_speed(struct ntb_softc *ntb) } static inline enum ntb_width -ntb_link_sta_width(struct ntb_softc *ntb) +intel_ntb_link_sta_width(struct ntb_softc *ntb) { if (!link_is_up(ntb)) @@ -2256,7 +2160,7 @@ SYSCTL_NODE(_hw_ntb, OID_AUTO, debug_info, CTLFLAG_RW, 0, #define NTB_REGFLAGS_MASK (NTB_REGSZ_MASK | NTB_DB_READ | NTB_PCI_REG) static void -ntb_sysctl_init(struct ntb_softc *ntb) +intel_ntb_sysctl_init(struct ntb_softc *ntb) { struct sysctl_oid_list *globals, *tree_par, *regpar, *statpar, *errpar; struct sysctl_ctx_list *ctx; @@ -2555,7 +2459,7 @@ sysctl_handle_link_admin(SYSCTL_HANDLER_ARGS) unsigned old, new; int error; - old = ntb_link_enabled(ntb->device); + old = intel_ntb_link_enabled(ntb->device); error = SYSCTL_OUT(req, &old, sizeof(old)); if (error != 0 || req->newptr == NULL) @@ -2565,13 +2469,13 @@ sysctl_handle_link_admin(SYSCTL_HANDLER_ARGS) if (error != 0) return (error); - ntb_printf(0, "Admin set interface state to '%sabled'\n", + intel_ntb_printf(0, "Admin set interface state to '%sabled'\n", (new != 0)? "en" : "dis"); if (new != 0) - error = ntb_link_enable(ntb->device, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); + error = intel_ntb_link_enable(ntb->device, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); else - error = ntb_link_disable(ntb->device); + error = intel_ntb_link_disable(ntb->device); return (error); } @@ -2586,7 +2490,7 @@ sysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS) sbuf_new_for_sysctl(&sb, NULL, 32, req); - if (ntb_link_is_up(ntb->device, &speed, &width)) + if (intel_ntb_link_is_up(ntb->device, &speed, &width)) sbuf_printf(&sb, "up / PCIe Gen %u / Width x%u", (unsigned)speed, (unsigned)width); else @@ -2607,7 +2511,7 @@ sysctl_handle_link_status(SYSCTL_HANDLER_ARGS) unsigned res; int error; - res = ntb_link_is_up(ntb->device, NULL, NULL); + res = intel_ntb_link_is_up(ntb->device, NULL, NULL); error = SYSCTL_OUT(req, &res, sizeof(res)); if (error || !req->newptr) @@ -2646,28 +2550,28 @@ sysctl_handle_register(SYSCTL_HANDLER_ARGS) if (pci) umv = pci_read_config(ntb->device, reg, 8); else - umv = ntb_reg_read(8, reg); + umv = intel_ntb_reg_read(8, reg); outsz = sizeof(uint64_t); break; case NTB_REG_32: if (pci) umv = pci_read_config(ntb->device, reg, 4); else - umv = ntb_reg_read(4, reg); + umv = intel_ntb_reg_read(4, reg); outsz = sizeof(uint32_t); break; case NTB_REG_16: if (pci) umv = pci_read_config(ntb->device, reg, 2); else - umv = ntb_reg_read(2, reg); + umv = intel_ntb_reg_read(2, reg); outsz = sizeof(uint16_t); break; case NTB_REG_8: if (pci) umv = pci_read_config(ntb->device, reg, 1); else - umv = ntb_reg_read(1, reg); + umv = intel_ntb_reg_read(1, reg); outsz = sizeof(uint8_t); break; default: @@ -2687,7 +2591,7 @@ sysctl_handle_register(SYSCTL_HANDLER_ARGS) } static unsigned -ntb_user_mw_to_idx(struct ntb_softc *ntb, unsigned uidx) +intel_ntb_user_mw_to_idx(struct ntb_softc *ntb, unsigned uidx) { if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && @@ -2701,8 +2605,21 @@ ntb_user_mw_to_idx(struct ntb_softc *ntb, unsigned uidx) return (uidx); } +#ifndef EARLY_AP_STARTUP +static int msix_ready; + static void -ntb_exchange_msix(void *ctx) +intel_ntb_msix_ready(void *arg __unused) +{ + + msix_ready = 1; +} +SYSINIT(intel_ntb_msix_ready, SI_SUB_SMP, SI_ORDER_ANY, + intel_ntb_msix_ready, NULL); +#endif + +static void +intel_ntb_exchange_msix(void *ctx) { struct ntb_softc *ntb; uint32_t val; @@ -2715,41 +2632,49 @@ ntb_exchange_msix(void *ctx) if (ntb->peer_msix_done) goto msix_done; +#ifndef EARLY_AP_STARTUP + /* Block MSIX negotiation until SMP started and IRQ reshuffled. */ + if (!msix_ready) + goto reschedule; +#endif + + intel_ntb_get_msix_info(ntb); for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { - ntb_peer_spad_write(ntb->device, NTB_MSIX_DATA0 + i, + intel_ntb_peer_spad_write(ntb->device, NTB_MSIX_DATA0 + i, ntb->msix_data[i].nmd_data); - ntb_peer_spad_write(ntb->device, NTB_MSIX_OFS0 + i, + intel_ntb_peer_spad_write(ntb->device, NTB_MSIX_OFS0 + i, ntb->msix_data[i].nmd_ofs - ntb->msix_xlat); } - ntb_peer_spad_write(ntb->device, NTB_MSIX_GUARD, NTB_MSIX_VER_GUARD); + intel_ntb_peer_spad_write(ntb->device, NTB_MSIX_GUARD, NTB_MSIX_VER_GUARD); - ntb_spad_read(ntb->device, NTB_MSIX_GUARD, &val); + intel_ntb_spad_read(ntb->device, NTB_MSIX_GUARD, &val); if (val != NTB_MSIX_VER_GUARD) goto reschedule; for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { - ntb_spad_read(ntb->device, NTB_MSIX_DATA0 + i, &val); - ntb_printf(2, "remote MSIX data(%u): 0x%x\n", i, val); + intel_ntb_spad_read(ntb->device, NTB_MSIX_DATA0 + i, &val); + intel_ntb_printf(2, "remote MSIX data(%u): 0x%x\n", i, val); ntb->peer_msix_data[i].nmd_data = val; - ntb_spad_read(ntb->device, NTB_MSIX_OFS0 + i, &val); - ntb_printf(2, "remote MSIX addr(%u): 0x%x\n", i, val); + intel_ntb_spad_read(ntb->device, NTB_MSIX_OFS0 + i, &val); + intel_ntb_printf(2, "remote MSIX addr(%u): 0x%x\n", i, val); ntb->peer_msix_data[i].nmd_ofs = val; } ntb->peer_msix_done = true; msix_done: - ntb_peer_spad_write(ntb->device, NTB_MSIX_DONE, NTB_MSIX_RECEIVED); - ntb_spad_read(ntb->device, NTB_MSIX_DONE, &val); + intel_ntb_peer_spad_write(ntb->device, NTB_MSIX_DONE, NTB_MSIX_RECEIVED); + intel_ntb_spad_read(ntb->device, NTB_MSIX_DONE, &val); if (val != NTB_MSIX_RECEIVED) goto reschedule; + intel_ntb_spad_clear(ntb->device); ntb->peer_msix_good = true; /* Give peer time to see our NTB_MSIX_RECEIVED. */ goto reschedule; msix_good: - ntb_poll_link(ntb); + intel_ntb_poll_link(ntb); ntb_link_event(ntb->device); return; @@ -2758,9 +2683,9 @@ reschedule: if (_xeon_link_is_up(ntb)) { callout_reset(&ntb->peer_msix_work, hz * (ntb->peer_msix_good ? 2 : 1) / 100, - ntb_exchange_msix, ntb); + intel_ntb_exchange_msix, ntb); } else - ntb_spad_clear(ntb->device); + intel_ntb_spad_clear(ntb->device); } /* @@ -2768,7 +2693,7 @@ reschedule: */ static uint8_t -ntb_spad_count(device_t dev) +intel_ntb_spad_count(device_t dev) { struct ntb_softc *ntb = device_get_softc(dev); @@ -2776,7 +2701,7 @@ ntb_spad_count(device_t dev) } static uint8_t -ntb_mw_count(device_t dev) +intel_ntb_mw_count(device_t dev) { struct ntb_softc *ntb = device_get_softc(dev); uint8_t res; @@ -2790,14 +2715,14 @@ ntb_mw_count(device_t dev) } static int -ntb_spad_write(device_t dev, unsigned int idx, uint32_t val) +intel_ntb_spad_write(device_t dev, unsigned int idx, uint32_t val) { struct ntb_softc *ntb = device_get_softc(dev); if (idx >= ntb->spad_count) return (EINVAL); - ntb_reg_write(4, ntb->self_reg->spad + idx * 4, val); + intel_ntb_reg_write(4, ntb->self_reg->spad + idx * 4, val); return (0); } @@ -2806,30 +2731,30 @@ ntb_spad_write(device_t dev, unsigned int idx, uint32_t val) * Zeros the local scratchpad. */ static void -ntb_spad_clear(device_t dev) +intel_ntb_spad_clear(device_t dev) { struct ntb_softc *ntb = device_get_softc(dev); unsigned i; for (i = 0; i < ntb->spad_count; i++) - ntb_spad_write(dev, i, 0); + intel_ntb_spad_write(dev, i, 0); } static int -ntb_spad_read(device_t dev, unsigned int idx, uint32_t *val) +intel_ntb_spad_read(device_t dev, unsigned int idx, uint32_t *val) { struct ntb_softc *ntb = device_get_softc(dev); if (idx >= ntb->spad_count) return (EINVAL); - *val = ntb_reg_read(4, ntb->self_reg->spad + idx * 4); + *val = intel_ntb_reg_read(4, ntb->self_reg->spad + idx * 4); return (0); } static int -ntb_peer_spad_write(device_t dev, unsigned int idx, uint32_t val) +intel_ntb_peer_spad_write(device_t dev, unsigned int idx, uint32_t val) { struct ntb_softc *ntb = device_get_softc(dev); @@ -2837,15 +2762,15 @@ ntb_peer_spad_write(device_t dev, unsigned int idx, uint32_t val) return (EINVAL); if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) - ntb_mw_write(4, XEON_SPAD_OFFSET + idx * 4, val); + intel_ntb_mw_write(4, XEON_SPAD_OFFSET + idx * 4, val); else - ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val); + intel_ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val); return (0); } static int -ntb_peer_spad_read(device_t dev, unsigned int idx, uint32_t *val) +intel_ntb_peer_spad_read(device_t dev, unsigned int idx, uint32_t *val) { struct ntb_softc *ntb = device_get_softc(dev); @@ -2853,15 +2778,15 @@ ntb_peer_spad_read(device_t dev, unsigned int idx, uint32_t *val) return (EINVAL); if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) - *val = ntb_mw_read(4, XEON_SPAD_OFFSET + idx * 4); + *val = intel_ntb_mw_read(4, XEON_SPAD_OFFSET + idx * 4); else - *val = ntb_reg_read(4, ntb->peer_reg->spad + idx * 4); + *val = intel_ntb_reg_read(4, ntb->peer_reg->spad + idx * 4); return (0); } static int -ntb_mw_get_range(device_t dev, unsigned mw_idx, vm_paddr_t *base, +intel_ntb_mw_get_range(device_t dev, unsigned mw_idx, vm_paddr_t *base, caddr_t *vbase, size_t *size, size_t *align, size_t *align_size, bus_addr_t *plimit) { @@ -2871,11 +2796,11 @@ ntb_mw_get_range(device_t dev, unsigned mw_idx, vm_paddr_t *base, size_t bar_b2b_off; enum ntb_bar bar_num; - if (mw_idx >= ntb_mw_count(dev)) + if (mw_idx >= intel_ntb_mw_count(dev)) return (EINVAL); - mw_idx = ntb_user_mw_to_idx(ntb, mw_idx); + mw_idx = intel_ntb_user_mw_to_idx(ntb, mw_idx); - bar_num = ntb_mw_to_bar(ntb, mw_idx); + bar_num = intel_ntb_mw_to_bar(ntb, mw_idx); bar = &ntb->bar_info[bar_num]; bar_b2b_off = 0; if (mw_idx == ntb->b2b_mw_idx) { @@ -2905,7 +2830,7 @@ ntb_mw_get_range(device_t dev, unsigned mw_idx, vm_paddr_t *base, } static int -ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr, size_t size) +intel_ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr, size_t size) { struct ntb_softc *ntb = device_get_softc(dev); struct ntb_pci_bar_info *bar; @@ -2914,11 +2839,11 @@ ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr, size_t size) uint32_t base_reg, xlat_reg, limit_reg; enum ntb_bar bar_num; - if (idx >= ntb_mw_count(dev)) + if (idx >= intel_ntb_mw_count(dev)) return (EINVAL); - idx = ntb_user_mw_to_idx(ntb, idx); + idx = intel_ntb_user_mw_to_idx(ntb, idx); - bar_num = ntb_mw_to_bar(ntb, idx); + bar_num = intel_ntb_mw_to_bar(ntb, idx); bar = &ntb->bar_info[bar_num]; bar_size = bar->size; @@ -2938,25 +2863,25 @@ ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr, size_t size) limit = 0; if (bar_is_64bit(ntb, bar_num)) { - base = ntb_reg_read(8, base_reg) & BAR_HIGH_MASK; + base = intel_ntb_reg_read(8, base_reg) & BAR_HIGH_MASK; if (limit_reg != 0 && size != mw_size) limit = base + size; /* Set and verify translation address */ - ntb_reg_write(8, xlat_reg, addr); - reg_val = ntb_reg_read(8, xlat_reg) & BAR_HIGH_MASK; + intel_ntb_reg_write(8, xlat_reg, addr); + reg_val = intel_ntb_reg_read(8, xlat_reg) & BAR_HIGH_MASK; if (reg_val != addr) { - ntb_reg_write(8, xlat_reg, 0); + intel_ntb_reg_write(8, xlat_reg, 0); return (EIO); } /* Set and verify the limit */ - ntb_reg_write(8, limit_reg, limit); - reg_val = ntb_reg_read(8, limit_reg) & BAR_HIGH_MASK; + intel_ntb_reg_write(8, limit_reg, limit); + reg_val = intel_ntb_reg_read(8, limit_reg) & BAR_HIGH_MASK; if (reg_val != limit) { - ntb_reg_write(8, limit_reg, base); - ntb_reg_write(8, xlat_reg, 0); + intel_ntb_reg_write(8, limit_reg, base); + intel_ntb_reg_write(8, xlat_reg, 0); return (EIO); } } else { @@ -2967,25 +2892,25 @@ ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr, size_t size) if (((addr + size) & UINT32_MAX) != (addr + size)) return (ERANGE); - base = ntb_reg_read(4, base_reg) & BAR_HIGH_MASK; + base = intel_ntb_reg_read(4, base_reg) & BAR_HIGH_MASK; if (limit_reg != 0 && size != mw_size) limit = base + size; /* Set and verify translation address */ - ntb_reg_write(4, xlat_reg, addr); - reg_val = ntb_reg_read(4, xlat_reg) & BAR_HIGH_MASK; + intel_ntb_reg_write(4, xlat_reg, addr); + reg_val = intel_ntb_reg_read(4, xlat_reg) & BAR_HIGH_MASK; if (reg_val != addr) { - ntb_reg_write(4, xlat_reg, 0); + intel_ntb_reg_write(4, xlat_reg, 0); return (EIO); } /* Set and verify the limit */ - ntb_reg_write(4, limit_reg, limit); - reg_val = ntb_reg_read(4, limit_reg) & BAR_HIGH_MASK; + intel_ntb_reg_write(4, limit_reg, limit); + reg_val = intel_ntb_reg_read(4, limit_reg) & BAR_HIGH_MASK; if (reg_val != limit) { - ntb_reg_write(4, limit_reg, base); - ntb_reg_write(4, xlat_reg, 0); + intel_ntb_reg_write(4, limit_reg, base); + intel_ntb_reg_write(4, xlat_reg, 0); return (EIO); } } @@ -2993,46 +2918,46 @@ ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr, size_t size) } static int -ntb_mw_clear_trans(device_t dev, unsigned mw_idx) +intel_ntb_mw_clear_trans(device_t dev, unsigned mw_idx) { - return (ntb_mw_set_trans(dev, mw_idx, 0, 0)); + return (intel_ntb_mw_set_trans(dev, mw_idx, 0, 0)); } static int -ntb_mw_get_wc(device_t dev, unsigned idx, vm_memattr_t *mode) +intel_ntb_mw_get_wc(device_t dev, unsigned idx, vm_memattr_t *mode) { struct ntb_softc *ntb = device_get_softc(dev); struct ntb_pci_bar_info *bar; - if (idx >= ntb_mw_count(dev)) + if (idx >= intel_ntb_mw_count(dev)) return (EINVAL); - idx = ntb_user_mw_to_idx(ntb, idx); + idx = intel_ntb_user_mw_to_idx(ntb, idx); - bar = &ntb->bar_info[ntb_mw_to_bar(ntb, idx)]; + bar = &ntb->bar_info[intel_ntb_mw_to_bar(ntb, idx)]; *mode = bar->map_mode; return (0); } static int -ntb_mw_set_wc(device_t dev, unsigned idx, vm_memattr_t mode) +intel_ntb_mw_set_wc(device_t dev, unsigned idx, vm_memattr_t mode) { struct ntb_softc *ntb = device_get_softc(dev); - if (idx >= ntb_mw_count(dev)) + if (idx >= intel_ntb_mw_count(dev)) return (EINVAL); - idx = ntb_user_mw_to_idx(ntb, idx); - return (ntb_mw_set_wc_internal(ntb, idx, mode)); + idx = intel_ntb_user_mw_to_idx(ntb, idx); + return (intel_ntb_mw_set_wc_internal(ntb, idx, mode)); } static int -ntb_mw_set_wc_internal(struct ntb_softc *ntb, unsigned idx, vm_memattr_t mode) +intel_ntb_mw_set_wc_internal(struct ntb_softc *ntb, unsigned idx, vm_memattr_t mode) { struct ntb_pci_bar_info *bar; int rc; - bar = &ntb->bar_info[ntb_mw_to_bar(ntb, idx)]; + bar = &ntb->bar_info[intel_ntb_mw_to_bar(ntb, idx)]; if (bar->map_mode == mode) return (0); @@ -3044,7 +2969,7 @@ ntb_mw_set_wc_internal(struct ntb_softc *ntb, unsigned idx, vm_memattr_t mode) } static void -ntb_peer_db_set(device_t dev, uint64_t bit) +intel_ntb_peer_db_set(device_t dev, uint64_t bit) { struct ntb_softc *ntb = device_get_softc(dev); @@ -3055,7 +2980,7 @@ ntb_peer_db_set(device_t dev, uint64_t bit) lapic = ntb->peer_lapic_bar; for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { - if ((bit & ntb_db_vector_mask(dev, i)) != 0) + if ((bit & intel_ntb_db_vector_mask(dev, i)) != 0) bus_space_write_4(lapic->pci_bus_tag, lapic->pci_bus_handle, ntb->peer_msix_data[i].nmd_ofs, @@ -3065,7 +2990,7 @@ ntb_peer_db_set(device_t dev, uint64_t bit) } if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) { - ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bit); + intel_ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bit); return; } @@ -3073,7 +2998,7 @@ ntb_peer_db_set(device_t dev, uint64_t bit) } static int -ntb_peer_db_addr(device_t dev, bus_addr_t *db_addr, vm_size_t *db_size) +intel_ntb_peer_db_addr(device_t dev, bus_addr_t *db_addr, vm_size_t *db_size) { struct ntb_softc *ntb = device_get_softc(dev); struct ntb_pci_bar_info *bar; @@ -3088,7 +3013,7 @@ ntb_peer_db_addr(device_t dev, bus_addr_t *db_addr, vm_size_t *db_size) KASSERT(ntb->b2b_mw_idx != B2B_MW_DISABLED, ("invalid b2b idx")); - bar = &ntb->bar_info[ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)]; + bar = &ntb->bar_info[intel_ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)]; regoff = XEON_PDOORBELL_OFFSET; } KASSERT(bar->pci_bus_tag != X86_BUS_SPACE_IO, ("uh oh")); @@ -3100,7 +3025,7 @@ ntb_peer_db_addr(device_t dev, bus_addr_t *db_addr, vm_size_t *db_size) } static uint64_t -ntb_db_valid_mask(device_t dev) +intel_ntb_db_valid_mask(device_t dev) { struct ntb_softc *ntb = device_get_softc(dev); @@ -3108,7 +3033,7 @@ ntb_db_valid_mask(device_t dev) } static int -ntb_db_vector_count(device_t dev) +intel_ntb_db_vector_count(device_t dev) { struct ntb_softc *ntb = device_get_softc(dev); @@ -3116,24 +3041,24 @@ ntb_db_vector_count(device_t dev) } static uint64_t -ntb_db_vector_mask(device_t dev, uint32_t vector) +intel_ntb_db_vector_mask(device_t dev, uint32_t vector) { struct ntb_softc *ntb = device_get_softc(dev); if (vector > ntb->db_vec_count) return (0); - return (ntb->db_valid_mask & ntb_vec_mask(ntb, vector)); + return (ntb->db_valid_mask & intel_ntb_vec_mask(ntb, vector)); } static bool -ntb_link_is_up(device_t dev, enum ntb_speed *speed, enum ntb_width *width) +intel_ntb_link_is_up(device_t dev, enum ntb_speed *speed, enum ntb_width *width) { struct ntb_softc *ntb = device_get_softc(dev); if (speed != NULL) - *speed = ntb_link_sta_speed(ntb); + *speed = intel_ntb_link_sta_speed(ntb); if (width != NULL) - *width = ntb_link_sta_width(ntb); + *width = intel_ntb_link_sta_width(ntb); return (link_is_up(ntb)); } @@ -3150,38 +3075,35 @@ save_bar_parameters(struct ntb_pci_bar_info *bar) static device_method_t ntb_intel_methods[] = { /* Device interface */ - DEVMETHOD(device_probe, ntb_probe), - DEVMETHOD(device_attach, ntb_attach), - DEVMETHOD(device_detach, ntb_detach), + DEVMETHOD(device_probe, intel_ntb_probe), + DEVMETHOD(device_attach, intel_ntb_attach), + DEVMETHOD(device_detach, intel_ntb_detach), /* NTB interface */ - DEVMETHOD(ntb_link_is_up, ntb_link_is_up), - DEVMETHOD(ntb_link_enable, ntb_link_enable), - DEVMETHOD(ntb_link_disable, ntb_link_disable), - DEVMETHOD(ntb_link_enabled, ntb_link_enabled), - DEVMETHOD(ntb_set_ctx, ntb_set_ctx), - DEVMETHOD(ntb_get_ctx, ntb_get_ctx), - DEVMETHOD(ntb_clear_ctx, ntb_clear_ctx), - DEVMETHOD(ntb_mw_count, ntb_mw_count), - DEVMETHOD(ntb_mw_get_range, ntb_mw_get_range), - DEVMETHOD(ntb_mw_set_trans, ntb_mw_set_trans), - DEVMETHOD(ntb_mw_clear_trans, ntb_mw_clear_trans), - DEVMETHOD(ntb_mw_get_wc, ntb_mw_get_wc), - DEVMETHOD(ntb_mw_set_wc, ntb_mw_set_wc), - DEVMETHOD(ntb_spad_count, ntb_spad_count), - DEVMETHOD(ntb_spad_clear, ntb_spad_clear), - DEVMETHOD(ntb_spad_write, ntb_spad_write), - DEVMETHOD(ntb_spad_read, ntb_spad_read), - DEVMETHOD(ntb_peer_spad_write, ntb_peer_spad_write), - DEVMETHOD(ntb_peer_spad_read, ntb_peer_spad_read), - DEVMETHOD(ntb_db_valid_mask, ntb_db_valid_mask), - DEVMETHOD(ntb_db_vector_count, ntb_db_vector_count), - DEVMETHOD(ntb_db_vector_mask, ntb_db_vector_mask), - DEVMETHOD(ntb_db_clear, ntb_db_clear), - DEVMETHOD(ntb_db_clear_mask, ntb_db_clear_mask), - DEVMETHOD(ntb_db_read, ntb_db_read), - DEVMETHOD(ntb_db_set_mask, ntb_db_set_mask), - DEVMETHOD(ntb_peer_db_addr, ntb_peer_db_addr), - DEVMETHOD(ntb_peer_db_set, ntb_peer_db_set), + DEVMETHOD(ntb_link_is_up, intel_ntb_link_is_up), + DEVMETHOD(ntb_link_enable, intel_ntb_link_enable), + DEVMETHOD(ntb_link_disable, intel_ntb_link_disable), + DEVMETHOD(ntb_link_enabled, intel_ntb_link_enabled), + DEVMETHOD(ntb_mw_count, intel_ntb_mw_count), + DEVMETHOD(ntb_mw_get_range, intel_ntb_mw_get_range), + DEVMETHOD(ntb_mw_set_trans, intel_ntb_mw_set_trans), + DEVMETHOD(ntb_mw_clear_trans, intel_ntb_mw_clear_trans), + DEVMETHOD(ntb_mw_get_wc, intel_ntb_mw_get_wc), + DEVMETHOD(ntb_mw_set_wc, intel_ntb_mw_set_wc), + DEVMETHOD(ntb_spad_count, intel_ntb_spad_count), + DEVMETHOD(ntb_spad_clear, intel_ntb_spad_clear), + DEVMETHOD(ntb_spad_write, intel_ntb_spad_write), + DEVMETHOD(ntb_spad_read, intel_ntb_spad_read), + DEVMETHOD(ntb_peer_spad_write, intel_ntb_peer_spad_write), + DEVMETHOD(ntb_peer_spad_read, intel_ntb_peer_spad_read), + DEVMETHOD(ntb_db_valid_mask, intel_ntb_db_valid_mask), + DEVMETHOD(ntb_db_vector_count, intel_ntb_db_vector_count), + DEVMETHOD(ntb_db_vector_mask, intel_ntb_db_vector_mask), + DEVMETHOD(ntb_db_clear, intel_ntb_db_clear), + DEVMETHOD(ntb_db_clear_mask, intel_ntb_db_clear_mask), + DEVMETHOD(ntb_db_read, intel_ntb_db_read), + DEVMETHOD(ntb_db_set_mask, intel_ntb_db_set_mask), + DEVMETHOD(ntb_peer_db_addr, intel_ntb_peer_db_addr), + DEVMETHOD(ntb_peer_db_set, intel_ntb_peer_db_set), DEVMETHOD_END }; diff --git a/sys/dev/ntb/ntb_if.m b/sys/dev/ntb/ntb_if.m index bb3b7bd1077b..d8ca22755839 100644 --- a/sys/dev/ntb/ntb_if.m +++ b/sys/dev/ntb/ntb_if.m @@ -60,133 +60,45 @@ HEADER { }; }; -# -# ntb_link_is_up() - get the current ntb link state -# @ntb: NTB device context -# @speed: OUT - The link speed expressed as PCIe generation number -# @width: OUT - The link width expressed as the number of PCIe lanes -# -# RETURNS: true or false based on the hardware link state -# METHOD bool link_is_up { device_t ntb; enum ntb_speed *speed; enum ntb_width *width; }; -# -# ntb_link_enable() - enable the link on the secondary side of the ntb -# @ntb: NTB device context -# @max_speed: The maximum link speed expressed as PCIe generation number[0] -# @max_width: The maximum link width expressed as the number of PCIe lanes[0] -# -# Enable the link on the secondary side of the ntb. This can only be done -# from the primary side of the ntb in primary or b2b topology. The ntb device -# should train the link to its maximum speed and width, or the requested speed -# and width, whichever is smaller, if supported. -# -# Return: Zero on success, otherwise an error number. -# -# [0]: Only NTB_SPEED_AUTO and NTB_WIDTH_AUTO are valid inputs; other speed -# and width input will be ignored. -#/ METHOD int link_enable { device_t ntb; enum ntb_speed speed; enum ntb_width width; }; -# -# ntb_link_disable() - disable the link on the secondary side of the ntb -# @ntb: NTB device context -# -# Disable the link on the secondary side of the ntb. This can only be done -# from the primary side of the ntb in primary or b2b topology. The ntb device -# should disable the link. Returning from this call must indicate that a -# barrier has passed, though with no more writes may pass in either direction -# across the link, except if this call returns an error number. -# -# Return: Zero on success, otherwise an error number. -# METHOD int link_disable { device_t ntb; }; -# -# get enable status of the link on the secondary side of the ntb -# METHOD bool link_enabled { device_t ntb; }; -# -# ntb_set_ctx() - associate a driver context with an ntb device -# @ntb: NTB device context -# @ctx: Driver context -# @ctx_ops: Driver context operations -# -# Associate a driver context and operations with a ntb device. The context is -# provided by the client driver, and the driver may associate a different -# context with each ntb device. -# -# Return: Zero if the context is associated, otherwise an error number. -# METHOD int set_ctx { device_t ntb; void *ctx; const struct ntb_ctx_ops *ctx_ops; }; -# -# ntb_set_ctx() - get a driver context associated with an ntb device -# @ntb: NTB device context -# @ctx_ops: Driver context operations -# -# Get a driver context and operations associated with a ntb device. -# METHOD void * get_ctx { device_t ntb; const struct ntb_ctx_ops **ctx_ops; }; -# -# ntb_clear_ctx() - disassociate any driver context from an ntb device -# @ntb: NTB device context -# -# Clear any association that may exist between a driver context and the ntb -# device. -# METHOD void clear_ctx { device_t ntb; }; -# -# ntb_mw_count() - Get the number of memory windows available for KPI -# consumers. -# -# (Excludes any MW wholly reserved for register access.) -# METHOD uint8_t mw_count { device_t ntb; }; -# -# ntb_mw_get_range() - get the range of a memory window -# @ntb: NTB device context -# @idx: Memory window number -# @base: OUT - the base address for mapping the memory window -# @size: OUT - the size for mapping the memory window -# @align: OUT - the base alignment for translating the memory window -# @align_size: OUT - the size alignment for translating the memory window -# -# Get the range of a memory window. NULL may be given for any output -# parameter if the value is not needed. The base and size may be used for -# mapping the memory window, to access the peer memory. The alignment and -# size may be used for translating the memory window, for the peer to access -# memory on the local system. -# -# Return: Zero on success, otherwise an error number. -# METHOD int mw_get_range { device_t ntb; unsigned mw_idx; @@ -198,22 +110,6 @@ METHOD int mw_get_range { bus_addr_t *plimit; }; -# -# ntb_mw_set_trans() - set the translation of a memory window -# @ntb: NTB device context -# @idx: Memory window number -# @addr: The dma address local memory to expose to the peer -# @size: The size of the local memory to expose to the peer -# -# Set the translation of a memory window. The peer may access local memory -# through the window starting at the address, up to the size. The address -# must be aligned to the alignment specified by ntb_mw_get_range(). The size -# must be aligned to the size alignment specified by ntb_mw_get_range(). The -# address must be below the plimit specified by ntb_mw_get_range() (i.e. for -# 32-bit BARs). -# -# Return: Zero on success, otherwise an error number. -# METHOD int mw_set_trans { device_t ntb; unsigned mw_idx; @@ -221,277 +117,94 @@ METHOD int mw_set_trans { size_t size; }; -# -# ntb_mw_clear_trans() - clear the translation of a memory window -# @ntb: NTB device context -# @idx: Memory window number -# -# Clear the translation of a memory window. The peer may no longer access -# local memory through the window. -# -# Return: Zero on success, otherwise an error number. -# METHOD int mw_clear_trans { device_t ntb; unsigned mw_idx; }; -# -# ntb_mw_get_wc - Get the write-combine status of a memory window -# -# Returns: Zero on success, setting *wc; otherwise an error number (e.g. if -# idx is an invalid memory window). -# -# Mode is a VM_MEMATTR_* type. -# METHOD int mw_get_wc { device_t ntb; unsigned mw_idx; vm_memattr_t *mode; }; -# -# ntb_mw_set_wc - Set the write-combine status of a memory window -# -# If 'mode' matches the current status, this does nothing and succeeds. Mode -# is a VM_MEMATTR_* type. -# -# Returns: Zero on success, setting the caching attribute on the virtual -# mapping of the BAR; otherwise an error number (e.g. if idx is an invalid -# memory window, or if changing the caching attribute fails). -# METHOD int mw_set_wc { device_t ntb; unsigned mw_idx; vm_memattr_t mode; }; -# -# ntb_spad_count() - get the total scratch regs usable -# @ntb: pointer to ntb_softc instance -# -# This function returns the max 32bit scratchpad registers usable by the -# upper layer. -# -# RETURNS: total number of scratch pad registers available -# METHOD uint8_t spad_count { device_t ntb; }; -# -# ntb_get_max_spads() - zero local scratch registers -# @ntb: pointer to ntb_softc instance -# -# This functions overwrites all local scratchpad registers with zeroes. -# METHOD void spad_clear { device_t ntb; }; -# -# ntb_spad_write() - write to the secondary scratchpad register -# @ntb: pointer to ntb_softc instance -# @idx: index to the scratchpad register, 0 based -# @val: the data value to put into the register -# -# This function allows writing of a 32bit value to the indexed scratchpad -# register. The register resides on the secondary (external) side. -# -# RETURNS: An appropriate ERRNO error value on error, or zero for success. -# METHOD int spad_write { device_t ntb; unsigned int idx; uint32_t val; }; -# -# ntb_spad_read() - read from the primary scratchpad register -# @ntb: pointer to ntb_softc instance -# @idx: index to scratchpad register, 0 based -# @val: pointer to 32bit integer for storing the register value -# -# This function allows reading of the 32bit scratchpad register on -# the primary (internal) side. -# -# RETURNS: An appropriate ERRNO error value on error, or zero for success. -# METHOD int spad_read { device_t ntb; unsigned int idx; uint32_t *val; }; -# -# ntb_peer_spad_write() - write to the secondary scratchpad register -# @ntb: pointer to ntb_softc instance -# @idx: index to the scratchpad register, 0 based -# @val: the data value to put into the register -# -# This function allows writing of a 32bit value to the indexed scratchpad -# register. The register resides on the secondary (external) side. -# -# RETURNS: An appropriate ERRNO error value on error, or zero for success. -# METHOD int peer_spad_write { device_t ntb; unsigned int idx; uint32_t val; }; -# -# ntb_peer_spad_read() - read from the primary scratchpad register -# @ntb: pointer to ntb_softc instance -# @idx: index to scratchpad register, 0 based -# @val: pointer to 32bit integer for storing the register value -# -# This function allows reading of the 32bit scratchpad register on -# the primary (internal) side. -# -# RETURNS: An appropriate ERRNO error value on error, or zero for success. -# METHOD int peer_spad_read { device_t ntb; unsigned int idx; uint32_t *val; }; -# -# ntb_db_valid_mask() - get a mask of doorbell bits supported by the ntb -# @ntb: NTB device context -# -# Hardware may support different number or arrangement of doorbell bits. -# -# Return: A mask of doorbell bits supported by the ntb. -# METHOD uint64_t db_valid_mask { device_t ntb; }; -# -# ntb_db_vector_count() - get the number of doorbell interrupt vectors -# @ntb: NTB device context. -# -# Hardware may support different number of interrupt vectors. -# -# Return: The number of doorbell interrupt vectors. -# METHOD int db_vector_count { device_t ntb; }; -# -# ntb_db_vector_mask() - get a mask of doorbell bits serviced by a vector -# @ntb: NTB device context -# @vector: Doorbell vector number -# -# Each interrupt vector may have a different number or arrangement of bits. -# -# Return: A mask of doorbell bits serviced by a vector. -# METHOD uint64_t db_vector_mask { device_t ntb; uint32_t vector; }; -# -# ntb_peer_db_addr() - address and size of the peer doorbell register -# @ntb: NTB device context. -# @db_addr: OUT - The address of the peer doorbell register. -# @db_size: OUT - The number of bytes to write the peer doorbell register. -# -# Return the address of the peer doorbell register. This may be used, for -# example, by drivers that offload memory copy operations to a dma engine. -# The drivers may wish to ring the peer doorbell at the completion of memory -# copy operations. For efficiency, and to simplify ordering of operations -# between the dma memory copies and the ringing doorbell, the driver may -# append one additional dma memory copy with the doorbell register as the -# destination, after the memory copy operations. -# -# Return: Zero on success, otherwise an error number. -# -# Note that writing the peer doorbell via a memory window will *not* generate -# an interrupt on the remote host; that must be done separately. -# METHOD int peer_db_addr { device_t ntb; bus_addr_t *db_addr; vm_size_t *db_size; }; -# -# ntb_db_clear() - clear bits in the local doorbell register -# @ntb: NTB device context. -# @db_bits: Doorbell bits to clear. -# -# Clear bits in the local doorbell register, arming the bits for the next -# doorbell. -# -# Return: Zero on success, otherwise an error number. -# METHOD void db_clear { device_t ntb; uint64_t bits; }; -# -# ntb_db_clear_mask() - clear bits in the local doorbell mask -# @ntb: NTB device context. -# @db_bits: Doorbell bits to clear. -# -# Clear bits in the local doorbell mask register, allowing doorbell interrupts -# from being generated for those doorbell bits. If a doorbell bit is already -# set at the time the mask is cleared, and the corresponding mask bit is -# changed from set to clear, then the ntb driver must ensure that -# ntb_db_event() is called. If the hardware does not generate the interrupt -# on clearing the mask bit, then the driver must call ntb_db_event() anyway. -# -# Return: Zero on success, otherwise an error number. -# METHOD void db_clear_mask { device_t ntb; uint64_t bits; }; -# -# ntb_db_read() - read the local doorbell register -# @ntb: NTB device context. -# -# Read the local doorbell register, and return the bits that are set. -# -# Return: The bits currently set in the local doorbell register. -# METHOD uint64_t db_read { device_t ntb; }; -# -# ntb_db_set_mask() - set bits in the local doorbell mask -# @ntb: NTB device context. -# @db_bits: Doorbell mask bits to set. -# -# Set bits in the local doorbell mask register, preventing doorbell interrupts -# from being generated for those doorbell bits. Bits that were already set -# must remain set. -# -# Return: Zero on success, otherwise an error number. -# METHOD void db_set_mask { device_t ntb; uint64_t bits; }; -# -# ntb_peer_db_set() - Set the doorbell on the secondary/external side -# @ntb: pointer to ntb_softc instance -# @bit: doorbell bits to ring -# -# This function allows triggering of a doorbell on the secondary/external -# side that will initiate an interrupt on the remote host -# METHOD void peer_db_set { device_t ntb; uint64_t bits; }; - diff --git a/sys/dev/ntb/ntb_transport.c b/sys/dev/ntb/ntb_transport.c index 6c1c5519fc28..c7bc4da2c388 100644 --- a/sys/dev/ntb/ntb_transport.c +++ b/sys/dev/ntb/ntb_transport.c @@ -43,7 +43,6 @@ __FBSDID("$FreeBSD$"); #include <sys/param.h> #include <sys/kernel.h> #include <sys/systm.h> -#include <sys/bitset.h> #include <sys/bus.h> #include <sys/ktr.h> #include <sys/limits.h> @@ -64,13 +63,6 @@ __FBSDID("$FreeBSD$"); #include "ntb.h" #include "ntb_transport.h" -#define QP_SETSIZE 64 -BITSET_DEFINE(_qpset, QP_SETSIZE); -#define test_bit(pos, addr) BIT_ISSET(QP_SETSIZE, (pos), (addr)) -#define set_bit(pos, addr) BIT_SET(QP_SETSIZE, (pos), (addr)) -#define clear_bit(pos, addr) BIT_CLR(QP_SETSIZE, (pos), (addr)) -#define ffs_bit(addr) BIT_FFS(QP_SETSIZE, (addr)) - #define KTR_NTB KTR_SPARE3 #define NTB_TRANSPORT_VERSION 4 @@ -94,12 +86,6 @@ SYSCTL_UQUAD(_hw_ntb_transport, OID_AUTO, max_mw_size, CTLFLAG_RDTUN, &max_mw_si "If enabled (non-zero), limit the size of large memory windows. " "Both sides of the NTB MUST set the same value here."); -static unsigned max_num_clients; -SYSCTL_UINT(_hw_ntb_transport, OID_AUTO, max_num_clients, CTLFLAG_RDTUN, - &max_num_clients, 0, "Maximum number of NTB transport clients. " - "0 (default) - use all available NTB memory windows; " - "positive integer N - Limit to N memory windows."); - static unsigned enable_xeon_watchdog; SYSCTL_UINT(_hw_ntb_transport, OID_AUTO, enable_xeon_watchdog, CTLFLAG_RDTUN, &enable_xeon_watchdog, 0, "If non-zero, write a register every second to " @@ -130,7 +116,7 @@ struct ntb_rx_info { struct ntb_transport_qp { struct ntb_transport_ctx *transport; - device_t ntb; + device_t dev; void *cb_data; @@ -200,15 +186,21 @@ struct ntb_transport_mw { bus_addr_t dma_addr; }; +struct ntb_transport_child { + device_t dev; + int qpoff; + int qpcnt; + struct ntb_transport_child *next; +}; + struct ntb_transport_ctx { device_t dev; - device_t ntb; + struct ntb_transport_child *child; struct ntb_transport_mw *mw_vec; struct ntb_transport_qp *qp_vec; - struct _qpset qp_bitmap; - struct _qpset qp_bitmap_free; unsigned mw_count; unsigned qp_count; + uint64_t qp_bitmap; volatile bool link_is_up; struct callout link_work; struct callout link_watchdog; @@ -243,7 +235,6 @@ enum { NTBT_MW0_SZ_LOW, NTBT_MW1_SZ_HIGH, NTBT_MW1_SZ_LOW, - NTBT_MAX_SPAD, /* * Some NTB-using hardware have a watchdog to work around NTB hangs; if @@ -317,7 +308,7 @@ xeon_link_watchdog_hb(void *arg) struct ntb_transport_ctx *nt; nt = arg; - NTB_SPAD_WRITE(nt->ntb, NTBT_WATCHDOG_SPAD, 0); + ntb_spad_write(nt->dev, NTBT_WATCHDOG_SPAD, 0); callout_reset(&nt->link_watchdog, 1 * hz, xeon_link_watchdog_hb, nt); } @@ -333,21 +324,50 @@ static int ntb_transport_attach(device_t dev) { struct ntb_transport_ctx *nt = device_get_softc(dev); - device_t ntb = device_get_parent(dev); + struct ntb_transport_child **cpp = &nt->child; + struct ntb_transport_child *nc; struct ntb_transport_mw *mw; - uint64_t qp_bitmap; - int rc; - unsigned i; + uint64_t db_bitmap; + int rc, i, db_count, spad_count, qp, qpu, qpo, qpt; + char cfg[128] = ""; + char buf[32]; + char *n, *np, *c, *name; nt->dev = dev; - nt->ntb = ntb; - nt->mw_count = NTB_MW_COUNT(ntb); + nt->mw_count = ntb_mw_count(dev); + spad_count = ntb_spad_count(dev); + db_bitmap = ntb_db_valid_mask(dev); + db_count = flsll(db_bitmap); + KASSERT(db_bitmap == (1 << db_count) - 1, + ("Doorbells are not sequential (%jx).\n", db_bitmap)); + + device_printf(dev, "%d memory windows, %d scratchpads, " + "%d doorbells\n", nt->mw_count, spad_count, db_count); + + if (nt->mw_count == 0) { + device_printf(dev, "At least 1 memory window required.\n"); + return (ENXIO); + } + if (spad_count < 6) { + device_printf(dev, "At least 6 scratchpads required.\n"); + return (ENXIO); + } + if (spad_count < 4 + 2 * nt->mw_count) { + nt->mw_count = (spad_count - 4) / 2; + device_printf(dev, "Scratchpads enough only for %d " + "memory windows.\n", nt->mw_count); + } + if (db_bitmap == 0) { + device_printf(dev, "At least one doorbell required.\n"); + return (ENXIO); + } + nt->mw_vec = malloc(nt->mw_count * sizeof(*nt->mw_vec), M_NTB_T, M_WAITOK | M_ZERO); for (i = 0; i < nt->mw_count; i++) { mw = &nt->mw_vec[i]; - rc = NTB_MW_GET_RANGE(ntb, i, &mw->phys_addr, &mw->vbase, + rc = ntb_mw_get_range(dev, i, &mw->phys_addr, &mw->vbase, &mw->phys_size, &mw->xlat_align, &mw->xlat_align_size, &mw->addr_limit); if (rc != 0) @@ -358,49 +378,80 @@ ntb_transport_attach(device_t dev) mw->virt_addr = NULL; mw->dma_addr = 0; - rc = NTB_MW_SET_WC(nt->ntb, i, VM_MEMATTR_WRITE_COMBINING); + rc = ntb_mw_set_wc(dev, i, VM_MEMATTR_WRITE_COMBINING); if (rc) ntb_printf(0, "Unable to set mw%d caching\n", i); } - qp_bitmap = NTB_DB_VALID_MASK(ntb); - nt->qp_count = flsll(qp_bitmap); - KASSERT(nt->qp_count != 0, ("bogus db bitmap")); - nt->qp_count -= 1; + qpu = 0; + qpo = imin(db_count, nt->mw_count); + qpt = db_count; + + snprintf(buf, sizeof(buf), "hint.%s.%d.config", device_get_name(dev), + device_get_unit(dev)); + TUNABLE_STR_FETCH(buf, cfg, sizeof(cfg)); + n = cfg; + i = 0; + while ((c = strsep(&n, ",")) != NULL) { + np = c; + name = strsep(&np, ":"); + if (name != NULL && name[0] == 0) + name = NULL; + qp = (np && np[0] != 0) ? strtol(np, NULL, 10) : qpo - qpu; + if (qp <= 0) + qp = 1; + + if (qp > qpt - qpu) { + device_printf(dev, "Not enough resources for config\n"); + break; + } + + nc = malloc(sizeof(*nc), M_DEVBUF, M_WAITOK | M_ZERO); + nc->qpoff = qpu; + nc->qpcnt = qp; + nc->dev = device_add_child(dev, name, -1); + if (nc->dev == NULL) { + device_printf(dev, "Can not add child.\n"); + break; + } + device_set_ivars(nc->dev, nc); + *cpp = nc; + cpp = &nc->next; + + if (bootverbose) { + device_printf(dev, "%d \"%s\": queues %d", + i, name, qpu); + if (qp > 1) + printf("-%d", qpu + qp - 1); + printf("\n"); + } - if (max_num_clients != 0 && max_num_clients < nt->qp_count) - nt->qp_count = max_num_clients; - else if (nt->mw_count < nt->qp_count) - nt->qp_count = nt->mw_count; - KASSERT(nt->qp_count <= QP_SETSIZE, ("invalid qp_count")); + qpu += qp; + i++; + } + nt->qp_count = qpu; nt->qp_vec = malloc(nt->qp_count * sizeof(*nt->qp_vec), M_NTB_T, M_WAITOK | M_ZERO); - for (i = 0; i < nt->qp_count; i++) { - set_bit(i, &nt->qp_bitmap); - set_bit(i, &nt->qp_bitmap_free); + for (i = 0; i < nt->qp_count; i++) ntb_transport_init_queue(nt, i); - } callout_init(&nt->link_work, 0); callout_init(&nt->link_watchdog, 0); TASK_INIT(&nt->link_cleanup, 0, ntb_transport_link_cleanup_work, nt); - rc = NTB_SET_CTX(ntb, nt, &ntb_transport_ops); + rc = ntb_set_ctx(dev, nt, &ntb_transport_ops); if (rc != 0) goto err; nt->link_is_up = false; - NTB_LINK_ENABLE(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); + ntb_link_enable(dev, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); if (enable_xeon_watchdog != 0) callout_reset(&nt->link_watchdog, 0, xeon_link_watchdog_hb, nt); - /* Attach children to this transport */ - device_add_child(dev, NULL, -1); bus_generic_attach(dev); - return (0); err: @@ -413,28 +464,27 @@ static int ntb_transport_detach(device_t dev) { struct ntb_transport_ctx *nt = device_get_softc(dev); - device_t ntb = nt->ntb; - struct _qpset qp_bitmap_alloc; - uint8_t i; - - /* Detach & delete all children */ - device_delete_children(dev); + struct ntb_transport_child **cpp = &nt->child; + struct ntb_transport_child *nc; + int error = 0, i; + + while ((nc = *cpp) != NULL) { + *cpp = (*cpp)->next; + error = device_delete_child(dev, nc->dev); + if (error) + break; + free(nc, M_DEVBUF); + } + KASSERT(nt->qp_bitmap == 0, + ("Some queues not freed on detach (%jx)", nt->qp_bitmap)); ntb_transport_link_cleanup(nt); taskqueue_drain(taskqueue_swi, &nt->link_cleanup); callout_drain(&nt->link_work); callout_drain(&nt->link_watchdog); - BIT_COPY(QP_SETSIZE, &nt->qp_bitmap, &qp_bitmap_alloc); - BIT_NAND(QP_SETSIZE, &qp_bitmap_alloc, &nt->qp_bitmap_free); - - /* Verify that all the QPs are freed */ - for (i = 0; i < nt->qp_count; i++) - if (test_bit(i, &qp_bitmap_alloc)) - ntb_transport_free_queue(&nt->qp_vec[i]); - - NTB_LINK_DISABLE(ntb); - NTB_CLEAR_CTX(ntb); + ntb_link_disable(dev); + ntb_clear_ctx(dev); for (i = 0; i < nt->mw_count; i++) ntb_free_mw(nt, i); @@ -444,6 +494,14 @@ ntb_transport_detach(device_t dev) return (0); } +int +ntb_transport_queue_count(device_t dev) +{ + struct ntb_transport_child *nc = device_get_ivars(dev); + + return (nc->qpcnt); +} + static void ntb_transport_init_queue(struct ntb_transport_ctx *nt, unsigned int qp_num) { @@ -461,7 +519,7 @@ ntb_transport_init_queue(struct ntb_transport_ctx *nt, unsigned int qp_num) qp = &nt->qp_vec[qp_num]; qp->qp_num = qp_num; qp->transport = nt; - qp->ntb = nt->ntb; + qp->dev = nt->dev; qp->client_ready = false; qp->event_handler = NULL; ntb_qp_link_down_reset(qp); @@ -511,14 +569,12 @@ ntb_transport_init_queue(struct ntb_transport_ctx *nt, unsigned int qp_num) void ntb_transport_free_queue(struct ntb_transport_qp *qp) { + struct ntb_transport_ctx *nt = qp->transport; struct ntb_queue_entry *entry; - if (qp == NULL) - return; - callout_drain(&qp->link_work); - NTB_DB_SET_MASK(qp->ntb, 1ull << qp->qp_num); + ntb_db_set_mask(qp->dev, 1ull << qp->qp_num); taskqueue_drain_all(qp->rxc_tq); taskqueue_free(qp->rxc_tq); @@ -536,7 +592,7 @@ ntb_transport_free_queue(struct ntb_transport_qp *qp) while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) free(entry, M_NTB_T); - set_bit(qp->qp_num, &qp->transport->qp_bitmap_free); + nt->qp_bitmap &= ~(1 << qp->qp_num); } /** @@ -554,25 +610,20 @@ ntb_transport_free_queue(struct ntb_transport_qp *qp) * RETURNS: pointer to newly created ntb_queue, NULL on error. */ struct ntb_transport_qp * -ntb_transport_create_queue(void *data, device_t dev, - const struct ntb_queue_handlers *handlers) +ntb_transport_create_queue(device_t dev, int q, + const struct ntb_queue_handlers *handlers, void *data) { - struct ntb_transport_ctx *nt = device_get_softc(dev); - device_t ntb = device_get_parent(dev); + struct ntb_transport_child *nc = device_get_ivars(dev); + struct ntb_transport_ctx *nt = device_get_softc(device_get_parent(dev)); struct ntb_queue_entry *entry; struct ntb_transport_qp *qp; - unsigned int free_queue; int i; - free_queue = ffs_bit(&nt->qp_bitmap_free); - if (free_queue == 0) + if (q < 0 || q >= nc->qpcnt) return (NULL); - /* decrement free_queue to make it zero based */ - free_queue--; - - qp = &nt->qp_vec[free_queue]; - clear_bit(qp->qp_num, &nt->qp_bitmap_free); + qp = &nt->qp_vec[nc->qpoff + q]; + nt->qp_bitmap |= (1 << qp->qp_num); qp->cb_data = data; qp->rx_handler = handlers->rx_handler; qp->tx_handler = handlers->tx_handler; @@ -593,7 +644,7 @@ ntb_transport_create_queue(void *data, device_t dev, ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q); } - NTB_DB_CLEAR(ntb, 1ull << qp->qp_num); + ntb_db_clear(dev, 1ull << qp->qp_num); return (qp); } @@ -640,7 +691,7 @@ ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, struct ntb_queue_entry *entry; int rc; - if (qp == NULL || !qp->link_is_up || len == 0) { + if (!qp->link_is_up || len == 0) { CTR0(KTR_NTB, "TX: link not up"); return (EINVAL); } @@ -680,7 +731,7 @@ ntb_tx_copy_callback(void *data) iowrite32(entry->flags | NTBT_DESC_DONE_FLAG, &hdr->flags); CTR1(KTR_NTB, "TX: hdr %p set DESC_DONE", hdr); - NTB_PEER_DB_SET(qp->ntb, 1ull << qp->qp_num); + ntb_peer_db_set(qp->dev, 1ull << qp->qp_num); /* * The entry length can only be zero if the packet is intended to be a @@ -790,9 +841,9 @@ again: ; CTR1(KTR_NTB, "RX: process_rxc returned %d", rc); - if ((NTB_DB_READ(qp->ntb) & (1ull << qp->qp_num)) != 0) { + if ((ntb_db_read(qp->dev) & (1ull << qp->qp_num)) != 0) { /* If db is set, clear it and check queue once more. */ - NTB_DB_CLEAR(qp->ntb, 1ull << qp->qp_num); + ntb_db_clear(qp->dev, 1ull << qp->qp_num); goto again; } } @@ -949,24 +1000,19 @@ ntb_transport_doorbell_callback(void *data, uint32_t vector) { struct ntb_transport_ctx *nt = data; struct ntb_transport_qp *qp; - struct _qpset db_bits; uint64_t vec_mask; unsigned qp_num; - BIT_COPY(QP_SETSIZE, &nt->qp_bitmap, &db_bits); - BIT_NAND(QP_SETSIZE, &db_bits, &nt->qp_bitmap_free); - - vec_mask = NTB_DB_VECTOR_MASK(nt->ntb, vector); + vec_mask = ntb_db_vector_mask(nt->dev, vector); + vec_mask &= nt->qp_bitmap; if ((vec_mask & (vec_mask - 1)) != 0) - vec_mask &= NTB_DB_READ(nt->ntb); + vec_mask &= ntb_db_read(nt->dev); while (vec_mask != 0) { qp_num = ffsll(vec_mask) - 1; - if (test_bit(qp_num, &db_bits)) { - qp = &nt->qp_vec[qp_num]; - if (qp->link_is_up) - taskqueue_enqueue(qp->rxc_tq, &qp->rxc_db_work); - } + qp = &nt->qp_vec[qp_num]; + if (qp->link_is_up) + taskqueue_enqueue(qp->rxc_tq, &qp->rxc_db_work); vec_mask &= ~(1ull << qp_num); } @@ -978,7 +1024,7 @@ ntb_transport_event_callback(void *data) { struct ntb_transport_ctx *nt = data; - if (NTB_LINK_IS_UP(nt->ntb, NULL, NULL)) { + if (ntb_link_is_up(nt->dev, NULL, NULL)) { ntb_printf(1, "HW link up\n"); callout_reset(&nt->link_work, 0, ntb_transport_link_work, nt); } else { @@ -992,7 +1038,7 @@ static void ntb_transport_link_work(void *arg) { struct ntb_transport_ctx *nt = arg; - device_t ntb = nt->ntb; + device_t dev = nt->dev; struct ntb_transport_qp *qp; uint64_t val64, size; uint32_t val; @@ -1006,36 +1052,34 @@ ntb_transport_link_work(void *arg) if (max_mw_size != 0 && size > max_mw_size) size = max_mw_size; - NTB_PEER_SPAD_WRITE(ntb, NTBT_MW0_SZ_HIGH + (i * 2), + ntb_peer_spad_write(dev, NTBT_MW0_SZ_HIGH + (i * 2), size >> 32); - NTB_PEER_SPAD_WRITE(ntb, NTBT_MW0_SZ_LOW + (i * 2), size); + ntb_peer_spad_write(dev, NTBT_MW0_SZ_LOW + (i * 2), size); } - - NTB_PEER_SPAD_WRITE(ntb, NTBT_NUM_MWS, nt->mw_count); - - NTB_PEER_SPAD_WRITE(ntb, NTBT_NUM_QPS, nt->qp_count); - - NTB_PEER_SPAD_WRITE(ntb, NTBT_VERSION, NTB_TRANSPORT_VERSION); + ntb_peer_spad_write(dev, NTBT_NUM_MWS, nt->mw_count); + ntb_peer_spad_write(dev, NTBT_NUM_QPS, nt->qp_count); + ntb_peer_spad_write(dev, NTBT_QP_LINKS, 0); + ntb_peer_spad_write(dev, NTBT_VERSION, NTB_TRANSPORT_VERSION); /* Query the remote side for its info */ val = 0; - NTB_SPAD_READ(ntb, NTBT_VERSION, &val); + ntb_spad_read(dev, NTBT_VERSION, &val); if (val != NTB_TRANSPORT_VERSION) goto out; - NTB_SPAD_READ(ntb, NTBT_NUM_QPS, &val); + ntb_spad_read(dev, NTBT_NUM_QPS, &val); if (val != nt->qp_count) goto out; - NTB_SPAD_READ(ntb, NTBT_NUM_MWS, &val); + ntb_spad_read(dev, NTBT_NUM_MWS, &val); if (val != nt->mw_count) goto out; for (i = 0; i < nt->mw_count; i++) { - NTB_SPAD_READ(ntb, NTBT_MW0_SZ_HIGH + (i * 2), &val); + ntb_spad_read(dev, NTBT_MW0_SZ_HIGH + (i * 2), &val); val64 = (uint64_t)val << 32; - NTB_SPAD_READ(ntb, NTBT_MW0_SZ_LOW + (i * 2), &val); + ntb_spad_read(dev, NTBT_MW0_SZ_LOW + (i * 2), &val); val64 |= val; rc = ntb_set_mw(nt, i, val64); @@ -1061,7 +1105,7 @@ free_mws: for (i = 0; i < nt->mw_count; i++) ntb_free_mw(nt, i); out: - if (NTB_LINK_IS_UP(ntb, NULL, NULL)) + if (ntb_link_is_up(dev, NULL, NULL)) callout_reset(&nt->link_work, NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_transport_link_work, nt); } @@ -1116,7 +1160,7 @@ ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw, size_t size) } /* Notify HW the memory location of the receive buffer */ - rc = NTB_MW_SET_TRANS(nt->ntb, num_mw, mw->dma_addr, mw->xlat_size); + rc = ntb_mw_set_trans(nt->dev, num_mw, mw->dma_addr, mw->xlat_size); if (rc) { ntb_printf(0, "Unable to set mw%d translation\n", num_mw); ntb_free_mw(nt, num_mw); @@ -1134,7 +1178,7 @@ ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw) if (mw->virt_addr == NULL) return; - NTB_MW_CLEAR_TRANS(nt->ntb, num_mw); + ntb_mw_clear_trans(nt->dev, num_mw); contigfree(mw->virt_addr, mw->xlat_size, M_NTB_T); mw->xlat_size = 0; mw->buff_size = 0; @@ -1194,18 +1238,20 @@ static void ntb_qp_link_work(void *arg) { struct ntb_transport_qp *qp = arg; - device_t ntb = qp->ntb; + device_t dev = qp->dev; struct ntb_transport_ctx *nt = qp->transport; - uint32_t val, dummy; - - NTB_SPAD_READ(ntb, NTBT_QP_LINKS, &val); - - NTB_PEER_SPAD_WRITE(ntb, NTBT_QP_LINKS, val | (1ull << qp->qp_num)); + int i; + uint32_t val; - /* query remote spad for qp ready bits */ - NTB_PEER_SPAD_READ(ntb, NTBT_QP_LINKS, &dummy); + /* Report queues that are up on our side */ + for (i = 0, val = 0; i < nt->qp_count; i++) { + if (nt->qp_vec[i].client_ready) + val |= (1 << i); + } + ntb_peer_spad_write(dev, NTBT_QP_LINKS, val); /* See if the remote side is up */ + ntb_spad_read(dev, NTBT_QP_LINKS, &val); if ((val & (1ull << qp->qp_num)) != 0) { ntb_printf(2, "qp %d link up\n", qp->qp_num); qp->link_is_up = true; @@ -1213,7 +1259,7 @@ ntb_qp_link_work(void *arg) if (qp->event_handler != NULL) qp->event_handler(qp->cb_data, NTB_LINK_UP); - NTB_DB_CLEAR_MASK(ntb, 1ull << qp->qp_num); + ntb_db_clear_mask(dev, 1ull << qp->qp_num); } else if (nt->link_is_up) callout_reset(&qp->link_work, NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_qp_link_work, qp); @@ -1224,19 +1270,16 @@ static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt) { struct ntb_transport_qp *qp; - struct _qpset qp_bitmap_alloc; - unsigned i; - - BIT_COPY(QP_SETSIZE, &nt->qp_bitmap, &qp_bitmap_alloc); - BIT_NAND(QP_SETSIZE, &qp_bitmap_alloc, &nt->qp_bitmap_free); + int i; /* Pass along the info to any clients */ - for (i = 0; i < nt->qp_count; i++) - if (test_bit(i, &qp_bitmap_alloc)) { + for (i = 0; i < nt->qp_count; i++) { + if ((nt->qp_bitmap & (1 << i)) != 0) { qp = &nt->qp_vec[i]; ntb_qp_link_cleanup(qp); callout_drain(&qp->link_work); } + } if (!nt->link_is_up) callout_drain(&nt->link_work); @@ -1246,8 +1289,7 @@ ntb_transport_link_cleanup(struct ntb_transport_ctx *nt) * goes down, blast them now to give them a sane value the next * time they are accessed */ - for (i = 0; i < NTBT_MAX_SPAD; i++) - NTB_SPAD_WRITE(nt->ntb, i, 0); + ntb_spad_clear(nt->dev); } static void @@ -1269,7 +1311,7 @@ ntb_qp_link_down_reset(struct ntb_transport_qp *qp) { qp->link_is_up = false; - NTB_DB_SET_MASK(qp->ntb, 1ull << qp->qp_num); + ntb_db_set_mask(qp->dev, 1ull << qp->qp_num); qp->tx_index = qp->rx_index = 0; qp->tx_bytes = qp->rx_bytes = 0; @@ -1305,17 +1347,16 @@ ntb_qp_link_cleanup(struct ntb_transport_qp *qp) void ntb_transport_link_down(struct ntb_transport_qp *qp) { + struct ntb_transport_ctx *nt = qp->transport; + int i; uint32_t val; - if (qp == NULL) - return; - qp->client_ready = false; - - NTB_SPAD_READ(qp->ntb, NTBT_QP_LINKS, &val); - - NTB_PEER_SPAD_WRITE(qp->ntb, NTBT_QP_LINKS, - val & ~(1 << qp->qp_num)); + for (i = 0, val = 0; i < nt->qp_count; i++) { + if (nt->qp_vec[i].client_ready) + val |= (1 << i); + } + ntb_peer_spad_write(qp->dev, NTBT_QP_LINKS, val); if (qp->link_is_up) ntb_send_link_down(qp); @@ -1334,8 +1375,6 @@ ntb_transport_link_down(struct ntb_transport_qp *qp) bool ntb_transport_link_query(struct ntb_transport_qp *qp) { - if (qp == NULL) - return (false); return (qp->link_is_up); } @@ -1434,8 +1473,6 @@ out: */ unsigned char ntb_transport_qp_num(struct ntb_transport_qp *qp) { - if (qp == NULL) - return 0; return (qp->qp_num); } @@ -1452,9 +1489,6 @@ unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp) { - if (qp == NULL) - return (0); - return (qp->tx_max_frame - sizeof(struct ntb_payload_header)); } diff --git a/sys/dev/ntb/ntb_transport.h b/sys/dev/ntb/ntb_transport.h index deb85688fe01..63cdbce96cb2 100644 --- a/sys/dev/ntb/ntb_transport.h +++ b/sys/dev/ntb/ntb_transport.h @@ -43,12 +43,13 @@ struct ntb_queue_handlers { void (*event_handler)(void *data, enum ntb_link_event status); }; -unsigned char ntb_transport_qp_num(struct ntb_transport_qp *qp); -unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp); +int ntb_transport_queue_count(device_t dev); struct ntb_transport_qp * -ntb_transport_create_queue(void *data, device_t dev, - const struct ntb_queue_handlers *handlers); +ntb_transport_create_queue(device_t dev, int q, + const struct ntb_queue_handlers *handlers, void *data); void ntb_transport_free_queue(struct ntb_transport_qp *qp); +unsigned char ntb_transport_qp_num(struct ntb_transport_qp *qp); +unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp); int ntb_transport_rx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, unsigned int len); int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, diff --git a/sys/dev/nvme/nvme_sim.c b/sys/dev/nvme/nvme_sim.c index 9c46e168ce52..4f545403fb4b 100644 --- a/sys/dev/nvme/nvme_sim.c +++ b/sys/dev/nvme/nvme_sim.c @@ -133,8 +133,6 @@ nvme_sim_action(struct cam_sim *sim, union ccb *ccb) ns = sim2ns(sim); ctrlr = sim2ctrlr(sim); - printf("Sim action: ctrlr %p ns %p\n", ctrlr, ns); - mtx_assert(&ctrlr->lock, MA_OWNED); switch (ccb->ccb_h.func_code) { @@ -310,8 +308,6 @@ nvme_sim_new_ns(struct nvme_namespace *ns, void *sc_arg) sc->s_ns = ns; - printf("Our SIM's softc %p ctrlr %p ns %p\n", sc, ctrlr, ns); - /* * XXX this is creating one bus per ns, but it should be one * XXX target per controller, and one LUN per namespace. @@ -349,7 +345,6 @@ nvme_sim_new_ns(struct nvme_namespace *ns, void *sc_arg) sc->s_path->device->nvme_cdata = nvme_ctrlr_get_data(ns->ctrlr); /* Scan bus */ - printf("Initiate rescan of the bus\n"); nvme_sim_rescan_target(ctrlr, sc->s_path); mtx_unlock(&ctrlr->lock); diff --git a/sys/dev/nvram2env/nvram2env.c b/sys/dev/nvram2env/nvram2env.c index a9423484fae8..d8da4dd1212e 100644 --- a/sys/dev/nvram2env/nvram2env.c +++ b/sys/dev/nvram2env/nvram2env.c @@ -46,50 +46,7 @@ __FBSDID("$FreeBSD$"); #include <machine/bus.h> -#include <dev/siba/siba_ids.h> -#include <dev/siba/sibareg.h> -#include <dev/siba/sibavar.h> - -#define nvram2env_read_1(sc, reg) \ - bus_space_read_1((sc)->sc_bt, (sc)->sc_bh,(reg)) - -#define nvram2env_read_2(sc, reg) \ - bus_space_read_2((sc)->sc_bt, (sc)->sc_bh,(reg)) - -#define nvram2env_read_4(sc, reg) \ - bus_space_read_4((sc)->sc_bt, (sc)->sc_bh,(reg)) - -#define nvram2env_write_1(sc, reg, val) \ - bus_space_write_1((sc)->sc_bt, (sc)->sc_bh, \ - (reg), (val)) - -#define nvram2env_write_2(sc, reg, val) \ - bus_space_write_2((sc)->sc_bt, (sc)->sc_bh, \ - (reg), (val)) - -#define nvram2env_write_4(sc, reg, val) \ - bus_space_write_4((sc)->sc_bt, (sc)->sc_bh, \ - (reg), (val)) - -struct nvram2env_softc { - bus_space_tag_t bst; - bus_space_handle_t bsh; - bus_addr_t addr; - int need_swap; - uint32_t sig; - uint32_t flags; -#define NVRAM_FLAGS_NOCHECK 0x0001 /* Do not check(CRC or somthing else)*/ -#define NVRAM_FLAGS_GENERIC 0x0002 /* Format Generic, skip 4b and read */ -#define NVRAM_FLAGS_BROADCOM 0x0004 /* Format Broadcom, use struct nvram */ -#define NVRAM_FLAGS_UBOOT 0x0008 /* Format Generic, skip 4b of CRC and read */ - uint32_t maxsize; - uint32_t crc; -}; - -static int nvram2env_attach(device_t); -static int nvram2env_probe(device_t); - -#define NVRAM_MAX_SIZE 0x10000 +#include "nvram2env.h" static void nvram2env_identify(driver_t * drv, device_t parent) @@ -100,34 +57,55 @@ nvram2env_identify(driver_t * drv, device_t parent) BUS_ADD_CHILD(parent, 0, "nvram2env", i); } -static int +int nvram2env_probe(device_t dev) { uint32_t i, ivar, sig; struct nvram2env_softc * sc = device_get_softc(dev); - sc->bst = mips_bus_space_generic; - if (resource_int_value("nvram", device_get_unit(dev), "sig", - &sc->sig) != 0 || sc->sig == 0) - sc->sig = 0x48534c46; + /* + * Please ensure that your implementation of NVRAM->ENV specifies + * bus tag + */ + if (sc->bst == NULL) + return (ENXIO); + + if (sc->sig == 0) + if (resource_int_value("nvram", device_get_unit(dev), "sig", + &sc->sig) != 0 || sc->sig == 0) + sc->sig = CFE_NVRAM_SIGNATURE; - if (resource_int_value("nvram", device_get_unit(dev), "maxsize", - &sc->maxsize) != 0 || sc->maxsize == 0) - sc->maxsize = NVRAM_MAX_SIZE; + if (sc->maxsize == 0) + if (resource_int_value("nvram", device_get_unit(dev), "maxsize", + &sc->maxsize) != 0 || sc->maxsize == 0) + sc->maxsize = NVRAM_MAX_SIZE; - if (resource_int_value("nvram", device_get_unit(dev), "flags", - &sc->flags) != 0 || sc->flags == 0) - sc->flags = NVRAM_FLAGS_GENERIC; + if (sc->flags == 0) + if (resource_int_value("nvram", device_get_unit(dev), "flags", + &sc->flags) != 0 || sc->flags == 0) + sc->flags = NVRAM_FLAGS_GENERIC; for (i = 0; i < 2; i ++) { - if (resource_int_value("nvram", device_get_unit(dev), - (!i)?"base":"fallbackbase", &ivar) != 0 || - ivar == 0) - continue; + switch (i) { + case 0: + break; + case 1: + case 2: + if (resource_int_value("nvram", device_get_unit(dev), + (i == 1) ? "base" : "fallbackbase", &ivar) != 0 || + ivar == 0) + continue; + + sc->addr = ivar; + break; + default: + break; + } - sc->addr = ivar; + if (sc->addr == 0) + continue; if (bootverbose) device_printf(dev, "base=0x%08x sig=0x%08x " @@ -172,15 +150,6 @@ unmap_done: } -struct nvram { - u_int32_t sig; - u_int32_t size; - u_int32_t unknown1; - u_int32_t unknown2; - u_int32_t unknown3; - char data[]; -}; - static uint32_t read_4(struct nvram2env_softc * sc, int offset) { if (sc->need_swap) @@ -190,7 +159,7 @@ static uint32_t read_4(struct nvram2env_softc * sc, int offset) } -static int +int nvram2env_attach(device_t dev) { struct nvram2env_softc *sc; @@ -209,10 +178,11 @@ nvram2env_attach(device_t dev) sig = read_4(sc, 0); size = read_4(sc, 4); -#if 1 + if (bootverbose) - device_printf(dev, " size=0x%05x maxsize=0x%05x\n", size, sc->maxsize); -#endif + device_printf(dev, " size=0x%05x maxsize=0x%05x\n", size, + sc->maxsize); + size = (size > sc->maxsize)?sc->maxsize:size; @@ -265,12 +235,12 @@ nvram2env_attach(device_t dev) assign = strchr(pair,'='); assign[0] = '\0'; value = assign+1; -#if 1 + if (bootverbose) - printf("ENV: %s=%s\n", pair, value); -#else - printf("ENV: %s\n", pair); -#endif + printf("ENV[%p]: %s=%s\n", + (void*)((char*)pair - (char*)nv), + pair, value); + kern_setenv(pair, value); if (strcasecmp(pair, "WAN_MAC_ADDR") == 0) { @@ -313,12 +283,10 @@ static device_method_t nvram2env_methods[] = { DEVMETHOD_END }; -static driver_t nvram2env_driver = { +driver_t nvram2env_driver = { "nvram2env", nvram2env_methods, sizeof(struct nvram2env_softc), }; -static devclass_t nvram2env_devclass; - -DRIVER_MODULE(nvram2env, nexus, nvram2env_driver, nvram2env_devclass, 0, 0); +devclass_t nvram2env_devclass; diff --git a/sys/dev/nvram2env/nvram2env.h b/sys/dev/nvram2env/nvram2env.h new file mode 100644 index 000000000000..8f81d99921ba --- /dev/null +++ b/sys/dev/nvram2env/nvram2env.h @@ -0,0 +1,88 @@ +/*- + * Copyright (c) 2010 Aleksandr Rybalko. + * Copyright (c) 2016 Michael Zhilin. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + + +#ifndef NVRAM2ENV_NVRAM2ENV_H_ +#define NVRAM2ENV_NVRAM2ENV_H_ + +#define nvram2env_read_1(sc, reg) \ + bus_space_read_1((sc)->sc_bt, (sc)->sc_bh,(reg)) + +#define nvram2env_read_2(sc, reg) \ + bus_space_read_2((sc)->sc_bt, (sc)->sc_bh,(reg)) + +#define nvram2env_read_4(sc, reg) \ + bus_space_read_4((sc)->sc_bt, (sc)->sc_bh,(reg)) + +#define nvram2env_write_1(sc, reg, val) \ + bus_space_write_1((sc)->sc_bt, (sc)->sc_bh, \ + (reg), (val)) + +#define nvram2env_write_2(sc, reg, val) \ + bus_space_write_2((sc)->sc_bt, (sc)->sc_bh, \ + (reg), (val)) + +#define nvram2env_write_4(sc, reg, val) \ + bus_space_write_4((sc)->sc_bt, (sc)->sc_bh, \ + (reg), (val)) + +struct nvram2env_softc { + bus_space_tag_t bst; + bus_space_handle_t bsh; + bus_addr_t addr; + int need_swap; + uint32_t sig; + uint32_t flags; +#define NVRAM_FLAGS_NOCHECK 0x0001 /* Do not check(CRC or somthing else)*/ +#define NVRAM_FLAGS_GENERIC 0x0002 /* Format Generic, skip 4b and read */ +#define NVRAM_FLAGS_BROADCOM 0x0004 /* Format Broadcom, use struct nvram */ +#define NVRAM_FLAGS_UBOOT 0x0008 /* Format Generic, skip 4b of CRC and read */ + uint32_t maxsize; + uint32_t crc; +}; + +#define NVRAM_MAX_SIZE 0x10000 +#define CFE_NVRAM_SIGNATURE 0x48534c46 + +struct nvram { + u_int32_t sig; + u_int32_t size; + u_int32_t unknown1; + u_int32_t unknown2; + u_int32_t unknown3; + char data[]; +}; + +int nvram2env_attach(device_t); +int nvram2env_probe(device_t); + +extern devclass_t nvram2env_devclass; +extern driver_t nvram2env_driver; + +#endif /* SYS_DEV_NVRAM2ENV_NVRAM2ENV_H_ */ diff --git a/sys/dev/nvram2env/nvram2env_mips.c b/sys/dev/nvram2env/nvram2env_mips.c new file mode 100644 index 000000000000..2a669a392f58 --- /dev/null +++ b/sys/dev/nvram2env/nvram2env_mips.c @@ -0,0 +1,69 @@ +/*- + * Copyright (c) 2010 Aleksandr Rybalko. + * Copyright (c) 2016 Michael Zhilin. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * Implementation of pseudo driver for MIPS to copy the NVRAM settings + * from various sources into the kernel environment. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/bus.h> +#include <sys/kernel.h> +#include <sys/module.h> + +#include <machine/bus.h> + +#include "nvram2env.h" + +static int +nvram2env_mips_probe(device_t dev) +{ + struct nvram2env_softc *sc; + + sc = device_get_softc(dev); + sc->bst = mips_bus_space_generic; + + return (nvram2env_probe(dev)); +} + +static device_method_t nvram2env_mips_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, nvram2env_mips_probe), + + DEVMETHOD_END +}; + +DEFINE_CLASS_1(nvram2env, nvram2env_mips_driver, nvram2env_mips_methods, + sizeof(struct nvram2env_softc), nvram2env_driver); +DRIVER_MODULE(nvram2env_mips, nexus, nvram2env_mips_driver, nvram2env_devclass, + NULL, NULL); + +MODULE_VERSION(nvram2env_mips, 1); +MODULE_DEPEND(nvram2env_mips, nvram2env, 1, 1, 1); diff --git a/sys/dev/ofw/ofw_fdt.c b/sys/dev/ofw/ofw_fdt.c index fc41d5bc6a3c..0f7cf07919b3 100644 --- a/sys/dev/ofw/ofw_fdt.c +++ b/sys/dev/ofw/ofw_fdt.c @@ -96,6 +96,27 @@ OFW_DEF(ofw_fdt); static void *fdtp = NULL; static int +sysctl_handle_dtb(SYSCTL_HANDLER_ARGS) +{ + + return (sysctl_handle_opaque(oidp, fdtp, fdt_totalsize(fdtp), req)); +} + +static void +sysctl_register_fdt_oid(void *arg) +{ + + /* If there is no FDT registered, skip adding the sysctl */ + if (fdtp == NULL) + return; + + SYSCTL_ADD_PROC(NULL, SYSCTL_STATIC_CHILDREN(_hw_fdt), OID_AUTO, "dtb", + CTLTYPE_OPAQUE | CTLFLAG_RD, NULL, 0, sysctl_handle_dtb, "", + "Device Tree Blob"); +} +SYSINIT(dtb_oid, SI_SUB_KMEM, SI_ORDER_ANY, sysctl_register_fdt_oid, 0); + +static int ofw_fdt_init(ofw_t ofw, void *data) { int err; diff --git a/sys/dev/ofw/ofwpci.c b/sys/dev/ofw/ofwpci.c index 3423050e5559..1daafa644cf6 100644 --- a/sys/dev/ofw/ofwpci.c +++ b/sys/dev/ofw/ofwpci.c @@ -414,14 +414,14 @@ ofw_pci_alloc_resource(device_t bus, device_t child, int type, int *rid, #if defined(NEW_PCIB) && defined(PCI_RES_BUS) if (type == PCI_RES_BUS) { return (pci_domain_alloc_bus(sc->sc_pci_domain, child, rid, - start, end, count, flags)); + start, end, count, flags | needactivate)); } #endif rm = ofw_pci_get_rman(sc, type, flags); if (rm == NULL) { return (bus_generic_alloc_resource(bus, child, type, rid, - start, end, count, flags)); + start, end, count, flags | needactivate)); } rv = rman_reserve_resource(rm, start, end, count, flags, child); diff --git a/sys/dev/ofw/openfirmio.c b/sys/dev/ofw/openfirmio.c index 5803ec1d6c4a..0079310dc83c 100644 --- a/sys/dev/ofw/openfirmio.c +++ b/sys/dev/ofw/openfirmio.c @@ -100,8 +100,6 @@ openfirm_getstr(int len, const char *user, char **cpp) return (ENAMETOOLONG); *cpp = cp = malloc(len + 1, M_TEMP, M_WAITOK); - if (cp == NULL) - return (ENOMEM); error = copyin(user, cp, len); cp[len] = '\0'; return (error); @@ -173,10 +171,6 @@ openfirm_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flags, if (len <= 0) break; value = malloc(len, M_TEMP, M_WAITOK); - if (value == NULL) { - error = ENOMEM; - break; - } len = OF_getprop(node, name, (void *)value, len); error = copyout(value, of->of_buf, len); break; @@ -199,10 +193,6 @@ openfirm_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flags, if (error) break; value = malloc(of->of_buflen, M_TEMP, M_WAITOK); - if (value == NULL) { - error = ENOMEM; - break; - } error = copyin(of->of_buf, value, of->of_buflen); if (error) break; diff --git a/sys/dev/ofw/openpromio.c b/sys/dev/ofw/openpromio.c index 8ba6d3acef3f..e2a345b1e9db 100644 --- a/sys/dev/ofw/openpromio.c +++ b/sys/dev/ofw/openpromio.c @@ -151,18 +151,10 @@ openprom_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flags, break; } prop = malloc(len, M_TEMP, M_WAITOK | M_ZERO); - if (prop == NULL) { - error = ENOMEM; - break; - } error = copyinstr(&oprom->oprom_array, prop, len, &done); if (error != 0) break; buf = malloc(OPROMMAXPARAM, M_TEMP, M_WAITOK | M_ZERO); - if (buf == NULL) { - error = ENOMEM; - break; - } node = openprom_node; switch (cmd) { case OPROMGETPROP: diff --git a/sys/dev/pci/pci_if.m b/sys/dev/pci/pci_if.m index 5f8415fddda7..11fa5d75d920 100644 --- a/sys/dev/pci/pci_if.m +++ b/sys/dev/pci/pci_if.m @@ -235,6 +235,7 @@ METHOD int iov_attach { device_t child; struct nvlist *pf_schema; struct nvlist *vf_schema; + const char *name; }; METHOD int iov_detach { diff --git a/sys/dev/pci/pci_iov.c b/sys/dev/pci/pci_iov.c index bba99f620a4e..db8a035f21aa 100644 --- a/sys/dev/pci/pci_iov.c +++ b/sys/dev/pci/pci_iov.c @@ -98,8 +98,22 @@ static nvlist_t *pci_iov_get_pf_subsystem_schema(void); static nvlist_t *pci_iov_get_vf_subsystem_schema(void); int +pci_iov_attach_name(device_t dev, struct nvlist *pf_schema, + struct nvlist *vf_schema, const char *fmt, ...) +{ + char buf[NAME_MAX + 1]; + va_list ap; + + va_start(ap, fmt); + vsnprintf(buf, sizeof(buf), fmt, ap); + va_end(ap); + return (PCI_IOV_ATTACH(device_get_parent(dev), dev, pf_schema, + vf_schema, buf)); +} + +int pci_iov_attach_method(device_t bus, device_t dev, nvlist_t *pf_schema, - nvlist_t *vf_schema) + nvlist_t *vf_schema, const char *name) { device_t pcib; struct pci_devinfo *dinfo; @@ -149,7 +163,7 @@ pci_iov_attach_method(device_t bus, device_t dev, nvlist_t *pf_schema, iov->iov_schema = schema; iov->iov_cdev = make_dev(&iov_cdevsw, device_get_unit(dev), - UID_ROOT, GID_WHEEL, 0600, "iov/%s", device_get_nameunit(dev)); + UID_ROOT, GID_WHEEL, 0600, "iov/%s", name); if (iov->iov_cdev == NULL) { error = ENOMEM; diff --git a/sys/dev/pci/pci_iov.h b/sys/dev/pci/pci_iov.h index fd2f8fba8876..297ae07f4c57 100644 --- a/sys/dev/pci/pci_iov.h +++ b/sys/dev/pci/pci_iov.h @@ -33,11 +33,14 @@ struct nvlist; +int pci_iov_attach_name(device_t dev, struct nvlist *pf_schema, + struct nvlist *vf_schema, const char *fmt, ...) __printflike(4, 5); + static __inline int pci_iov_attach(device_t dev, struct nvlist *pf_schema, struct nvlist *vf_schema) { return (PCI_IOV_ATTACH(device_get_parent(dev), dev, pf_schema, - vf_schema)); + vf_schema, device_get_nameunit(dev))); } static __inline int diff --git a/sys/dev/pci/pci_pci.c b/sys/dev/pci/pci_pci.c index 06e9e1c919ab..6e8d076d2cad 100644 --- a/sys/dev/pci/pci_pci.c +++ b/sys/dev/pci/pci_pci.c @@ -128,9 +128,11 @@ static devclass_t pcib_devclass; DEFINE_CLASS_0(pcib, pcib_driver, pcib_methods, sizeof(struct pcib_softc)); DRIVER_MODULE(pcib, pci, pcib_driver, pcib_devclass, NULL, NULL); -#ifdef NEW_PCIB +#if defined(NEW_PCIB) || defined(PCI_HP) SYSCTL_DECL(_hw_pci); +#endif +#ifdef NEW_PCIB static int pci_clear_pcib; SYSCTL_INT(_hw_pci, OID_AUTO, clear_pcib, CTLFLAG_RDTUN, &pci_clear_pcib, 0, "Clear firmware-assigned resources for PCI-PCI bridge I/O windows."); @@ -907,11 +909,19 @@ pcib_set_mem_decode(struct pcib_softc *sc) /* * PCI-express HotPlug support. */ +static int pci_enable_pcie_hp = 1; +SYSCTL_INT(_hw_pci, OID_AUTO, enable_pcie_hp, CTLFLAG_RDTUN, + &pci_enable_pcie_hp, 0, + "Enable support for native PCI-express HotPlug."); + static void pcib_probe_hotplug(struct pcib_softc *sc) { device_t dev; + if (!pci_enable_pcie_hp) + return; + dev = sc->dev; if (pci_find_cap(dev, PCIY_EXPRESS, NULL) != 0) return; @@ -922,6 +932,13 @@ pcib_probe_hotplug(struct pcib_softc *sc) sc->pcie_link_cap = pcie_read_config(dev, PCIER_LINK_CAP, 4); sc->pcie_slot_cap = pcie_read_config(dev, PCIER_SLOT_CAP, 4); + /* + * XXX: Handling of slots with a power controller needs to be + * reexamined. Ignore hotplug on such slots for now. + */ + if (sc->pcie_slot_cap & PCIEM_SLOT_CAP_PCP) + return; + if (sc->pcie_slot_cap & PCIEM_SLOT_CAP_HPC) sc->flags |= PCIB_HOTPLUG; } @@ -1040,7 +1057,7 @@ static void pcib_pcie_hotplug_update(struct pcib_softc *sc, uint16_t val, uint16_t mask, bool schedule_task) { - bool card_inserted; + bool card_inserted, ei_engaged; /* Clear DETACHING if Present Detect has cleared. */ if ((sc->pcie_slot_sta & (PCIEM_SLOT_STA_PDC | PCIEM_SLOT_STA_PDS)) == @@ -1077,8 +1094,8 @@ pcib_pcie_hotplug_update(struct pcib_softc *sc, uint16_t val, uint16_t mask, */ if (sc->pcie_slot_cap & PCIEM_SLOT_CAP_EIP) { mask |= PCIEM_SLOT_CTL_EIC; - if (card_inserted != - !(sc->pcie_slot_sta & PCIEM_SLOT_STA_EIS)) + ei_engaged = (sc->pcie_slot_sta & PCIEM_SLOT_STA_EIS) != 0; + if (card_inserted != ei_engaged) val |= PCIEM_SLOT_CTL_EIC; } @@ -1105,7 +1122,7 @@ pcib_pcie_hotplug_update(struct pcib_softc *sc, uint16_t val, uint16_t mask, pcib_pcie_hotplug_command(sc, val, mask); /* - * During attach the child "pci" device is added sychronously; + * During attach the child "pci" device is added synchronously; * otherwise, the task is scheduled to manage the child * device. */ diff --git a/sys/dev/pci/pci_private.h b/sys/dev/pci/pci_private.h index a9958ca627db..b0f1481810c7 100644 --- a/sys/dev/pci/pci_private.h +++ b/sys/dev/pci/pci_private.h @@ -158,7 +158,8 @@ struct resource *pci_alloc_multi_resource(device_t dev, device_t child, rman_res_t count, u_long num, u_int flags); int pci_iov_attach_method(device_t bus, device_t dev, - struct nvlist *pf_schema, struct nvlist *vf_schema); + struct nvlist *pf_schema, struct nvlist *vf_schema, + const char *name); int pci_iov_detach_method(device_t bus, device_t dev); device_t pci_create_iov_child_method(device_t bus, device_t pf, diff --git a/sys/dev/pci/pci_user.c b/sys/dev/pci/pci_user.c index f5a921a75ff1..dba76d130aa5 100644 --- a/sys/dev/pci/pci_user.c +++ b/sys/dev/pci/pci_user.c @@ -708,10 +708,9 @@ pci_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag, struct thread *t * Go through the list of devices and copy out the devices * that match the user's criteria. */ - for (cio->num_matches = 0, error = 0, i = 0, + for (cio->num_matches = 0, i = 0, dinfo = STAILQ_FIRST(devlist_head); - (dinfo != NULL) && (cio->num_matches < ionum) && - (error == 0) && (i < pci_numdevs); + dinfo != NULL; dinfo = STAILQ_NEXT(dinfo, pci_links), i++) { if (i < cio->offset) @@ -833,11 +832,12 @@ pci_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag, struct thread *t } else #endif /* PRE7_COMPAT */ confdata = &dinfo->conf; - /* Only if we can copy it out do we count it. */ - if (!(error = copyout(confdata, + error = copyout(confdata, (caddr_t)cio->matches + - confsz * cio->num_matches, confsz))) - cio->num_matches++; + confsz * cio->num_matches, confsz); + if (error) + break; + cio->num_matches++; } } diff --git a/sys/dev/pci/pcivar.h b/sys/dev/pci/pcivar.h index 3180b4ef5de7..2e50af66c1c2 100644 --- a/sys/dev/pci/pcivar.h +++ b/sys/dev/pci/pcivar.h @@ -174,7 +174,7 @@ struct pcicfg_ea { /* config header information common to all header types */ typedef struct pcicfg { - struct device *dev; /* device which owns this */ + device_t dev; /* device which owns this */ STAILQ_HEAD(, pci_map) maps; /* BARs */ diff --git a/sys/dev/sound/sbus/cs4231.c b/sys/dev/sound/sbus/cs4231.c index 83f70f3a4813..7b80afcf0f9b 100644 --- a/sys/dev/sound/sbus/cs4231.c +++ b/sys/dev/sound/sbus/cs4231.c @@ -113,7 +113,7 @@ struct cs4231_channel { #define CS4231_RES_MEM_MAX 4 #define CS4231_RES_IRQ_MAX 2 struct cs4231_softc { - struct device *sc_dev; + device_t sc_dev; int sc_rid[CS4231_RES_MEM_MAX]; struct resource *sc_res[CS4231_RES_MEM_MAX]; bus_space_handle_t sc_regh[CS4231_RES_MEM_MAX]; diff --git a/sys/dev/syscons/syscons.c b/sys/dev/syscons/syscons.c index 063083e0b41d..58d6a0c43efc 100644 --- a/sys/dev/syscons/syscons.c +++ b/sys/dev/syscons/syscons.c @@ -172,8 +172,6 @@ SYSCTL_INT(_machdep, OID_AUTO, enable_panic_key, CTLFLAG_RW, &enable_panic_key, #define VTY_WCHAN(sc, vty) (&SC_DEV(sc, vty)) -static int debugger; - /* prototypes */ static int sc_allocate_keyboard(sc_softc_t *sc, int unit); static int scvidprobe(int unit, int flags, int cons); @@ -1647,56 +1645,67 @@ sc_cnterm(struct consdev *cp) sc_console = NULL; } +struct sc_cnstate; /* not used yet */ +static void sccnclose(sc_softc_t *sc, struct sc_cnstate *sp); +static void sccnopen(sc_softc_t *sc, struct sc_cnstate *sp, int flags); + static void -sc_cngrab(struct consdev *cp) +sccnopen(sc_softc_t *sc, struct sc_cnstate *sp, int flags) { - scr_stat *scp; + int kbd_mode; if (!cold && - sc_console->sc->cur_scp->index != sc_console->index && - sc_console->sc->cur_scp->smode.mode == VT_AUTO && + sc->cur_scp->index != sc_console->index && + sc->cur_scp->smode.mode == VT_AUTO && sc_console->smode.mode == VT_AUTO) - sc_switch_scr(sc_console->sc, sc_console->index); + sc_switch_scr(sc, sc_console->index); - scp = sc_console->sc->cur_scp; - - if (scp->sc->kbd == NULL) - return; - - if (scp->grabbed++ > 0) + if (sc->kbd == NULL) return; /* * Make sure the keyboard is accessible even when the kbd device * driver is disabled. */ - kbdd_enable(scp->sc->kbd); + kbdd_enable(sc->kbd); - /* we shall always use the keyboard in the XLATE mode here */ - scp->kbd_prev_mode = scp->kbd_mode; - scp->kbd_mode = K_XLATE; - (void)kbdd_ioctl(scp->sc->kbd, KDSKBMODE, (caddr_t)&scp->kbd_mode); - - kbdd_poll(scp->sc->kbd, TRUE); + /* Switch the keyboard to console mode (K_XLATE, polled) on all scp's. */ + kbd_mode = K_XLATE; + (void)kbdd_ioctl(sc->kbd, KDSKBMODE, (caddr_t)&kbd_mode); + kbdd_poll(sc->kbd, TRUE); } static void -sc_cnungrab(struct consdev *cp) +sccnclose(sc_softc_t *sc, struct sc_cnstate *sp) { - scr_stat *scp; - - scp = sc_console->sc->cur_scp; /* XXX */ - if (scp->sc->kbd == NULL) + if (sc->kbd == NULL) return; - if (--scp->grabbed > 0) - return; + /* Restore keyboard mode (for the current, possibly-changed scp). */ + kbdd_poll(sc->kbd, FALSE); + (void)kbdd_ioctl(sc->kbd, KDSKBMODE, (caddr_t)&sc->cur_scp->kbd_mode); - kbdd_poll(scp->sc->kbd, FALSE); + kbdd_disable(sc->kbd); +} + +static void +sc_cngrab(struct consdev *cp) +{ + sc_softc_t *sc; - scp->kbd_mode = scp->kbd_prev_mode; - (void)kbdd_ioctl(scp->sc->kbd, KDSKBMODE, (caddr_t)&scp->kbd_mode); - kbdd_disable(scp->sc->kbd); + sc = sc_console->sc; + if (sc->grab_level++ == 0) + sccnopen(sc, NULL, 0); +} + +static void +sc_cnungrab(struct consdev *cp) +{ + sc_softc_t *sc; + + sc = sc_console->sc; + if (--sc->grab_level == 0) + sccnclose(sc, NULL); } static void @@ -1815,7 +1824,7 @@ sccnupdate(scr_stat *scp) if (suspend_in_progress || scp->sc->font_loading_in_progress) return; - if (debugger > 0 || panicstr || shutdown_in_progress) { + if (kdb_active || panicstr || shutdown_in_progress) { sc_touch_scrn_saver(); } else if (scp != scp->sc->cur_scp) { return; @@ -1884,7 +1893,7 @@ scrn_timer(void *arg) #endif /* PC98 */ /* should we stop the screen saver? */ - if (debugger > 0 || panicstr || shutdown_in_progress) + if (kdb_active || panicstr || shutdown_in_progress) sc_touch_scrn_saver(); if (run_scrn_saver) { if (time_uptime > sc->scrn_time_stamp + scrn_blank_time) @@ -2279,7 +2288,7 @@ stop_scrn_saver(sc_softc_t *sc, void (*saver)(sc_softc_t *, int)) mark_all(sc->cur_scp); if (sc->delayed_next_scr) sc_switch_scr(sc, sc->delayed_next_scr - 1); - if (debugger == 0) + if (!kdb_active) wakeup(&scrn_blanked); } @@ -2474,7 +2483,7 @@ sc_switch_scr(sc_softc_t *sc, u_int next_scr) DPRINTF(5, ("error 2, requested vty isn't open!\n")); return EINVAL; } - if ((debugger > 0) && (SC_STAT(tp)->smode.mode == VT_PROCESS)) { + if (kdb_active && SC_STAT(tp)->smode.mode == VT_PROCESS) { splx(s); DPRINTF(5, ("error 3, requested vty is in the VT_PROCESS mode\n")); return EINVAL; @@ -2495,7 +2504,7 @@ sc_switch_scr(sc_softc_t *sc, u_int next_scr) * is supposed to be locked by splhigh(), but the debugger may * be invoked at splhigh(). */ - if (debugger == 0) + if (!kdb_active) wakeup(VTY_WCHAN(sc,next_scr)); splx(s); DPRINTF(5, ("switch done (new == old)\n")); @@ -2518,7 +2527,7 @@ sc_switch_scr(sc_softc_t *sc, u_int next_scr) s = spltty(); /* wake up processes waiting for this vty */ - if (debugger == 0) + if (!kdb_active) wakeup(VTY_WCHAN(sc,next_scr)); /* wait for the controlling process to acknowledge, if necessary */ @@ -2669,7 +2678,7 @@ exchange_scr(sc_softc_t *sc) sc_set_border(scp, scp->border); /* set up the keyboard for the new screen */ - if (sc->old_scp->kbd_mode != scp->kbd_mode) + if (sc->grab_level == 0 && sc->old_scp->kbd_mode != scp->kbd_mode) (void)kbdd_ioctl(sc->kbd, KDSKBMODE, (caddr_t)&scp->kbd_mode); update_kbd_state(scp, scp->status, LOCK_MASK); @@ -2688,13 +2697,13 @@ sc_puts(scr_stat *scp, u_char *buf, int len, int kernel) #endif if (scp->tsw) { - if (!kdb_active && !mtx_owned(&scp->scr_lock)) { + if (!kdb_active && !mtx_owned(&scp->sc->scr_lock)) { need_unlock = 1; - mtx_lock_spin(&scp->scr_lock); + mtx_lock_spin(&scp->sc->scr_lock); } (*scp->tsw->te_puts)(scp, buf, len, kernel); if (need_unlock) - mtx_unlock_spin(&scp->scr_lock); + mtx_unlock_spin(&scp->sc->scr_lock); } if (scp->sc->delayed_next_scr) @@ -2859,8 +2868,10 @@ scinit(int unit, int flags) * disappeared... */ sc = sc_get_softc(unit, flags & SC_KERNEL_CONSOLE); - if ((sc->flags & SC_INIT_DONE) == 0) + if ((sc->flags & SC_INIT_DONE) == 0) { + mtx_init(&sc->scr_lock, "scrlock", NULL, MTX_SPIN); SC_VIDEO_LOCKINIT(sc); + } adp = NULL; if (sc->adapter >= 0) { @@ -3077,7 +3088,8 @@ scterm(int unit, int flags) (*scp->tsw->te_term)(scp, &scp->ts); if (scp->ts != NULL) free(scp->ts, M_DEVBUF); - mtx_destroy(&scp->scr_lock); + mtx_destroy(&sc->scr_lock); + mtx_destroy(&sc->video_mtx); /* clear the structure */ if (!(flags & SC_KERNEL_CONSOLE)) { @@ -3302,8 +3314,6 @@ init_scp(sc_softc_t *sc, int vty, scr_stat *scp) scp->history = NULL; scp->history_pos = 0; scp->history_size = 0; - - mtx_init(&scp->scr_lock, "scrlock", NULL, MTX_SPIN); } int @@ -3413,7 +3423,7 @@ next_code: if (!(flags & SCGETC_CN)) random_harvest_queue(&c, sizeof(c), 1, RANDOM_KEYBOARD); - if (scp->kbd_mode != K_XLATE) + if (sc->grab_level == 0 && scp->kbd_mode != K_XLATE) return KEYCHAR(c); /* if scroll-lock pressed allow history browsing */ @@ -3506,8 +3516,9 @@ next_code: scp->status |= CURSOR_ENABLED; sc_draw_cursor_image(scp); } + /* Only safe in Giant-locked context. */ tp = SC_DEV(sc, scp->index); - if (!kdb_active && tty_opened_ns(tp)) + if (!(flags & SCGETC_CN) && tty_opened_ns(tp)) sctty_outwakeup(tp); #endif } @@ -3558,21 +3569,21 @@ next_code: case RBT: #ifndef SC_DISABLE_REBOOT - if (enable_reboot) + if (enable_reboot && !(flags & SCGETC_CN)) shutdown_nice(0); #endif break; case HALT: #ifndef SC_DISABLE_REBOOT - if (enable_reboot) + if (enable_reboot && !(flags & SCGETC_CN)) shutdown_nice(RB_HALT); #endif break; case PDWN: #ifndef SC_DISABLE_REBOOT - if (enable_reboot) + if (enable_reboot && !(flags & SCGETC_CN)) shutdown_nice(RB_HALT|RB_POWEROFF); #endif break; @@ -3843,7 +3854,7 @@ sc_respond(scr_stat *scp, const u_char *p, int count, int wakeup) void sc_bell(scr_stat *scp, int pitch, int duration) { - if (cold || shutdown_in_progress || !enable_bell) + if (cold || kdb_active || shutdown_in_progress || !enable_bell) return; if (scp != scp->sc->cur_scp && (scp->sc->flags & SC_QUIET_BELL)) diff --git a/sys/dev/syscons/syscons.h b/sys/dev/syscons/syscons.h index 06e104595c2e..e0aa0beb4328 100644 --- a/sys/dev/syscons/syscons.h +++ b/sys/dev/syscons/syscons.h @@ -230,6 +230,8 @@ typedef struct sc_softc { char switch_in_progress; char write_in_progress; char blink_in_progress; + int grab_level; + struct mtx scr_lock; /* mutex for sc_puts() */ struct mtx video_mtx; long scrn_time_stamp; @@ -303,9 +305,7 @@ typedef struct scr_stat { void *ts; int status; /* status (bitfield) */ - int grabbed; int kbd_mode; /* keyboard I/O mode */ - int kbd_prev_mode; /* keyboard I/O mode */ int cursor_pos; /* cursor buffer position */ int cursor_oldpos; /* cursor old buffer position */ @@ -344,7 +344,6 @@ typedef struct scr_stat { int splash_save_mode; /* saved mode for splash screen */ int splash_save_status; /* saved status for splash screen */ - struct mtx scr_lock; /* mutex for sc_puts() */ #ifdef _SCR_MD_STAT_DECLARED_ scr_md_stat_t md; /* machine dependent vars */ #endif diff --git a/sys/dev/tpm/tpm.c b/sys/dev/tpm/tpm.c index 3ac52d6e7691..3779c09f8193 100644 --- a/sys/dev/tpm/tpm.c +++ b/sys/dev/tpm/tpm.c @@ -175,8 +175,8 @@ struct cfdriver tpm_cd = { NULL, "tpm", DV_DULL }; -int tpm_match(struct device *, void *, void *); -void tpm_attach(struct device *, struct device *, void *); +int tpm_match(device_t , void *, void *); +void tpm_attach(device_t , device_t , void *); struct cfattach tpm_ca = { sizeof(struct tpm_softc), tpm_match, tpm_attach @@ -337,7 +337,7 @@ tpm_detach(device_t dev) * OpenBSD specific code for probing and attaching TPM to device tree. */ int -tpm_match(struct device *parent, void *match, void *aux) +tpm_match(device_t parent, void *match, void *aux) { struct isa_attach_args *ia = aux; struct cfdata *cf = match; @@ -370,7 +370,7 @@ tpm_match(struct device *parent, void *match, void *aux) } void -tpm_attach(struct device *parent, struct device *self, void *aux) +tpm_attach(device_t parent, device_t self, void *aux) { struct tpm_softc *sc = (struct tpm_softc *)self; struct isa_attach_args *ia = aux; diff --git a/sys/dev/tws/tws.c b/sys/dev/tws/tws.c index bf9e143c72f5..480f6f954893 100644 --- a/sys/dev/tws/tws.c +++ b/sys/dev/tws/tws.c @@ -606,21 +606,9 @@ tws_init(struct tws_softc *sc) sc->reqs = malloc(sizeof(struct tws_request) * tws_queue_depth, M_TWS, M_WAITOK | M_ZERO); - if ( sc->reqs == NULL ) { - TWS_TRACE_DEBUG(sc, "malloc failed", 0, sc->is64bit); - return(ENOMEM); - } sc->sense_bufs = malloc(sizeof(struct tws_sense) * tws_queue_depth, M_TWS, M_WAITOK | M_ZERO); - if ( sc->sense_bufs == NULL ) { - TWS_TRACE_DEBUG(sc, "sense malloc failed", 0, sc->is64bit); - return(ENOMEM); - } sc->scan_ccb = malloc(sizeof(union ccb), M_TWS, M_WAITOK | M_ZERO); - if ( sc->scan_ccb == NULL ) { - TWS_TRACE_DEBUG(sc, "ccb malloc failed", 0, sc->is64bit); - return(ENOMEM); - } if (bus_dmamem_alloc(sc->data_tag, (void **)&sc->ioctl_data_mem, (BUS_DMA_NOWAIT | BUS_DMA_ZERO), &sc->ioctl_data_map)) { device_printf(sc->tws_dev, "Cannot allocate ioctl data mem\n"); @@ -668,8 +656,6 @@ tws_init_aen_q(struct tws_softc *sc) sc->aen_q.overflow=0; sc->aen_q.q = malloc(sizeof(struct tws_event_packet)*sc->aen_q.depth, M_TWS, M_WAITOK | M_ZERO); - if ( ! sc->aen_q.q ) - return(FAILURE); return(SUCCESS); } @@ -682,8 +668,6 @@ tws_init_trace_q(struct tws_softc *sc) sc->trace_q.overflow=0; sc->trace_q.q = malloc(sizeof(struct tws_trace_rec)*sc->trace_q.depth, M_TWS, M_WAITOK | M_ZERO); - if ( ! sc->trace_q.q ) - return(FAILURE); return(SUCCESS); } diff --git a/sys/dev/uart/uart_cpu_fdt.c b/sys/dev/uart/uart_cpu_fdt.c index e73ff6a63bd9..daa4d581177a 100644 --- a/sys/dev/uart/uart_cpu_fdt.c +++ b/sys/dev/uart/uart_cpu_fdt.c @@ -70,53 +70,6 @@ uart_cpu_eqres(struct uart_bas *b1, struct uart_bas *b2) return ((pmap_kextract(b1->bsh) == pmap_kextract(b2->bsh)) ? 1 : 0); } -static int -phandle_chosen_propdev(phandle_t chosen, const char *name, phandle_t *node) -{ - char buf[64]; - - if (OF_getprop(chosen, name, buf, sizeof(buf)) <= 0) - return (ENXIO); - if ((*node = OF_finddevice(buf)) == -1) - return (ENXIO); - - return (0); -} - -static const struct ofw_compat_data * -uart_fdt_find_compatible(phandle_t node, const struct ofw_compat_data *cd) -{ - const struct ofw_compat_data *ocd; - - for (ocd = cd; ocd->ocd_str != NULL; ocd++) { - if (fdt_is_compatible(node, ocd->ocd_str)) - return (ocd); - } - return (NULL); -} - -static uintptr_t -uart_fdt_find_by_node(phandle_t node, int class_list) -{ - struct ofw_compat_data **cd; - const struct ofw_compat_data *ocd; - - if (class_list) { - SET_FOREACH(cd, uart_fdt_class_set) { - ocd = uart_fdt_find_compatible(node, *cd); - if ((ocd != NULL) && (ocd->ocd_data != 0)) - return (ocd->ocd_data); - } - } else { - SET_FOREACH(cd, uart_fdt_class_and_device_set) { - ocd = uart_fdt_find_compatible(node, *cd); - if ((ocd != NULL) && (ocd->ocd_data != 0)) - return (ocd->ocd_data); - } - } - return (0); -} - int uart_cpu_getdev(int devtype, struct uart_devinfo *di) { diff --git a/sys/dev/usb/controller/ehci_ixp4xx.c b/sys/dev/usb/controller/ehci_ixp4xx.c index 05efbb51c129..1b2f6409d053 100644 --- a/sys/dev/usb/controller/ehci_ixp4xx.c +++ b/sys/dev/usb/controller/ehci_ixp4xx.c @@ -147,13 +147,13 @@ ehci_ixp_attach(device_t self) isc->iot = rman_get_bustag(sc->sc_io_res); isc->tag.bs_privdata = isc->iot; /* read single */ - isc->tag.bs_r_1 = ehci_bs_r_1, - isc->tag.bs_r_2 = ehci_bs_r_2, - isc->tag.bs_r_4 = ehci_bs_r_4, + isc->tag.bs_r_1 = ehci_bs_r_1; + isc->tag.bs_r_2 = ehci_bs_r_2; + isc->tag.bs_r_4 = ehci_bs_r_4; /* write (single) */ - isc->tag.bs_w_1 = ehci_bs_w_1, - isc->tag.bs_w_2 = ehci_bs_w_2, - isc->tag.bs_w_4 = ehci_bs_w_4, + isc->tag.bs_w_1 = ehci_bs_w_1; + isc->tag.bs_w_2 = ehci_bs_w_2; + isc->tag.bs_w_4 = ehci_bs_w_4; sc->sc_io_tag = &isc->tag; sc->sc_io_hdl = rman_get_bushandle(sc->sc_io_res); diff --git a/sys/dev/usb/controller/generic_ehci.c b/sys/dev/usb/controller/generic_ehci.c new file mode 100644 index 000000000000..fcae7dd41a99 --- /dev/null +++ b/sys/dev/usb/controller/generic_ehci.c @@ -0,0 +1,220 @@ +/*- + * Copyright (c) 2012 Ganbold Tsagaankhuu <ganbold@freebsd.org> + * Copyright (c) 2016 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Andrew Turner under + * sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * Generic EHCI driver based on the Allwinner A10 EHCI driver + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include "opt_bus.h" + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/bus.h> +#include <sys/rman.h> +#include <sys/condvar.h> +#include <sys/kernel.h> +#include <sys/module.h> + +#include <machine/bus.h> + +#include <dev/usb/usb.h> +#include <dev/usb/usbdi.h> + +#include <dev/usb/usb_core.h> +#include <dev/usb/usb_busdma.h> +#include <dev/usb/usb_process.h> +#include <dev/usb/usb_util.h> + +#include <dev/usb/usb_controller.h> +#include <dev/usb/usb_bus.h> +#include <dev/usb/controller/ehci.h> +#include <dev/usb/controller/ehcireg.h> + +#include <contrib/dev/acpica/include/acpi.h> +#include <contrib/dev/acpica/include/accommon.h> +#include <dev/acpica/acpivar.h> + +static device_attach_t generic_ehci_attach; +static device_detach_t generic_ehci_detach; + +static int +generic_ehci_probe(device_t self) +{ + ACPI_HANDLE h; + + if ((h = acpi_get_handle(self)) == NULL || + !acpi_MatchHid(h, "PNP0D20")) + return (ENXIO); + + device_set_desc(self, "Generic EHCI Controller"); + return (BUS_PROBE_DEFAULT); +} + +static int +generic_ehci_attach(device_t self) +{ + ehci_softc_t *sc = device_get_softc(self); + int err; + int rid; + + /* initialise some bus fields */ + sc->sc_bus.parent = self; + sc->sc_bus.devices = sc->sc_devices; + sc->sc_bus.devices_max = EHCI_MAX_DEVICES; + sc->sc_bus.dma_bits = 32; + + /* get all DMA memory */ + if (usb_bus_mem_alloc_all(&sc->sc_bus, + USB_GET_DMA_TAG(self), &ehci_iterate_hw_softc)) { + return (ENOMEM); + } + + sc->sc_bus.usbrev = USB_REV_2_0; + + rid = 0; + sc->sc_io_res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, + RF_ACTIVE); + if (!sc->sc_io_res) { + device_printf(self, "Could not map memory\n"); + goto error; + } + + sc->sc_io_tag = rman_get_bustag(sc->sc_io_res); + sc->sc_io_hdl = rman_get_bushandle(sc->sc_io_res); + sc->sc_io_size = rman_get_size(sc->sc_io_res); + + rid = 0; + sc->sc_irq_res = bus_alloc_resource_any(self, SYS_RES_IRQ, &rid, + RF_SHAREABLE | RF_ACTIVE); + if (sc->sc_irq_res == NULL) { + device_printf(self, "Could not allocate irq\n"); + goto error; + } + sc->sc_bus.bdev = device_add_child(self, "usbus", -1); + if (!sc->sc_bus.bdev) { + device_printf(self, "Could not add USB device\n"); + goto error; + } + device_set_ivars(sc->sc_bus.bdev, &sc->sc_bus); + + strlcpy(sc->sc_vendor, "Generic", sizeof(sc->sc_vendor)); + + err = bus_setup_intr(self, sc->sc_irq_res, INTR_TYPE_BIO | INTR_MPSAFE, + NULL, (driver_intr_t *)ehci_interrupt, sc, &sc->sc_intr_hdl); + if (err) { + device_printf(self, "Could not setup irq, %d\n", err); + sc->sc_intr_hdl = NULL; + goto error; + } + + sc->sc_flags |= EHCI_SCFLG_DONTRESET; + + err = ehci_init(sc); + if (!err) + err = device_probe_and_attach(sc->sc_bus.bdev); + if (err) + goto error; + + return (0); + +error: + generic_ehci_detach(self); + return (ENXIO); +} + +static int +generic_ehci_detach(device_t self) +{ + ehci_softc_t *sc = device_get_softc(self); + device_t bdev; + int err; + + if (sc->sc_bus.bdev) { + bdev = sc->sc_bus.bdev; + device_detach(bdev); + device_delete_child(self, bdev); + } + /* during module unload there are lots of children leftover */ + device_delete_children(self); + + if (sc->sc_irq_res && sc->sc_intr_hdl) { + /* + * only call ehci_detach() after ehci_init() + */ + ehci_detach(sc); + + err = bus_teardown_intr(self, sc->sc_irq_res, sc->sc_intr_hdl); + + if (err) + /* XXX or should we panic? */ + device_printf(self, "Could not tear down irq, %d\n", + err); + sc->sc_intr_hdl = NULL; + } + + if (sc->sc_irq_res) { + bus_release_resource(self, SYS_RES_IRQ, 0, sc->sc_irq_res); + sc->sc_irq_res = NULL; + } + if (sc->sc_io_res) { + bus_release_resource(self, SYS_RES_MEMORY, 0, + sc->sc_io_res); + sc->sc_io_res = NULL; + } + usb_bus_mem_free_all(&sc->sc_bus, &ehci_iterate_hw_softc); + + return (0); +} + +static device_method_t ehci_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, generic_ehci_probe), + DEVMETHOD(device_attach, generic_ehci_attach), + DEVMETHOD(device_detach, generic_ehci_detach), + DEVMETHOD(device_suspend, bus_generic_suspend), + DEVMETHOD(device_resume, bus_generic_resume), + DEVMETHOD(device_shutdown, bus_generic_shutdown), + + DEVMETHOD_END +}; + +static driver_t ehci_driver = { + .name = "ehci", + .methods = ehci_methods, + .size = sizeof(ehci_softc_t), +}; + +static devclass_t ehci_devclass; + +DRIVER_MODULE(ehci, acpi, ehci_driver, ehci_devclass, 0, 0); +MODULE_DEPEND(ehci, usb, 1, 1, 1); diff --git a/sys/dev/usb/input/ukbd.c b/sys/dev/usb/input/ukbd.c index 02a6ff37e8d4..42d96cfa64db 100644 --- a/sys/dev/usb/input/ukbd.c +++ b/sys/dev/usb/input/ukbd.c @@ -198,6 +198,7 @@ struct ukbd_softc { int sc_mode; /* input mode (K_XLATE,K_RAW,K_CODE) */ int sc_state; /* shift/lock key state */ int sc_accents; /* accent key index (> 0) */ + int sc_polling; /* polling recursion count */ int sc_led_size; int sc_kbd_size; @@ -1983,7 +1984,16 @@ ukbd_poll(keyboard_t *kbd, int on) struct ukbd_softc *sc = kbd->kb_data; UKBD_LOCK(); - if (on) { + /* + * Keep a reference count on polling to allow recursive + * cngrab() during a panic for example. + */ + if (on) + sc->sc_polling++; + else + sc->sc_polling--; + + if (sc->sc_polling != 0) { sc->sc_flags |= UKBD_FLAG_POLLING; sc->sc_poll_thread = curthread; } else { diff --git a/sys/dev/usb/serial/u3g.c b/sys/dev/usb/serial/u3g.c index 9988347243c4..0aa26bbffe0b 100644 --- a/sys/dev/usb/serial/u3g.c +++ b/sys/dev/usb/serial/u3g.c @@ -1100,6 +1100,7 @@ u3g_cfg_get_status(struct ucom_softc *ucom, uint8_t *lsr, uint8_t *msr) { struct u3g_softc *sc = ucom->sc_parent; + /* XXX Note: sc_lsr is always zero */ *lsr = sc->sc_lsr[ucom->sc_subunit]; *msr = sc->sc_msr[ucom->sc_subunit]; } diff --git a/sys/dev/usb/serial/uark.c b/sys/dev/usb/serial/uark.c index e0e6d0bc9049..a1c8976494b4 100644 --- a/sys/dev/usb/serial/uark.c +++ b/sys/dev/usb/serial/uark.c @@ -427,6 +427,7 @@ uark_cfg_get_status(struct ucom_softc *ucom, uint8_t *lsr, uint8_t *msr) { struct uark_softc *sc = ucom->sc_parent; + /* XXX Note: sc_lsr is always zero */ *lsr = sc->sc_lsr; *msr = sc->sc_msr; } diff --git a/sys/dev/usb/serial/ubsa.c b/sys/dev/usb/serial/ubsa.c index 5fac56a27666..74159687e76c 100644 --- a/sys/dev/usb/serial/ubsa.c +++ b/sys/dev/usb/serial/ubsa.c @@ -650,11 +650,19 @@ ubsa_intr_callback(struct usb_xfer *xfer, usb_error_t error) usbd_copy_out(pc, 0, buf, sizeof(buf)); /* - * incidentally, Belkin adapter status bits match - * UART 16550 bits + * MSR bits need translation from ns16550 to SER_* values. + * LSR bits are ns16550 in hardware and ucom. */ + sc->sc_msr = 0; + if (buf[3] & UBSA_MSR_CTS) + sc->sc_msr |= SER_CTS; + if (buf[3] & UBSA_MSR_DCD) + sc->sc_msr |= SER_DCD; + if (buf[3] & UBSA_MSR_RI) + sc->sc_msr |= SER_RI; + if (buf[3] & UBSA_MSR_DSR) + sc->sc_msr |= SER_DSR; sc->sc_lsr = buf[2]; - sc->sc_msr = buf[3]; DPRINTF("lsr = 0x%02x, msr = 0x%02x\n", sc->sc_lsr, sc->sc_msr); @@ -663,7 +671,7 @@ ubsa_intr_callback(struct usb_xfer *xfer, usb_error_t error) } else { DPRINTF("ignoring short packet, %d bytes\n", actlen); } - + /* FALLTHROUGH */ case USB_ST_SETUP: tr_setup: usbd_xfer_set_frame_len(xfer, 0, usbd_xfer_max_len(xfer)); diff --git a/sys/dev/usb/serial/uchcom.c b/sys/dev/usb/serial/uchcom.c index d1063fd4f634..e3fdeedc8a23 100644 --- a/sys/dev/usb/serial/uchcom.c +++ b/sys/dev/usb/serial/uchcom.c @@ -625,6 +625,7 @@ uchcom_cfg_get_status(struct ucom_softc *ucom, uint8_t *lsr, uint8_t *msr) DPRINTF("\n"); + /* XXX Note: sc_lsr is always zero */ *lsr = sc->sc_lsr; *msr = sc->sc_msr; } diff --git a/sys/dev/usb/serial/ufoma.c b/sys/dev/usb/serial/ufoma.c index fb526cd57e6d..dfb41e1871d0 100644 --- a/sys/dev/usb/serial/ufoma.c +++ b/sys/dev/usb/serial/ufoma.c @@ -900,6 +900,7 @@ ufoma_cfg_get_status(struct ucom_softc *ucom, uint8_t *lsr, uint8_t *msr) { struct ufoma_softc *sc = ucom->sc_parent; + /* XXX Note: sc_lsr is always zero */ *lsr = sc->sc_lsr; *msr = sc->sc_msr; } diff --git a/sys/dev/usb/serial/umcs.c b/sys/dev/usb/serial/umcs.c index 6b55b5a8450b..c5ac0d6c52aa 100644 --- a/sys/dev/usb/serial/umcs.c +++ b/sys/dev/usb/serial/umcs.c @@ -743,15 +743,26 @@ umcs7840_cfg_get_status(struct ucom_softc *ucom, uint8_t *lsr, uint8_t *msr) { struct umcs7840_softc *sc = ucom->sc_parent; uint8_t pn = ucom->sc_portno; - uint8_t hw_lsr = 0; /* local line status register */ uint8_t hw_msr = 0; /* local modem status register */ - /* Read LSR & MSR */ - umcs7840_get_UART_reg_sync(sc, pn, MCS7840_UART_REG_LSR, &hw_lsr); + /* + * Read status registers. MSR bits need translation from ns16550 to + * SER_* values. LSR bits are ns16550 in hardware and ucom. + */ + umcs7840_get_UART_reg_sync(sc, pn, MCS7840_UART_REG_LSR, lsr); umcs7840_get_UART_reg_sync(sc, pn, MCS7840_UART_REG_MSR, &hw_msr); - *lsr = hw_lsr; - *msr = hw_msr; + if (hw_msr & MCS7840_UART_MSR_NEGCTS) + *msr |= SER_CTS; + + if (hw_msr & MCS7840_UART_MSR_NEGDCD) + *msr |= SER_DCD; + + if (hw_msr & MCS7840_UART_MSR_NEGRI) + *msr |= SER_RI; + + if (hw_msr & MCS7840_UART_MSR_NEGDSR) + *msr |= SER_DSR; DPRINTF("Port %d status: LSR=%02x MSR=%02x\n", ucom->sc_portno, *lsr, *msr); } diff --git a/sys/dev/usb/serial/umct.c b/sys/dev/usb/serial/umct.c index 32c789743723..251e0790fce7 100644 --- a/sys/dev/usb/serial/umct.c +++ b/sys/dev/usb/serial/umct.c @@ -86,6 +86,15 @@ __FBSDID("$FreeBSD$"); #define UMCT_SET_MCR 10 /* Set Modem Control Register */ #define UMCT_SET_MCR_SIZE 1 +#define UMCT_MSR_CTS_CHG 0x01 +#define UMCT_MSR_DSR_CHG 0x02 +#define UMCT_MSR_RI_CHG 0x04 +#define UMCT_MSR_CD_CHG 0x08 +#define UMCT_MSR_CTS 0x10 +#define UMCT_MSR_RTS 0x20 +#define UMCT_MSR_RI 0x40 +#define UMCT_MSR_CD 0x80 + #define UMCT_INTR_INTERVAL 100 #define UMCT_IFACE_INDEX 0 #define UMCT_CONFIG_INDEX 0 @@ -384,11 +393,23 @@ umct_intr_callback_sub(struct usb_xfer *xfer, usb_error_t error) pc = usbd_xfer_get_frame(xfer, 0); usbd_copy_out(pc, 0, buf, sizeof(buf)); - sc->sc_msr = buf[0]; + /* + * MSR bits need translation from ns16550 to SER_* values. + * LSR bits are ns16550 in hardware and ucom. + */ + sc->sc_msr = 0; + if (buf[0] & UMCT_MSR_CTS) + sc->sc_msr |= SER_CTS; + if (buf[0] & UMCT_MSR_CD) + sc->sc_msr |= SER_DCD; + if (buf[0] & UMCT_MSR_RI) + sc->sc_msr |= SER_RI; + if (buf[0] & UMCT_MSR_RTS) + sc->sc_msr |= SER_DSR; sc->sc_lsr = buf[1]; ucom_status_change(&sc->sc_ucom); - + /* FALLTHROUGH */ case USB_ST_SETUP: tr_setup: usbd_xfer_set_frame_len(xfer, 0, usbd_xfer_max_len(xfer)); diff --git a/sys/dev/usb/serial/umodem.c b/sys/dev/usb/serial/umodem.c index 5e65c12e1cb7..b9f798b479ae 100644 --- a/sys/dev/usb/serial/umodem.c +++ b/sys/dev/usb/serial/umodem.c @@ -558,6 +558,7 @@ umodem_cfg_get_status(struct ucom_softc *ucom, uint8_t *lsr, uint8_t *msr) DPRINTF("\n"); + /* XXX Note: sc_lsr is always zero */ *lsr = sc->sc_lsr; *msr = sc->sc_msr; } diff --git a/sys/dev/usb/serial/umoscom.c b/sys/dev/usb/serial/umoscom.c index 8c580dabb59c..daa83c54feab 100644 --- a/sys/dev/usb/serial/umoscom.c +++ b/sys/dev/usb/serial/umoscom.c @@ -523,14 +523,16 @@ static void umoscom_cfg_get_status(struct ucom_softc *ucom, uint8_t *p_lsr, uint8_t *p_msr) { struct umoscom_softc *sc = ucom->sc_parent; - uint8_t lsr; uint8_t msr; DPRINTFN(5, "\n"); - /* read status registers */ + /* + * Read status registers. MSR bits need translation from ns16550 to + * SER_* values. LSR bits are ns16550 in hardware and ucom. + */ - lsr = umoscom_cfg_read(sc, UMOSCOM_LSR); + *p_lsr = umoscom_cfg_read(sc, UMOSCOM_LSR); msr = umoscom_cfg_read(sc, UMOSCOM_MSR); /* translate bits */ diff --git a/sys/dev/usb/serial/uplcom.c b/sys/dev/usb/serial/uplcom.c index 6e238758fa26..dfc0933d4f09 100644 --- a/sys/dev/usb/serial/uplcom.c +++ b/sys/dev/usb/serial/uplcom.c @@ -807,6 +807,7 @@ uplcom_cfg_get_status(struct ucom_softc *ucom, uint8_t *lsr, uint8_t *msr) DPRINTF("\n"); + /* XXX Note: sc_lsr is always zero */ *lsr = sc->sc_lsr; *msr = sc->sc_msr; } diff --git a/sys/dev/usb/serial/uslcom.c b/sys/dev/usb/serial/uslcom.c index b516bdec3641..063f715eb8f3 100644 --- a/sys/dev/usb/serial/uslcom.c +++ b/sys/dev/usb/serial/uslcom.c @@ -704,6 +704,7 @@ uslcom_get_status(struct ucom_softc *ucom, uint8_t *lsr, uint8_t *msr) DPRINTF("\n"); + /* XXX Note: sc_lsr is always zero */ *lsr = sc->sc_lsr; *msr = sc->sc_msr; } diff --git a/sys/dev/usb/template/usb_template_mtp.c b/sys/dev/usb/template/usb_template_mtp.c index f0528a52aa89..d428e992f062 100644 --- a/sys/dev/usb/template/usb_template_mtp.c +++ b/sys/dev/usb/template/usb_template_mtp.c @@ -26,7 +26,7 @@ */ /* - * This file contains the USB templates for an USB Message Transfer + * This file contains the USB templates for an USB Media Transfer * Protocol device. * * NOTE: It is common practice that MTP devices use some dummy diff --git a/sys/dev/usb/usb_device.c b/sys/dev/usb/usb_device.c index a5c2444ec859..c1b21df58119 100644 --- a/sys/dev/usb/usb_device.c +++ b/sys/dev/usb/usb_device.c @@ -1724,8 +1724,8 @@ usb_alloc_device(device_t parent_dev, struct usb_bus *bus, /* Setup USB descriptors */ err = (usb_temp_setup_by_index_p) (udev, usb_template); if (err) { - DPRINTFN(0, "setting up USB template failed maybe the USB " - "template module has not been loaded\n"); + DPRINTFN(0, "setting up USB template failed - " + "usb_template(4) not loaded?\n"); goto done; } } diff --git a/sys/dev/usb/usbdevs b/sys/dev/usb/usbdevs index 44d93212bfeb..5cd917476fb5 100644 --- a/sys/dev/usb/usbdevs +++ b/sys/dev/usb/usbdevs @@ -2729,6 +2729,7 @@ product LOGITECH BB13 0xc401 USB-PS/2 Trackball product LOGITECH RK53 0xc501 Cordless mouse product LOGITECH RB6 0xc503 Cordless keyboard product LOGITECH MX700 0xc506 Cordless optical mouse +product LOGITECH UNIFYING 0xc52b Logitech Unifying Receiver product LOGITECH QUICKCAMPRO2 0xd001 QuickCam Pro /* Logitec Corp. products */ diff --git a/sys/dev/virtio/network/if_vtnet.c b/sys/dev/virtio/network/if_vtnet.c index 6b89b869242e..37189753012e 100644 --- a/sys/dev/virtio/network/if_vtnet.c +++ b/sys/dev/virtio/network/if_vtnet.c @@ -230,18 +230,32 @@ static void vtnet_disable_interrupts(struct vtnet_softc *); static int vtnet_tunable_int(struct vtnet_softc *, const char *, int); /* Tunables. */ +static SYSCTL_NODE(_hw, OID_AUTO, vtnet, CTLFLAG_RD, 0, "VNET driver parameters"); static int vtnet_csum_disable = 0; TUNABLE_INT("hw.vtnet.csum_disable", &vtnet_csum_disable); +SYSCTL_INT(_hw_vtnet, OID_AUTO, csum_disable, CTLFLAG_RDTUN, + &vtnet_csum_disable, 0, "Disables receive and send checksum offload"); static int vtnet_tso_disable = 0; TUNABLE_INT("hw.vtnet.tso_disable", &vtnet_tso_disable); +SYSCTL_INT(_hw_vtnet, OID_AUTO, tso_disable, CTLFLAG_RDTUN, &vtnet_tso_disable, + 0, "Disables TCP Segmentation Offload"); static int vtnet_lro_disable = 0; TUNABLE_INT("hw.vtnet.lro_disable", &vtnet_lro_disable); +SYSCTL_INT(_hw_vtnet, OID_AUTO, lro_disable, CTLFLAG_RDTUN, &vtnet_lro_disable, + 0, "Disables TCP Large Receive Offload"); static int vtnet_mq_disable = 0; TUNABLE_INT("hw.vtnet.mq_disable", &vtnet_mq_disable); -static int vtnet_mq_max_pairs = 0; +SYSCTL_INT(_hw_vtnet, OID_AUTO, mq_disable, CTLFLAG_RDTUN, &vtnet_mq_disable, + 0, "Disables Multi Queue support"); +static int vtnet_mq_max_pairs = VTNET_MAX_QUEUE_PAIRS; TUNABLE_INT("hw.vtnet.mq_max_pairs", &vtnet_mq_max_pairs); +SYSCTL_INT(_hw_vtnet, OID_AUTO, mq_max_pairs, CTLFLAG_RDTUN, + &vtnet_mq_max_pairs, 0, "Sets the maximum number of Multi Queue pairs"); static int vtnet_rx_process_limit = 512; TUNABLE_INT("hw.vtnet.rx_process_limit", &vtnet_rx_process_limit); +SYSCTL_INT(_hw_vtnet, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN, + &vtnet_rx_process_limit, 0, + "Limits the number RX segments processed in a single pass"); static uma_zone_t vtnet_tx_header_zone; @@ -597,7 +611,6 @@ static void vtnet_setup_features(struct vtnet_softc *sc) { device_t dev; - int max_pairs, max; dev = sc->vtnet_dev; @@ -646,32 +659,31 @@ vtnet_setup_features(struct vtnet_softc *sc) if (virtio_with_feature(dev, VIRTIO_NET_F_MQ) && sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) { - max_pairs = virtio_read_dev_config_2(dev, + sc->vtnet_max_vq_pairs = virtio_read_dev_config_2(dev, offsetof(struct virtio_net_config, max_virtqueue_pairs)); - if (max_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN || - max_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX) - max_pairs = 1; } else - max_pairs = 1; + sc->vtnet_max_vq_pairs = 1; - if (max_pairs > 1) { + if (sc->vtnet_max_vq_pairs > 1) { /* - * Limit the maximum number of queue pairs to the number of - * CPUs or the configured maximum. The actual number of - * queues that get used may be less. + * Limit the maximum number of queue pairs to the lower of + * the number of CPUs and the configured maximum. + * The actual number of queues that get used may be less. */ + int max; + max = vtnet_tunable_int(sc, "mq_max_pairs", vtnet_mq_max_pairs); - if (max > 0 && max_pairs > max) - max_pairs = max; - if (max_pairs > mp_ncpus) - max_pairs = mp_ncpus; - if (max_pairs > VTNET_MAX_QUEUE_PAIRS) - max_pairs = VTNET_MAX_QUEUE_PAIRS; - if (max_pairs > 1) - sc->vtnet_flags |= VTNET_FLAG_MULTIQ; + if (max > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN) { + if (max > mp_ncpus) + max = mp_ncpus; + if (max > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX) + max = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX; + if (max > 1) { + sc->vtnet_requested_vq_pairs = max; + sc->vtnet_flags |= VTNET_FLAG_MULTIQ; + } + } } - - sc->vtnet_max_vq_pairs = max_pairs; } static int @@ -2982,13 +2994,11 @@ vtnet_set_active_vq_pairs(struct vtnet_softc *sc) dev = sc->vtnet_dev; if ((sc->vtnet_flags & VTNET_FLAG_MULTIQ) == 0) { - MPASS(sc->vtnet_max_vq_pairs == 1); sc->vtnet_act_vq_pairs = 1; return; } - /* BMV: Just use the maximum configured for now. */ - npairs = sc->vtnet_max_vq_pairs; + npairs = sc->vtnet_requested_vq_pairs; if (vtnet_ctrl_mq_cmd(sc, npairs) != 0) { device_printf(dev, @@ -3852,6 +3862,9 @@ vtnet_setup_sysctl(struct vtnet_softc *sc) SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_vq_pairs", CTLFLAG_RD, &sc->vtnet_max_vq_pairs, 0, "Maximum number of supported virtqueue pairs"); + SYSCTL_ADD_INT(ctx, child, OID_AUTO, "requested_vq_pairs", + CTLFLAG_RD, &sc->vtnet_requested_vq_pairs, 0, + "Requested number of virtqueue pairs"); SYSCTL_ADD_INT(ctx, child, OID_AUTO, "act_vq_pairs", CTLFLAG_RD, &sc->vtnet_act_vq_pairs, 0, "Number of active virtqueue pairs"); diff --git a/sys/dev/virtio/network/if_vtnetvar.h b/sys/dev/virtio/network/if_vtnetvar.h index f89f6b11fb21..15436d983ca9 100644 --- a/sys/dev/virtio/network/if_vtnetvar.h +++ b/sys/dev/virtio/network/if_vtnetvar.h @@ -155,6 +155,7 @@ struct vtnet_softc { int vtnet_if_flags; int vtnet_act_vq_pairs; int vtnet_max_vq_pairs; + int vtnet_requested_vq_pairs; struct virtqueue *vtnet_ctrl_vq; struct vtnet_mac_filter *vtnet_mac_filter; diff --git a/sys/dev/vt/vt_core.c b/sys/dev/vt/vt_core.c index 32ac08048fea..e09e8a786b3c 100644 --- a/sys/dev/vt/vt_core.c +++ b/sys/dev/vt/vt_core.c @@ -2228,9 +2228,11 @@ skip_thunk: return (EINVAL); if (vw == vd->vd_curwindow) { + mtx_lock(&Giant); kbd = kbd_get_keyboard(vd->vd_keyboard); if (kbd != NULL) vt_save_kbd_state(vw, kbd); + mtx_unlock(&Giant); } vi->m_num = vd->vd_curwindow->vw_number + 1; diff --git a/sys/dev/xen/netfront/netfront.c b/sys/dev/xen/netfront/netfront.c index 3d02a52d0ac0..a68bc9675da5 100644 --- a/sys/dev/xen/netfront/netfront.c +++ b/sys/dev/xen/netfront/netfront.c @@ -1760,7 +1760,7 @@ xn_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) #ifdef INET struct ifaddr *ifa = (struct ifaddr *)data; #endif - int mask, error = 0; + int mask, error = 0, reinit; dev = sc->xbdev; @@ -1809,41 +1809,36 @@ xn_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) break; case SIOCSIFCAP: mask = ifr->ifr_reqcap ^ ifp->if_capenable; + reinit = 0; + if (mask & IFCAP_TXCSUM) { - if (IFCAP_TXCSUM & ifp->if_capenable) { - ifp->if_capenable &= ~(IFCAP_TXCSUM|IFCAP_TSO4); - ifp->if_hwassist &= ~(CSUM_TCP | CSUM_UDP - | CSUM_IP | CSUM_TSO); - } else { - ifp->if_capenable |= IFCAP_TXCSUM; - ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP - | CSUM_IP); - } - } - if (mask & IFCAP_RXCSUM) { - ifp->if_capenable ^= IFCAP_RXCSUM; + ifp->if_capenable ^= IFCAP_TXCSUM; + ifp->if_hwassist ^= XN_CSUM_FEATURES; } if (mask & IFCAP_TSO4) { - if (IFCAP_TSO4 & ifp->if_capenable) { - ifp->if_capenable &= ~IFCAP_TSO4; - ifp->if_hwassist &= ~CSUM_TSO; - } else if (IFCAP_TXCSUM & ifp->if_capenable) { - ifp->if_capenable |= IFCAP_TSO4; - ifp->if_hwassist |= CSUM_TSO; - } else { - IPRINTK("Xen requires tx checksum offload" - " be enabled to use TSO\n"); - error = EINVAL; - } + ifp->if_capenable ^= IFCAP_TSO4; + ifp->if_hwassist ^= CSUM_TSO; } - if (mask & IFCAP_LRO) { - ifp->if_capenable ^= IFCAP_LRO; + if (mask & (IFCAP_RXCSUM | IFCAP_LRO)) { + /* These Rx features require us to renegotiate. */ + reinit = 1; + + if (mask & IFCAP_RXCSUM) + ifp->if_capenable ^= IFCAP_RXCSUM; + if (mask & IFCAP_LRO) + ifp->if_capenable ^= IFCAP_LRO; } + + if (reinit == 0) + break; + /* * We must reset the interface so the backend picks up the * new features. */ + device_printf(sc->xbdev, + "performing interface reset due to feature change\n"); XN_LOCK(sc); netfront_carrier_off(sc); sc->xn_reset = true; @@ -1865,6 +1860,13 @@ xn_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) xs_rm(XST_NIL, xenbus_get_node(dev), "feature-gso-tcpv4"); xs_rm(XST_NIL, xenbus_get_node(dev), "feature-no-csum-offload"); xenbus_set_state(dev, XenbusStateClosing); + + /* + * Wait for the frontend to reconnect before returning + * from the ioctl. 30s should be more than enough for any + * sane backend to reconnect. + */ + error = tsleep(sc, 0, "xn_rst", 30*hz); break; case SIOCADDMULTI: case SIOCDELMULTI: @@ -1971,6 +1973,7 @@ xn_connect(struct netfront_info *np) * packets. */ netfront_carrier_on(np); + wakeup(np); return (0); } @@ -2085,7 +2088,7 @@ xn_configure_features(struct netfront_info *np) #endif if ((ifp->if_capabilities & cap_enabled & IFCAP_TXCSUM) != 0) { ifp->if_capenable |= IFCAP_TXCSUM; - ifp->if_hwassist |= CSUM_TCP|CSUM_UDP; + ifp->if_hwassist |= XN_CSUM_FEATURES; } if ((ifp->if_capabilities & cap_enabled & IFCAP_RXCSUM) != 0) ifp->if_capenable |= IFCAP_RXCSUM; @@ -2157,6 +2160,9 @@ xn_txq_mq_start(struct ifnet *ifp, struct mbuf *m) np = ifp->if_softc; npairs = np->num_queues; + if (!netfront_carrier_ok(np)) + return (ENOBUFS); + KASSERT(npairs != 0, ("called with 0 available queues")); /* check if flowid is set */ |