- Don't set CAS_PCS_DATAPATH to anything except CAS_PCS_DATAPATH_SERDES

  on Cassini using the external PCS SERDES otherwise unaligned access
  traps and other strange effects happen with some machines. Don't touch
  the MIF which is unused in that case either. These changes require the
  PHY type to use to be determined via the OFW device tree or from the
  VPD in machines without the former.
- Disable the SERDES pins of Saturn when not used in order to save power
  and ensure they are enabled otherwise.
- In cas_attach() use the correct register offset for CAS_PCS_CONF_EN.
- Add some bus space barriers missing in the PCS code path.

These changes make the Sun GigaSwift Ethernet 1.0 MMF cards as well as
the on-board interfaces found in Sun Fire B100s Blade Server work.

PR:	144867

2 files changed, 174 insertions, 70 deletions

diff --git a/sys/dev/cas/if_cas.c b/sys/dev/cas/if_cas.c
index edcfec47cddd..3522f7ca3b74 100644
--- a/sys/dev/cas/if_cas.c
+++ b/sys/dev/cas/if_cas.c
@@ -76,6 +76,7 @@ __FBSDID("$FreeBSD$");
 
 #include <machine/bus.h>
 #if defined(__powerpc__) || defined(__sparc64__)
+#include <dev/ofw/ofw_bus.h>
 #include <dev/ofw/openfirm.h>
 #include <machine/ofw_machdep.h>
 #endif
@@ -321,55 +322,82 @@ cas_attach(struct cas_softc *sc)
 		}
 	}
 
-	CAS_WRITE_4(sc, CAS_PCS_DATAPATH, CAS_PCS_DATAPATH_MII);
-
-	cas_mifinit(sc);
-
-	/*
-	 * Look for an external PHY.
-	 */
-	error = ENXIO;
-	v = CAS_READ_4(sc, CAS_MIF_CONF);
-	if ((v & CAS_MIF_CONF_MDI1) != 0) {
-		v |= CAS_MIF_CONF_PHY_SELECT;
-		CAS_WRITE_4(sc, CAS_MIF_CONF, v);
-		switch (sc->sc_variant) {
-		default:
-			sc->sc_phyad = -1;
-			break;
+	if ((sc->sc_flags & CAS_SERDES) == 0) {
+		CAS_WRITE_4(sc, CAS_PCS_DATAPATH, CAS_PCS_DATAPATH_MII);
+		CAS_BARRIER(sc, CAS_PCS_DATAPATH, 4,
+		    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
+		cas_mifinit(sc);
+		/*
+		 * Look for an external PHY.
+		 */
+		error = ENXIO;
+		v = CAS_READ_4(sc, CAS_MIF_CONF);
+		if ((v & CAS_MIF_CONF_MDI1) != 0) {
+			v |= CAS_MIF_CONF_PHY_SELECT;
+			CAS_WRITE_4(sc, CAS_MIF_CONF, v);
+			CAS_BARRIER(sc, CAS_MIF_CONF, 4,
+			    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
+			/* Enable/unfreeze the GMII pins of Saturn. */
+			if (sc->sc_variant == CAS_SATURN) {
+				CAS_WRITE_4(sc, CAS_SATURN_PCFG, 0);
+				CAS_BARRIER(sc, CAS_SATURN_PCFG, 4,
+				    BUS_SPACE_BARRIER_READ |
+				    BUS_SPACE_BARRIER_WRITE);
+			}
+			switch (sc->sc_variant) {
+			default:
+				sc->sc_phyad = -1;
+				break;
+			}
+			error = mii_phy_probe(sc->sc_dev, &sc->sc_miibus,
+			    cas_mediachange, cas_mediastatus);
 		}
-		error = mii_phy_probe(sc->sc_dev, &sc->sc_miibus,
-		    cas_mediachange, cas_mediastatus);
-	}
-
-	/*
-	 * Fall back on an internal PHY if no external PHY was found.
-	 */
-	if (error != 0 && (v & CAS_MIF_CONF_MDI0) != 0) {
-		v &= ~CAS_MIF_CONF_PHY_SELECT;
-		CAS_WRITE_4(sc, CAS_MIF_CONF, v);
-		switch (sc->sc_variant) {
-		default:
-			sc->sc_phyad = -1;
-			break;
+		/*
+		 * Fall back on an internal PHY if no external PHY was found.
+		 */
+		if (error != 0 && (v & CAS_MIF_CONF_MDI0) != 0) {
+			v &= ~CAS_MIF_CONF_PHY_SELECT;
+			CAS_WRITE_4(sc, CAS_MIF_CONF, v);
+			CAS_BARRIER(sc, CAS_MIF_CONF, 4,
+			    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
+			/* Freeze the GMII pins of Saturn for saving power. */
+			if (sc->sc_variant == CAS_SATURN) {
+				CAS_WRITE_4(sc, CAS_SATURN_PCFG,
+				    CAS_SATURN_PCFG_FSI);
+				CAS_BARRIER(sc, CAS_SATURN_PCFG, 4,
+				    BUS_SPACE_BARRIER_READ |
+				    BUS_SPACE_BARRIER_WRITE);
+			}
+			switch (sc->sc_variant) {
+			default:
+				sc->sc_phyad = -1;
+				break;
+			}
+			error = mii_phy_probe(sc->sc_dev, &sc->sc_miibus,
+			    cas_mediachange, cas_mediastatus);
 		}
-		error = mii_phy_probe(sc->sc_dev, &sc->sc_miibus,
-		    cas_mediachange, cas_mediastatus);
-	}
-
-	/*
-	 * Try the external PCS SERDES if we didn't find any PHYs.
-	 */
-	if (error != 0) {
+	} else {
+		/*
+		 * Use the external PCS SERDES.
+		 */
 		CAS_WRITE_4(sc, CAS_PCS_DATAPATH, CAS_PCS_DATAPATH_SERDES);
+		CAS_BARRIER(sc, CAS_PCS_DATAPATH, 4, BUS_SPACE_BARRIER_WRITE);
+		/* Enable/unfreeze the SERDES pins of Saturn. */
+		if (sc->sc_variant == CAS_SATURN) {
+			CAS_WRITE_4(sc, CAS_SATURN_PCFG, 0);
+			CAS_BARRIER(sc, CAS_SATURN_PCFG, 4,
+			    BUS_SPACE_BARRIER_WRITE);
+		}
 		CAS_WRITE_4(sc, CAS_PCS_SERDES_CTRL, CAS_PCS_SERDES_CTRL_ESD);
-		CAS_WRITE_4(sc, CAS_PCS_CONF_EN, CAS_PCS_CONF_EN);
-		sc->sc_flags |= CAS_SERDES;
+		CAS_BARRIER(sc, CAS_PCS_SERDES_CTRL, 4,
+		    BUS_SPACE_BARRIER_WRITE);
+		CAS_WRITE_4(sc, CAS_PCS_CONF, CAS_PCS_CONF_EN);
+		CAS_BARRIER(sc, CAS_PCS_CONF, 4,
+		    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 		sc->sc_phyad = CAS_PHYAD_EXTERNAL;
 		error = mii_phy_probe(sc->sc_dev, &sc->sc_miibus,
 		    cas_mediachange, cas_mediastatus);
 	}
-
 	if (error != 0) {
 		device_printf(sc->sc_dev, "PHY probe failed: %d\n", error);
 		goto fail_rxmap;
@@ -956,8 +984,9 @@ cas_init_locked(struct cas_softc *sc)
 	    __func__);
 #endif
 
-	/* Re-initialize the MIF. */
-	cas_mifinit(sc);
+	if ((sc->sc_flags & CAS_SERDES) == 0)
+		/* Re-initialize the MIF. */
+		cas_mifinit(sc);
 
 	/* step 3.  Setup data structures in host memory. */
 	cas_meminit(sc);
@@ -2105,6 +2134,8 @@ cas_mifinit(struct cas_softc *sc)
 	/* Configure the MIF in frame mode. */
 	CAS_WRITE_4(sc, CAS_MIF_CONF,
 	    CAS_READ_4(sc, CAS_MIF_CONF) & ~CAS_MIF_CONF_BB_MODE);
+	CAS_BARRIER(sc, CAS_MIF_CONF, 4,
+	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 }
 
 /*
@@ -2219,10 +2250,16 @@ cas_mii_writereg(device_t dev, int phy, int reg, int val)
 			CAS_BARRIER(sc, CAS_PCS_CONF, 4,
 			    BUS_SPACE_BARRIER_WRITE);
 			CAS_WRITE_4(sc, CAS_PCS_ANAR, val);
+			CAS_BARRIER(sc, CAS_PCS_ANAR, 4,
+			    BUS_SPACE_BARRIER_WRITE);
 			CAS_WRITE_4(sc, CAS_PCS_SERDES_CTRL,
 			    CAS_PCS_SERDES_CTRL_ESD);
+			CAS_BARRIER(sc, CAS_PCS_CONF, 4,
+			    BUS_SPACE_BARRIER_WRITE);
 			CAS_WRITE_4(sc, CAS_PCS_CONF,
 			    CAS_PCS_CONF_EN);
+			CAS_BARRIER(sc, CAS_PCS_CONF, 4,
+			    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 			return (0);
 		case MII_ANLPAR:
 			reg = CAS_PCS_ANLPAR;
@@ -2233,6 +2270,8 @@ cas_mii_writereg(device_t dev, int phy, int reg, int val)
 			return (0);
 		}
 		CAS_WRITE_4(sc, reg, val);
+		CAS_BARRIER(sc, reg, 4,
+		    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 		return (0);
 	}
 
@@ -2630,15 +2669,20 @@ static struct resource_spec cas_pci_res_spec[] = {
 	{ -1, 0 }
 };
 
+#define	CAS_LOCAL_MAC_ADDRESS	"local-mac-address"
+#define	CAS_PHY_INTERFACE	"phy-interface"
+#define	CAS_PHY_TYPE		"phy-type"
+#define	CAS_PHY_TYPE_PCS	"pcs"
+
 static int
 cas_pci_attach(device_t dev)
 {
+	char buf[sizeof(CAS_LOCAL_MAC_ADDRESS)];
 	struct cas_softc *sc;
 	int i;
 #if !(defined(__powerpc__) || defined(__sparc64__))
 	u_char enaddr[4][ETHER_ADDR_LEN];
-	char lma[sizeof("local-mac-address")];
-	int found, j;
+	u_int j, k, lma, pcs[4], phy;
 #endif
 
 	sc = device_get_softc(dev);
@@ -2679,13 +2723,20 @@ cas_pci_attach(device_t dev)
 
 #if defined(__powerpc__) || defined(__sparc64__)
 	OF_getetheraddr(dev, sc->sc_enaddr);
+	if (OF_getprop(ofw_bus_get_node(dev), CAS_PHY_INTERFACE, buf,
+	    sizeof(buf)) > 0 || OF_getprop(ofw_bus_get_node(dev),
+	    CAS_PHY_TYPE, buf, sizeof(buf)) > 0) {
+		buf[sizeof(buf) - 1] = '\0';
+		if (strcmp(buf, CAS_PHY_TYPE_PCS) == 0)
+			sc->sc_flags |= CAS_SERDES;
+	}
 #else
 	/*
-	 * Dig out VPD (vital product data) and read the MAX address.
-	 * The VPD resides in the PCI Expansion ROM (PCI FCode) and
-	 * can't be accessed via the PCI capability pointer.
-	 * SUNW,pci-ce and SUNW,pci-qge use the Enhanced VPD format
-	 * described in US Patent 7149820.
+	 * Dig out VPD (vital product data) and read the MAC address as well
+	 * as the PHY type.  The VPD resides in the PCI Expansion ROM (PCI
+	 * FCode) and can't be accessed via the PCI capability pointer.
+	 * SUNW,pci-ce and SUNW,pci-qge use the Enhanced VPD format described
+	 * in the free US Patent 7149820.
 	 */
 
 #define	PCI_ROMHDR_SIZE			0x1c
@@ -2719,7 +2770,10 @@ cas_pci_attach(device_t dev)
 #define	CAS_ROM_READ_4(sc, offs)					\
 	CAS_READ_4((sc), CAS_PCI_ROM_OFFSET + (offs))
 
-	found = 0;
+	lma = phy = 0;
+	memset(enaddr, 0, sizeof(enaddr));
+	memset(pcs, 0, sizeof(pcs));
+
 	/* Enable PCI Expansion ROM access. */
 	CAS_WRITE_4(sc, CAS_BIM_LDEV_OEN,
 	    CAS_BIM_LDEV_OEN_PAD | CAS_BIM_LDEV_OEN_PROM);
@@ -2768,23 +2822,51 @@ cas_pci_attach(device_t dev)
 			if (CAS_ROM_READ_1(sc, j + PCI_VPD_SIZE) != 'I')
 				/* no instance property */
 				continue;
-			if (CAS_ROM_READ_1(sc, j + PCI_VPD_SIZE + 3) != 'B')
-				/* no byte array */
-				continue;
-			if (CAS_ROM_READ_1(sc, j + PCI_VPD_SIZE + 4) !=
-			    ETHER_ADDR_LEN)
-				continue;
-			bus_read_region_1(sc->sc_res[CAS_RES_MEM],
-			    CAS_PCI_ROM_OFFSET + j + PCI_VPD_SIZE + 5,
-			    lma, sizeof(lma));
-			if (strcmp(lma, "local-mac-address") != 0)
-				continue;
-			bus_read_region_1(sc->sc_res[CAS_RES_MEM],
-			    CAS_PCI_ROM_OFFSET + j + PCI_VPD_SIZE + 5 +
-			    sizeof(lma), enaddr[found],
-			    sizeof(enaddr[found]));
-			if (found++ == 4)
-				break;
+			if (CAS_ROM_READ_1(sc, j + PCI_VPD_SIZE + 3) == 'B') {
+				/* byte array */
+				if (CAS_ROM_READ_1(sc,
+				    j + PCI_VPD_SIZE + 4) != ETHER_ADDR_LEN)
+					continue;
+				bus_read_region_1(sc->sc_res[CAS_RES_MEM],
+				    CAS_PCI_ROM_OFFSET + j + PCI_VPD_SIZE + 5,
+				    buf, sizeof(buf));
+				buf[sizeof(buf) - 1] = '\0';
+				if (strcmp(buf, CAS_LOCAL_MAC_ADDRESS) != 0)
+					continue;
+				bus_read_region_1(sc->sc_res[CAS_RES_MEM],
+				    CAS_PCI_ROM_OFFSET + j + PCI_VPD_SIZE +
+				    5 + sizeof(CAS_LOCAL_MAC_ADDRESS),
+				    enaddr[lma], sizeof(enaddr[lma]));
+				lma++;
+				if (lma == 4 && phy == 4)
+					break;
+			} else if (CAS_ROM_READ_1(sc, j + PCI_VPD_SIZE + 3) ==
+			   'S') {
+				/* string */
+				if (CAS_ROM_READ_1(sc,
+				    j + PCI_VPD_SIZE + 4) !=
+				    sizeof(CAS_PHY_TYPE_PCS))
+					continue;
+				bus_read_region_1(sc->sc_res[CAS_RES_MEM],
+				    CAS_PCI_ROM_OFFSET + j + PCI_VPD_SIZE + 5,
+				    buf, sizeof(buf));
+				buf[sizeof(buf) - 1] = '\0';
+				if (strcmp(buf, CAS_PHY_INTERFACE) == 0)
+					k = sizeof(CAS_PHY_INTERFACE);
+				else if (strcmp(buf, CAS_PHY_TYPE) == 0)
+					k = sizeof(CAS_PHY_TYPE);
+				else
+					continue;
+				bus_read_region_1(sc->sc_res[CAS_RES_MEM],
+				    CAS_PCI_ROM_OFFSET + j + PCI_VPD_SIZE +
+				    5 + k, buf, sizeof(buf));
+				buf[sizeof(buf) - 1] = '\0';
+				if (strcmp(buf, CAS_PHY_TYPE_PCS) == 0)
+					pcs[phy] = 1;
+				phy++;
+				if (lma == 4 && phy == 4)
+					break;
+			}
 		}
 		break;
 	default:
@@ -2795,14 +2877,24 @@ cas_pci_attach(device_t dev)
  fail_prom:
 	CAS_WRITE_4(sc, CAS_BIM_LDEV_OEN, 0);
 
-	if (found == 0) {
+	if (lma == 0) {
 		device_printf(dev, "could not determine Ethernet address\n");
 		goto fail;
 	}
 	i = 0;
-	if (found > 1 && pci_get_slot(dev) < sizeof(enaddr) / sizeof(*enaddr))
+	if (lma > 1 && pci_get_slot(dev) < sizeof(enaddr) / sizeof(*enaddr))
 		i = pci_get_slot(dev);
 	memcpy(sc->sc_enaddr, enaddr[i], ETHER_ADDR_LEN);
+
+	if (phy == 0) {
+		device_printf(dev, "could not determine PHY type\n");
+		goto fail;
+	}
+	i = 0;
+	if (phy > 1 && pci_get_slot(dev) < sizeof(pcs) / sizeof(*pcs))
+		i = pci_get_slot(dev);
+	if (pcs[i] != 0)
+		sc->sc_flags |= CAS_SERDES;
 #endif
 
 	if (cas_attach(sc) != 0) {
diff --git a/sys/dev/cas/if_casreg.h b/sys/dev/cas/if_casreg.h
index 97250e027913..0d1c1d822b9b 100644
--- a/sys/dev/cas/if_casreg.h
+++ b/sys/dev/cas/if_casreg.h
@@ -68,6 +68,7 @@
 #define	CAS_STATUS4		0x105c	/* interrupt status 4 for INTD */
 #define	CAS_CLEAR_ALIAS4	0x1060	/* clear mask alias 4 for INTD */
 #define	CAS_STATUS_ALIAS4	0x1064	/* interrupt status alias 4 for INTD */
+#define	CAS_SATURN_PCFG		0x106c	/* internal MACPHY pin configuration */
 
 #define	CAS_CAW_RX_WGHT_MASK	0x00000003	/* RX DMA factor for... */
 #define	CAS_CAW_RX_WGHT_SHFT	0		/* ...weighted round robin */
@@ -171,6 +172,17 @@
 /* INTn enable bit for CAS_INTMASK[2-4] */
 #define	CAS_INTMASKN_EN		0x00000080	/* INT[B-D] enable */
 
+#define	CAS_SATURN_PCFG_TLA	0x00000001	/* PHY activity LED */
+#define	CAS_SATURN_PCFG_FLA	0x00000002	/* PHY 10MBit/sec LED */
+#define	CAS_SATURN_PCFG_CLA	0x00000004	/* PHY 100MBit/sec LED */
+#define	CAS_SATURN_PCFG_LLA	0x00000008	/* PHY 1000MBit/sec LED */
+#define	CAS_SATURN_PCFG_RLA	0x00000010	/* PHY full-duplex LED */
+#define	CAS_SATURN_PCFG_PDS	0x00000020	/* PHY debug mode */
+#define	CAS_SATURN_PCFG_MTP	0x00000080	/* test point select */
+#define	CAS_SATURN_PCFG_GMO	0x00000100	/* GMII observe */
+#define	CAS_SATURN_PCFG_FSI	0x00000200	/* freeze GMII/SERDES */
+#define	CAS_SATURN_PCFG_LAD	0x00000800	/* MAC LED control active low */
+
 /* TX DMA registers */
 #define	CAS_TX_CONF		0x2004	/* TX configuration */
 #define	CAS_TX_FIFO_WR		0x2014	/* FIFO write pointer */