On sparc64 machines with multiple host-PCI-bridges these bridges

have separate configuration spaces so by definition they implement
different PCI domains. Thus change psycho(4) to use PCI domains
instead of reenumerating all PCI busses so they have globally unique
bus numbers and drop support for reenumerating busses in the OFW PCI
code.
According to CVS history reenumeration was also required in order to
get some E450 to boot but given that no other open source kernel
changes the PCI bus numbers assigned by the firmware I believe the
real problem was that the old code used the bus number as the device
number for the PCI busses and unlike most of the other machines the
firmwares of the problematic ones don't use disjoint PCI bus numbers
across the host-PCI-bridges.

MFC after:	1 month

8 files changed, 21 insertions, 149 deletions

diff --git a/sys/sparc64/pci/apb.c b/sys/sparc64/pci/apb.c
index 9455882d5dbe..d908c178df56 100644
--- a/sys/sparc64/pci/apb.c
+++ b/sys/sparc64/pci/apb.c
@@ -106,9 +106,6 @@ static device_method_t apb_methods[] = {
 	/* ofw_bus interface */
 	DEVMETHOD(ofw_bus_get_node,	ofw_pcib_gen_get_node),
 
-	/* ofw_pci interface */
-	DEVMETHOD(ofw_pci_adjust_busrange,	ofw_pcib_gen_adjust_busrange),
-
 	{ 0, 0 }
 };
 
@@ -179,6 +176,11 @@ apb_attach(device_t dev)
 	/*
 	 * Get current bridge configuration.
 	 */
+	sc->sc_bsc.ops_pcib_sc.domain = pci_get_domain(dev);
+	sc->sc_bsc.ops_pcib_sc.secbus =
+	    pci_read_config(dev, PCIR_SECBUS_1, 1);
+	sc->sc_bsc.ops_pcib_sc.subbus =
+	    pci_read_config(dev, PCIR_SUBBUS_1, 1);
 	sc->sc_iomap = pci_read_config(dev, APBR_IOMAP, 1);
 	sc->sc_memmap = pci_read_config(dev, APBR_MEMMAP, 1);
 	ofw_pcib_gen_setup(dev);
@@ -198,7 +200,7 @@ apb_attach(device_t dev)
 		printf("\n");
 	}
 
-	device_add_child(dev, "pci", sc->sc_bsc.ops_pcib_sc.secbus);
+	device_add_child(dev, "pci", -1);
 	return (bus_generic_attach(dev));
 }
 
@@ -207,7 +209,7 @@ apb_attach(device_t dev)
  * is set up to, or capable of handling them.
  */
 static struct resource *
-apb_alloc_resource(device_t dev, device_t child, int type, int *rid, 
+apb_alloc_resource(device_t dev, device_t child, int type, int *rid,
     u_long start, u_long end, u_long count, u_int flags)
 {
 	struct apb_softc *sc;
diff --git a/sys/sparc64/pci/ofw_pci_if.m b/sys/sparc64/pci/ofw_pci_if.m
index d93f9784e2cc..ee0e99c9a600 100644
--- a/sys/sparc64/pci/ofw_pci_if.m
+++ b/sys/sparc64/pci/ofw_pci_if.m
@@ -34,8 +34,6 @@ INTERFACE ofw_pci;
 
 CODE {
 	static ofw_pci_intr_pending_t ofw_pci_default_intr_pending;
-	static ofw_pci_alloc_busno_t ofw_pci_default_alloc_busno;
-	static ofw_pci_adjust_busrange_t ofw_pci_default_adjust_busrange;
 
 	static int
 	ofw_pci_default_intr_pending(device_t dev, ofw_pci_intr_t intr)
@@ -46,24 +44,6 @@ CODE {
 			    intr));
 		return (0);
 	}
-
-	static int
-	ofw_pci_default_alloc_busno(device_t dev)
-	{
-
-		if (device_get_parent(dev) != NULL)
-			return (OFW_PCI_ALLOC_BUSNO(device_get_parent(dev)));
-		return (-1);
-	}
-
-	static void
-	ofw_pci_default_adjust_busrange(device_t dev, u_int busno)
-	{
-
-		if (device_get_parent(dev) != NULL)
-			return (OFW_PCI_ADJUST_BUSRANGE(device_get_parent(dev),
-			    busno));
-	}
 };
 
 # Return whether an interrupt request is pending for the INO intr.
@@ -71,18 +51,3 @@ METHOD int intr_pending {
 	device_t dev;
 	ofw_pci_intr_t intr;
 } DEFAULT ofw_pci_default_intr_pending;
-
-# Allocate a bus number for reenumerating a PCI bus. A return value of -1
-# means that reenumeration is generally not supported, otherwise all PCI
-# busses must be reenumerated using bus numbers obtained via this method.
-METHOD int alloc_busno {
-	device_t dev;
-} DEFAULT ofw_pci_default_alloc_busno;
-
-# Make sure that all PCI bridges up in the hierarchy contain this bus in
-# their subordinate bus range. This is required when reenumerating the PCI
-# buses.
-METHOD void adjust_busrange {
-	device_t dev;
-	u_int subbus;
-} DEFAULT ofw_pci_default_adjust_busrange;
diff --git a/sys/sparc64/pci/ofw_pcib.c b/sys/sparc64/pci/ofw_pcib.c
index aaa2c530cc7a..966d77f6d2c7 100644
--- a/sys/sparc64/pci/ofw_pcib.c
+++ b/sys/sparc64/pci/ofw_pcib.c
@@ -87,9 +87,6 @@ static device_method_t ofw_pcib_methods[] = {
 	/* ofw_bus interface */
 	DEVMETHOD(ofw_bus_get_node,	ofw_pcib_gen_get_node),
 
-	/* ofw_pci interface */
-	DEVMETHOD(ofw_pci_adjust_busrange,	ofw_pcib_gen_adjust_busrange),
-
 	{ 0, 0 }
 };
 
diff --git a/sys/sparc64/pci/ofw_pcib_subr.c b/sys/sparc64/pci/ofw_pcib_subr.c
index 7cc867bfa303..9996b7e63665 100644
--- a/sys/sparc64/pci/ofw_pcib_subr.c
+++ b/sys/sparc64/pci/ofw_pcib_subr.c
@@ -52,33 +52,13 @@ void
 ofw_pcib_gen_setup(device_t bridge)
 {
 	struct ofw_pcib_gen_softc *sc;
-#ifndef SUN4V
-	int secbus;
 
-#endif
 	sc = device_get_softc(bridge);
 	sc->ops_pcib_sc.dev = bridge;
 	sc->ops_node = ofw_bus_get_node(bridge);
 	KASSERT(sc->ops_node != 0,
 	    ("ofw_pcib_gen_setup: no ofw pci parent bus!"));
 
-	/*
-	 * Setup the secondary bus number register, if supported, by
-	 * allocating a new unique bus number for it; the firmware
-	 * preset does not always seem to be correct in that case.
-	 */
-#ifndef SUN4V
-	secbus = OFW_PCI_ALLOC_BUSNO(bridge);
-	if (secbus != -1) {
-		pci_write_config(bridge, PCIR_PRIBUS_1, pci_get_bus(bridge), 1);
-		pci_write_config(bridge, PCIR_SECBUS_1, secbus, 1);
-		pci_write_config(bridge, PCIR_SUBBUS_1, secbus, 1);
-		sc->ops_pcib_sc.subbus = sc->ops_pcib_sc.secbus = secbus;
-		/* Notify parent bridges. */
-		OFW_PCI_ADJUST_BUSRANGE(device_get_parent(bridge), secbus);
-	}
-
-#endif
 	ofw_bus_setup_iinfo(sc->ops_node, &sc->ops_iinfo,
 	    sizeof(ofw_pci_intr_t));
 }
@@ -126,22 +106,3 @@ ofw_pcib_gen_get_node(device_t bridge, device_t dev)
 	sc = device_get_softc(bridge);
 	return (sc->ops_node);
 }
-
-void
-ofw_pcib_gen_adjust_busrange(device_t bridge, u_int subbus)
-{
-	struct ofw_pcib_gen_softc *sc;
-
-	sc = device_get_softc(bridge);
-	if (subbus > sc->ops_pcib_sc.subbus) {
-#ifdef OFW_PCI_DEBUG
-		device_printf(bridge,
-		    "adjusting subordinate bus number from %d to %d\n",
-		    sc->ops_pcib_sc.subbus, subbus);
-#endif
-		pci_write_config(bridge, PCIR_SUBBUS_1, subbus, 1);
-		sc->ops_pcib_sc.subbus = subbus;
-		/* Notify parent bridges. */
-		OFW_PCI_ADJUST_BUSRANGE(device_get_parent(bridge), subbus);
-	}
-}
diff --git a/sys/sparc64/pci/ofw_pcib_subr.h b/sys/sparc64/pci/ofw_pcib_subr.h
index 8c17e2caeae3..d4e90d589b47 100644
--- a/sys/sparc64/pci/ofw_pcib_subr.h
+++ b/sys/sparc64/pci/ofw_pcib_subr.h
@@ -42,6 +42,5 @@ struct ofw_pcib_gen_softc {
 void ofw_pcib_gen_setup(device_t);
 pcib_route_interrupt_t ofw_pcib_gen_route_interrupt;
 ofw_bus_get_node_t ofw_pcib_gen_get_node;
-ofw_pci_adjust_busrange_t ofw_pcib_gen_adjust_busrange;
 
 #endif /* !_SPARC64_PCI_OFW_PCI_SUBR_H */
diff --git a/sys/sparc64/pci/ofw_pcibus.c b/sys/sparc64/pci/ofw_pcibus.c
index f77138958dc3..b9b9fed24085 100644
--- a/sys/sparc64/pci/ofw_pcibus.c
+++ b/sys/sparc64/pci/ofw_pcibus.c
@@ -179,10 +179,6 @@ ofw_pcibus_attach(device_t dev)
 	pcib = device_get_parent(dev);
 
 	domain = pcib_get_domain(dev);
-	/*
-	 * Ask the bridge for the bus number - in some cases, we need to
-	 * renumber buses, so the firmware information cannot be trusted.
-	 */
 	busno = pcib_get_bus(dev);
 	if (bootverbose)
 		device_printf(dev, "domain=%d, physical bus=%d\n",
diff --git a/sys/sparc64/pci/psycho.c b/sys/sparc64/pci/psycho.c
index 846afdd57f6a..8ffeb4894b66 100644
--- a/sys/sparc64/pci/psycho.c
+++ b/sys/sparc64/pci/psycho.c
@@ -122,8 +122,6 @@ static pcib_write_config_t psycho_write_config;
 static pcib_route_interrupt_t psycho_route_interrupt;
 static ofw_pci_intr_pending_t psycho_intr_pending;
 static ofw_bus_get_node_t psycho_get_node;
-static ofw_pci_alloc_busno_t psycho_alloc_busno;
-static ofw_pci_adjust_busrange_t psycho_adjust_busrange;
 
 static device_method_t psycho_methods[] = {
 	/* Device interface */
@@ -155,8 +153,6 @@ static device_method_t psycho_methods[] = {
 
 	/* ofw_pci interface */
 	DEVMETHOD(ofw_pci_intr_pending,	psycho_intr_pending),
-	DEVMETHOD(ofw_pci_alloc_busno,	psycho_alloc_busno),
-	DEVMETHOD(ofw_pci_adjust_busrange,	psycho_adjust_busrange),
 
 	{ 0, 0 }
 };
@@ -170,8 +166,6 @@ DRIVER_MODULE(psycho, nexus, psycho_driver, psycho_devclass, 0, 0);
 static SLIST_HEAD(, psycho_softc) psycho_softcs =
     SLIST_HEAD_INITIALIZER(psycho_softcs);
 
-static uint8_t psycho_pci_bus_cnt;
-
 static const struct intr_controller psycho_ic = {
 	psycho_intr_enable,
 	psycho_intr_disable,
@@ -304,7 +298,7 @@ psycho_attach(device_t dev)
 	bus_addr_t intrclr, intrmap;
 	uint64_t csr, dr;
 	phandle_t child, node;
-	uint32_t dvmabase, psycho_br[2];
+	uint32_t dvmabase, prop_array[2];
 	int32_t rev;
 	u_int ver;
 	int i, n, nrange, rid;
@@ -636,18 +630,19 @@ psycho_attach(device_t dev)
 	sc->sc_pci_dmat->dt_cookie = sc->sc_is;
 	sc->sc_pci_dmat->dt_mt = &iommu_dma_methods;
 
-	/*
-	 * Get the bus range from the firmware; it is used solely for obtaining
-	 * the inital bus number, and cannot be trusted on all machines.
-	 */
-	n = OF_getprop(node, "bus-range", (void *)psycho_br, sizeof(psycho_br));
+	n = OF_getprop(node, "bus-range", (void *)prop_array,
+	    sizeof(prop_array));
 	if (n == -1)
 		panic("%s: could not get bus-range", __func__);
-	if (n != sizeof(psycho_br))
+	if (n != sizeof(prop_array))
 		panic("%s: broken bus-range (%d)", __func__, n);
+	if (bootverbose)
+		device_printf(dev, "bus range %u to %u; PCI bus %d\n",
+		    prop_array[0], prop_array[1], prop_array[0]);
+	sc->sc_pci_secbus = prop_array[0];
 
 	/* Clear PCI status error bits. */
-	PCIB_WRITE_CONFIG(dev, psycho_br[0], PCS_DEVICE, PCS_FUNC,
+	PCIB_WRITE_CONFIG(dev, sc->sc_pci_secbus, PCS_DEVICE, PCS_FUNC,
 	    PCIR_STATUS, PCIM_STATUS_PERR | PCIM_STATUS_RMABORT |
 	    PCIM_STATUS_RTABORT | PCIM_STATUS_STABORT |
 	    PCIM_STATUS_PERRREPORT, 2);
@@ -656,23 +651,9 @@ psycho_attach(device_t dev)
 	 * Set the latency timer register as this isn't always done by the
 	 * firmware.
 	 */
-	PCIB_WRITE_CONFIG(dev, psycho_br[0], PCS_DEVICE, PCS_FUNC,
+	PCIB_WRITE_CONFIG(dev, sc->sc_pci_secbus, PCS_DEVICE, PCS_FUNC,
 	    PCIR_LATTIMER, 64, 1);
 
-	sc->sc_pci_secbus = sc->sc_pci_subbus = psycho_alloc_busno(dev);
-	/*
-	 * Program the bus range registers.
-	 * NOTE: for the Psycho, the second write changes the bus number the
-	 * Psycho itself uses for it's configuration space, so these
-	 * writes must be kept in this order!
-	 * The Hummingbird/Sabre always uses bus 0, but there only can be one
-	 * Hummingbird/Sabre per machine.
-	 */
-	PCIB_WRITE_CONFIG(dev, psycho_br[0], PCS_DEVICE, PCS_FUNC, PCSR_SUBBUS,
-	    sc->sc_pci_subbus, 1);
-	PCIB_WRITE_CONFIG(dev, psycho_br[0], PCS_DEVICE, PCS_FUNC, PCSR_SECBUS,
-	    sc->sc_pci_secbus, 1);
-
 	for (n = PCIR_VENDOR; n < PCIR_STATUS; n += sizeof(uint16_t))
 		le16enc(&sc->sc_pci_hpbcfg[n], bus_space_read_2(
 		    sc->sc_pci_cfgt, sc->sc_pci_bh[OFW_PCI_CS_CONFIG],
@@ -698,7 +679,7 @@ psycho_attach(device_t dev)
 		*(ofw_pci_intr_t *)(&sc->sc_pci_iinfo.opi_imapmsk[
 		    sc->sc_pci_iinfo.opi_addrc]) = INTMAP_INO_MASK;
 
-	device_add_child(dev, "pci", sc->sc_pci_secbus);
+	device_add_child(dev, "pci", -1);
 	return (bus_generic_attach(dev));
 }
 
@@ -1078,7 +1059,7 @@ psycho_read_ivar(device_t dev, device_t child, int which, uintptr_t *result)
 	sc = device_get_softc(dev);
 	switch (which) {
 	case PCIB_IVAR_DOMAIN:
-		*result = 0;
+		*result = device_get_unit(dev);
 		return (0);
 	case PCIB_IVAR_BUS:
 		*result = sc->sc_pci_secbus;
@@ -1405,34 +1386,6 @@ psycho_get_node(device_t bus, device_t dev)
 	return (sc->sc_node);
 }
 
-static int
-psycho_alloc_busno(device_t dev)
-{
-
-	if (psycho_pci_bus_cnt == PCI_BUSMAX)
-		panic("%s: out of PCI bus numbers", __func__);
-	return (psycho_pci_bus_cnt++);
-}
-
-static void
-psycho_adjust_busrange(device_t dev, u_int subbus)
-{
-	struct psycho_softc *sc;
-
-	sc = device_get_softc(dev);
-	/* If necessary, adjust the subordinate bus number register. */
-	if (subbus > sc->sc_pci_subbus) {
-#ifdef PSYCHO_DEBUG
-		device_printf(dev,
-		    "adjusting subordinate bus number from %d to %d\n",
-		    sc->sc_pci_subbus, subbus);
-#endif
-		sc->sc_pci_subbus = subbus;
-		PCIB_WRITE_CONFIG(dev, sc->sc_pci_secbus, PCS_DEVICE, PCS_FUNC,
-		    PCSR_SUBBUS, subbus, 1);
-	}
-}
-
 static bus_space_tag_t
 psycho_alloc_bus_tag(struct psycho_softc *sc, int type)
 {
diff --git a/sys/sparc64/pci/psychovar.h b/sys/sparc64/pci/psychovar.h
index b99181d97f28..ef4286bfbcc1 100644
--- a/sys/sparc64/pci/psychovar.h
+++ b/sys/sparc64/pci/psychovar.h
@@ -71,12 +71,11 @@ struct psycho_softc {
 
 	bus_space_handle_t		sc_pci_bh[PSYCHO_NRANGE];
 
-	u_int				sc_pci_secbus;
-	u_int				sc_pci_subbus;
-
 	struct rman			sc_pci_mem_rman;
 	struct rman			sc_pci_io_rman;
 
+	uint8_t				sc_pci_secbus;
+
 	uint8_t				sc_pci_hpbcfg[16];
 
 	SLIST_ENTRY(psycho_softc)	sc_link;