cxgbe(4): Stop and restart the L2T allocator with the LLD.

L2T entries are used by both filters and TOE and the L2T is shared
between the base driver (LLD) and the TOM ULD.  Add a flag to indicate
that the L2T is stopped, which means:
* t4_alloc_l2e and t4_l2t_alloc_switching will not allocate new entries.
* t4_tom will ignore all ARP/NDP updates from the kernel.
* Previously allocated L2T entries can still be freed.

MFC after:      1 month
Sponsored by:   Chelsio Communications

4 files changed, 47 insertions, 5 deletions

diff --git a/sys/dev/cxgbe/t4_l2t.c b/sys/dev/cxgbe/t4_l2t.c
index 90f184f5e666..b210003cfac0 100644
--- a/sys/dev/cxgbe/t4_l2t.c
+++ b/sys/dev/cxgbe/t4_l2t.c
@@ -73,7 +73,8 @@ t4_alloc_l2e(struct l2t_data *d)
 	struct l2t_entry *end, *e, **p;
 
 	rw_assert(&d->lock, RA_WLOCKED);
-
+	if (__predict_false(d->l2t_stopped))
+		return (NULL);
 	if (!atomic_load_acq_int(&d->nfree))
 		return (NULL);
 
@@ -291,7 +292,10 @@ t4_l2t_alloc_switching(struct adapter *sc, uint16_t vlan, uint8_t port,
 	int rc;
 
 	rw_wlock(&d->lock);
-	e = find_or_alloc_l2e(d, vlan, port, eth_addr);
+	if (__predict_false(d->l2t_stopped))
+		e = NULL;
+	else
+		e = find_or_alloc_l2e(d, vlan, port, eth_addr);
 	if (e) {
 		if (atomic_load_acq_int(&e->refcnt) == 0) {
 			mtx_lock(&e->lock);    /* avoid race with t4_l2t_free */
@@ -333,6 +337,7 @@ t4_init_l2t(struct adapter *sc, int flags)
 		return (ENOMEM);
 
 	d->l2t_size = l2t_size;
+	d->l2t_stopped = false;
 	d->rover = d->l2tab;
 	atomic_store_rel_int(&d->nfree, l2t_size);
 	rw_init(&d->lock, "L2T");
@@ -353,8 +358,9 @@ t4_init_l2t(struct adapter *sc, int flags)
 }
 
 int
-t4_free_l2t(struct l2t_data *d)
+t4_free_l2t(struct adapter *sc)
 {
+	struct l2t_data *d = sc->l2t;
 	int i;
 
 	for (i = 0; i < d->l2t_size; i++)
@@ -366,6 +372,30 @@ t4_free_l2t(struct l2t_data *d)
 }
 
 int
+t4_stop_l2t(struct adapter *sc)
+{
+	struct l2t_data *d = sc->l2t;
+
+	rw_wlock(&d->lock);
+	d->l2t_stopped = true;
+	rw_wunlock(&d->lock);
+
+	return (0);
+}
+
+int
+t4_restart_l2t(struct adapter *sc)
+{
+	struct l2t_data *d = sc->l2t;
+
+	rw_wlock(&d->lock);
+	d->l2t_stopped = false;
+	rw_wunlock(&d->lock);
+
+	return (0);
+}
+
+int
 do_l2t_write_rpl(struct sge_iq *iq, const struct rss_header *rss,
     struct mbuf *m)
 {
diff --git a/sys/dev/cxgbe/t4_l2t.h b/sys/dev/cxgbe/t4_l2t.h
index 991b4476eb61..f8681a4ab556 100644
--- a/sys/dev/cxgbe/t4_l2t.h
+++ b/sys/dev/cxgbe/t4_l2t.h
@@ -79,6 +79,7 @@ struct l2t_entry {
 struct l2t_data {
 	struct rwlock lock;
 	u_int l2t_size;
+	bool l2t_stopped;
 	volatile int nfree;	/* number of free entries */
 	struct l2t_entry *rover;/* starting point for next allocation */
 	struct l2t_entry l2tab[];
@@ -86,7 +87,9 @@ struct l2t_data {
 
 
 int t4_init_l2t(struct adapter *, int);
-int t4_free_l2t(struct l2t_data *);
+int t4_free_l2t(struct adapter *);
+int t4_stop_l2t(struct adapter *);
+int t4_restart_l2t(struct adapter *);
 struct l2t_entry *t4_alloc_l2e(struct l2t_data *);
 struct l2t_entry *t4_l2t_alloc_switching(struct adapter *, uint16_t, uint8_t,
     uint8_t *);
diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c
index f1f4b2d26fdf..3baa2d5ac0f7 100644
--- a/sys/dev/cxgbe/t4_main.c
+++ b/sys/dev/cxgbe/t4_main.c
@@ -1847,7 +1847,7 @@ t4_detach_common(device_t dev)
 		    sc->msix_res);
 
 	if (sc->l2t)
-		t4_free_l2t(sc->l2t);
+		t4_free_l2t(sc);
 	if (sc->smt)
 		t4_free_smt(sc->smt);
 	t4_free_atid_table(sc);
@@ -2101,6 +2101,7 @@ stop_lld(struct adapter *sc)
 	end_synchronized_op(sc, 0);
 
 	stop_atid_allocator(sc);
+	t4_stop_l2t(sc);
 
 	return (rc);
 }
@@ -2454,6 +2455,7 @@ done:
 	free(old_state, M_CXGBE);
 
 	restart_atid_allocator(sc);
+	t4_restart_l2t(sc);
 
 	return (rc);
 }
diff --git a/sys/dev/cxgbe/tom/t4_tom_l2t.c b/sys/dev/cxgbe/tom/t4_tom_l2t.c
index ee442f023970..749e5704863e 100644
--- a/sys/dev/cxgbe/tom/t4_tom_l2t.c
+++ b/sys/dev/cxgbe/tom/t4_tom_l2t.c
@@ -380,6 +380,10 @@ t4_l2t_get(struct port_info *pi, if_t ifp, struct sockaddr *sa)
 
 	hash = l2_hash(d, sa, if_getindex(ifp));
 	rw_wlock(&d->lock);
+	if (__predict_false(d->l2t_stopped)) {
+		e = NULL;
+		goto done;
+	}
 	for (e = d->l2tab[hash].first; e; e = e->next) {
 		if (l2_cmp(sa, e) == 0 && e->ifp == ifp && e->vlan == vtag &&
 		    e->smt_idx == smt_idx) {
@@ -429,6 +433,8 @@ t4_l2_update(struct toedev *tod, if_t ifp, struct sockaddr *sa,
 
 	hash = l2_hash(d, sa, if_getindex(ifp));
 	rw_rlock(&d->lock);
+	if (__predict_false(d->l2t_stopped))
+		goto done;
 	for (e = d->l2tab[hash].first; e; e = e->next) {
 		if (l2_cmp(sa, e) == 0 && e->ifp == ifp) {
 			mtx_lock(&e->lock);
@@ -439,6 +445,7 @@ t4_l2_update(struct toedev *tod, if_t ifp, struct sockaddr *sa,
 			break;
 		}
 	}
+done:
 	rw_runlock(&d->lock);
 
 	/*