Hoist 802.11 encapsulation up into net80211:

o call ieee80211_encap in ieee80211_start so frames passed down to drivers
  are already encapsulated
o remove ieee80211_encap calls in drivers
o fixup wi so it recreates the 802.3 head it requires from the 802.11
  header contents
o move fast-frame aggregation from ath to net80211 (conditional on
  IEEE80211_SUPPORT_SUPERG):
  - aggregation is now done in ieee80211_start; it is enabled when the
    packets/sec exceeds ieee80211_ffppsmin (net.wlan.ffppsmin) and frames
    are held on a staging queue according to ieee80211_ffagemax
    (net.wlan.ffagemax) to wait for a frame to combine with
  - drivers must call back to age/flush the staging queue (ath does this
    on tx done, at swba, and on rx according to the state of the tx queues
    and/or the contents of the staging queue)
  - remove fast-frame-related data structures from ath
  - add ieee80211_ff_node_init and ieee80211_ff_node_cleanup to handle
    per-node fast-frames state (we reuse 11n tx ampdu state)
o change ieee80211_encap calling convention to include an explicit vap
  so frames coming through a WDS vap are recognized w/o setting M_WDS

With these changes any device able to tx/rx 3Kbyte+ frames can use fast-frames.

Reviewed by:	thompsa, rpaulo, avatar, imp, sephe

10 files changed, 402 insertions, 24 deletions

diff --git a/sys/net80211/ieee80211_freebsd.c b/sys/net80211/ieee80211_freebsd.c
index 47a9e00cd550..8d386145a55d 100644
--- a/sys/net80211/ieee80211_freebsd.c
+++ b/sys/net80211/ieee80211_freebsd.c
@@ -189,6 +189,15 @@ SYSCTL_PROC(_net_wlan, OID_AUTO, addba_backoff, CTLFLAG_RW,
 extern int ieee80211_addba_maxtries;
 SYSCTL_INT(_net_wlan, OID_AUTO, addba_maxtries, CTLFLAG_RW,
 	&ieee80211_addba_maxtries, 0, "max ADDBA requests sent before backoff");
+#ifdef IEEE80211_SUPPORT_SUPERG
+extern int ieee80211_ffppsmin;
+SYSCTL_INT(_net_wlan, OID_AUTO, ffppsmin, CTLFLAG_RW,
+	&ieee80211_ffppsmin, 0, "min packet rate before fast-frame staging");
+extern int ieee80211_ffagemax;
+SYSCTL_PROC(_net_wlan, OID_AUTO, ffagemax, CTLFLAG_RW,
+	&ieee80211_ffagemax, 0, ieee80211_sysctl_msecs_ticks, "I",
+	"max hold time for fast-frame staging (ms)");
+#endif /* IEEE80211_SUPPORT_SUPERG */
 
 static int
 ieee80211_sysctl_inact(SYSCTL_HANDLER_ARGS)
diff --git a/sys/net80211/ieee80211_hostap.c b/sys/net80211/ieee80211_hostap.c
index c47262a7080b..5b16a8dde952 100644
--- a/sys/net80211/ieee80211_hostap.c
+++ b/sys/net80211/ieee80211_hostap.c
@@ -2038,6 +2038,10 @@ hostap_recv_mgmt(struct ieee80211_node *ni, struct mbuf *m0,
 			ieee80211_ht_updatehtcap(ni, htcap);
 		} else if (ni->ni_flags & IEEE80211_NODE_HT)
 			ieee80211_ht_node_cleanup(ni);
+#ifdef IEEE80211_SUPPORT_SUPERG
+		else if (ni->ni_ath_flags & IEEE80211_NODE_ATH)
+			ieee80211_ff_node_cleanup(ni);
+#endif
 		/*
 		 * Allow AMPDU operation only with unencrypted traffic
 		 * or AES-CCM; the 11n spec only specifies these ciphers
diff --git a/sys/net80211/ieee80211_ioctl.h b/sys/net80211/ieee80211_ioctl.h
index 367f505e88ce..6a81375bda9a 100644
--- a/sys/net80211/ieee80211_ioctl.h
+++ b/sys/net80211/ieee80211_ioctl.h
@@ -217,7 +217,8 @@ struct ieee80211_stats {
 	uint8_t		is_rx_authfail_code;	/* last rx'd auth fail reason */
 	uint32_t	is_beacon_miss;		/* beacon miss notification */
 	uint32_t	is_rx_badstate;		/* rx discard state != RUN */
-	uint32_t	is_spare[12];
+	uint32_t	is_ff_flush;		/* ff's flush'd from stageq */
+	uint32_t	is_spare[11];
 };
 
 /*
diff --git a/sys/net80211/ieee80211_node.c b/sys/net80211/ieee80211_node.c
index 5796c2ce9fbf..1632b199beed 100644
--- a/sys/net80211/ieee80211_node.c
+++ b/sys/net80211/ieee80211_node.c
@@ -929,6 +929,10 @@ node_cleanup(struct ieee80211_node *ni)
 	 */
 	if (ni->ni_flags & IEEE80211_NODE_HT)
 		ieee80211_ht_node_cleanup(ni);
+#ifdef IEEE80211_SUPPORT_SUPERG
+	else if (ni->ni_ath_flags & IEEE80211_NODE_ATH)
+		ieee80211_ff_node_cleanup(ni);
+#endif
 	/*
 	 * Clear AREF flag that marks the authorization refcnt bump
 	 * has happened.  This is probably not needed as the node
diff --git a/sys/net80211/ieee80211_output.c b/sys/net80211/ieee80211_output.c
index 3928e8695f52..4ad1f8c931ec 100644
--- a/sys/net80211/ieee80211_output.c
+++ b/sys/net80211/ieee80211_output.c
@@ -217,6 +217,7 @@ ieee80211_start(struct ifnet *ifp)
 			ieee80211_free_node(ni);
 			continue;
 		}
+
 		if ((ni->ni_flags & IEEE80211_NODE_PWR_MGT) &&
 		    (m->m_flags & M_PWR_SAV) == 0) {
 			/*
@@ -241,23 +242,26 @@ ieee80211_start(struct ifnet *ifp)
 			continue;
 		}
 
-		BPF_MTAP(ifp, m);		/* 802.11 tx path */
-
+		BPF_MTAP(ifp, m);		/* 802.3 tx */
+ 
+#ifdef IEEE80211_SUPPORT_SUPERG
+		if (IEEE80211_ATH_CAP(vap, ni, IEEE80211_NODE_FF)) {
+			m = ieee80211_ff_check(ni, m);
+			if (m == NULL) {
+				/* NB: any ni ref held on stageq */
+				continue;
+			}
+		}
+#endif /* IEEE80211_SUPPORT_SUPERG */
 		/*
-		 * XXX When ni is associated with a WDS link then
-		 * the vap will be the WDS vap but ni_vap will point
-		 * to the ap vap the station associated to.  Once
-		 * we handoff the packet to the driver the callback
-		 * to ieee80211_encap won't be able to tell if the
-		 * packet should be encapsulated for WDS or not (e.g.
-		 * multicast frames will not be handled correctly).
-		 * We hack this by marking the mbuf so ieee80211_encap
-		 * can do the right thing.
+		 * Encapsulate the packet in prep for transmission.
 		 */
-		if (vap->iv_opmode == IEEE80211_M_WDS)
-			m->m_flags |= M_WDS;
-		else
-			m->m_flags &= ~M_WDS;
+		m = ieee80211_encap(vap, ni, m);
+		if (m == NULL) {
+			/* NB: stat+msg handled in ieee80211_encap */
+			ieee80211_free_node(ni);
+			continue;
+		}
 
 		/*
 		 * Stash the node pointer and hand the frame off to
@@ -267,7 +271,6 @@ ieee80211_start(struct ifnet *ifp)
 		 */
 		m->m_pkthdr.rcvif = (void *)ni;
 
-		/* XXX defer if_start calls? */
 		error = parent->if_transmit(parent, m);
 		if (error != 0) {
 			/* NB: IFQ_HANDOFF reclaims mbuf */
@@ -852,10 +855,10 @@ ieee80211_crypto_getmcastkey(struct ieee80211vap *vap,
  *     marked EAPOL frames w/ M_EAPOL.
  */
 struct mbuf *
-ieee80211_encap(struct ieee80211_node *ni, struct mbuf *m)
+ieee80211_encap(struct ieee80211vap *vap, struct ieee80211_node *ni,
+    struct mbuf *m)
 {
 #define	WH4(wh)	((struct ieee80211_frame_addr4 *)(wh))
-	struct ieee80211vap *vap = ni->ni_vap;
 	struct ieee80211com *ic = ni->ni_ic;
 	struct ether_header eh;
 	struct ieee80211_frame *wh;
@@ -955,6 +958,9 @@ ieee80211_encap(struct ieee80211_node *ni, struct mbuf *m)
 		llc->llc_snap.ether_type = eh.ether_type;
 	} else {
 #ifdef IEEE80211_SUPPORT_SUPERG
+		/*
+		 * Aggregated frame.
+		 */
 		m = ieee80211_ff_encap(vap, m, hdrspace, key);
 		if (m == NULL)
 #endif
diff --git a/sys/net80211/ieee80211_proto.h b/sys/net80211/ieee80211_proto.h
index e82ff13d4d6e..2d32d1e058b3 100644
--- a/sys/net80211/ieee80211_proto.h
+++ b/sys/net80211/ieee80211_proto.h
@@ -75,8 +75,9 @@ void	ieee80211_start(struct ifnet *);
 int	ieee80211_send_nulldata(struct ieee80211_node *);
 int	ieee80211_classify(struct ieee80211_node *, struct mbuf *m);
 struct mbuf *ieee80211_mbuf_adjust(struct ieee80211vap *, int,
-	struct ieee80211_key *, struct mbuf *);
-struct mbuf *ieee80211_encap(struct ieee80211_node *, struct mbuf *);
+		struct ieee80211_key *, struct mbuf *);
+struct mbuf *ieee80211_encap(struct ieee80211vap *, struct ieee80211_node *,
+		struct mbuf *);
 int	ieee80211_send_mgmt(struct ieee80211_node *, int, int);
 struct ieee80211_appie;
 int	ieee80211_send_probereq(struct ieee80211_node *ni,
diff --git a/sys/net80211/ieee80211_sta.c b/sys/net80211/ieee80211_sta.c
index 3eff98ec69ba..ff3365fe2246 100644
--- a/sys/net80211/ieee80211_sta.c
+++ b/sys/net80211/ieee80211_sta.c
@@ -1459,6 +1459,11 @@ sta_recv_mgmt(struct ieee80211_node *ni, struct mbuf *m0,
 			ieee80211_setup_htrates(ni, htcap,
 			     IEEE80211_F_JOIN | IEEE80211_F_DOBRS);
 			ieee80211_setup_basic_htrates(ni, htinfo);
+		} else {
+#ifdef IEEE80211_SUPPORT_SUPERG
+			if (IEEE80211_ATH_CAP(vap, ni, IEEE80211_NODE_ATH))
+				ieee80211_ff_node_init(ni);
+#endif
 		}
 		/*
 		 * Configure state now that we are associated.
diff --git a/sys/net80211/ieee80211_superg.c b/sys/net80211/ieee80211_superg.c
index 5a6e41a92a27..4bae8852195d 100644
--- a/sys/net80211/ieee80211_superg.c
+++ b/sys/net80211/ieee80211_superg.c
@@ -76,12 +76,21 @@ __FBSDID("$FreeBSD$");
 #define	ATH_FF_SNAP_ORGCODE_1	0x03
 #define	ATH_FF_SNAP_ORGCODE_2	0x7f
 
+#define	ATH_FF_TXQMIN	2		/* min txq depth for staging */
+#define	ATH_FF_TXQMAX	50		/* maximum # of queued frames allowed */
+#define	ATH_FF_STAGEMAX	5		/* max waiting period for staged frame*/
+
 #define	ETHER_HEADER_COPY(dst, src) \
 	memcpy(dst, src, sizeof(struct ether_header))
 
+/* XXX public for sysctl hookup */
+int	ieee80211_ffppsmin = 2;		/* pps threshold for ff aggregation */
+int	ieee80211_ffagemax = -1;	/* max time frames held on stage q */
+
 void
 ieee80211_superg_attach(struct ieee80211com *ic)
 {
+	ieee80211_ffagemax = msecs_to_ticks(150);
 }
 
 void
@@ -354,10 +363,10 @@ ieee80211_ff_encap(struct ieee80211vap *vap, struct mbuf *m1, int hdrspace,
 	}
 	m1->m_nextpkt = NULL;
 	/*
-	 * Include fast frame headers in adjusting header
-	 * layout; this allocates space according to what
-	 * ff_encap will do.
+	 * Include fast frame headers in adjusting header layout.
 	 */
+	KASSERT(m1->m_len >= sizeof(eh1), ("no ethernet header!"));
+	ETHER_HEADER_COPY(&eh1, mtod(m1, caddr_t));
 	m1 = ieee80211_mbuf_adjust(vap,
 		hdrspace + sizeof(struct llc) + sizeof(uint32_t) + 2 +
 		    sizeof(struct ether_header),
@@ -461,6 +470,314 @@ bad:
 	return NULL;
 }
 
+static void
+ff_transmit(struct ieee80211_node *ni, struct mbuf *m)
+{
+	struct ieee80211vap *vap = ni->ni_vap;
+	int error;
+
+	/* encap and xmit */
+	m = ieee80211_encap(vap, ni, m);
+	if (m != NULL) {
+		struct ifnet *ifp = vap->iv_ifp;
+		struct ifnet *parent = ni->ni_ic->ic_ifp;
+
+		error = parent->if_transmit(parent, m);
+		if (error != 0) {
+			/* NB: IFQ_HANDOFF reclaims mbuf */
+			ieee80211_free_node(ni);
+		} else {
+			ifp->if_opackets++;
+		}
+	} else
+		ieee80211_free_node(ni);
+}
+
+/*
+ * Flush frames to device; note we re-use the linked list
+ * the frames were stored on and use the sentinel (unchanged)
+ * which may be non-NULL.
+ */
+static void
+ff_flush(struct mbuf *head, struct mbuf *last)
+{
+	struct mbuf *m, *next;
+	struct ieee80211_node *ni;
+	struct ieee80211vap *vap;
+
+	for (m = head; m != last; m = next) {
+		next = m->m_nextpkt;
+		m->m_nextpkt = NULL;
+
+		ni = (struct ieee80211_node *) m->m_pkthdr.rcvif;
+		vap = ni->ni_vap;
+
+		IEEE80211_NOTE(vap, IEEE80211_MSG_SUPERG, ni,
+		    "%s: flush frame, age %u", __func__, M_AGE_GET(m));
+		vap->iv_stats.is_ff_flush++;
+
+		ff_transmit(ni, m);
+	}
+}
+
+/*
+ * Age frames on the staging queue.
+ */
+void
+ieee80211_ff_age(struct ieee80211com *ic, struct ieee80211_stageq *sq, int quanta)
+{
+	struct mbuf *m, *head;
+	struct ieee80211_node *ni;
+	struct ieee80211_tx_ampdu *tap;
+
+	KASSERT(sq->head != NULL, ("stageq empty"));
+
+	IEEE80211_LOCK(ic);
+	head = sq->head;
+	while ((m = sq->head) != NULL && M_AGE_GET(m) < quanta) {
+		/* clear tap ref to frame */
+		ni = (struct ieee80211_node *) m->m_pkthdr.rcvif;
+		tap = &ni->ni_tx_ampdu[M_WME_GETAC(m)];
+		KASSERT(tap->txa_private == m, ("staging queue empty"));
+		tap->txa_private = NULL;
+
+		sq->head = m->m_nextpkt;
+		sq->depth--;
+		ic->ic_stageqdepth--;
+	}
+	if (m == NULL)
+		sq->tail = NULL;
+	else
+		M_AGE_SUB(m, quanta);
+	IEEE80211_UNLOCK(ic);
+
+	ff_flush(head, m);
+}
+
+static void
+stageq_add(struct ieee80211_stageq *sq, struct mbuf *m)
+{
+	int age = ieee80211_ffagemax;
+	if (sq->tail != NULL) {
+		sq->tail->m_nextpkt = m;
+		age -= M_AGE_GET(sq->head);
+	} else
+		sq->head = m;
+	KASSERT(age >= 0, ("age %d", age));
+	M_AGE_SET(m, age);
+	m->m_nextpkt = NULL;
+	sq->tail = m;
+	sq->depth++;
+}
+
+static void
+stageq_remove(struct ieee80211_stageq *sq, struct mbuf *mstaged)
+{
+	struct mbuf *m, *mprev;
+
+	mprev = NULL;
+	for (m = sq->head; m != NULL; m = m->m_nextpkt) {
+		if (m == mstaged) {
+			if (mprev == NULL)
+				sq->head = m->m_nextpkt;
+			else
+				mprev->m_nextpkt = m->m_nextpkt;
+			if (sq->tail == m)
+				sq->tail = mprev;
+			sq->depth--;
+			return;
+		}
+		mprev = m;
+	}
+	printf("%s: packet not found\n", __func__);
+}
+
+static uint32_t
+ff_approx_txtime(struct ieee80211_node *ni,
+	const struct mbuf *m1, const struct mbuf *m2)
+{
+	struct ieee80211com *ic = ni->ni_ic;
+	struct ieee80211vap *vap = ni->ni_vap;
+	uint32_t framelen;
+
+	/*
+	 * Approximate the frame length to be transmitted. A swag to add
+	 * the following maximal values to the skb payload:
+	 *   - 32: 802.11 encap + CRC
+	 *   - 24: encryption overhead (if wep bit)
+	 *   - 4 + 6: fast-frame header and padding
+	 *   - 16: 2 LLC FF tunnel headers
+	 *   - 14: 1 802.3 FF tunnel header (mbuf already accounts for 2nd)
+	 */
+	framelen = m1->m_pkthdr.len + 32 +
+	    ATH_FF_MAX_HDR_PAD + ATH_FF_MAX_SEP_PAD + ATH_FF_MAX_HDR;
+	if (vap->iv_flags & IEEE80211_F_PRIVACY)
+		framelen += 24;
+	if (m2 != NULL)
+		framelen += m2->m_pkthdr.len;
+	return ieee80211_compute_duration(ic->ic_rt, framelen, ni->ni_txrate, 0);
+}
+
+/*
+ * Check if the supplied frame can be partnered with an existing
+ * or pending frame.  Return a reference to any frame that should be
+ * sent on return; otherwise return NULL.
+ */
+struct mbuf *
+ieee80211_ff_check(struct ieee80211_node *ni, struct mbuf *m)
+{
+	struct ieee80211vap *vap = ni->ni_vap;
+	struct ieee80211com *ic = ni->ni_ic;
+	const int pri = M_WME_GETAC(m);
+	struct ieee80211_stageq *sq;
+	struct ieee80211_tx_ampdu *tap;
+	struct mbuf *mstaged;
+	uint32_t txtime, limit;
+
+	/*
+	 * Check if the supplied frame can be aggregated.
+	 *
+	 * NB: we allow EAPOL frames to be aggregated with other ucast traffic.
+	 *     Do 802.1x EAPOL frames proceed in the clear? Then they couldn't
+	 *     be aggregated with other types of frames when encryption is on?
+	 */
+	IEEE80211_LOCK(ic);
+	tap = &ni->ni_tx_ampdu[pri];
+	mstaged = tap->txa_private;		/* NB: we reuse AMPDU state */
+	ieee80211_txampdu_count_packet(tap);
+
+	/*
+	 * When not in station mode never aggregate a multicast
+	 * frame; this insures, for example, that a combined frame
+	 * does not require multiple encryption keys.
+	 */
+	if (vap->iv_opmode != IEEE80211_M_STA &&
+	    ETHER_IS_MULTICAST(mtod(m, struct ether_header *)->ether_dhost)) {
+		/* XXX flush staged frame? */
+		IEEE80211_UNLOCK(ic);
+		return m;
+	}
+	/*
+	 * If there is no frame to combine with and the pps is
+	 * too low; then do not attempt to aggregate this frame.
+	 */
+	if (mstaged == NULL &&
+	    ieee80211_txampdu_getpps(tap) < ieee80211_ffppsmin) {
+		IEEE80211_UNLOCK(ic);
+		return m;
+	}
+	sq = &ic->ic_ff_stageq[pri];
+	/*
+	 * Check the txop limit to insure the aggregate fits.
+	 */
+	limit = IEEE80211_TXOP_TO_US(
+		ic->ic_wme.wme_chanParams.cap_wmeParams[pri].wmep_txopLimit);
+	if (limit != 0 &&
+	    (txtime = ff_approx_txtime(ni, m, mstaged)) > limit) {
+		/*
+		 * Aggregate too long, return to the caller for direct
+		 * transmission.  In addition, flush any pending frame
+		 * before sending this one.
+		 */
+		IEEE80211_DPRINTF(vap, IEEE80211_MSG_SUPERG,
+		    "%s: txtime %u exceeds txop limit %u\n",
+		    __func__, txtime, limit);
+
+		tap->txa_private = NULL;
+		if (mstaged != NULL)
+			stageq_remove(sq, mstaged);
+		IEEE80211_UNLOCK(ic);
+
+		if (mstaged != NULL) {
+			IEEE80211_NOTE(vap, IEEE80211_MSG_SUPERG, ni,
+			    "%s: flush staged frame", __func__);
+			/* encap and xmit */
+			ff_transmit(ni, mstaged);
+		}
+		return m;		/* NB: original frame */
+	}
+	/*
+	 * An aggregation candidate.  If there's a frame to partner
+	 * with then combine and return for processing.  Otherwise
+	 * save this frame and wait for a partner to show up (or
+	 * the frame to be flushed).  Note that staged frames also
+	 * hold their node reference.
+	 */
+	if (mstaged != NULL) {
+		tap->txa_private = NULL;
+		stageq_remove(sq, mstaged);
+		IEEE80211_UNLOCK(ic);
+
+		IEEE80211_NOTE(vap, IEEE80211_MSG_SUPERG, ni,
+		    "%s: aggregate fast-frame", __func__);
+		/*
+		 * Release the node reference; we only need
+		 * the one already in mstaged.
+		 */
+		KASSERT(mstaged->m_pkthdr.rcvif == (void *)ni,
+		    ("rcvif %p ni %p", mstaged->m_pkthdr.rcvif, ni));
+		ieee80211_free_node(ni);
+
+		m->m_nextpkt = NULL;
+		mstaged->m_nextpkt = m;
+		mstaged->m_flags |= M_FF; /* NB: mark for encap work */
+	} else {
+		m->m_pkthdr.rcvif = (void *)ni;	/* NB: hold node reference */
+
+		KASSERT(tap->txa_private == NULL,
+		    ("txa_private %p", tap->txa_private));
+		tap->txa_private = m;
+
+		stageq_add(sq, m);
+		ic->ic_stageqdepth++;
+		IEEE80211_UNLOCK(ic);
+
+		IEEE80211_NOTE(vap, IEEE80211_MSG_SUPERG, ni,
+		    "%s: stage frame, %u queued", __func__, sq->depth);
+		/* NB: mstaged is NULL */
+	}
+	return mstaged;
+}
+
+void
+ieee80211_ff_node_init(struct ieee80211_node *ni)
+{
+	/*
+	 * Clean FF state on re-associate.  This handles the case
+	 * where a station leaves w/o notifying us and then returns
+	 * before node is reaped for inactivity.
+	 */
+	ieee80211_ff_node_cleanup(ni);
+}
+
+void
+ieee80211_ff_node_cleanup(struct ieee80211_node *ni)
+{
+	struct ieee80211com *ic = ni->ni_ic;
+	struct ieee80211_tx_ampdu *tap;
+	struct mbuf *m, *head;
+	int ac;
+
+	IEEE80211_LOCK(ic);
+	head = NULL;
+	for (ac = 0; ac < WME_NUM_AC; ac++) {
+		tap = &ni->ni_tx_ampdu[ac];
+		m = tap->txa_private;
+		if (m != NULL) {
+			tap->txa_private = NULL;
+			stageq_remove(&ic->ic_ff_stageq[ac], m);
+			m->m_nextpkt = head;
+			head = m;
+		}
+	}
+	IEEE80211_UNLOCK(ic);
+
+	for (m = head; m != NULL; m = m->m_nextpkt) {
+		m_freem(m);
+		ieee80211_free_node(ni);
+	}
+}
+
 /*
  * Switch between turbo and non-turbo operating modes.
  * Use the specified channel flags to locate the new
diff --git a/sys/net80211/ieee80211_superg.h b/sys/net80211/ieee80211_superg.h
index 9364453e5d84..d627ab0eafad 100644
--- a/sys/net80211/ieee80211_superg.h
+++ b/sys/net80211/ieee80211_superg.h
@@ -68,6 +68,26 @@ void	ieee80211_parse_ath(struct ieee80211_node *, uint8_t *);
 int	ieee80211_parse_athparams(struct ieee80211_node *, uint8_t *,
 	    const struct ieee80211_frame *);
 
+void	ieee80211_ff_node_init(struct ieee80211_node *);
+void	ieee80211_ff_node_cleanup(struct ieee80211_node *);
+
+struct mbuf *ieee80211_ff_check(struct ieee80211_node *, struct mbuf *);
+void	ieee80211_ff_age(struct ieee80211com *, struct ieee80211_stageq *, int);
+
+static __inline void
+ieee80211_flush_stageq(struct ieee80211com *ic, int ac)
+{
+	if (ic->ic_ff_stageq[ac].depth)
+		ieee80211_ff_age(ic, &ic->ic_ff_stageq[ac], 0x7fffffff);
+}
+
+static __inline void
+ieee80211_age_stageq(struct ieee80211com *ic, int ac, int quanta)
+{
+	if (ic->ic_ff_stageq[ac].depth)
+		ieee80211_ff_age(ic, &ic->ic_ff_stageq[ac], quanta);
+}
+
 struct mbuf *ieee80211_ff_encap(struct ieee80211vap *, struct mbuf *,
 	    int, struct ieee80211_key *);
 
diff --git a/sys/net80211/ieee80211_var.h b/sys/net80211/ieee80211_var.h
index 33eb92aff5d3..db0485cbf551 100644
--- a/sys/net80211/ieee80211_var.h
+++ b/sys/net80211/ieee80211_var.h
@@ -107,6 +107,13 @@ struct ieee80211_appie {
 };
 
 struct ieee80211_tdma_param;
+struct ieee80211_rate_table;
+
+struct ieee80211_stageq {
+	struct mbuf		*head;		/* frames linked w/ m_nextpkt */
+	struct mbuf		*tail;		/* last frame in queue */
+	int			depth;		/* # items on head */
+};
 
 struct ieee80211com {
 	struct ifnet		*ic_ifp;	/* associated device */
@@ -197,6 +204,10 @@ struct ieee80211com {
 	int			ic_lastnonerp;	/* last time non-ERP sta noted*/
 	int			ic_lastnonht;	/* last time non-HT sta noted */
 
+	/* fast-frames staging q */
+	struct ieee80211_stageq	ic_ff_stageq[WME_NUM_AC];
+	int			ic_stageqdepth;	/* cumulative depth */
+
 	/* virtual ap create/delete */
 	struct ieee80211vap*	(*ic_vap_create)(struct ieee80211com *,
 				    const char name[IFNAMSIZ], int unit,