diff options
Diffstat (limited to 'sys')
49 files changed, 411 insertions, 458 deletions
diff --git a/sys/contrib/ipfilter/netinet/ip_fil_freebsd.c b/sys/contrib/ipfilter/netinet/ip_fil_freebsd.c index ef03cf678ed7..5f16d8b84c93 100644 --- a/sys/contrib/ipfilter/netinet/ip_fil_freebsd.c +++ b/sys/contrib/ipfilter/netinet/ip_fil_freebsd.c @@ -1312,8 +1312,10 @@ ipf_inject(fin, m) fr_info_t *fin; mb_t *m; { + struct epoch_tracker et; int error = 0; + NET_EPOCH_ENTER(et); if (fin->fin_out == 0) { netisr_dispatch(NETISR_IP, m); } else { @@ -1321,6 +1323,7 @@ ipf_inject(fin, m) fin->fin_ip->ip_off = ntohs(fin->fin_ip->ip_off); error = ip_output(m, NULL, NULL, IP_FORWARDING, NULL, NULL); } + NET_EPOCH_EXIT(et); return error; } diff --git a/sys/dev/firewire/if_fwip.c b/sys/dev/firewire/if_fwip.c index 35a58c013707..6d2b3fb59221 100644 --- a/sys/dev/firewire/if_fwip.c +++ b/sys/dev/firewire/if_fwip.c @@ -708,6 +708,7 @@ fwip_start_send (void *arg, int count) static void fwip_stream_input(struct fw_xferq *xferq) { + struct epoch_tracker et; struct mbuf *m, *m0; struct m_tag *mtag; struct ifnet *ifp; @@ -720,6 +721,7 @@ fwip_stream_input(struct fw_xferq *xferq) fwip = (struct fwip_softc *)xferq->sc; ifp = fwip->fw_softc.fwip_ifp; + NET_EPOCH_ENTER(et); while ((sxfer = STAILQ_FIRST(&xferq->stvalid)) != NULL) { STAILQ_REMOVE_HEAD(&xferq->stvalid, link); fp = mtod(sxfer->mbuf, struct fw_pkt *); @@ -808,6 +810,7 @@ fwip_stream_input(struct fw_xferq *xferq) firewire_input(ifp, m, src); if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); } + NET_EPOCH_EXIT(et); if (STAILQ_FIRST(&xferq->stfree) != NULL) fwip->fd.fc->irx_enable(fwip->fd.fc, fwip->dma_ch); } diff --git a/sys/dev/iicbus/if_ic.c b/sys/dev/iicbus/if_ic.c index ca50fbb11c33..4dac86141230 100644 --- a/sys/dev/iicbus/if_ic.c +++ b/sys/dev/iicbus/if_ic.c @@ -309,9 +309,13 @@ icintr(device_t dev, int event, char *ptr) BPF_TAP(sc->ic_ifp, sc->ic_ifbuf, len + ICHDRLEN); top = m_devget(sc->ic_ifbuf + ICHDRLEN, len, 0, sc->ic_ifp, 0); if (top) { + struct epoch_tracker et; + mtx_unlock(&sc->ic_lock); M_SETFIB(top, sc->ic_ifp->if_fib); + NET_EPOCH_ENTER(et); netisr_dispatch(NETISR_IP, top); + NET_EPOCH_EXIT(et); mtx_lock(&sc->ic_lock); } break; diff --git a/sys/dev/usb/net/if_usie.c b/sys/dev/usb/net/if_usie.c index a6e9c62f2537..eb92f4aabe4b 100644 --- a/sys/dev/usb/net/if_usie.c +++ b/sys/dev/usb/net/if_usie.c @@ -773,6 +773,7 @@ tr_setup: static void usie_if_rx_callback(struct usb_xfer *xfer, usb_error_t error) { + struct epoch_tracker et; struct usie_softc *sc = usbd_xfer_softc(xfer); struct ifnet *ifp = sc->sc_ifp; struct mbuf *m0; @@ -852,6 +853,7 @@ tr_setup: err = pkt = 0; /* HW can aggregate multiple frames in a single USB xfer */ + NET_EPOCH_ENTER(et); for (;;) { rxd = mtod(m, struct usie_desc *); @@ -918,6 +920,7 @@ tr_setup: m->m_data += diff; m->m_pkthdr.len = (m->m_len -= diff); } + NET_EPOCH_EXIT(et); mtx_lock(&sc->sc_mtx); diff --git a/sys/dev/usb/net/uhso.c b/sys/dev/usb/net/uhso.c index 8e826fd9e6a7..127379fe22b0 100644 --- a/sys/dev/usb/net/uhso.c +++ b/sys/dev/usb/net/uhso.c @@ -1664,6 +1664,7 @@ tr_setup: static void uhso_if_rxflush(void *arg) { + struct epoch_tracker et; struct uhso_softc *sc = arg; struct ifnet *ifp = sc->sc_ifp; uint8_t *cp; @@ -1677,6 +1678,7 @@ uhso_if_rxflush(void *arg) m = NULL; mwait = sc->sc_mwait; + NET_EPOCH_ENTER(et); for (;;) { if (m == NULL) { if ((m = mbufq_dequeue(&sc->sc_rxq)) == NULL) @@ -1787,6 +1789,7 @@ uhso_if_rxflush(void *arg) m = m0 != NULL ? m0 : NULL; mtx_lock(&sc->sc_mtx); } + NET_EPOCH_EXIT(et); sc->sc_mwait = mwait; } diff --git a/sys/dev/usb/net/usb_ethernet.c b/sys/dev/usb/net/usb_ethernet.c index b637f1ace0e1..b1d1ff202fc4 100644 --- a/sys/dev/usb/net/usb_ethernet.c +++ b/sys/dev/usb/net/usb_ethernet.c @@ -645,22 +645,21 @@ void uether_rxflush(struct usb_ether *ue) { struct ifnet *ifp = ue->ue_ifp; - struct mbuf *m; + struct epoch_tracker et; + struct mbuf *m, *n; UE_LOCK_ASSERT(ue, MA_OWNED); - for (;;) { - m = mbufq_dequeue(&ue->ue_rxq); - if (m == NULL) - break; - - /* - * The USB xfer has been resubmitted so its safe to unlock now. - */ - UE_UNLOCK(ue); + n = mbufq_flush(&ue->ue_rxq); + UE_UNLOCK(ue); + NET_EPOCH_ENTER(et); + while ((m = n) != NULL) { + n = STAILQ_NEXT(m, m_stailqpkt); + m->m_nextpkt = NULL; ifp->if_input(ifp, m); - UE_LOCK(ue); } + NET_EPOCH_EXIT(et); + UE_LOCK(ue); } /* diff --git a/sys/kern/uipc_socket.c b/sys/kern/uipc_socket.c index 1f480462db2a..dbb6ab4d261a 100644 --- a/sys/kern/uipc_socket.c +++ b/sys/kern/uipc_socket.c @@ -146,6 +146,8 @@ __FBSDID("$FreeBSD$"); #include <netinet/tcp.h> #include <net/vnet.h> +#include <net/if.h> /* XXXGL: net_epoch should move out there */ +#include <net/if_var.h> /* XXXGL: net_epoch should move out there */ #include <security/mac/mac_framework.h> diff --git a/sys/net/if.c b/sys/net/if.c index 406c57c8efb4..089a481c3b1d 100644 --- a/sys/net/if.c +++ b/sys/net/if.c @@ -351,17 +351,14 @@ ifnet_byindex(u_short idx) struct ifnet * ifnet_byindex_ref(u_short idx) { - struct epoch_tracker et; struct ifnet *ifp; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + ifp = ifnet_byindex_locked(idx); - if (ifp == NULL || (ifp->if_flags & IFF_DYING)) { - NET_EPOCH_EXIT(et); + if (ifp == NULL || (ifp->if_flags & IFF_DYING)) return (NULL); - } if_ref(ifp); - NET_EPOCH_EXIT(et); return (ifp); } @@ -425,15 +422,14 @@ ifnet_setbyindex(u_short idx, struct ifnet *ifp) struct ifaddr * ifaddr_byindex(u_short idx) { - struct epoch_tracker et; struct ifnet *ifp; struct ifaddr *ifa = NULL; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + ifp = ifnet_byindex_locked(idx); if (ifp != NULL && (ifa = ifp->if_addr) != NULL) ifa_ref(ifa); - NET_EPOCH_EXIT(et); return (ifa); } @@ -1640,39 +1636,32 @@ ifgr_groups_get(void *ifgrp) static int if_getgroup(struct ifgroupreq *ifgr, struct ifnet *ifp) { - struct epoch_tracker et; int len, error; struct ifg_list *ifgl; struct ifg_req ifgrq, *ifgp; + NET_EPOCH_ASSERT(); + if (ifgr->ifgr_len == 0) { - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) ifgr->ifgr_len += sizeof(struct ifg_req); - NET_EPOCH_EXIT(et); return (0); } len = ifgr->ifgr_len; ifgp = ifgr_groups_get(ifgr); /* XXX: wire */ - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) { - if (len < sizeof(ifgrq)) { - NET_EPOCH_EXIT(et); + if (len < sizeof(ifgrq)) return (EINVAL); - } bzero(&ifgrq, sizeof ifgrq); strlcpy(ifgrq.ifgrq_group, ifgl->ifgl_group->ifg_group, sizeof(ifgrq.ifgrq_group)); - if ((error = copyout(&ifgrq, ifgp, sizeof(struct ifg_req)))) { - NET_EPOCH_EXIT(et); + if ((error = copyout(&ifgrq, ifgp, sizeof(struct ifg_req)))) return (error); - } len -= sizeof(ifgrq); ifgp++; } - NET_EPOCH_EXIT(et); return (0); } @@ -1972,7 +1961,8 @@ ifa_ifwithaddr(const struct sockaddr *addr) struct ifnet *ifp; struct ifaddr *ifa; - MPASS(in_epoch(net_epoch_preempt)); + NET_EPOCH_ASSERT(); + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != addr->sa_family) @@ -2324,17 +2314,22 @@ if_link_state_change(struct ifnet *ifp, int link_state) ifp->if_link_state = link_state; + /* XXXGL: reference ifp? */ taskqueue_enqueue(taskqueue_swi, &ifp->if_linktask); } static void do_link_state_change(void *arg, int pending) { - struct ifnet *ifp = (struct ifnet *)arg; - int link_state = ifp->if_link_state; - CURVNET_SET(ifp->if_vnet); + struct epoch_tracker et; + struct ifnet *ifp; + int link_state; + + NET_EPOCH_ENTER(et); + ifp = arg; + link_state = ifp->if_link_state; - /* Notify that the link state has changed. */ + CURVNET_SET(ifp->if_vnet); rt_ifmsg(ifp); if (ifp->if_vlantrunk != NULL) (*vlan_link_state_p)(ifp); @@ -2360,6 +2355,7 @@ do_link_state_change(void *arg, int pending) (link_state == LINK_STATE_UP) ? "UP" : "DOWN" ); EVENTHANDLER_INVOKE(ifnet_link_event, ifp, link_state); CURVNET_RESTORE(); + NET_EPOCH_EXIT(et); } /* @@ -2912,9 +2908,14 @@ ifhwioctl(u_long cmd, struct ifnet *ifp, caddr_t data, struct thread *td) break; case CASE_IOC_IFGROUPREQ(SIOCGIFGROUP): - if ((error = if_getgroup((struct ifgroupreq *)data, ifp))) - return (error); + { + struct epoch_tracker et; + + NET_EPOCH_ENTER(et); + error = if_getgroup((struct ifgroupreq *)data, ifp); + NET_EPOCH_EXIT(et); break; + } case CASE_IOC_IFGROUPREQ(SIOCDIFGROUP): error = priv_check(td, PRIV_NET_DELIFGROUP); @@ -2983,7 +2984,7 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct thread *td) #ifdef COMPAT_FREEBSD32 caddr_t saved_data = NULL; struct ifmediareq ifmr; - struct ifmediareq *ifmrp; + struct ifmediareq *ifmrp = NULL; #endif struct ifnet *ifp; struct ifreq *ifr; @@ -2999,12 +3000,10 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct thread *td) } #endif - switch (cmd) { case SIOCGIFCONF: error = ifconf(cmd, data); - CURVNET_RESTORE(); - return (error); + goto out_noref; #ifdef COMPAT_FREEBSD32 case SIOCGIFCONF32: @@ -3017,16 +3016,14 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct thread *td) ifc.ifc_buf = PTRIN(ifc32->ifc_buf); error = ifconf(SIOCGIFCONF, (void *)&ifc); - CURVNET_RESTORE(); if (error == 0) ifc32->ifc_len = ifc.ifc_len; - return (error); + goto out_noref; } #endif } #ifdef COMPAT_FREEBSD32 - ifmrp = NULL; switch (cmd) { case SIOCGIFMEDIA32: case SIOCGIFXMEDIA32: @@ -3618,16 +3615,15 @@ if_delmulti(struct ifnet *ifp, struct sockaddr *sa) struct ifmultiaddr *ifma; int lastref; #ifdef INVARIANTS - struct epoch_tracker et; struct ifnet *oifp; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + CK_STAILQ_FOREACH(oifp, &V_ifnet, if_link) if (ifp == oifp) break; if (ifp != oifp) ifp = NULL; - NET_EPOCH_EXIT(et); KASSERT(ifp != NULL, ("%s: ifnet went away", __func__)); #endif @@ -3693,16 +3689,15 @@ if_delmulti_ifma_flags(struct ifmultiaddr *ifma, int flags) if (ifp == NULL) { printf("%s: ifma_ifp seems to be detached\n", __func__); } else { - struct epoch_tracker et; struct ifnet *oifp; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + CK_STAILQ_FOREACH(oifp, &V_ifnet, if_link) if (ifp == oifp) break; if (ifp != oifp) ifp = NULL; - NET_EPOCH_EXIT(et); } #endif /* @@ -3826,11 +3821,11 @@ if_setlladdr(struct ifnet *ifp, const u_char *lladdr, int len) struct sockaddr_dl *sdl; struct ifaddr *ifa; struct ifreq ifr; - struct epoch_tracker et; int rc; + NET_EPOCH_ASSERT(); + rc = 0; - NET_EPOCH_ENTER(et); ifa = ifp->if_addr; if (ifa == NULL) { rc = EINVAL; @@ -3864,7 +3859,6 @@ if_setlladdr(struct ifnet *ifp, const u_char *lladdr, int len) * to re-init it in order to reprogram its * address filter. */ - NET_EPOCH_EXIT(et); if ((ifp->if_flags & IFF_UP) != 0) { if (ifp->if_ioctl) { ifp->if_flags &= ~IFF_UP; @@ -3879,8 +3873,7 @@ if_setlladdr(struct ifnet *ifp, const u_char *lladdr, int len) } EVENTHANDLER_INVOKE(iflladdr_event, ifp); return (0); - out: - NET_EPOCH_EXIT(et); +out: return (rc); } diff --git a/sys/net/if_ethersubr.c b/sys/net/if_ethersubr.c index 991ee0882625..7a750558890a 100644 --- a/sys/net/if_ethersubr.c +++ b/sys/net/if_ethersubr.c @@ -800,6 +800,7 @@ VNET_SYSUNINIT(vnet_ether_uninit, SI_SUB_PROTO_IF, SI_ORDER_ANY, static void ether_input(struct ifnet *ifp, struct mbuf *m) { + struct epoch_tracker et; struct mbuf *mn; /* @@ -807,22 +808,24 @@ ether_input(struct ifnet *ifp, struct mbuf *m) * m_nextpkt. We split them up into separate packets here and pass * them up. This allows the drivers to amortize the receive lock. */ + CURVNET_SET_QUIET(ifp->if_vnet); + NET_EPOCH_ENTER(et); while (m) { mn = m->m_nextpkt; m->m_nextpkt = NULL; /* - * We will rely on rcvif being set properly in the deferred context, - * so assert it is correct here. + * We will rely on rcvif being set properly in the deferred + * context, so assert it is correct here. */ MPASS((m->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0); KASSERT(m->m_pkthdr.rcvif == ifp, ("%s: ifnet mismatch m %p " "rcvif %p ifp %p", __func__, m, m->m_pkthdr.rcvif, ifp)); - CURVNET_SET_QUIET(ifp->if_vnet); netisr_dispatch(NETISR_ETHER, m); - CURVNET_RESTORE(); m = mn; } + NET_EPOCH_EXIT(et); + CURVNET_RESTORE(); } /* @@ -835,6 +838,7 @@ ether_demux(struct ifnet *ifp, struct mbuf *m) int i, isr; u_short ether_type; + NET_EPOCH_ASSERT(); KASSERT(ifp != NULL, ("%s: NULL interface pointer", __func__)); /* Do not grab PROMISC frames in case we are re-entered. */ diff --git a/sys/net/if_gif.c b/sys/net/if_gif.c index 6a5571a996f5..af911d09c945 100644 --- a/sys/net/if_gif.c +++ b/sys/net/if_gif.c @@ -415,6 +415,8 @@ gif_input(struct mbuf *m, struct ifnet *ifp, int proto, uint8_t ecn) struct ifnet *oldifp; int isr, n, af; + NET_EPOCH_ASSERT(); + if (ifp == NULL) { /* just in case */ m_freem(m); diff --git a/sys/net/if_me.c b/sys/net/if_me.c index c9679d23e67b..05bc50cfb917 100644 --- a/sys/net/if_me.c +++ b/sys/net/if_me.c @@ -451,6 +451,8 @@ me_input(struct mbuf *m, int off, int proto, void *arg) struct ip *ip; int hlen; + NET_EPOCH_ASSERT(); + ifp = ME2IFP(sc); /* checks for short packets */ hlen = sizeof(struct mobhdr); diff --git a/sys/net/if_stf.c b/sys/net/if_stf.c index 225bd30b6bd3..aadc58ec5a48 100644 --- a/sys/net/if_stf.c +++ b/sys/net/if_stf.c @@ -613,6 +613,8 @@ in_stf_input(struct mbuf *m, int off, int proto, void *arg) u_int8_t otos, itos; struct ifnet *ifp; + NET_EPOCH_ASSERT(); + if (proto != IPPROTO_IPV6) { m_freem(m); return (IPPROTO_DONE); diff --git a/sys/net/if_tuntap.c b/sys/net/if_tuntap.c index ff6c3291d7f4..eb197207ca46 100644 --- a/sys/net/if_tuntap.c +++ b/sys/net/if_tuntap.c @@ -1662,6 +1662,7 @@ tunwrite_l2(struct tuntap_softc *tp, struct mbuf *m) static int tunwrite_l3(struct tuntap_softc *tp, struct mbuf *m) { + struct epoch_tracker et; struct ifnet *ifp; int family, isr; @@ -1702,7 +1703,9 @@ tunwrite_l3(struct tuntap_softc *tp, struct mbuf *m) if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); CURVNET_SET(ifp->if_vnet); M_SETFIB(m, ifp->if_fib); + NET_EPOCH_ENTER(et); netisr_dispatch(isr, m); + NET_EPOCH_EXIT(et); CURVNET_RESTORE(); return (0); } diff --git a/sys/net/if_vlan.c b/sys/net/if_vlan.c index 76a0f29d63c3..2f6aa69ca000 100644 --- a/sys/net/if_vlan.c +++ b/sys/net/if_vlan.c @@ -255,7 +255,6 @@ static struct sx _VLAN_SX_ID; #define TRUNK_LOCK_DESTROY(trunk) mtx_destroy(&(trunk)->lock) #define TRUNK_WLOCK(trunk) mtx_lock(&(trunk)->lock) #define TRUNK_WUNLOCK(trunk) mtx_unlock(&(trunk)->lock) -#define TRUNK_LOCK_ASSERT(trunk) MPASS(in_epoch(net_epoch_preempt) || mtx_owned(&(trunk)->lock)) #define TRUNK_WLOCK_ASSERT(trunk) mtx_assert(&(trunk)->lock, MA_OWNED); /* @@ -704,18 +703,17 @@ vlan_ifdetach(void *arg __unused, struct ifnet *ifp) static struct ifnet * vlan_trunkdev(struct ifnet *ifp) { - struct epoch_tracker et; struct ifvlan *ifv; + NET_EPOCH_ASSERT(); + if (ifp->if_type != IFT_L2VLAN) return (NULL); - NET_EPOCH_ENTER(et); ifv = ifp->if_softc; ifp = NULL; if (ifv->ifv_trunk) ifp = PARENT(ifv); - NET_EPOCH_EXIT(et); return (ifp); } @@ -787,21 +785,18 @@ vlan_setcookie(struct ifnet *ifp, void *cookie) static struct ifnet * vlan_devat(struct ifnet *ifp, uint16_t vid) { - struct epoch_tracker et; struct ifvlantrunk *trunk; struct ifvlan *ifv; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + trunk = ifp->if_vlantrunk; - if (trunk == NULL) { - NET_EPOCH_EXIT(et); + if (trunk == NULL) return (NULL); - } ifp = NULL; ifv = vlan_gethash(trunk, vid); if (ifv) ifp = ifv->ifv_ifp; - NET_EPOCH_EXIT(et); return (ifp); } @@ -1140,16 +1135,15 @@ vlan_init(void *foo __unused) static int vlan_transmit(struct ifnet *ifp, struct mbuf *m) { - struct epoch_tracker et; struct ifvlan *ifv; struct ifnet *p; int error, len, mcast; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + ifv = ifp->if_softc; if (TRUNK(ifv) == NULL) { if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); - NET_EPOCH_EXIT(et); m_freem(m); return (ENETDOWN); } @@ -1169,7 +1163,6 @@ vlan_transmit(struct ifnet *ifp, struct mbuf *m) vst = mst_to_vst(mst); if (vst->tag->ifp != p) { if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); - NET_EPOCH_EXIT(et); m_freem(m); return (EAGAIN); } @@ -1185,14 +1178,12 @@ vlan_transmit(struct ifnet *ifp, struct mbuf *m) */ if (!UP_AND_RUNNING(p)) { if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); - NET_EPOCH_EXIT(et); m_freem(m); return (ENETDOWN); } if (!ether_8021q_frame(&m, ifp, p, ifv->ifv_vid, ifv->ifv_pcp)) { if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); - NET_EPOCH_EXIT(et); return (0); } @@ -1206,7 +1197,6 @@ vlan_transmit(struct ifnet *ifp, struct mbuf *m) if_inc_counter(ifp, IFCOUNTER_OMCASTS, mcast); } else if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); - NET_EPOCH_EXIT(et); return (error); } @@ -1214,19 +1204,17 @@ static int vlan_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, struct route *ro) { - struct epoch_tracker et; struct ifvlan *ifv; struct ifnet *p; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + ifv = ifp->if_softc; if (TRUNK(ifv) == NULL) { - NET_EPOCH_EXIT(et); m_freem(m); return (ENETDOWN); } p = PARENT(ifv); - NET_EPOCH_EXIT(et); return p->if_output(ifp, m, dst, ro); } @@ -1242,16 +1230,15 @@ vlan_qflush(struct ifnet *ifp __unused) static void vlan_input(struct ifnet *ifp, struct mbuf *m) { - struct epoch_tracker et; struct ifvlantrunk *trunk; struct ifvlan *ifv; struct m_tag *mtag; uint16_t vid, tag; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + trunk = ifp->if_vlantrunk; if (trunk == NULL) { - NET_EPOCH_EXIT(et); m_freem(m); return; } @@ -1274,7 +1261,6 @@ vlan_input(struct ifnet *ifp, struct mbuf *m) if (m->m_len < sizeof(*evl) && (m = m_pullup(m, sizeof(*evl))) == NULL) { if_printf(ifp, "cannot pullup VLAN header\n"); - NET_EPOCH_EXIT(et); return; } evl = mtod(m, struct ether_vlan_header *); @@ -1297,7 +1283,6 @@ vlan_input(struct ifnet *ifp, struct mbuf *m) __func__, ifp->if_xname, ifp->if_type); #endif if_inc_counter(ifp, IFCOUNTER_NOPROTO, 1); - NET_EPOCH_EXIT(et); m_freem(m); return; } @@ -1307,7 +1292,6 @@ vlan_input(struct ifnet *ifp, struct mbuf *m) ifv = vlan_gethash(trunk, vid); if (ifv == NULL || !UP_AND_RUNNING(ifv->ifv_ifp)) { - NET_EPOCH_EXIT(et); if_inc_counter(ifp, IFCOUNTER_NOPROTO, 1); m_freem(m); return; @@ -1327,7 +1311,6 @@ vlan_input(struct ifnet *ifp, struct mbuf *m) sizeof(uint8_t), M_NOWAIT); if (mtag == NULL) { if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); - NET_EPOCH_EXIT(et); m_freem(m); return; } @@ -1338,7 +1321,6 @@ vlan_input(struct ifnet *ifp, struct mbuf *m) m->m_pkthdr.rcvif = ifv->ifv_ifp; if_inc_counter(ifv->ifv_ifp, IFCOUNTER_IPACKETS, 1); - NET_EPOCH_EXIT(et); /* Pass it back through the parent's input routine. */ (*ifv->ifv_ifp->if_input)(ifv->ifv_ifp, m); @@ -1364,11 +1346,12 @@ vlan_lladdr_fn(void *arg, int pending __unused) static int vlan_config(struct ifvlan *ifv, struct ifnet *p, uint16_t vid) { - struct epoch_tracker et; struct ifvlantrunk *trunk; struct ifnet *ifp; int error = 0; + NET_EPOCH_ASSERT(); + /* * We can handle non-ethernet hardware types as long as * they handle the tagging and headers themselves. @@ -1469,9 +1452,7 @@ vlan_config(struct ifvlan *ifv, struct ifnet *p, uint16_t vid) ifp->if_link_state = p->if_link_state; - NET_EPOCH_ENTER(et); vlan_capabilities(ifv); - NET_EPOCH_EXIT(et); /* * Set up our interface address to reflect the underlying @@ -1643,17 +1624,14 @@ vlan_setflags(struct ifnet *ifp, int status) static void vlan_link_state(struct ifnet *ifp) { - struct epoch_tracker et; struct ifvlantrunk *trunk; struct ifvlan *ifv; - /* Called from a taskqueue_swi task, so we cannot sleep. */ - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + trunk = ifp->if_vlantrunk; - if (trunk == NULL) { - NET_EPOCH_EXIT(et); + if (trunk == NULL) return; - } TRUNK_WLOCK(trunk); VLAN_FOREACH(ifv, trunk) { @@ -1662,7 +1640,6 @@ vlan_link_state(struct ifnet *ifp) trunk->parent->if_link_state); } TRUNK_WUNLOCK(trunk); - NET_EPOCH_EXIT(et); } static void @@ -1674,8 +1651,9 @@ vlan_capabilities(struct ifvlan *ifv) int cap = 0, ena = 0, mena; u_long hwa = 0; - VLAN_SXLOCK_ASSERT(); NET_EPOCH_ASSERT(); + VLAN_SXLOCK_ASSERT(); + p = PARENT(ifv); ifp = ifv->ifv_ifp; @@ -1791,7 +1769,6 @@ vlan_capabilities(struct ifvlan *ifv) static void vlan_trunk_capabilities(struct ifnet *ifp) { - struct epoch_tracker et; struct ifvlantrunk *trunk; struct ifvlan *ifv; @@ -1801,11 +1778,8 @@ vlan_trunk_capabilities(struct ifnet *ifp) VLAN_SUNLOCK(); return; } - NET_EPOCH_ENTER(et); - VLAN_FOREACH(ifv, trunk) { + VLAN_FOREACH(ifv, trunk) vlan_capabilities(ifv); - } - NET_EPOCH_EXIT(et); VLAN_SUNLOCK(); } @@ -1820,6 +1794,8 @@ vlan_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) struct vlanreq vlr; int error = 0; + NET_EPOCH_ASSERT(); + ifr = (struct ifreq *)data; ifa = (struct ifaddr *) data; ifv = ifp->if_softc; @@ -1996,13 +1972,8 @@ vlan_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) VLAN_SLOCK(); ifv->ifv_capenable = ifr->ifr_reqcap; trunk = TRUNK(ifv); - if (trunk != NULL) { - struct epoch_tracker et; - - NET_EPOCH_ENTER(et); + if (trunk != NULL) vlan_capabilities(ifv); - NET_EPOCH_EXIT(et); - } VLAN_SUNLOCK(); break; diff --git a/sys/net/netisr.c b/sys/net/netisr.c index 049fe0fc32d8..ca269815d2fe 100644 --- a/sys/net/netisr.c +++ b/sys/net/netisr.c @@ -861,6 +861,7 @@ static u_int netisr_process_workstream_proto(struct netisr_workstream *nwsp, u_int proto) { struct netisr_work local_npw, *npwp; + struct epoch_tracker et; u_int handled; struct mbuf *m; @@ -890,6 +891,7 @@ netisr_process_workstream_proto(struct netisr_workstream *nwsp, u_int proto) npwp->nw_len = 0; nwsp->nws_pendingbits &= ~(1 << proto); NWS_UNLOCK(nwsp); + NET_EPOCH_ENTER(et); while ((m = local_npw.nw_head) != NULL) { local_npw.nw_head = m->m_nextpkt; m->m_nextpkt = NULL; @@ -902,6 +904,7 @@ netisr_process_workstream_proto(struct netisr_workstream *nwsp, u_int proto) netisr_proto[proto].np_handler(m); CURVNET_RESTORE(); } + NET_EPOCH_EXIT(et); KASSERT(local_npw.nw_len == 0, ("%s(%u): len %u", __func__, proto, local_npw.nw_len)); if (netisr_proto[proto].np_drainedcpu) @@ -1088,6 +1091,7 @@ netisr_dispatch_src(u_int proto, uintptr_t source, struct mbuf *m) int dosignal, error; u_int cpuid, dispatch_policy; + NET_EPOCH_ASSERT(); KASSERT(proto < NETISR_MAXPROT, ("%s: invalid proto %u", __func__, proto)); #ifdef NETISR_LOCKING diff --git a/sys/net/route.c b/sys/net/route.c index 1dd12da6ded1..ee3550441cf9 100644 --- a/sys/net/route.c +++ b/sys/net/route.c @@ -593,12 +593,12 @@ rtredirect_fib(struct sockaddr *dst, int error = 0; short *stat = NULL; struct rt_addrinfo info; - struct epoch_tracker et; struct ifaddr *ifa; struct rib_head *rnh; + NET_EPOCH_ASSERT(); + ifa = NULL; - NET_EPOCH_ENTER(et); rnh = rt_tables_get_rnh(fibnum, dst->sa_family); if (rnh == NULL) { error = EAFNOSUPPORT; @@ -693,7 +693,6 @@ done: if (rt) RTFREE_LOCKED(rt); out: - NET_EPOCH_EXIT(et); if (error) V_rtstat.rts_badredirect++; else if (stat != NULL) diff --git a/sys/net/rtsock.c b/sys/net/rtsock.c index 63dd1dedb481..cec807eb6155 100644 --- a/sys/net/rtsock.c +++ b/sys/net/rtsock.c @@ -560,6 +560,7 @@ route_output(struct mbuf *m, struct socket *so, ...) struct rib_head *rnh; struct rt_addrinfo info; struct sockaddr_storage ss; + struct epoch_tracker et; #ifdef INET6 struct sockaddr_in6 *sin6; int i, rti_need_deembed = 0; @@ -579,6 +580,7 @@ route_output(struct mbuf *m, struct socket *so, ...) return (ENOBUFS); if ((m->m_flags & M_PKTHDR) == 0) panic("route_output"); + NET_EPOCH_ENTER(et); len = m->m_pkthdr.len; if (len < sizeof(*rtm) || len != mtod(m, struct rt_msghdr *)->rtm_msglen) @@ -803,11 +805,11 @@ route_output(struct mbuf *m, struct socket *so, ...) NET_EPOCH_ENTER(et); ifa = ifa_ifwithnet(info.rti_info[RTAX_DST], 1, RT_ALL_FIBS); + NET_EPOCH_EXIT(et); if (ifa != NULL) rt_maskedcopy(ifa->ifa_addr, &laddr, ifa->ifa_netmask); - NET_EPOCH_EXIT(et); } else rt_maskedcopy(rt->rt_ifa->ifa_addr, &laddr, @@ -898,6 +900,7 @@ report: } flush: + NET_EPOCH_EXIT(et); if (rt != NULL) RTFREE(rt); /* @@ -1761,11 +1764,9 @@ sysctl_iflist(int af, struct walkarg *w) struct rt_addrinfo info; int len, error = 0; struct sockaddr_storage ss; - struct epoch_tracker et; bzero((caddr_t)&info, sizeof(info)); bzero(&ifd, sizeof(ifd)); - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if (w->w_arg && w->w_arg != ifp->if_index) continue; @@ -1815,7 +1816,6 @@ sysctl_iflist(int af, struct walkarg *w) info.rti_info[RTAX_BRD] = NULL; } done: - NET_EPOCH_EXIT(et); return (error); } @@ -1823,16 +1823,16 @@ static int sysctl_ifmalist(int af, struct walkarg *w) { struct rt_addrinfo info; - struct epoch_tracker et; struct ifaddr *ifa; struct ifmultiaddr *ifma; struct ifnet *ifp; int error, len; + NET_EPOCH_ASSERT(); + error = 0; bzero((caddr_t)&info, sizeof(info)); - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if (w->w_arg && w->w_arg != ifp->if_index) continue; @@ -1867,7 +1867,6 @@ sysctl_ifmalist(int af, struct walkarg *w) if (error != 0) break; } - NET_EPOCH_EXIT(et); return (error); } @@ -1875,6 +1874,7 @@ static int sysctl_rtsock(SYSCTL_HANDLER_ARGS) { RIB_RLOCK_TRACKER; + struct epoch_tracker et; int *name = (int *)arg1; u_int namelen = arg2; struct rib_head *rnh = NULL; /* silence compiler. */ @@ -1918,8 +1918,8 @@ sysctl_rtsock(SYSCTL_HANDLER_ARGS) w.w_tmemsize = 65536; w.w_tmem = malloc(w.w_tmemsize, M_TEMP, M_WAITOK); + NET_EPOCH_ENTER(et); switch (w.w_op) { - case NET_RT_DUMP: case NET_RT_FLAGS: if (af == 0) { /* dump all tables */ @@ -1946,13 +1946,9 @@ sysctl_rtsock(SYSCTL_HANDLER_ARGS) for (error = 0; error == 0 && i <= lim; i++) { rnh = rt_tables_get_rnh(fib, i); if (rnh != NULL) { - struct epoch_tracker et; - RIB_RLOCK(rnh); - NET_EPOCH_ENTER(et); error = rnh->rnh_walktree(&rnh->head, sysctl_dumpentry, &w); - NET_EPOCH_EXIT(et); RIB_RUNLOCK(rnh); } else if (af != 0) error = EAFNOSUPPORT; @@ -1968,6 +1964,7 @@ sysctl_rtsock(SYSCTL_HANDLER_ARGS) error = sysctl_ifmalist(af, &w); break; } + NET_EPOCH_EXIT(et); free(w.w_tmem, M_TEMP); return (error); diff --git a/sys/netgraph/ng_ether.c b/sys/netgraph/ng_ether.c index 575f3740a6f1..3b1a9efc849e 100644 --- a/sys/netgraph/ng_ether.c +++ b/sys/netgraph/ng_ether.c @@ -711,6 +711,7 @@ ng_ether_rcv_lower(hook_p hook, item_p item) static int ng_ether_rcv_upper(hook_p hook, item_p item) { + struct epoch_tracker et; struct mbuf *m; const node_p node = NG_HOOK_NODE(hook); const priv_p priv = NG_NODE_PRIVATE(node); @@ -738,7 +739,9 @@ ng_ether_rcv_upper(hook_p hook, item_p item) } /* Route packet back in */ + NET_EPOCH_ENTER(et); ether_demux(ifp, m); + NET_EPOCH_EXIT(et); return (0); } diff --git a/sys/netgraph/ng_iface.c b/sys/netgraph/ng_iface.c index f1ddce70c86d..605a1f7d9e24 100644 --- a/sys/netgraph/ng_iface.c +++ b/sys/netgraph/ng_iface.c @@ -690,6 +690,7 @@ ng_iface_rcvdata(hook_p hook, item_p item) const priv_p priv = NG_NODE_PRIVATE(NG_HOOK_NODE(hook)); const iffam_p iffam = get_iffam_from_hook(priv, hook); struct ifnet *const ifp = priv->ifp; + struct epoch_tracker et; struct mbuf *m; int isr; @@ -731,7 +732,9 @@ ng_iface_rcvdata(hook_p hook, item_p item) } random_harvest_queue(m, sizeof(*m), RANDOM_NET_NG); M_SETFIB(m, ifp->if_fib); + NET_EPOCH_ENTER(et); netisr_dispatch(isr, m); + NET_EPOCH_EXIT(et); return (0); } diff --git a/sys/netgraph/ng_ip_input.c b/sys/netgraph/ng_ip_input.c index 5e6dce38f6d1..88e0e11c08d3 100644 --- a/sys/netgraph/ng_ip_input.c +++ b/sys/netgraph/ng_ip_input.c @@ -125,8 +125,13 @@ ngipi_rcvdata(hook_p hook, item_p item) NG_FREE_ITEM(item); if (curthread->td_ng_outbound) netisr_queue(NETISR_IP, m); - else + else { + struct epoch_tracker et; + + NET_EPOCH_ENTER(et); netisr_dispatch(NETISR_IP, m); + NET_EPOCH_EXIT(et); + } return 0; } diff --git a/sys/netinet/if_ether.c b/sys/netinet/if_ether.c index 0c85349f88a8..cb3c335d0897 100644 --- a/sys/netinet/if_ether.c +++ b/sys/netinet/if_ether.c @@ -258,12 +258,16 @@ arptimer(void *arg) if (r_skip_req == 0 && lle->la_preempt > 0) { /* Entry was used, issue refresh request */ + struct epoch_tracker et; struct in_addr dst; + dst = lle->r_l3addr.addr4; lle->la_preempt--; callout_schedule(&lle->lle_timer, hz * V_arpt_rexmit); LLE_WUNLOCK(lle); + NET_EPOCH_ENTER(et); arprequest(ifp, NULL, &dst, NULL); + NET_EPOCH_EXIT(et); CURVNET_RESTORE(); return; } @@ -362,15 +366,15 @@ arprequest_internal(struct ifnet *ifp, const struct in_addr *sip, struct route ro; int error; + NET_EPOCH_ASSERT(); + if (sip == NULL) { /* * The caller did not supply a source address, try to find * a compatible one among those assigned to this interface. */ - struct epoch_tracker et; struct ifaddr *ifa; - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET) continue; @@ -388,7 +392,6 @@ arprequest_internal(struct ifnet *ifp, const struct in_addr *sip, IA_MASKSIN(ifa)->sin_addr.s_addr)) break; /* found it. */ } - NET_EPOCH_EXIT(et); if (sip == NULL) { printf("%s: cannot find matching address\n", __func__); return (EADDRNOTAVAIL); @@ -475,18 +478,15 @@ arpresolve_full(struct ifnet *ifp, int is_gw, int flags, struct mbuf *m, char *lladdr; int ll_len; + NET_EPOCH_ASSERT(); + if (pflags != NULL) *pflags = 0; if (plle != NULL) *plle = NULL; - if ((flags & LLE_CREATE) == 0) { - struct epoch_tracker et; - - NET_EPOCH_ENTER(et); + if ((flags & LLE_CREATE) == 0) la = lla_lookup(LLTABLE(ifp), LLE_EXCLUSIVE, dst); - NET_EPOCH_EXIT(et); - } if (la == NULL && (ifp->if_flags & (IFF_NOARP | IFF_STATICARP)) == 0) { la = lltable_alloc_entry(LLTABLE(ifp), 0, dst); if (la == NULL) { @@ -623,9 +623,10 @@ arpresolve(struct ifnet *ifp, int is_gw, struct mbuf *m, const struct sockaddr *dst, u_char *desten, uint32_t *pflags, struct llentry **plle) { - struct epoch_tracker et; struct llentry *la = NULL; + NET_EPOCH_ASSERT(); + if (pflags != NULL) *pflags = 0; if (plle != NULL) @@ -645,7 +646,6 @@ arpresolve(struct ifnet *ifp, int is_gw, struct mbuf *m, } } - NET_EPOCH_ENTER(et); la = lla_lookup(LLTABLE(ifp), plle ? LLE_EXCLUSIVE : LLE_UNLOCKED, dst); if (la != NULL && (la->r_flags & RLLE_VALID) != 0) { /* Entry found, let's copy lle info */ @@ -659,12 +659,10 @@ arpresolve(struct ifnet *ifp, int is_gw, struct mbuf *m, *plle = la; LLE_WUNLOCK(la); } - NET_EPOCH_EXIT(et); return (0); } if (plle && la) LLE_WUNLOCK(la); - NET_EPOCH_EXIT(et); return (arpresolve_full(ifp, is_gw, la == NULL ? LLE_CREATE : 0, m, dst, desten, pflags, plle)); @@ -809,7 +807,8 @@ in_arpinput(struct mbuf *m) int lladdr_off; int error; char addrbuf[INET_ADDRSTRLEN]; - struct epoch_tracker et; + + NET_EPOCH_ASSERT(); sin.sin_len = sizeof(struct sockaddr_in); sin.sin_family = AF_INET; @@ -902,17 +901,14 @@ in_arpinput(struct mbuf *m) * No match, use the first inet address on the receive interface * as a dummy address for the rest of the function. */ - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) if (ifa->ifa_addr->sa_family == AF_INET && (ifa->ifa_carp == NULL || (*carp_iamatch_p)(ifa, &enaddr))) { ia = ifatoia(ifa); ifa_ref(ifa); - NET_EPOCH_EXIT(et); goto match; } - NET_EPOCH_EXIT(et); /* * If bridging, fall back to using any inet address. @@ -969,9 +965,7 @@ match: sin.sin_family = AF_INET; sin.sin_addr = isaddr; dst = (struct sockaddr *)&sin; - NET_EPOCH_ENTER(et); la = lla_lookup(LLTABLE(ifp), LLE_EXCLUSIVE, dst); - NET_EPOCH_EXIT(et); if (la != NULL) arp_check_update_lle(ah, isaddr, ifp, bridged, la); else if (itaddr.s_addr == myaddr.s_addr) { @@ -1049,9 +1043,7 @@ reply: struct llentry *lle = NULL; sin.sin_addr = itaddr; - NET_EPOCH_ENTER(et); lle = lla_lookup(LLTABLE(ifp), 0, (struct sockaddr *)&sin); - NET_EPOCH_EXIT(et); if ((lle != NULL) && (lle->la_flags & LLE_PUB)) { (void)memcpy(ar_tha(ah), ar_sha(ah), ah->ar_hln); @@ -1430,6 +1422,7 @@ garp_timer_start(struct ifaddr *ifa) void arp_ifinit(struct ifnet *ifp, struct ifaddr *ifa) { + struct epoch_tracker et; const struct sockaddr_in *dst_in; const struct sockaddr *dst; @@ -1441,7 +1434,9 @@ arp_ifinit(struct ifnet *ifp, struct ifaddr *ifa) if (ntohl(dst_in->sin_addr.s_addr) == INADDR_ANY) return; + NET_EPOCH_ENTER(et); arp_announce_ifaddr(ifp, dst_in->sin_addr, IF_LLADDR(ifp)); + NET_EPOCH_EXIT(et); if (garp_rexmit_count > 0) { garp_timer_start(ifa); } diff --git a/sys/netinet/igmp.c b/sys/netinet/igmp.c index beb0f98209be..81d09c926998 100644 --- a/sys/netinet/igmp.c +++ b/sys/netinet/igmp.c @@ -487,8 +487,10 @@ out_locked: static void igmp_dispatch_queue(struct mbufq *mq, int limit, const int loop) { + struct epoch_tracker et; struct mbuf *m; + NET_EPOCH_ENTER(et); while ((m = mbufq_dequeue(mq)) != NULL) { CTR3(KTR_IGMPV3, "%s: dispatch %p from %p", __func__, mq, m); if (loop) @@ -497,6 +499,7 @@ igmp_dispatch_queue(struct mbufq *mq, int limit, const int loop) if (--limit == 0) break; } + NET_EPOCH_EXIT(et); } /* @@ -692,11 +695,12 @@ static int igmp_input_v1_query(struct ifnet *ifp, const struct ip *ip, const struct igmp *igmp) { - struct epoch_tracker et; struct ifmultiaddr *ifma; struct igmp_ifsoftc *igi; struct in_multi *inm; + NET_EPOCH_ASSERT(); + /* * IGMPv1 Host Mmembership Queries SHOULD always be addressed to * 224.0.0.1. They are always treated as General Queries. @@ -734,7 +738,6 @@ igmp_input_v1_query(struct ifnet *ifp, const struct ip *ip, * for the interface on which the query arrived, * except those which are already running. */ - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_INET || ifma->ifma_protospec == NULL) @@ -762,7 +765,6 @@ igmp_input_v1_query(struct ifnet *ifp, const struct ip *ip, break; } } - NET_EPOCH_EXIT(et); out_locked: IGMP_UNLOCK(); @@ -778,13 +780,14 @@ static int igmp_input_v2_query(struct ifnet *ifp, const struct ip *ip, const struct igmp *igmp) { - struct epoch_tracker et; struct ifmultiaddr *ifma; struct igmp_ifsoftc *igi; struct in_multi *inm; int is_general_query; uint16_t timer; + NET_EPOCH_ASSERT(); + is_general_query = 0; /* @@ -836,7 +839,6 @@ igmp_input_v2_query(struct ifnet *ifp, const struct ip *ip, */ CTR2(KTR_IGMPV3, "process v2 general query on ifp %p(%s)", ifp, ifp->if_xname); - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_INET || ifma->ifma_protospec == NULL) @@ -844,7 +846,6 @@ igmp_input_v2_query(struct ifnet *ifp, const struct ip *ip, inm = (struct in_multi *)ifma->ifma_protospec; igmp_v2_update_group(inm, timer); } - NET_EPOCH_EXIT(et); } else { /* * Group-specific IGMPv2 query, we need only @@ -1220,13 +1221,9 @@ igmp_input_v1_report(struct ifnet *ifp, /*const*/ struct ip *ip, * Replace 0.0.0.0 with the subnet address if told to do so. */ if (V_igmp_recvifkludge && in_nullhost(ip->ip_src)) { - struct epoch_tracker et; - - NET_EPOCH_ENTER(et); IFP_TO_IA(ifp, ia, &in_ifa_tracker); if (ia != NULL) ip->ip_src.s_addr = htonl(ia->ia_subnet); - NET_EPOCH_EXIT(et); } CTR3(KTR_IGMPV3, "process v1 report 0x%08x on ifp %p(%s)", @@ -1311,7 +1308,6 @@ igmp_input_v2_report(struct ifnet *ifp, /*const*/ struct ip *ip, /*const*/ struct igmp *igmp) { struct rm_priotracker in_ifa_tracker; - struct epoch_tracker et; struct in_ifaddr *ia; struct in_multi *inm; @@ -1320,23 +1316,19 @@ igmp_input_v2_report(struct ifnet *ifp, /*const*/ struct ip *ip, * leave requires knowing that we are the only member of a * group. */ - NET_EPOCH_ENTER(et); IFP_TO_IA(ifp, ia, &in_ifa_tracker); if (ia != NULL && in_hosteq(ip->ip_src, IA_SIN(ia)->sin_addr)) { - NET_EPOCH_EXIT(et); return (0); } IGMPSTAT_INC(igps_rcv_reports); if (ifp->if_flags & IFF_LOOPBACK) { - NET_EPOCH_EXIT(et); return (0); } if (!IN_MULTICAST(ntohl(igmp->igmp_group.s_addr)) || !in_hosteq(igmp->igmp_group, ip->ip_dst)) { - NET_EPOCH_EXIT(et); IGMPSTAT_INC(igps_rcv_badreports); return (EINVAL); } @@ -1352,7 +1344,6 @@ igmp_input_v2_report(struct ifnet *ifp, /*const*/ struct ip *ip, if (ia != NULL) ip->ip_src.s_addr = htonl(ia->ia_subnet); } - NET_EPOCH_EXIT(et); CTR3(KTR_IGMPV3, "process v2 report 0x%08x on ifp %p(%s)", ntohl(igmp->igmp_group.s_addr), ifp, ifp->if_xname); @@ -1994,13 +1985,14 @@ igmp_v3_cancel_link_timers(struct igmp_ifsoftc *igi) struct ifnet *ifp; struct in_multi *inm; struct in_multi_head inm_free_tmp; - struct epoch_tracker et; CTR3(KTR_IGMPV3, "%s: cancel v3 timers on ifp %p(%s)", __func__, igi->igi_ifp, igi->igi_ifp->if_xname); IN_MULTI_LIST_LOCK_ASSERT(); IGMP_LOCK_ASSERT(); + NET_EPOCH_ASSERT(); + SLIST_INIT(&inm_free_tmp); /* @@ -2015,7 +2007,6 @@ igmp_v3_cancel_link_timers(struct igmp_ifsoftc *igi) * for all memberships scoped to this link. */ ifp = igi->igi_ifp; - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_INET || ifma->ifma_protospec == NULL) @@ -2060,7 +2051,6 @@ igmp_v3_cancel_link_timers(struct igmp_ifsoftc *igi) inm->inm_timer = 0; mbufq_drain(&inm->inm_scq); } - NET_EPOCH_EXIT(et); inm_release_list_deferred(&inm_free_tmp); } @@ -2189,6 +2179,7 @@ igmp_v1v2_queue_report(struct in_multi *inm, const int type) struct ip *ip; struct mbuf *m; + NET_EPOCH_ASSERT(); IN_MULTI_LIST_LOCK_ASSERT(); IGMP_LOCK_ASSERT(); @@ -3303,7 +3294,6 @@ igmp_v3_merge_state_changes(struct in_multi *inm, struct mbufq *scq) static void igmp_v3_dispatch_general_query(struct igmp_ifsoftc *igi) { - struct epoch_tracker et; struct ifmultiaddr *ifma; struct ifnet *ifp; struct in_multi *inm; @@ -3311,6 +3301,7 @@ igmp_v3_dispatch_general_query(struct igmp_ifsoftc *igi) IN_MULTI_LIST_LOCK_ASSERT(); IGMP_LOCK_ASSERT(); + NET_EPOCH_ASSERT(); KASSERT(igi->igi_version == IGMP_VERSION_3, ("%s: called when version %d", __func__, igi->igi_version)); @@ -3326,7 +3317,6 @@ igmp_v3_dispatch_general_query(struct igmp_ifsoftc *igi) ifp = igi->igi_ifp; - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_INET || ifma->ifma_protospec == NULL) @@ -3357,7 +3347,6 @@ igmp_v3_dispatch_general_query(struct igmp_ifsoftc *igi) break; } } - NET_EPOCH_EXIT(et); send: loop = (igi->igi_flags & IGIF_LOOPBACK) ? 1 : 0; @@ -3529,14 +3518,11 @@ igmp_v3_encap_report(struct ifnet *ifp, struct mbuf *m) ip->ip_src.s_addr = INADDR_ANY; if (m->m_flags & M_IGMP_LOOP) { - struct epoch_tracker et; struct in_ifaddr *ia; - NET_EPOCH_ENTER(et); IFP_TO_IA(ifp, ia, &in_ifa_tracker); if (ia != NULL) ip->ip_src = ia->ia_addr.sin_addr; - NET_EPOCH_EXIT(et); } ip->ip_dst.s_addr = htonl(INADDR_ALLRPTS_GROUP); diff --git a/sys/netinet/in.c b/sys/netinet/in.c index cbb171395662..c83f0ae83fc4 100644 --- a/sys/netinet/in.c +++ b/sys/netinet/in.c @@ -139,21 +139,18 @@ in_localip(struct in_addr in) int in_ifhasaddr(struct ifnet *ifp, struct in_addr in) { - struct epoch_tracker et; struct ifaddr *ifa; struct in_ifaddr *ia; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET) continue; ia = (struct in_ifaddr *)ifa; - if (ia->ia_addr.sin_addr.s_addr == in.s_addr) { - NET_EPOCH_EXIT(et); + if (ia->ia_addr.sin_addr.s_addr == in.s_addr) return (1); - } } - NET_EPOCH_EXIT(et); return (0); } @@ -965,10 +962,11 @@ in_ifaddr_broadcast(struct in_addr in, struct in_ifaddr *ia) int in_broadcast(struct in_addr in, struct ifnet *ifp) { - struct epoch_tracker et; struct ifaddr *ifa; int found; + NET_EPOCH_ASSERT(); + if (in.s_addr == INADDR_BROADCAST || in.s_addr == INADDR_ANY) return (1); @@ -979,14 +977,12 @@ in_broadcast(struct in_addr in, struct ifnet *ifp) * Look through the list of addresses for a match * with a broadcast address. */ - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) if (ifa->ifa_addr->sa_family == AF_INET && in_ifaddr_broadcast(in, (struct in_ifaddr *)ifa)) { found = 1; break; } - NET_EPOCH_EXIT(et); return (found); } diff --git a/sys/netinet/in_mcast.c b/sys/netinet/in_mcast.c index ac5c6a03ab54..a91d88277fce 100644 --- a/sys/netinet/in_mcast.c +++ b/sys/netinet/in_mcast.c @@ -388,13 +388,12 @@ inm_lookup_locked(struct ifnet *ifp, const struct in_addr ina) struct in_multi * inm_lookup(struct ifnet *ifp, const struct in_addr ina) { - struct epoch_tracker et; struct in_multi *inm; IN_MULTI_LIST_LOCK_ASSERT(); - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + inm = inm_lookup_locked(ifp, ina); - NET_EPOCH_EXIT(et); return (inm); } @@ -1199,10 +1198,13 @@ int in_joingroup(struct ifnet *ifp, const struct in_addr *gina, /*const*/ struct in_mfilter *imf, struct in_multi **pinm) { + struct epoch_tracker et; int error; IN_MULTI_LOCK(); + NET_EPOCH_ENTER(et); error = in_joingroup_locked(ifp, gina, imf, pinm); + NET_EPOCH_EXIT(et); IN_MULTI_UNLOCK(); return (error); @@ -1225,6 +1227,7 @@ in_joingroup_locked(struct ifnet *ifp, const struct in_addr *gina, struct in_multi *inm; int error; + NET_EPOCH_ASSERT(); IN_MULTI_LOCK_ASSERT(); IN_MULTI_LIST_UNLOCK_ASSERT(); @@ -1282,11 +1285,14 @@ in_joingroup_locked(struct ifnet *ifp, const struct in_addr *gina, int in_leavegroup(struct in_multi *inm, /*const*/ struct in_mfilter *imf) { + struct epoch_tracker et; int error; + NET_EPOCH_ENTER(et); IN_MULTI_LOCK(); error = in_leavegroup_locked(inm, imf); IN_MULTI_UNLOCK(); + NET_EPOCH_EXIT(et); return (error); } @@ -1310,11 +1316,12 @@ in_leavegroup_locked(struct in_multi *inm, /*const*/ struct in_mfilter *imf) struct in_mfilter timf; int error; - error = 0; - + NET_EPOCH_ASSERT(); IN_MULTI_LOCK_ASSERT(); IN_MULTI_LIST_UNLOCK_ASSERT(); + error = 0; + CTR5(KTR_IGMPV3, "%s: leave inm %p, 0x%08x/%s, imf %p", __func__, inm, ntohl(inm->inm_addr.s_addr), (inm_is_ifp_detached(inm) ? "null" : inm->inm_ifp->if_xname), @@ -1811,15 +1818,11 @@ inp_getmoptions(struct inpcb *inp, struct sockopt *sopt) if (!in_nullhost(imo->imo_multicast_addr)) { mreqn.imr_address = imo->imo_multicast_addr; } else if (ifp != NULL) { - struct epoch_tracker et; - mreqn.imr_ifindex = ifp->if_index; - NET_EPOCH_ENTER(et); IFP_TO_IA(ifp, ia, &in_ifa_tracker); if (ia != NULL) mreqn.imr_address = IA_SIN(ia)->sin_addr; - NET_EPOCH_EXIT(et); } } INP_WUNLOCK(inp); @@ -2908,8 +2911,10 @@ sysctl_ip_mcast_filters(SYSCTL_HANDLER_ARGS) return (EINVAL); } + NET_EPOCH_ENTER(et); ifp = ifnet_byindex(ifindex); if (ifp == NULL) { + NET_EPOCH_EXIT(et); CTR2(KTR_IGMPV3, "%s: no ifp for ifindex %u", __func__, ifindex); return (ENOENT); @@ -2917,12 +2922,13 @@ sysctl_ip_mcast_filters(SYSCTL_HANDLER_ARGS) retval = sysctl_wire_old_buffer(req, sizeof(uint32_t) + (in_mcast_maxgrpsrc * sizeof(struct in_addr))); - if (retval) + if (retval) { + NET_EPOCH_EXIT(et); return (retval); + } IN_MULTI_LIST_LOCK(); - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_INET || ifma->ifma_protospec == NULL) @@ -2951,9 +2957,9 @@ sysctl_ip_mcast_filters(SYSCTL_HANDLER_ARGS) break; } } - NET_EPOCH_EXIT(et); IN_MULTI_LIST_UNLOCK(); + NET_EPOCH_EXIT(et); return (retval); } diff --git a/sys/netinet/in_rmx.c b/sys/netinet/in_rmx.c index 2cface409d7f..cc866e33cfaa 100644 --- a/sys/netinet/in_rmx.c +++ b/sys/netinet/in_rmx.c @@ -80,12 +80,17 @@ in_addroute(void *v_arg, void *n_arg, struct radix_head *head, * dubious since it's so easy to inspect the address). */ if (rt->rt_flags & RTF_HOST) { - if (in_broadcast(sin->sin_addr, rt->rt_ifp)) { + struct epoch_tracker et; + bool bcast; + + NET_EPOCH_ENTER(et); + bcast = in_broadcast(sin->sin_addr, rt->rt_ifp); + NET_EPOCH_EXIT(et); + if (bcast) rt->rt_flags |= RTF_BROADCAST; - } else if (satosin(rt->rt_ifa->ifa_addr)->sin_addr.s_addr == - sin->sin_addr.s_addr) { + else if (satosin(rt->rt_ifa->ifa_addr)->sin_addr.s_addr == + sin->sin_addr.s_addr) rt->rt_flags |= RTF_LOCAL; - } } if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) rt->rt_flags |= RTF_MULTICAST; diff --git a/sys/netinet/in_var.h b/sys/netinet/in_var.h index 50112481f236..96879db0abb7 100644 --- a/sys/netinet/in_var.h +++ b/sys/netinet/in_var.h @@ -171,6 +171,7 @@ do { \ /* struct in_ifaddr *ia; */ \ /* struct rm_priotracker *t; */ \ do { \ + NET_EPOCH_ASSERT(); \ IN_IFADDR_RLOCK((t)); \ for ((ia) = CK_STAILQ_FIRST(&V_in_ifaddrhead); \ (ia) != NULL && (ia)->ia_ifp != (ifp); \ diff --git a/sys/netinet/ip_carp.c b/sys/netinet/ip_carp.c index 819a0f9c75e6..ceccd5a02f3a 100644 --- a/sys/netinet/ip_carp.c +++ b/sys/netinet/ip_carp.c @@ -644,9 +644,10 @@ carp_input_c(struct mbuf *m, struct carp_header *ch, sa_family_t af) struct carp_softc *sc; uint64_t tmp_counter; struct timeval sc_tv, ch_tv; - struct epoch_tracker et; int error; + NET_EPOCH_ASSERT(); + /* * Verify that the VHID is valid on the receiving interface. * @@ -658,7 +659,6 @@ carp_input_c(struct mbuf *m, struct carp_header *ch, sa_family_t af) * (these should never happen, and as noted above, we may * miss real loops; this is just a double-check). */ - NET_EPOCH_ENTER(et); error = 0; match = NULL; IFNET_FOREACH_IFA(ifp, ifa) { @@ -672,7 +672,6 @@ carp_input_c(struct mbuf *m, struct carp_header *ch, sa_family_t af) ifa = error ? NULL : match; if (ifa != NULL) ifa_ref(ifa); - NET_EPOCH_EXIT(et); if (ifa == NULL) { if (error == ELOOP) { @@ -880,19 +879,18 @@ carp_send_ad_error(struct carp_softc *sc, int error) static struct ifaddr * carp_best_ifa(int af, struct ifnet *ifp) { - struct epoch_tracker et; struct ifaddr *ifa, *best; + NET_EPOCH_ASSERT(); + if (af >= AF_MAX) return (NULL); best = NULL; - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family == af && (best == NULL || ifa_preferred(best, ifa))) best = ifa; } - NET_EPOCH_EXIT(et); if (best != NULL) ifa_ref(best); return (best); @@ -1169,11 +1167,11 @@ carp_send_na(struct carp_softc *sc) struct ifaddr * carp_iamatch6(struct ifnet *ifp, struct in6_addr *taddr) { - struct epoch_tracker et; struct ifaddr *ifa; + NET_EPOCH_ASSERT(); + ifa = NULL; - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; @@ -1185,7 +1183,6 @@ carp_iamatch6(struct ifnet *ifp, struct in6_addr *taddr) ifa_ref(ifa); break; } - NET_EPOCH_EXIT(et); return (ifa); } @@ -1193,18 +1190,16 @@ carp_iamatch6(struct ifnet *ifp, struct in6_addr *taddr) caddr_t carp_macmatch6(struct ifnet *ifp, struct mbuf *m, const struct in6_addr *taddr) { - struct epoch_tracker et; struct ifaddr *ifa; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + IFNET_FOREACH_IFA(ifp, ifa) if (ifa->ifa_addr->sa_family == AF_INET6 && IN6_ARE_ADDR_EQUAL(taddr, IFA_IN6(ifa))) { struct carp_softc *sc = ifa->ifa_carp; struct m_tag *mtag; - NET_EPOCH_EXIT(et); - mtag = m_tag_get(PACKET_TAG_CARP, sizeof(struct carp_softc *), M_NOWAIT); if (mtag == NULL) @@ -1216,7 +1211,6 @@ carp_macmatch6(struct ifnet *ifp, struct mbuf *m, const struct in6_addr *taddr) return (LLADDR(&sc->sc_addr)); } - NET_EPOCH_EXIT(et); return (NULL); } diff --git a/sys/netinet/ip_encap.c b/sys/netinet/ip_encap.c index ebcce2958a32..15ba528e7e0c 100644 --- a/sys/netinet/ip_encap.c +++ b/sys/netinet/ip_encap.c @@ -125,11 +125,9 @@ MTX_SYSINIT(srcaddrmtx, &srcaddrmtx, "srcaddrmtx", MTX_DEF); #define ENCAP_WLOCK() mtx_lock(&encapmtx) #define ENCAP_WUNLOCK() mtx_unlock(&encapmtx) #define ENCAP_RLOCK_TRACKER struct epoch_tracker encap_et -#define ENCAP_RLOCK() \ - epoch_enter_preempt(net_epoch_preempt, &encap_et) -#define ENCAP_RUNLOCK() \ - epoch_exit_preempt(net_epoch_preempt, &encap_et) -#define ENCAP_WAIT() epoch_wait_preempt(net_epoch_preempt) +#define ENCAP_RLOCK() NET_EPOCH_ENTER(encap_et) +#define ENCAP_RUNLOCK() NET_EPOCH_EXIT(encap_et) +#define ENCAP_WAIT() NET_EPOCH_WAIT() #define SRCADDR_WLOCK() mtx_lock(&srcaddrmtx) #define SRCADDR_WUNLOCK() mtx_unlock(&srcaddrmtx) diff --git a/sys/netinet/ip_icmp.c b/sys/netinet/ip_icmp.c index 3e5ce6283f0c..0bf9b86756f8 100644 --- a/sys/netinet/ip_icmp.c +++ b/sys/netinet/ip_icmp.c @@ -393,7 +393,6 @@ freeit: int icmp_input(struct mbuf **mp, int *offp, int proto) { - struct epoch_tracker et; struct icmp *icp; struct in_ifaddr *ia; struct mbuf *m = *mp; @@ -405,6 +404,8 @@ icmp_input(struct mbuf **mp, int *offp, int proto) void (*ctlfunc)(int, struct sockaddr *, void *); int fibnum; + NET_EPOCH_ASSERT(); + *mp = NULL; /* @@ -421,7 +422,6 @@ icmp_input(struct mbuf **mp, int *offp, int proto) inet_ntoa_r(ip->ip_dst, dstbuf), icmplen); } #endif - NET_EPOCH_ENTER(et); if (icmplen < ICMP_MINLEN) { ICMPSTAT_INC(icps_tooshort); goto freeit; @@ -429,7 +429,6 @@ icmp_input(struct mbuf **mp, int *offp, int proto) i = hlen + min(icmplen, ICMP_ADVLENMIN); if (m->m_len < i && (m = m_pullup(m, i)) == NULL) { ICMPSTAT_INC(icps_tooshort); - NET_EPOCH_EXIT(et); return (IPPROTO_DONE); } ip = mtod(m, struct ip *); @@ -547,7 +546,6 @@ icmp_input(struct mbuf **mp, int *offp, int proto) if (m->m_len < i && (m = m_pullup(m, i)) == NULL) { /* This should actually not happen */ ICMPSTAT_INC(icps_tooshort); - NET_EPOCH_EXIT(et); return (IPPROTO_DONE); } ip = mtod(m, struct ip *); @@ -640,7 +638,6 @@ reflect: ICMPSTAT_INC(icps_reflect); ICMPSTAT_INC(icps_outhist[icp->icmp_type]); icmp_reflect(m); - NET_EPOCH_EXIT(et); return (IPPROTO_DONE); case ICMP_REDIRECT: @@ -717,13 +714,11 @@ reflect: } raw: - NET_EPOCH_EXIT(et); *mp = m; rip_input(mp, offp, proto); return (IPPROTO_DONE); freeit: - NET_EPOCH_EXIT(et); m_freem(m); return (IPPROTO_DONE); } @@ -735,7 +730,6 @@ static void icmp_reflect(struct mbuf *m) { struct rm_priotracker in_ifa_tracker; - struct epoch_tracker et; struct ip *ip = mtod(m, struct ip *); struct ifaddr *ifa; struct ifnet *ifp; @@ -745,6 +739,8 @@ icmp_reflect(struct mbuf *m) struct mbuf *opts = NULL; int optlen = (ip->ip_hl << 2) - sizeof(struct ip); + NET_EPOCH_ASSERT(); + if (IN_MULTICAST(ntohl(ip->ip_src.s_addr)) || IN_EXPERIMENTAL(ntohl(ip->ip_src.s_addr)) || IN_ZERONET(ntohl(ip->ip_src.s_addr)) ) { @@ -779,7 +775,6 @@ icmp_reflect(struct mbuf *m) */ ifp = m->m_pkthdr.rcvif; if (ifp != NULL && ifp->if_flags & IFF_BROADCAST) { - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET) continue; @@ -787,11 +782,9 @@ icmp_reflect(struct mbuf *m) if (satosin(&ia->ia_broadaddr)->sin_addr.s_addr == t.s_addr) { t = IA_SIN(ia)->sin_addr; - NET_EPOCH_EXIT(et); goto match; } } - NET_EPOCH_EXIT(et); } /* * If the packet was transiting through us, use the address of @@ -800,16 +793,13 @@ icmp_reflect(struct mbuf *m) * criteria apply. */ if (V_icmp_rfi && ifp != NULL) { - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET) continue; ia = ifatoia(ifa); t = IA_SIN(ia)->sin_addr; - NET_EPOCH_EXIT(et); goto match; } - NET_EPOCH_EXIT(et); } /* * If the incoming packet was not addressed directly to us, use @@ -818,16 +808,13 @@ icmp_reflect(struct mbuf *m) * with normal source selection. */ if (V_reply_src[0] != '\0' && (ifp = ifunit(V_reply_src))) { - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET) continue; ia = ifatoia(ifa); t = IA_SIN(ia)->sin_addr; - NET_EPOCH_EXIT(et); goto match; } - NET_EPOCH_EXIT(et); } /* * If the packet was transiting through us, use the address of diff --git a/sys/netinet/ip_input.c b/sys/netinet/ip_input.c index 03f5fed3e82b..8991262cc120 100644 --- a/sys/netinet/ip_input.c +++ b/sys/netinet/ip_input.c @@ -457,6 +457,7 @@ ip_input(struct mbuf *m) struct in_addr odst; /* original dst address */ M_ASSERTPKTHDR(m); + NET_EPOCH_ASSERT(); if (m->m_flags & M_FASTFWD_OURS) { m->m_flags &= ~M_FASTFWD_OURS; @@ -708,9 +709,6 @@ passin: * into the stack for SIMPLEX interfaces handled by ether_output(). */ if (ifp != NULL && ifp->if_flags & IFF_BROADCAST) { - struct epoch_tracker et; - - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET) continue; @@ -720,7 +718,6 @@ passin: counter_u64_add(ia->ia_ifa.ifa_ipackets, 1); counter_u64_add(ia->ia_ifa.ifa_ibytes, m->m_pkthdr.len); - NET_EPOCH_EXIT(et); goto ours; } #ifdef BOOTP_COMPAT @@ -728,12 +725,10 @@ passin: counter_u64_add(ia->ia_ifa.ifa_ipackets, 1); counter_u64_add(ia->ia_ifa.ifa_ibytes, m->m_pkthdr.len); - NET_EPOCH_EXIT(et); goto ours; } #endif } - NET_EPOCH_EXIT(et); ia = NULL; } /* RFC 3927 2.7: Do not forward datagrams for 169.254.0.0/16. */ @@ -953,9 +948,10 @@ ip_forward(struct mbuf *m, int srcrt) struct sockaddr_in *sin; struct in_addr dest; struct route ro; - struct epoch_tracker et; int error, type = 0, code = 0, mtu = 0; + NET_EPOCH_ASSERT(); + if (m->m_flags & (M_BCAST|M_MCAST) || in_canforward(ip->ip_dst) == 0) { IPSTAT_INC(ips_cantforward); m_freem(m); @@ -982,7 +978,6 @@ ip_forward(struct mbuf *m, int srcrt) #else in_rtalloc_ign(&ro, 0, M_GETFIB(m)); #endif - NET_EPOCH_ENTER(et); if (ro.ro_rt != NULL) { ia = ifatoia(ro.ro_rt->rt_ifa); } else @@ -1030,7 +1025,7 @@ ip_forward(struct mbuf *m, int srcrt) m_freem(mcopy); if (error != EINPROGRESS) IPSTAT_INC(ips_cantforward); - goto out; + return; } /* No IPsec processing required */ } @@ -1083,11 +1078,11 @@ ip_forward(struct mbuf *m, int srcrt) else { if (mcopy) m_freem(mcopy); - goto out; + return; } } if (mcopy == NULL) - goto out; + return; switch (error) { @@ -1130,11 +1125,9 @@ ip_forward(struct mbuf *m, int srcrt) case ENOBUFS: case EACCES: /* ipfw denied packet */ m_freem(mcopy); - goto out; + return; } icmp_error(mcopy, type, code, dest.s_addr, mtu); - out: - NET_EPOCH_EXIT(et); } #define CHECK_SO_CT(sp, ct) \ diff --git a/sys/netinet/ip_mroute.c b/sys/netinet/ip_mroute.c index 14ebb6ef1007..dd43b467f1f3 100644 --- a/sys/netinet/ip_mroute.c +++ b/sys/netinet/ip_mroute.c @@ -874,18 +874,13 @@ add_vif(struct vifctl *vifcp) */ ifp = NULL; } else { - struct epoch_tracker et; - sin.sin_addr = vifcp->vifc_lcl_addr; - NET_EPOCH_ENTER(et); ifa = ifa_ifwithaddr((struct sockaddr *)&sin); if (ifa == NULL) { - NET_EPOCH_EXIT(et); VIF_UNLOCK(); return EADDRNOTAVAIL; } ifp = ifa->ifa_ifp; - NET_EPOCH_EXIT(et); } if ((vifcp->vifc_flags & VIFF_TUNNEL) != 0) { diff --git a/sys/netinet/ip_options.c b/sys/netinet/ip_options.c index 87202eb806c4..cb9920d86d9e 100644 --- a/sys/netinet/ip_options.c +++ b/sys/netinet/ip_options.c @@ -109,7 +109,8 @@ ip_dooptions(struct mbuf *m, int pass) uint32_t ntime; struct nhop4_extended nh_ext; struct sockaddr_in ipaddr = { sizeof(ipaddr), AF_INET }; - struct epoch_tracker et; + + NET_EPOCH_ASSERT(); /* Ignore or reject packets with IP options. */ if (V_ip_doopts == 0) @@ -117,10 +118,9 @@ ip_dooptions(struct mbuf *m, int pass) else if (V_ip_doopts == 2) { type = ICMP_UNREACH; code = ICMP_UNREACH_FILTER_PROHIB; - goto bad_unlocked; + goto bad; } - NET_EPOCH_ENTER(et); dst = ip->ip_dst; cp = (u_char *)(ip + 1); cnt = (ip->ip_hl << 2) - sizeof (struct ip); @@ -226,7 +226,6 @@ dropit: #endif IPSTAT_INC(ips_cantforward); m_freem(m); - NET_EPOCH_EXIT(et); return (1); } } @@ -381,15 +380,12 @@ dropit: cp[IPOPT_OFFSET] += sizeof(uint32_t); } } - NET_EPOCH_EXIT(et); if (forward && V_ipforwarding) { ip_forward(m, 1); return (1); } return (0); bad: - NET_EPOCH_EXIT(et); -bad_unlocked: icmp_error(m, type, code, 0, 0); IPSTAT_INC(ips_badoptions); return (1); diff --git a/sys/netinet/ip_output.c b/sys/netinet/ip_output.c index cbd2d72188fa..9266dce3f7c2 100644 --- a/sys/netinet/ip_output.c +++ b/sys/netinet/ip_output.c @@ -1078,7 +1078,8 @@ in_delayed_cksum(struct mbuf *m) int ip_ctloutput(struct socket *so, struct sockopt *sopt) { - struct inpcb *inp = sotoinpcb(so); + struct inpcb *inp = sotoinpcb(so); + struct epoch_tracker et; int error, optval; #ifdef RSS uint32_t rss_bucket; @@ -1517,7 +1518,9 @@ ip_ctloutput(struct socket *so, struct sockopt *sopt) case IP_MULTICAST_TTL: case IP_MULTICAST_LOOP: case IP_MSFILTER: + NET_EPOCH_ENTER(et); error = inp_getmoptions(inp, sopt); + NET_EPOCH_EXIT(et); break; #if defined(IPSEC) || defined(IPSEC_SUPPORT) diff --git a/sys/netinet/sctp_bsd_addr.c b/sys/netinet/sctp_bsd_addr.c index e0432fb6b689..df8841c767c3 100644 --- a/sys/netinet/sctp_bsd_addr.c +++ b/sys/netinet/sctp_bsd_addr.c @@ -198,6 +198,7 @@ sctp_init_ifns_for_vrf(int vrfid) * make sure we lock any IFA that exists as we float through the * list of IFA's */ + struct epoch_tracker et; struct ifnet *ifn; struct ifaddr *ifa; struct sctp_ifa *sctp_ifa; @@ -207,14 +208,12 @@ sctp_init_ifns_for_vrf(int vrfid) #endif IFNET_RLOCK(); + NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifn, &MODULE_GLOBAL(ifnet), if_link) { - struct epoch_tracker et; - if (sctp_is_desired_interface_type(ifn) == 0) { /* non desired type */ continue; } - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifn->if_addrhead, ifa_link) { if (ifa->ifa_addr == NULL) { continue; @@ -267,8 +266,8 @@ sctp_init_ifns_for_vrf(int vrfid) sctp_ifa->localifa_flags &= ~SCTP_ADDR_DEFER_USE; } } - NET_EPOCH_EXIT(et); } + NET_EPOCH_EXIT(et); IFNET_RUNLOCK(); } diff --git a/sys/netinet6/icmp6.c b/sys/netinet6/icmp6.c index b18de150fd33..f2d0321ffdab 100644 --- a/sys/netinet6/icmp6.c +++ b/sys/netinet6/icmp6.c @@ -1660,13 +1660,14 @@ static int ni6_addrs(struct icmp6_nodeinfo *ni6, struct mbuf *m, struct ifnet **ifpp, struct in6_addr *subj) { - struct epoch_tracker et; struct ifnet *ifp; struct in6_ifaddr *ifa6; struct ifaddr *ifa; int addrs = 0, addrsofif, iffound = 0; int niflags = ni6->ni_flags; + NET_EPOCH_ASSERT(); + if ((niflags & NI_NODEADDR_FLAG_ALL) == 0) { switch (ni6->ni_code) { case ICMP6_NI_SUBJ_IPV6: @@ -1682,7 +1683,6 @@ ni6_addrs(struct icmp6_nodeinfo *ni6, struct mbuf *m, struct ifnet **ifpp, } } - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { addrsofif = 0; CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { @@ -1737,13 +1737,11 @@ ni6_addrs(struct icmp6_nodeinfo *ni6, struct mbuf *m, struct ifnet **ifpp, } if (iffound) { *ifpp = ifp; - NET_EPOCH_EXIT(et); return (addrsofif); } addrs += addrsofif; } - NET_EPOCH_EXIT(et); return (addrs); } @@ -1752,7 +1750,6 @@ static int ni6_store_addrs(struct icmp6_nodeinfo *ni6, struct icmp6_nodeinfo *nni6, struct ifnet *ifp0, int resid) { - struct epoch_tracker et; struct ifnet *ifp; struct in6_ifaddr *ifa6; struct ifaddr *ifa; @@ -1762,10 +1759,11 @@ ni6_store_addrs(struct icmp6_nodeinfo *ni6, struct icmp6_nodeinfo *nni6, int niflags = ni6->ni_flags; u_int32_t ltime; + NET_EPOCH_ASSERT(); + if (ifp0 == NULL && !(niflags & NI_NODEADDR_FLAG_ALL)) return (0); /* needless to copy */ - NET_EPOCH_ENTER(et); ifp = ifp0 ? ifp0 : CK_STAILQ_FIRST(&V_ifnet); again: @@ -1829,7 +1827,6 @@ ni6_store_addrs(struct icmp6_nodeinfo *ni6, struct icmp6_nodeinfo *nni6, * Set the truncate flag and return. */ nni6->ni_flags |= NI_NODEADDR_FLAG_TRUNCATE; - NET_EPOCH_EXIT(et); return (copied); } @@ -1881,8 +1878,6 @@ ni6_store_addrs(struct icmp6_nodeinfo *ni6, struct icmp6_nodeinfo *nni6, goto again; } - NET_EPOCH_EXIT(et); - return (copied); } @@ -2563,14 +2558,11 @@ icmp6_redirect_output(struct mbuf *m0, struct rtentry *rt) { /* target lladdr option */ - struct epoch_tracker et; int len; struct nd_opt_hdr *nd_opt; char *lladdr; - NET_EPOCH_ENTER(et); ln = nd6_lookup(router_ll6, 0, ifp); - NET_EPOCH_EXIT(et); if (ln == NULL) goto nolladdropt; diff --git a/sys/netinet6/in6.c b/sys/netinet6/in6.c index 4de6e8d2dfe0..c92ee171b784 100644 --- a/sys/netinet6/in6.c +++ b/sys/netinet6/in6.c @@ -1474,10 +1474,10 @@ done: struct in6_ifaddr * in6ifa_ifpforlinklocal(struct ifnet *ifp, int ignoreflags) { - struct epoch_tracker et; struct ifaddr *ifa; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; @@ -1489,7 +1489,6 @@ in6ifa_ifpforlinklocal(struct ifnet *ifp, int ignoreflags) break; } } - NET_EPOCH_EXIT(et); return ((struct in6_ifaddr *)ifa); } @@ -1549,13 +1548,13 @@ in6ifa_ifpwithaddr(struct ifnet *ifp, const struct in6_addr *addr) struct in6_ifaddr * in6ifa_llaonifp(struct ifnet *ifp) { - struct epoch_tracker et; struct sockaddr_in6 *sin6; struct ifaddr *ifa; + NET_EPOCH_ASSERT(); + if (ND_IFINFO(ifp)->flags & ND6_IFF_IFDISABLED) return (NULL); - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; @@ -1565,7 +1564,6 @@ in6ifa_llaonifp(struct ifnet *ifp) IN6_IS_ADDR_MC_NODELOCAL(&sin6->sin6_addr)) break; } - NET_EPOCH_EXIT(et); return ((struct in6_ifaddr *)ifa); } @@ -1702,26 +1700,23 @@ int in6_ifhasaddr(struct ifnet *ifp, struct in6_addr *addr) { struct in6_addr in6; - struct epoch_tracker et; struct ifaddr *ifa; struct in6_ifaddr *ia6; + NET_EPOCH_ASSERT(); + in6 = *addr; if (in6_clearscope(&in6)) return (0); in6_setscope(&in6, ifp, NULL); - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; ia6 = (struct in6_ifaddr *)ifa; - if (IN6_ARE_ADDR_EQUAL(&ia6->ia_addr.sin6_addr, &in6)) { - NET_EPOCH_EXIT(et); + if (IN6_ARE_ADDR_EQUAL(&ia6->ia_addr.sin6_addr, &in6)) return (1); - } } - NET_EPOCH_EXIT(et); return (0); } @@ -1825,12 +1820,13 @@ in6_prefixlen2mask(struct in6_addr *maskp, int len) struct in6_ifaddr * in6_ifawithifp(struct ifnet *ifp, struct in6_addr *dst) { - struct epoch_tracker et; int dst_scope = in6_addrscope(dst), blen = -1, tlen; struct ifaddr *ifa; struct in6_ifaddr *besta = NULL; struct in6_ifaddr *dep[2]; /* last-resort: deprecated */ + NET_EPOCH_ASSERT(); + dep[0] = dep[1] = NULL; /* @@ -1839,7 +1835,6 @@ in6_ifawithifp(struct ifnet *ifp, struct in6_addr *dst) * If two or more, return one which matches the dst longest. * If none, return one of global addresses assigned other ifs. */ - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; @@ -1873,7 +1868,6 @@ in6_ifawithifp(struct ifnet *ifp, struct in6_addr *dst) } if (besta) { ifa_ref(&besta->ia_ifa); - NET_EPOCH_EXIT(et); return (besta); } @@ -1894,23 +1888,19 @@ in6_ifawithifp(struct ifnet *ifp, struct in6_addr *dst) if (ifa != NULL) ifa_ref(ifa); - NET_EPOCH_EXIT(et); return (struct in6_ifaddr *)ifa; } /* use the last-resort values, that are, deprecated addresses */ if (dep[0]) { ifa_ref((struct ifaddr *)dep[0]); - NET_EPOCH_EXIT(et); return dep[0]; } if (dep[1]) { ifa_ref((struct ifaddr *)dep[1]); - NET_EPOCH_EXIT(et); return dep[1]; } - NET_EPOCH_EXIT(et); return NULL; } @@ -1940,12 +1930,12 @@ in6_if_up(struct ifnet *ifp) arc4random() % (MAX_RTR_SOLICITATION_DELAY * hz)); } } - NET_EPOCH_EXIT(et); /* * special cases, like 6to4, are handled in in6_ifattach */ in6_ifattach(ifp, NULL); + NET_EPOCH_EXIT(et); } int @@ -2152,6 +2142,7 @@ in6_lltable_rtcheck(struct ifnet *ifp, char ip6buf[INET6_ADDRSTRLEN]; int fibnum; + NET_EPOCH_ASSERT(); KASSERT(l3addr->sa_family == AF_INET6, ("sin_family %d", l3addr->sa_family)); @@ -2160,19 +2151,15 @@ in6_lltable_rtcheck(struct ifnet *ifp, fibnum = V_rt_add_addr_allfibs ? RT_DEFAULT_FIB : ifp->if_fib; error = fib6_lookup_nh_basic(fibnum, &dst, scopeid, 0, 0, &nh6); if (error != 0 || (nh6.nh_flags & NHF_GATEWAY) || nh6.nh_ifp != ifp) { - struct epoch_tracker et; struct ifaddr *ifa; /* * Create an ND6 cache for an IPv6 neighbor * that is not covered by our own prefix. */ - NET_EPOCH_ENTER(et); ifa = ifaof_ifpforaddr(l3addr, ifp); if (ifa != NULL) { - NET_EPOCH_EXIT(et); return 0; } - NET_EPOCH_EXIT(et); log(LOG_INFO, "IPv6 address: \"%s\" is not on the network\n", ip6_sprintf(ip6buf, &sin6->sin6_addr)); return EINVAL; diff --git a/sys/netinet6/in6_ifattach.c b/sys/netinet6/in6_ifattach.c index 7ce680f2f91a..fd46f4494a43 100644 --- a/sys/netinet6/in6_ifattach.c +++ b/sys/netinet6/in6_ifattach.c @@ -240,7 +240,6 @@ generate_tmp_ifid(u_int8_t *seed0, const u_int8_t *seed1, u_int8_t *ret) int in6_get_hw_ifid(struct ifnet *ifp, struct in6_addr *in6) { - struct epoch_tracker et; struct ifaddr *ifa; struct sockaddr_dl *sdl; u_int8_t *addr; @@ -249,7 +248,8 @@ in6_get_hw_ifid(struct ifnet *ifp, struct in6_addr *in6) static u_int8_t allone[8] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_LINK) continue; @@ -261,12 +261,10 @@ in6_get_hw_ifid(struct ifnet *ifp, struct in6_addr *in6) goto found; } - NET_EPOCH_EXIT(et); return -1; found: - IF_ADDR_LOCK_ASSERT(ifp); addr = LLADDR(sdl); addrlen = sdl->sdl_alen; @@ -283,24 +281,18 @@ found: addrlen = 8; /* look at IEEE802/EUI64 only */ - if (addrlen != 8 && addrlen != 6) { - NET_EPOCH_EXIT(et); + if (addrlen != 8 && addrlen != 6) return -1; - } /* * check for invalid MAC address - on bsdi, we see it a lot * since wildboar configures all-zero MAC on pccard before * card insertion. */ - if (bcmp(addr, allzero, addrlen) == 0) { - NET_EPOCH_EXIT(et); + if (bcmp(addr, allzero, addrlen) == 0) return -1; - } - if (bcmp(addr, allone, addrlen) == 0) { - NET_EPOCH_EXIT(et); + if (bcmp(addr, allone, addrlen) == 0) return -1; - } /* make EUI64 address */ if (addrlen == 8) @@ -325,27 +317,21 @@ found: * identifier source (can be renumbered). * we don't do this. */ - NET_EPOCH_EXIT(et); return -1; case IFT_INFINIBAND: - if (addrlen != 20) { - NET_EPOCH_EXIT(et); + if (addrlen != 20) return -1; - } bcopy(addr + 12, &in6->s6_addr[8], 8); break; default: - NET_EPOCH_EXIT(et); return -1; } /* sanity check: g bit must not indicate "group" */ - if (EUI64_GROUP(in6)) { - NET_EPOCH_EXIT(et); + if (EUI64_GROUP(in6)) return -1; - } /* convert EUI64 into IPv6 interface identifier */ EUI64_TO_IFID(in6); @@ -355,12 +341,9 @@ found: * subnet router anycast */ if ((in6->s6_addr[8] & ~(EUI64_GBIT | EUI64_UBIT)) == 0x00 && - bcmp(&in6->s6_addr[9], allzero, 7) == 0) { - NET_EPOCH_EXIT(et); + bcmp(&in6->s6_addr[9], allzero, 7) == 0) return -1; - } - NET_EPOCH_EXIT(et); return 0; } @@ -375,9 +358,10 @@ static int get_ifid(struct ifnet *ifp0, struct ifnet *altifp, struct in6_addr *in6) { - struct epoch_tracker et; struct ifnet *ifp; + NET_EPOCH_ASSERT(); + /* first, try to get it from the interface itself */ if (in6_get_hw_ifid(ifp0, in6) == 0) { nd6log((LOG_DEBUG, "%s: got interface identifier from itself\n", @@ -393,7 +377,6 @@ get_ifid(struct ifnet *ifp0, struct ifnet *altifp, } /* next, try to get it from some other hardware interface */ - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if (ifp == ifp0) continue; @@ -408,11 +391,9 @@ get_ifid(struct ifnet *ifp0, struct ifnet *altifp, nd6log((LOG_DEBUG, "%s: borrow interface identifier from %s\n", if_name(ifp0), if_name(ifp))); - NET_EPOCH_EXIT(et); goto success; } } - NET_EPOCH_EXIT(et); /* last resort: get from random number source */ if (get_rand_ifid(ifp, in6) == 0) { @@ -686,6 +667,8 @@ in6_ifattach(struct ifnet *ifp, struct ifnet *altifp) { struct in6_ifaddr *ia; + NET_EPOCH_ASSERT(); + if (ifp->if_afdata[AF_INET6] == NULL) return; /* diff --git a/sys/netinet6/in6_mcast.c b/sys/netinet6/in6_mcast.c index 904154f56da5..5e7969d2a6da 100644 --- a/sys/netinet6/in6_mcast.c +++ b/sys/netinet6/in6_mcast.c @@ -2770,8 +2770,10 @@ sysctl_ip6_mcast_filters(SYSCTL_HANDLER_ARGS) return (EINVAL); } + NET_EPOCH_ENTER(et); ifp = ifnet_byindex(ifindex); if (ifp == NULL) { + NET_EPOCH_EXIT(et); CTR2(KTR_MLD, "%s: no ifp for ifindex %u", __func__, ifindex); return (ENOENT); @@ -2783,12 +2785,13 @@ sysctl_ip6_mcast_filters(SYSCTL_HANDLER_ARGS) retval = sysctl_wire_old_buffer(req, sizeof(uint32_t) + (in6_mcast_maxgrpsrc * sizeof(struct in6_addr))); - if (retval) + if (retval) { + NET_EPOCH_EXIT(et); return (retval); + } IN6_MULTI_LOCK(); IN6_MULTI_LIST_LOCK(); - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { inm = in6m_ifmultiaddr_get_inm(ifma); if (inm == NULL) @@ -2816,10 +2819,9 @@ sysctl_ip6_mcast_filters(SYSCTL_HANDLER_ARGS) break; } } - NET_EPOCH_EXIT(et); - IN6_MULTI_LIST_UNLOCK(); IN6_MULTI_UNLOCK(); + NET_EPOCH_EXIT(et); return (retval); } diff --git a/sys/netinet6/in6_var.h b/sys/netinet6/in6_var.h index 3e535310b29e..9856acd87429 100644 --- a/sys/netinet6/in6_var.h +++ b/sys/netinet6/in6_var.h @@ -785,6 +785,8 @@ in6m_ifmultiaddr_get_inm(struct ifmultiaddr *ifma) * Look up an in6_multi record for an IPv6 multicast address * on the interface ifp. * If no record found, return NULL. + * + * SMPng: The IN6_MULTI_LOCK and must be held and must be in network epoch. */ static __inline struct in6_multi * in6m_lookup_locked(struct ifnet *ifp, const struct in6_addr *mcaddr) @@ -805,18 +807,17 @@ in6m_lookup_locked(struct ifnet *ifp, const struct in6_addr *mcaddr) /* * Wrapper for in6m_lookup_locked(). * - * SMPng: Assumes that neithr the IN6_MULTI_LOCK() or IF_ADDR_LOCK() are held. + * SMPng: Assumes network epoch entered and that IN6_MULTI_LOCK() isn't held. */ static __inline struct in6_multi * in6m_lookup(struct ifnet *ifp, const struct in6_addr *mcaddr) { - struct epoch_tracker et; struct in6_multi *inm; + NET_EPOCH_ASSERT(); + IN6_MULTI_LIST_LOCK(); - NET_EPOCH_ENTER(et); inm = in6m_lookup_locked(ifp, mcaddr); - NET_EPOCH_EXIT(et); IN6_MULTI_LIST_UNLOCK(); return (inm); diff --git a/sys/netinet6/ip6_output.c b/sys/netinet6/ip6_output.c index 08dffde61721..4cf4288b1b5b 100644 --- a/sys/netinet6/ip6_output.c +++ b/sys/netinet6/ip6_output.c @@ -384,6 +384,7 @@ ip6_output(struct mbuf *m0, struct ip6_pktopts *opt, struct mbuf *m = m0; struct mbuf *mprev = NULL; int hlen, tlen, len; + struct epoch_tracker et; struct route_in6 ip6route; struct rtentry *rt = NULL; struct sockaddr_in6 *dst, src_sa, dst_sa; @@ -586,6 +587,7 @@ ip6_output(struct mbuf *m0, struct ip6_pktopts *opt, ro = &opt->ip6po_route; dst = (struct sockaddr_in6 *)&ro->ro_dst; fibnum = (inp != NULL) ? inp->inp_inc.inc_fibnum : M_GETFIB(m); + NET_EPOCH_ENTER(et); again: /* * if specified, try to fill in the traffic class field. @@ -1186,6 +1188,7 @@ sendorfree: IP6STAT_INC(ip6s_fragmented); done: + NET_EPOCH_EXIT(et); if (ro == &ip6route) RO_RTFREE(ro); return (error); diff --git a/sys/netinet6/mld6.c b/sys/netinet6/mld6.c index 6ea63b47b47a..868edeaff8c5 100644 --- a/sys/netinet6/mld6.c +++ b/sys/netinet6/mld6.c @@ -636,7 +636,6 @@ static int mld_v1_input_query(struct ifnet *ifp, const struct ip6_hdr *ip6, /*const*/ struct mld_hdr *mld) { - struct epoch_tracker et; struct ifmultiaddr *ifma; struct mld_ifsoftc *mli; struct in6_multi *inm; @@ -646,6 +645,8 @@ mld_v1_input_query(struct ifnet *ifp, const struct ip6_hdr *ip6, char ip6tbuf[INET6_ADDRSTRLEN]; #endif + NET_EPOCH_ASSERT(); + is_general_query = 0; if (!mld_v1enable) { @@ -704,7 +705,6 @@ mld_v1_input_query(struct ifnet *ifp, const struct ip6_hdr *ip6, if (timer == 0) timer = 1; - NET_EPOCH_ENTER(et); if (is_general_query) { /* * For each reporting group joined on this @@ -735,7 +735,6 @@ mld_v1_input_query(struct ifnet *ifp, const struct ip6_hdr *ip6, in6_clearscope(&mld->mld_addr); } - NET_EPOCH_EXIT(et); MLD_UNLOCK(); IN6_MULTI_LIST_UNLOCK(); @@ -824,6 +823,8 @@ mld_v2_input_query(struct ifnet *ifp, const struct ip6_hdr *ip6, char ip6tbuf[INET6_ADDRSTRLEN]; #endif + NET_EPOCH_ASSERT(); + if (!mld_v2enable) { CTR3(KTR_MLD, "ignore v2 query src %s on ifp %p(%s)", ip6_sprintf(ip6tbuf, &ip6->ip6_src), @@ -937,8 +938,6 @@ mld_v2_input_query(struct ifnet *ifp, const struct ip6_hdr *ip6, V_interface_timers_running6 = 1; } } else { - struct epoch_tracker et; - /* * MLDv2 Group-specific or Group-and-source-specific Query. * @@ -947,18 +946,14 @@ mld_v2_input_query(struct ifnet *ifp, const struct ip6_hdr *ip6, * Queries for groups we are not a member of on this * link are simply ignored. */ - NET_EPOCH_ENTER(et); inm = in6m_lookup_locked(ifp, &mld->mld_addr); - if (inm == NULL) { - NET_EPOCH_EXIT(et); + if (inm == NULL) goto out_locked; - } if (nsrc > 0) { if (!ratecheck(&inm->in6m_lastgsrtv, &V_mld_gsrdelay)) { CTR1(KTR_MLD, "%s: GS query throttled.", __func__); - NET_EPOCH_EXIT(et); goto out_locked; } } @@ -976,7 +971,6 @@ mld_v2_input_query(struct ifnet *ifp, const struct ip6_hdr *ip6, /* XXX Clear embedded scope ID as userland won't expect it. */ in6_clearscope(&mld->mld_addr); - NET_EPOCH_EXIT(et); } out_locked: @@ -1110,13 +1104,14 @@ mld_v1_input_report(struct ifnet *ifp, const struct ip6_hdr *ip6, /*const*/ struct mld_hdr *mld) { struct in6_addr src, dst; - struct epoch_tracker et; struct in6_ifaddr *ia; struct in6_multi *inm; #ifdef KTR char ip6tbuf[INET6_ADDRSTRLEN]; #endif + NET_EPOCH_ASSERT(); + if (!mld_v1enable) { CTR3(KTR_MLD, "ignore v1 report %s on ifp %p(%s)", ip6_sprintf(ip6tbuf, &mld->mld_addr), @@ -1186,7 +1181,6 @@ mld_v1_input_report(struct ifnet *ifp, const struct ip6_hdr *ip6, IN6_MULTI_LIST_LOCK(); MLD_LOCK(); - NET_EPOCH_ENTER(et); /* * MLDv1 report suppression. @@ -1234,7 +1228,6 @@ mld_v1_input_report(struct ifnet *ifp, const struct ip6_hdr *ip6, } out_locked: - NET_EPOCH_EXIT(et); MLD_UNLOCK(); IN6_MULTI_LIST_UNLOCK(); @@ -1412,8 +1405,8 @@ mld_fasttimo_vnet(struct in6_multi_head *inmh) mbufq_init(&scq, MLD_MAX_STATE_CHANGE_PACKETS); } - IF_ADDR_WLOCK(ifp); NET_EPOCH_ENTER(et); + IF_ADDR_WLOCK(ifp); CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { inm = in6m_ifmultiaddr_get_inm(ifma); if (inm == NULL) @@ -2995,12 +2988,12 @@ mld_v2_merge_state_changes(struct in6_multi *inm, struct mbufq *scq) static void mld_v2_dispatch_general_query(struct mld_ifsoftc *mli) { - struct epoch_tracker et; struct ifmultiaddr *ifma; struct ifnet *ifp; struct in6_multi *inm; int retval; + NET_EPOCH_ASSERT(); IN6_MULTI_LIST_LOCK_ASSERT(); MLD_LOCK_ASSERT(); @@ -3018,7 +3011,6 @@ mld_v2_dispatch_general_query(struct mld_ifsoftc *mli) ifp = mli->mli_ifp; - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { inm = in6m_ifmultiaddr_get_inm(ifma); if (inm == NULL) @@ -3047,7 +3039,6 @@ mld_v2_dispatch_general_query(struct mld_ifsoftc *mli) break; } } - NET_EPOCH_EXIT(et); send: mld_dispatch_queue(&mli->mli_gq, MLD_MAX_RESPONSE_BURST); diff --git a/sys/netinet6/nd6.c b/sys/netinet6/nd6.c index e359b72cb455..fae88877a2f5 100644 --- a/sys/netinet6/nd6.c +++ b/sys/netinet6/nd6.c @@ -732,6 +732,7 @@ nd6_llinfo_setstate(struct llentry *lle, int newstate) static __noinline void nd6_llinfo_timer(void *arg) { + struct epoch_tracker et; struct llentry *ln; struct in6_addr *dst, *pdst, *psrc, src; struct ifnet *ifp; @@ -768,6 +769,7 @@ nd6_llinfo_timer(void *arg) CURVNET_RESTORE(); return; } + NET_EPOCH_ENTER(et); ndi = ND_IFINFO(ifp); send_ns = 0; dst = &ln->r_l3addr.addr6; @@ -886,6 +888,7 @@ done: if (ln != NULL) LLE_FREE_LOCKED(ln); + NET_EPOCH_EXIT(et); CURVNET_RESTORE(); } @@ -897,6 +900,7 @@ void nd6_timer(void *arg) { CURVNET_SET((struct vnet *) arg); + struct epoch_tracker et; struct nd_drhead drq; struct nd_prhead prl; struct nd_defrouter *dr, *ndr; @@ -927,6 +931,7 @@ nd6_timer(void *arg) * * XXXRW: in6_ifaddrhead locking. */ + NET_EPOCH_ENTER(et); addrloop: CK_STAILQ_FOREACH_SAFE(ia6, &V_in6_ifaddrhead, ia_link, nia6) { /* check address lifetime */ @@ -1014,6 +1019,7 @@ nd6_timer(void *arg) ia6->ia6_flags &= ~IN6_IFF_DEPRECATED; } } + NET_EPOCH_EXIT(et); ND6_WLOCK(); restart: @@ -1069,13 +1075,13 @@ restart: static int regen_tmpaddr(struct in6_ifaddr *ia6) { - struct epoch_tracker et; struct ifaddr *ifa; struct ifnet *ifp; struct in6_ifaddr *public_ifa6 = NULL; + NET_EPOCH_ASSERT(); + ifp = ia6->ia_ifa.ifa_ifp; - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { struct in6_ifaddr *it6; @@ -1116,7 +1122,6 @@ regen_tmpaddr(struct in6_ifaddr *ia6) } if (public_ifa6 != NULL) ifa_ref(&public_ifa6->ia_ifa); - NET_EPOCH_EXIT(et); if (public_ifa6 != NULL) { int e; @@ -1387,10 +1392,10 @@ restart: int nd6_is_addr_neighbor(const struct sockaddr_in6 *addr, struct ifnet *ifp) { - struct epoch_tracker et; struct llentry *lle; int rc = 0; + NET_EPOCH_ASSERT(); IF_AFDATA_UNLOCK_ASSERT(ifp); if (nd6_is_new_addr_neighbor(addr, ifp)) return (1); @@ -1399,12 +1404,10 @@ nd6_is_addr_neighbor(const struct sockaddr_in6 *addr, struct ifnet *ifp) * Even if the address matches none of our addresses, it might be * in the neighbor cache. */ - NET_EPOCH_ENTER(et); if ((lle = nd6_lookup(&addr->sin6_addr, 0, ifp)) != NULL) { LLE_RUNLOCK(lle); rc = 1; } - NET_EPOCH_EXIT(et); return (rc); } @@ -1688,6 +1691,7 @@ nd6_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp) struct ifaddr *ifa; struct in6_ifaddr *ia; + NET_EPOCH_ENTER(et); if ((ND_IFINFO(ifp)->flags & ND6_IFF_IFDISABLED) && !(ND.flags & ND6_IFF_IFDISABLED)) { /* ifdisabled 1->0 transision */ @@ -1698,7 +1702,6 @@ nd6_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp) * do not clear ND6_IFF_IFDISABLED. * See RFC 4862, Section 5.4.5. */ - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; @@ -1707,7 +1710,6 @@ nd6_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp) IN6_IS_ADDR_LINKLOCAL(IA6_IN6(ia))) break; } - NET_EPOCH_EXIT(et); if (ifa != NULL) { /* LLA is duplicated. */ @@ -1728,7 +1730,6 @@ nd6_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp) ND_IFINFO(ifp)->flags |= ND6_IFF_IFDISABLED; if (V_ip6_dad_count > 0 && (ND_IFINFO(ifp)->flags & ND6_IFF_NO_DAD) == 0) { - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != @@ -1737,7 +1738,6 @@ nd6_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp) ia = (struct in6_ifaddr *)ifa; ia->ia6_flags |= IN6_IFF_TENTATIVE; } - NET_EPOCH_EXIT(et); } } @@ -1756,7 +1756,6 @@ nd6_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp) * address is assigned, and IFF_UP, try to * assign one. */ - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != @@ -1766,15 +1765,15 @@ nd6_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp) if (IN6_IS_ADDR_LINKLOCAL(IA6_IN6(ia))) break; } - NET_EPOCH_EXIT(et); if (ifa != NULL) /* No LLA is configured. */ in6_ifattach(ifp, NULL); } } - } ND_IFINFO(ifp)->flags = ND.flags; + NET_EPOCH_EXIT(et); break; + } #undef ND case SIOCSNDFLUSH_IN6: /* XXX: the ioctl name is confusing... */ /* sync kernel routing table with the default router list */ @@ -1970,12 +1969,12 @@ nd6_cache_lladdr(struct ifnet *ifp, struct in6_addr *from, char *lladdr, int flags; uint16_t router = 0; struct sockaddr_in6 sin6; - struct epoch_tracker et; struct mbuf *chain = NULL; u_char linkhdr[LLE_MAX_LINKHDR]; size_t linkhdrsize; int lladdr_off; + NET_EPOCH_ASSERT(); IF_AFDATA_UNLOCK_ASSERT(ifp); KASSERT(ifp != NULL, ("%s: ifp == NULL", __func__)); @@ -1995,9 +1994,7 @@ nd6_cache_lladdr(struct ifnet *ifp, struct in6_addr *from, char *lladdr, * description on it in NS section (RFC 2461 7.2.3). */ flags = lladdr ? LLE_EXCLUSIVE : 0; - NET_EPOCH_ENTER(et); ln = nd6_lookup(from, flags, ifp); - NET_EPOCH_EXIT(et); is_newentry = 0; if (ln == NULL) { flags |= LLE_EXCLUSIVE; @@ -2256,10 +2253,11 @@ nd6_resolve(struct ifnet *ifp, int is_gw, struct mbuf *m, const struct sockaddr *sa_dst, u_char *desten, uint32_t *pflags, struct llentry **plle) { - struct epoch_tracker et; struct llentry *ln = NULL; const struct sockaddr_in6 *dst6; + NET_EPOCH_ASSERT(); + if (pflags != NULL) *pflags = 0; @@ -2285,7 +2283,6 @@ nd6_resolve(struct ifnet *ifp, int is_gw, struct mbuf *m, } } - NET_EPOCH_ENTER(et); ln = nd6_lookup(&dst6->sin6_addr, plle ? LLE_EXCLUSIVE : LLE_UNLOCKED, ifp); if (ln != NULL && (ln->r_flags & RLLE_VALID) != 0) { @@ -2305,11 +2302,9 @@ nd6_resolve(struct ifnet *ifp, int is_gw, struct mbuf *m, *plle = ln; LLE_WUNLOCK(ln); } - NET_EPOCH_EXIT(et); return (0); } else if (plle && ln) LLE_WUNLOCK(ln); - NET_EPOCH_EXIT(et); return (nd6_resolve_slow(ifp, 0, m, dst6, desten, pflags, plle)); } @@ -2336,6 +2331,8 @@ nd6_resolve_slow(struct ifnet *ifp, int flags, struct mbuf *m, int send_ns, ll_len; char *lladdr; + NET_EPOCH_ASSERT(); + /* * Address resolution or Neighbor Unreachability Detection * for the next hop. @@ -2343,11 +2340,7 @@ nd6_resolve_slow(struct ifnet *ifp, int flags, struct mbuf *m, * or an anycast address(i.e. not a multicast). */ if (lle == NULL) { - struct epoch_tracker et; - - NET_EPOCH_ENTER(et); lle = nd6_lookup(&dst->sin6_addr, LLE_EXCLUSIVE, ifp); - NET_EPOCH_EXIT(et); if ((lle == NULL) && nd6_is_addr_neighbor(dst, ifp)) { /* * Since nd6_is_addr_neighbor() internally calls nd6_lookup(), diff --git a/sys/netinet6/nd6_nbr.c b/sys/netinet6/nd6_nbr.c index cbac0dcb724e..3624909cc164 100644 --- a/sys/netinet6/nd6_nbr.c +++ b/sys/netinet6/nd6_nbr.c @@ -614,7 +614,6 @@ nd6_ns_output(struct ifnet *ifp, const struct in6_addr *saddr6, void nd6_na_input(struct mbuf *m, int off, int icmp6len) { - struct epoch_tracker et; struct ifnet *ifp = m->m_pkthdr.rcvif; struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *); struct nd_neighbor_advert *nd_na; @@ -637,6 +636,8 @@ nd6_na_input(struct mbuf *m, int off, int icmp6len) int lladdr_off; char ip6bufs[INET6_ADDRSTRLEN], ip6bufd[INET6_ADDRSTRLEN]; + NET_EPOCH_ASSERT(); + /* RFC 6980: Nodes MUST silently ignore fragments */ if(m->m_flags & M_FRAGMENTED) goto freeit; @@ -742,9 +743,7 @@ nd6_na_input(struct mbuf *m, int off, int icmp6len) * If no neighbor cache entry is found, NA SHOULD silently be * discarded. */ - NET_EPOCH_ENTER(et); ln = nd6_lookup(&taddr6, LLE_EXCLUSIVE, ifp); - NET_EPOCH_EXIT(et); if (ln == NULL) { goto freeit; } @@ -1193,8 +1192,13 @@ static void nd6_dad_starttimer(struct dadq *dp, int ticks, int send_ns) { - if (send_ns != 0) + if (send_ns != 0) { + struct epoch_tracker et; + + NET_EPOCH_ENTER(et); nd6_dad_ns_output(dp); + NET_EPOCH_EXIT(et); + } callout_reset(&dp->dad_timer_ch, ticks, (void (*)(void *))nd6_dad_timer, (void *)dp); } diff --git a/sys/netinet6/nd6_rtr.c b/sys/netinet6/nd6_rtr.c index d0b8dc51eba7..03442745f96a 100644 --- a/sys/netinet6/nd6_rtr.c +++ b/sys/netinet6/nd6_rtr.c @@ -1260,7 +1260,10 @@ nd6_prelist_add(struct nd_prefixctl *pr, struct nd_defrouter *dr, /* ND_OPT_PI_FLAG_ONLINK processing */ if (new->ndpr_raf_onlink) { + struct epoch_tracker et; + ND6_ONLINK_LOCK(); + NET_EPOCH_ENTER(et); if ((error = nd6_prefix_onlink(new)) != 0) { nd6log((LOG_ERR, "nd6_prelist_add: failed to make " "the prefix %s/%d on-link on %s (errno=%d)\n", @@ -1268,6 +1271,7 @@ nd6_prelist_add(struct nd_prefixctl *pr, struct nd_defrouter *dr, pr->ndpr_plen, if_name(pr->ndpr_ifp), error)); /* proceed anyway. XXX: is it correct? */ } + NET_EPOCH_EXIT(et); ND6_ONLINK_UNLOCK(); } @@ -1351,7 +1355,8 @@ prelist_update(struct nd_prefixctl *new, struct nd_defrouter *dr, int auth; struct in6_addrlifetime lt6_tmp; char ip6buf[INET6_ADDRSTRLEN]; - struct epoch_tracker et; + + NET_EPOCH_ASSERT(); auth = 0; if (m) { @@ -1465,7 +1470,6 @@ prelist_update(struct nd_prefixctl *new, struct nd_defrouter *dr, * consider autoconfigured addresses while RFC2462 simply said * "address". */ - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { struct in6_ifaddr *ifa6; u_int32_t remaininglifetime; @@ -1588,7 +1592,6 @@ prelist_update(struct nd_prefixctl *new, struct nd_defrouter *dr, ifa6->ia6_lifetime = lt6_tmp; ifa6->ia6_updatetime = time_uptime; } - NET_EPOCH_EXIT(et); if (ia6_match == NULL && new->ndpr_vltime) { int ifidlen; @@ -1684,10 +1687,9 @@ find_pfxlist_reachable_router(struct nd_prefix *pr) ND6_LOCK_ASSERT(); + NET_EPOCH_ENTER(et); LIST_FOREACH(pfxrtr, &pr->ndpr_advrtrs, pfr_entry) { - NET_EPOCH_ENTER(et); ln = nd6_lookup(&pfxrtr->router->rtaddr, 0, pfxrtr->router->ifp); - NET_EPOCH_EXIT(et); if (ln == NULL) continue; canreach = ND6_IS_LLINFO_PROBREACH(ln); @@ -1695,6 +1697,7 @@ find_pfxlist_reachable_router(struct nd_prefix *pr) if (canreach) break; } + NET_EPOCH_EXIT(et); return (pfxrtr); } diff --git a/sys/netinet6/raw_ip6.c b/sys/netinet6/raw_ip6.c index a0d06230e852..ef903bc6bd66 100644 --- a/sys/netinet6/raw_ip6.c +++ b/sys/netinet6/raw_ip6.c @@ -734,12 +734,12 @@ rip6_disconnect(struct socket *so) static int rip6_bind(struct socket *so, struct sockaddr *nam, struct thread *td) { - struct epoch_tracker et; struct inpcb *inp; struct sockaddr_in6 *addr = (struct sockaddr_in6 *)nam; struct ifaddr *ifa = NULL; int error = 0; + NET_EPOCH_ASSERT(); inp = sotoinpcb(so); KASSERT(inp != NULL, ("rip6_bind: inp == NULL")); @@ -752,20 +752,14 @@ rip6_bind(struct socket *so, struct sockaddr *nam, struct thread *td) if ((error = sa6_embedscope(addr, V_ip6_use_defzone)) != 0) return (error); - NET_EPOCH_ENTER(et); if (!IN6_IS_ADDR_UNSPECIFIED(&addr->sin6_addr) && - (ifa = ifa_ifwithaddr((struct sockaddr *)addr)) == NULL) { - NET_EPOCH_EXIT(et); + (ifa = ifa_ifwithaddr((struct sockaddr *)addr)) == NULL) return (EADDRNOTAVAIL); - } if (ifa != NULL && ((struct in6_ifaddr *)ifa)->ia6_flags & (IN6_IFF_ANYCAST|IN6_IFF_NOTREADY| - IN6_IFF_DETACHED|IN6_IFF_DEPRECATED)) { - NET_EPOCH_EXIT(et); + IN6_IFF_DETACHED|IN6_IFF_DEPRECATED)) return (EADDRNOTAVAIL); - } - NET_EPOCH_EXIT(et); INP_INFO_WLOCK(&V_ripcbinfo); INP_WLOCK(inp); inp->in6p_laddr = addr->sin6_addr; diff --git a/sys/netipsec/xform_ipcomp.c b/sys/netipsec/xform_ipcomp.c index df09eb36cfa3..96cffd6305a4 100644 --- a/sys/netipsec/xform_ipcomp.c +++ b/sys/netipsec/xform_ipcomp.c @@ -52,6 +52,8 @@ #include <net/netisr.h> #include <net/vnet.h> +#include <net/if.h> /* XXXGL: net_epoch should move out there */ +#include <net/if_var.h> /* XXXGL: net_epoch should move out there */ #include <netipsec/ipsec.h> #include <netipsec/xform.h> @@ -122,6 +124,8 @@ ipcomp_nonexp_input(struct mbuf *m, int off, int proto, void *arg __unused) { int isr; + NET_EPOCH_ASSERT(); + switch (proto) { #ifdef INET case IPPROTO_IPV4: diff --git a/sys/netpfil/ipfw/ip_dn_io.c b/sys/netpfil/ipfw/ip_dn_io.c index 0bd869076779..c5f0f1518751 100644 --- a/sys/netpfil/ipfw/ip_dn_io.c +++ b/sys/netpfil/ipfw/ip_dn_io.c @@ -50,6 +50,7 @@ __FBSDID("$FreeBSD$"); #include <sys/sysctl.h> #include <net/if.h> /* IFNAMSIZ, struct ifaddr, ifq head, lock.h mutex.h */ +#include <net/if_var.h> /* NET_EPOCH_... */ #include <net/netisr.h> #include <net/vnet.h> @@ -741,6 +742,8 @@ dummynet_send(struct mbuf *m) { struct mbuf *n; + NET_EPOCH_ASSERT(); + for (; m != NULL; m = n) { struct ifnet *ifp = NULL; /* gcc 3.4.6 complains */ struct m_tag *tag; diff --git a/sys/netpfil/pf/pf_if.c b/sys/netpfil/pf/pf_if.c index f187db2a224c..62653c877122 100644 --- a/sys/netpfil/pf/pf_if.c +++ b/sys/netpfil/pf/pf_if.c @@ -76,8 +76,8 @@ eventhandler_tag pfi_change_group_cookie; eventhandler_tag pfi_detach_group_cookie; eventhandler_tag pfi_ifaddr_event_cookie; -static void pfi_attach_ifnet(struct ifnet *); -static void pfi_attach_ifgroup(struct ifg_group *); +static void pfi_attach_ifnet(struct ifnet *, struct pfi_kif *); +static void pfi_attach_ifgroup(struct ifg_group *, struct pfi_kif *); static void pfi_kif_update(struct pfi_kif *); static void pfi_dynaddr_update(struct pfi_dynaddr *dyn); @@ -114,25 +114,49 @@ MTX_SYSINIT(pfi_unlnkdkifs_mtx, &pfi_unlnkdkifs_mtx, "pf unlinked interfaces", void pfi_initialize_vnet(void) { + struct pfi_list kifs = LIST_HEAD_INITIALIZER(); + struct epoch_tracker et; + struct pfi_kif *kif; struct ifg_group *ifg; struct ifnet *ifp; - struct pfi_kif *kif; + int nkifs; V_pfi_buffer_max = 64; V_pfi_buffer = malloc(V_pfi_buffer_max * sizeof(*V_pfi_buffer), PFI_MTYPE, M_WAITOK); - kif = malloc(sizeof(*kif), PFI_MTYPE, M_WAITOK); - PF_RULES_WLOCK(); - V_pfi_all = pfi_kif_attach(kif, IFG_ALL); - PF_RULES_WUNLOCK(); - + nkifs = 1; /* one for V_pfi_all */ IFNET_RLOCK(); CK_STAILQ_FOREACH(ifg, &V_ifg_head, ifg_next) - pfi_attach_ifgroup(ifg); + nkifs++; CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) - pfi_attach_ifnet(ifp); + nkifs++; + + for (int n = 0; n < nkifs; n++) { + kif = malloc(sizeof(*kif), PFI_MTYPE, M_WAITOK); + LIST_INSERT_HEAD(&kifs, kif, pfik_list); + } + + NET_EPOCH_ENTER(et); + PF_RULES_WLOCK(); + kif = LIST_FIRST(&kifs); + LIST_REMOVE(kif, pfik_list); + V_pfi_all = pfi_kif_attach(kif, IFG_ALL); + CK_STAILQ_FOREACH(ifg, &V_ifg_head, ifg_next) { + kif = LIST_FIRST(&kifs); + LIST_REMOVE(kif, pfik_list); + pfi_attach_ifgroup(ifg, kif); + } + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { + kif = LIST_FIRST(&kifs); + LIST_REMOVE(kif, pfik_list); + pfi_attach_ifnet(ifp, kif); + } + PF_RULES_WUNLOCK(); + NET_EPOCH_EXIT(et); IFNET_RUNLOCK(); + + MPASS(LIST_EMPTY(&kifs)); } void @@ -296,59 +320,44 @@ pfi_kif_match(struct pfi_kif *rule_kif, struct pfi_kif *packet_kif) { struct ifg_list *p; + NET_EPOCH_ASSERT(); + if (rule_kif == NULL || rule_kif == packet_kif) return (1); if (rule_kif->pfik_group != NULL) { - struct epoch_tracker et; - - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(p, &packet_kif->pfik_ifp->if_groups, ifgl_next) - if (p->ifgl_group == rule_kif->pfik_group) { - NET_EPOCH_EXIT(et); + if (p->ifgl_group == rule_kif->pfik_group) return (1); - } - NET_EPOCH_EXIT(et); } - return (0); } static void -pfi_attach_ifnet(struct ifnet *ifp) +pfi_attach_ifnet(struct ifnet *ifp, struct pfi_kif *kif) { - struct pfi_kif *kif; - kif = malloc(sizeof(*kif), PFI_MTYPE, M_WAITOK); + PF_RULES_WASSERT(); - PF_RULES_WLOCK(); V_pfi_update++; kif = pfi_kif_attach(kif, ifp->if_xname); - if_ref(ifp); - kif->pfik_ifp = ifp; ifp->if_pf_kif = kif; - pfi_kif_update(kif); - PF_RULES_WUNLOCK(); } static void -pfi_attach_ifgroup(struct ifg_group *ifg) +pfi_attach_ifgroup(struct ifg_group *ifg, struct pfi_kif *kif) { - struct pfi_kif *kif; - kif = malloc(sizeof(*kif), PFI_MTYPE, M_WAITOK); + PF_RULES_WASSERT(); - PF_RULES_WLOCK(); V_pfi_update++; kif = pfi_kif_attach(kif, ifg->ifg_group); - kif->pfik_group = ifg; ifg->ifg_pf_kif = kif; - PF_RULES_WUNLOCK(); } int @@ -389,6 +398,7 @@ pfi_match_addr(struct pfi_dynaddr *dyn, struct pf_addr *a, sa_family_t af) int pfi_dynaddr_setup(struct pf_addr_wrap *aw, sa_family_t af) { + struct epoch_tracker et; struct pfi_dynaddr *dyn; char tblname[PF_TABLE_NAME_SIZE]; struct pf_ruleset *ruleset = NULL; @@ -445,7 +455,9 @@ pfi_dynaddr_setup(struct pf_addr_wrap *aw, sa_family_t af) TAILQ_INSERT_TAIL(&dyn->pfid_kif->pfik_dynaddrs, dyn, entry); aw->p.dyn = dyn; + NET_EPOCH_ENTER(et); pfi_kif_update(dyn->pfid_kif); + NET_EPOCH_EXIT(et); return (0); @@ -467,6 +479,7 @@ pfi_kif_update(struct pfi_kif *kif) struct ifg_list *ifgl; struct pfi_dynaddr *p; + NET_EPOCH_ASSERT(); PF_RULES_WASSERT(); /* update all dynaddr */ @@ -475,13 +488,9 @@ pfi_kif_update(struct pfi_kif *kif) /* again for all groups kif is member of */ if (kif->pfik_ifp != NULL) { - struct epoch_tracker et; - - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifgl, &kif->pfik_ifp->if_groups, ifgl_next) pfi_kif_update((struct pfi_kif *) ifgl->ifgl_group->ifg_pf_kif); - NET_EPOCH_EXIT(et); } } @@ -512,17 +521,15 @@ pfi_table_update(struct pfr_ktable *kt, struct pfi_kif *kif, int net, int flags) int e, size2 = 0; struct ifg_member *ifgm; + NET_EPOCH_ASSERT(); + V_pfi_buffer_cnt = 0; if (kif->pfik_ifp != NULL) pfi_instance_add(kif->pfik_ifp, net, flags); else if (kif->pfik_group != NULL) { - struct epoch_tracker et; - - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifgm, &kif->pfik_group->ifg_members, ifgm_next) pfi_instance_add(ifgm->ifgm_ifp, net, flags); - NET_EPOCH_EXIT(et); } if ((e = pfr_set_addrs(&kt->pfrkt_t, V_pfi_buffer, V_pfi_buffer_cnt, &size2, @@ -534,12 +541,12 @@ pfi_table_update(struct pfr_ktable *kt, struct pfi_kif *kif, int net, int flags) static void pfi_instance_add(struct ifnet *ifp, int net, int flags) { - struct epoch_tracker et; struct ifaddr *ia; int got4 = 0, got6 = 0; int net2, af; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + CK_STAILQ_FOREACH(ia, &ifp->if_addrhead, ifa_link) { if (ia->ifa_addr == NULL) continue; @@ -597,7 +604,6 @@ pfi_instance_add(struct ifnet *ifp, int net, int flags) else pfi_address_add(ia->ifa_addr, af, net2); } - NET_EPOCH_EXIT(et); } static void @@ -732,9 +738,11 @@ pfi_update_status(const char *name, struct pf_status *pfs) void pfi_get_ifaces(const char *name, struct pfi_kif *buf, int *size) { + struct epoch_tracker et; struct pfi_kif *p, *nextp; int n = 0; + NET_EPOCH_ENTER(et); for (p = RB_MIN(pfi_ifhead, &V_pfi_ifs); p; p = nextp) { nextp = RB_NEXT(pfi_ifhead, &V_pfi_ifs, p); if (pfi_skip_if(name, p)) @@ -747,6 +755,7 @@ pfi_get_ifaces(const char *name, struct pfi_kif *buf, int *size) nextp = RB_NEXT(pfi_ifhead, &V_pfi_ifs, p); } *size = n; + NET_EPOCH_EXIT(et); } static int @@ -755,6 +764,8 @@ pfi_skip_if(const char *filter, struct pfi_kif *p) struct ifg_list *i; int n; + NET_EPOCH_ASSERT(); + if (filter == NULL || !*filter) return (0); if (!strcmp(p->pfik_name, filter)) @@ -764,45 +775,43 @@ pfi_skip_if(const char *filter, struct pfi_kif *p) return (1); /* sanity check */ if (filter[n-1] >= '0' && filter[n-1] <= '9') return (1); /* group names may not end in a digit */ - if (p->pfik_ifp != NULL) { - struct epoch_tracker et; - - NET_EPOCH_ENTER(et); - CK_STAILQ_FOREACH(i, &p->pfik_ifp->if_groups, ifgl_next) { - if (!strncmp(i->ifgl_group->ifg_group, filter, - IFNAMSIZ)) { - NET_EPOCH_EXIT(et); - return (0); /* iface is in group "filter" */ - } - } - NET_EPOCH_EXIT(et); - } + if (p->pfik_ifp == NULL) + return (1); + CK_STAILQ_FOREACH(i, &p->pfik_ifp->if_groups, ifgl_next) + if (!strncmp(i->ifgl_group->ifg_group, filter, IFNAMSIZ)) + return (0); /* iface is in group "filter" */ return (1); } int pfi_set_flags(const char *name, int flags) { + struct epoch_tracker et; struct pfi_kif *p; + NET_EPOCH_ENTER(et); RB_FOREACH(p, pfi_ifhead, &V_pfi_ifs) { if (pfi_skip_if(name, p)) continue; p->pfik_flags |= flags; } + NET_EPOCH_EXIT(et); return (0); } int pfi_clear_flags(const char *name, int flags) { + struct epoch_tracker et; struct pfi_kif *p; + NET_EPOCH_ENTER(et); RB_FOREACH(p, pfi_ifhead, &V_pfi_ifs) { if (pfi_skip_if(name, p)) continue; p->pfik_flags &= ~flags; } + NET_EPOCH_EXIT(et); return (0); } @@ -829,22 +838,28 @@ pfi_unmask(void *addr) static void pfi_attach_ifnet_event(void *arg __unused, struct ifnet *ifp) { + struct epoch_tracker et; + struct pfi_kif *kif; if (V_pf_vnet_active == 0) { /* Avoid teardown race in the least expensive way. */ return; } - pfi_attach_ifnet(ifp); -#ifdef ALTQ + kif = malloc(sizeof(*kif), PFI_MTYPE, M_WAITOK); + NET_EPOCH_ENTER(et); PF_RULES_WLOCK(); + pfi_attach_ifnet(ifp, kif); +#ifdef ALTQ pf_altq_ifnet_event(ifp, 0); - PF_RULES_WUNLOCK(); #endif + PF_RULES_WUNLOCK(); + NET_EPOCH_EXIT(et); } static void pfi_detach_ifnet_event(void *arg __unused, struct ifnet *ifp) { + struct epoch_tracker et; struct pfi_kif *kif = (struct pfi_kif *)ifp->if_pf_kif; if (pfsync_detach_ifnet_ptr) @@ -858,6 +873,7 @@ pfi_detach_ifnet_event(void *arg __unused, struct ifnet *ifp) return; } + NET_EPOCH_ENTER(et); PF_RULES_WLOCK(); V_pfi_update++; pfi_kif_update(kif); @@ -871,22 +887,31 @@ pfi_detach_ifnet_event(void *arg __unused, struct ifnet *ifp) pf_altq_ifnet_event(ifp, 1); #endif PF_RULES_WUNLOCK(); + NET_EPOCH_EXIT(et); } static void pfi_attach_group_event(void *arg __unused, struct ifg_group *ifg) { + struct epoch_tracker et; + struct pfi_kif *kif; if (V_pf_vnet_active == 0) { /* Avoid teardown race in the least expensive way. */ return; } - pfi_attach_ifgroup(ifg); + kif = malloc(sizeof(*kif), PFI_MTYPE, M_WAITOK); + NET_EPOCH_ENTER(et); + PF_RULES_WLOCK(); + pfi_attach_ifgroup(ifg, kif); + PF_RULES_WUNLOCK(); + NET_EPOCH_EXIT(et); } static void pfi_change_group_event(void *arg __unused, char *gname) { + struct epoch_tracker et; struct pfi_kif *kif; if (V_pf_vnet_active == 0) { @@ -895,11 +920,13 @@ pfi_change_group_event(void *arg __unused, char *gname) } kif = malloc(sizeof(*kif), PFI_MTYPE, M_WAITOK); + NET_EPOCH_ENTER(et); PF_RULES_WLOCK(); V_pfi_update++; kif = pfi_kif_attach(kif, gname); pfi_kif_update(kif); PF_RULES_WUNLOCK(); + NET_EPOCH_EXIT(et); } static void @@ -937,8 +964,12 @@ pfi_ifaddr_event(void *arg __unused, struct ifnet *ifp) } PF_RULES_WLOCK(); if (ifp->if_pf_kif) { + struct epoch_tracker et; + V_pfi_update++; + NET_EPOCH_ENTER(et); pfi_kif_update(ifp->if_pf_kif); + NET_EPOCH_EXIT(et); } PF_RULES_WUNLOCK(); } diff --git a/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c b/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c index bb1efcc666a1..e411000f84b3 100644 --- a/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c +++ b/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c @@ -1591,6 +1591,7 @@ bad: void ipoib_demux(struct ifnet *ifp, struct mbuf *m, u_short proto) { + struct epoch_tracker et; int isr; #ifdef MAC @@ -1632,7 +1633,9 @@ ipoib_demux(struct ifnet *ifp, struct mbuf *m, u_short proto) default: goto discard; } + NET_EPOCH_ENTER(et); netisr_dispatch(isr, m); + NET_EPOCH_EXIT(et); return; discard: |