aboutsummaryrefslogtreecommitdiff
path: root/sys/kern/uipc_sockbuf.c
diff options
context:
space:
mode:
Diffstat (limited to 'sys/kern/uipc_sockbuf.c')
-rw-r--r--sys/kern/uipc_sockbuf.c153
1 files changed, 125 insertions, 28 deletions
diff --git a/sys/kern/uipc_sockbuf.c b/sys/kern/uipc_sockbuf.c
index 00ffee326323..b5a5185d46ce 100644
--- a/sys/kern/uipc_sockbuf.c
+++ b/sys/kern/uipc_sockbuf.c
@@ -69,6 +69,43 @@ static struct mbuf *sbcut_internal(struct sockbuf *sb, int len);
static void sbflush_internal(struct sockbuf *sb);
/*
+ * Mark ready "count" mbufs starting with "m".
+ */
+int
+sbready(struct sockbuf *sb, struct mbuf *m, int count)
+{
+ u_int blocker;
+
+ SOCKBUF_LOCK_ASSERT(sb);
+ KASSERT(sb->sb_fnrdy != NULL, ("%s: sb %p NULL fnrdy", __func__, sb));
+
+ blocker = (sb->sb_fnrdy == m) ? M_BLOCKED : 0;
+
+ for (int i = 0; i < count; i++, m = m->m_next) {
+ KASSERT(m->m_flags & M_NOTREADY,
+ ("%s: m %p !M_NOTREADY", __func__, m));
+ m->m_flags &= ~(M_NOTREADY | blocker);
+ if (blocker)
+ sb->sb_acc += m->m_len;
+ }
+
+ if (!blocker)
+ return (EINPROGRESS);
+
+ /* This one was blocking all the queue. */
+ for (; m && (m->m_flags & M_NOTREADY) == 0; m = m->m_next) {
+ KASSERT(m->m_flags & M_BLOCKED,
+ ("%s: m %p !M_BLOCKED", __func__, m));
+ m->m_flags &= ~M_BLOCKED;
+ sb->sb_acc += m->m_len;
+ }
+
+ sb->sb_fnrdy = m;
+
+ return (0);
+}
+
+/*
* Adjust sockbuf state reflecting allocation of m.
*/
void
@@ -77,7 +114,15 @@ sballoc(struct sockbuf *sb, struct mbuf *m)
SOCKBUF_LOCK_ASSERT(sb);
- sb->sb_cc += m->m_len;
+ sb->sb_ccc += m->m_len;
+
+ if (sb->sb_fnrdy == NULL) {
+ if (m->m_flags & M_NOTREADY)
+ sb->sb_fnrdy = m;
+ else
+ sb->sb_acc += m->m_len;
+ } else
+ m->m_flags |= M_BLOCKED;
if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA)
sb->sb_ctl += m->m_len;
@@ -102,7 +147,25 @@ sbfree(struct sockbuf *sb, struct mbuf *m)
SOCKBUF_LOCK_ASSERT(sb);
#endif
- sb->sb_cc -= m->m_len;
+ sb->sb_ccc -= m->m_len;
+
+ if (!(m->m_flags & M_NOTAVAIL))
+ sb->sb_acc -= m->m_len;
+
+ if (m == sb->sb_fnrdy) {
+ struct mbuf *n;
+
+ KASSERT(m->m_flags & M_NOTREADY,
+ ("%s: m %p !M_NOTREADY", __func__, m));
+
+ n = m->m_next;
+ while (n != NULL && !(n->m_flags & M_NOTREADY)) {
+ n->m_flags &= ~M_BLOCKED;
+ sb->sb_acc += n->m_len;
+ n = n->m_next;
+ }
+ sb->sb_fnrdy = n;
+ }
if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA)
sb->sb_ctl -= m->m_len;
@@ -181,7 +244,7 @@ sbwait(struct sockbuf *sb)
SOCKBUF_LOCK_ASSERT(sb);
sb->sb_flags |= SB_WAIT;
- return (msleep_sbt(&sb->sb_cc, &sb->sb_mtx,
+ return (msleep_sbt(&sb->sb_acc, &sb->sb_mtx,
(sb->sb_flags & SB_NOINTR) ? PSOCK : PSOCK | PCATCH, "sbwait",
sb->sb_timeo, 0, 0));
}
@@ -238,7 +301,7 @@ sowakeup(struct socket *so, struct sockbuf *sb)
sb->sb_flags &= ~SB_SEL;
if (sb->sb_flags & SB_WAIT) {
sb->sb_flags &= ~SB_WAIT;
- wakeup(&sb->sb_cc);
+ wakeup(&sb->sb_acc);
}
KNOTE_LOCKED(&sb->sb_sel.si_note, 0);
if (sb->sb_upcall != NULL) {
@@ -609,12 +672,13 @@ sbappendstream(struct sockbuf *sb, struct mbuf *m)
void
sbcheck(struct sockbuf *sb, const char *file, int line)
{
- struct mbuf *m, *n;
- u_long cc, mbcnt;
+ struct mbuf *m, *n, *fnrdy;
+ u_long acc, ccc, mbcnt;
SOCKBUF_LOCK_ASSERT(sb);
- cc = mbcnt = 0;
+ acc = ccc = mbcnt = 0;
+ fnrdy = NULL;
for (m = sb->sb_mb; m; m = n) {
n = m->m_nextpkt;
@@ -623,15 +687,31 @@ sbcheck(struct sockbuf *sb, const char *file, int line)
printf("sb %p empty mbuf %p\n", sb, m);
goto fail;
}
- cc += m->m_len;
+ if ((m->m_flags & M_NOTREADY) && fnrdy == NULL) {
+ if (m != sb->sb_fnrdy) {
+ printf("sb %p: fnrdy %p != m %p\n",
+ sb, sb->sb_fnrdy, m);
+ goto fail;
+ }
+ fnrdy = m;
+ }
+ if (fnrdy) {
+ if (!(m->m_flags & M_NOTAVAIL)) {
+ printf("sb %p: fnrdy %p, m %p is avail\n",
+ sb, sb->sb_fnrdy, m);
+ goto fail;
+ }
+ } else
+ acc += m->m_len;
+ ccc += m->m_len;
mbcnt += MSIZE;
if (m->m_flags & M_EXT) /*XXX*/ /* pretty sure this is bogus */
mbcnt += m->m_ext.ext_size;
}
}
- if (cc != sb->sb_cc || mbcnt != sb->sb_mbcnt) {
- printf("cc %ld != %u || mbcnt %ld != %u\n", cc, sb->sb_cc,
- mbcnt, sb->sb_mbcnt);
+ if (acc != sb->sb_acc || ccc != sb->sb_ccc || mbcnt != sb->sb_mbcnt) {
+ printf("acc %ld/%u ccc %ld/%u mbcnt %ld/%u\n",
+ acc, sb->sb_acc, ccc, sb->sb_ccc, mbcnt, sb->sb_mbcnt);
goto fail;
}
return;
@@ -832,8 +912,8 @@ sbappendcontrol(struct sockbuf *sb, struct mbuf *m0, struct mbuf *control)
*
* (2) The mbuf may be coalesced -- i.e., data in the mbuf may be copied into
* an mbuf already in the socket buffer. This can occur if an
- * appropriate mbuf exists, there is room, and no merging of data types
- * will occur.
+ * appropriate mbuf exists, there is room, both mbufs are not marked as
+ * not ready, and no merging of data types will occur.
*
* (3) The mbuf may be appended to the end of the existing mbuf chain.
*
@@ -862,13 +942,17 @@ sbcompress(struct sockbuf *sb, struct mbuf *m, struct mbuf *n)
if (n && (n->m_flags & M_EOR) == 0 &&
M_WRITABLE(n) &&
((sb->sb_flags & SB_NOCOALESCE) == 0) &&
+ !(m->m_flags & M_NOTREADY) &&
+ !(n->m_flags & M_NOTREADY) &&
m->m_len <= MCLBYTES / 4 && /* XXX: Don't copy too much */
m->m_len <= M_TRAILINGSPACE(n) &&
n->m_type == m->m_type) {
bcopy(mtod(m, caddr_t), mtod(n, caddr_t) + n->m_len,
(unsigned)m->m_len);
n->m_len += m->m_len;
- sb->sb_cc += m->m_len;
+ sb->sb_ccc += m->m_len;
+ if (sb->sb_fnrdy == NULL)
+ sb->sb_acc += m->m_len;
if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA)
/* XXX: Probably don't need.*/
sb->sb_ctl += m->m_len;
@@ -905,13 +989,13 @@ sbflush_internal(struct sockbuf *sb)
* Don't call sbcut(sb, 0) if the leading mbuf is non-empty:
* we would loop forever. Panic instead.
*/
- if (!sb->sb_cc && (sb->sb_mb == NULL || sb->sb_mb->m_len))
+ if (sb->sb_ccc == 0 && (sb->sb_mb == NULL || sb->sb_mb->m_len))
break;
- m_freem(sbcut_internal(sb, (int)sb->sb_cc));
+ m_freem(sbcut_internal(sb, (int)sb->sb_ccc));
}
- if (sb->sb_cc || sb->sb_mb || sb->sb_mbcnt)
- panic("sbflush_internal: cc %u || mb %p || mbcnt %u",
- sb->sb_cc, (void *)sb->sb_mb, sb->sb_mbcnt);
+ KASSERT(sb->sb_ccc == 0 && sb->sb_mb == 0 && sb->sb_mbcnt == 0,
+ ("%s: ccc %u mb %p mbcnt %u", __func__,
+ sb->sb_ccc, (void *)sb->sb_mb, sb->sb_mbcnt));
}
void
@@ -937,7 +1021,7 @@ sbflush(struct sockbuf *sb)
static struct mbuf *
sbcut_internal(struct sockbuf *sb, int len)
{
- struct mbuf *m, *n, *next, *mfree;
+ struct mbuf *m, *next, *mfree;
next = (m = sb->sb_mb) ? m->m_nextpkt : 0;
mfree = NULL;
@@ -949,9 +1033,12 @@ sbcut_internal(struct sockbuf *sb, int len)
next = m->m_nextpkt;
}
if (m->m_len > len) {
+ KASSERT(!(m->m_flags & M_NOTAVAIL),
+ ("%s: m %p M_NOTAVAIL", __func__, m));
m->m_len -= len;
m->m_data += len;
- sb->sb_cc -= len;
+ sb->sb_ccc -= len;
+ sb->sb_acc -= len;
if (sb->sb_sndptroff != 0)
sb->sb_sndptroff -= len;
if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA)
@@ -960,10 +1047,20 @@ sbcut_internal(struct sockbuf *sb, int len)
}
len -= m->m_len;
sbfree(sb, m);
- n = m->m_next;
- m->m_next = mfree;
- mfree = m;
- m = n;
+ /*
+ * Do not put M_NOTREADY buffers to the free list, they
+ * are referenced from outside.
+ */
+ if (m->m_flags & M_NOTREADY)
+ m = m->m_next;
+ else {
+ struct mbuf *n;
+
+ n = m->m_next;
+ m->m_next = mfree;
+ mfree = m;
+ m = n;
+ }
}
if (m) {
sb->sb_mb = m;
@@ -1030,8 +1127,8 @@ sbsndptr(struct sockbuf *sb, u_int off, u_int len, u_int *moff)
struct mbuf *m, *ret;
KASSERT(sb->sb_mb != NULL, ("%s: sb_mb is NULL", __func__));
- KASSERT(off + len <= sb->sb_cc, ("%s: beyond sb", __func__));
- KASSERT(sb->sb_sndptroff <= sb->sb_cc, ("%s: sndptroff broken", __func__));
+ KASSERT(off + len <= sb->sb_acc, ("%s: beyond sb", __func__));
+ KASSERT(sb->sb_sndptroff <= sb->sb_acc, ("%s: sndptroff broken", __func__));
/*
* Is off below stored offset? Happens on retransmits.
@@ -1180,7 +1277,7 @@ void
sbtoxsockbuf(struct sockbuf *sb, struct xsockbuf *xsb)
{
- xsb->sb_cc = sb->sb_cc;
+ xsb->sb_cc = sb->sb_ccc;
xsb->sb_hiwat = sb->sb_hiwat;
xsb->sb_mbcnt = sb->sb_mbcnt;
xsb->sb_mcnt = sb->sb_mcnt;
generated by cgit v1.2.3 (git 2.25.1) at 2025-04-10 05:39:08 +0000