Set default stack size for Linux apps to 8MB. This matches Linux'

defaults, makes core files smaller, and fixes applications which use
pthread_join(3) in a wrong way, namely Steam.

This is based on a patch submitted by Jason Yang, which I've reworked
to set the limit instead of only changing the value reported (which
is enough to fix the bug for Linux pthreads, but could be confusing).

PR:		248225
Submitted by:	Jason_YH_Yang at wistron.com (earlier version)
Analyzed by:	Alex S <iwtcex@gmail.com>
Reviewed by:	emaste
MFC after:	2 weeks
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D26778

3 files changed, 30 insertions, 0 deletions

diff --git a/sys/compat/linux/linux_emul.c b/sys/compat/linux/linux_emul.c
index 4326ef5b16cb..605abfaacbfd 100644
--- a/sys/compat/linux/linux_emul.c
+++ b/sys/compat/linux/linux_emul.c
@@ -115,6 +115,29 @@ linux_set_default_openfiles(struct thread *td, struct proc *p)
 	KASSERT(error == 0, ("kern_proc_setrlimit failed"));
 }
 
+/*
+ * The default stack size limit in Linux is 8MB.
+ */
+static void
+linux_set_default_stacksize(struct thread *td, struct proc *p)
+{
+	struct rlimit rlim;
+	int error;
+
+	if (linux_default_stacksize < 0)
+		return;
+
+	PROC_LOCK(p);
+	lim_rlimit_proc(p, RLIMIT_STACK, &rlim);
+	PROC_UNLOCK(p);
+	if (rlim.rlim_cur != rlim.rlim_max ||
+	    rlim.rlim_cur <= linux_default_stacksize)
+		return;
+	rlim.rlim_cur = linux_default_stacksize;
+	error = kern_proc_setrlimit(td, p, RLIMIT_STACK, &rlim);
+	KASSERT(error == 0, ("kern_proc_setrlimit failed"));
+}
+
 void
 linux_proc_init(struct thread *td, struct thread *newtd, int flags)
 {
@@ -145,6 +168,7 @@ linux_proc_init(struct thread *td, struct thread *newtd, int flags)
 		newtd->td_emuldata = em;
 
 		linux_set_default_openfiles(td, p);
+		linux_set_default_stacksize(td, p);
 	} else {
 		p = td->td_proc;
 
diff --git a/sys/compat/linux/linux_mib.c b/sys/compat/linux/linux_mib.c
index 3d7f7b40bf15..68fc65db31a7 100644
--- a/sys/compat/linux/linux_mib.c
+++ b/sys/compat/linux/linux_mib.c
@@ -72,6 +72,11 @@ SYSCTL_INT(_compat_linux, OID_AUTO, default_openfiles, CTLFLAG_RWTUN,
     &linux_default_openfiles, 0,
     "Default soft openfiles resource limit, or -1 for unlimited");
 
+int linux_default_stacksize = 8 * 1024 * 1024;
+SYSCTL_INT(_compat_linux, OID_AUTO, default_stacksize, CTLFLAG_RWTUN,
+    &linux_default_stacksize, 0,
+    "Default soft stack size resource limit, or -1 for unlimited");
+
 int linux_ignore_ip_recverr = 1;
 SYSCTL_INT(_compat_linux, OID_AUTO, ignore_ip_recverr, CTLFLAG_RWTUN,
     &linux_ignore_ip_recverr, 0, "Ignore enabling IP_RECVERR");
diff --git a/sys/compat/linux/linux_mib.h b/sys/compat/linux/linux_mib.h
index 8606e995bd76..b3f16622d81c 100644
--- a/sys/compat/linux/linux_mib.h
+++ b/sys/compat/linux/linux_mib.h
@@ -64,6 +64,7 @@ int	linux_kernver(struct thread *td);
 
 extern int linux_debug;
 extern int linux_default_openfiles;
+extern int linux_default_stacksize;
 extern int linux_ignore_ip_recverr;
 extern int linux_preserve_vstatus;
 extern bool linux_map_sched_prio;