Optimize libc to get and set TLS using the RDFSBASE and RDGSBASE

instructions, if supported both by CPU and kernel.

Reviewed by:	jhb (previous version)
Tested by:	pho (previous version)
Sponsored by:	The FreeBSD Foundation
MFC after:	3 weeks
Differential revision:	https://reviews.freebsd.org/D12023

7 files changed, 155 insertions, 1 deletions

diff --git a/lib/libc/amd64/sys/Makefile.inc b/lib/libc/amd64/sys/Makefile.inc
index d208f0fcc111..d49bab7daf18 100644
--- a/lib/libc/amd64/sys/Makefile.inc
+++ b/lib/libc/amd64/sys/Makefile.inc
@@ -1,7 +1,11 @@
 #	from: Makefile.inc,v 1.1 1993/09/03 19:04:23 jtc Exp
 # $FreeBSD$
 
-SRCS+=	amd64_get_fsbase.c amd64_get_gsbase.c amd64_set_fsbase.c \
+SRCS+=	\
+	amd64_detect_rdfsgsbase.c \
+	amd64_get_fsbase.c \
+	amd64_get_gsbase.c \
+	amd64_set_fsbase.c \
 	amd64_set_gsbase.c
 
 MDASM=	vfork.S brk.S cerror.S exect.S getcontext.S \
diff --git a/lib/libc/amd64/sys/amd64_detect_rdfsgsbase.c b/lib/libc/amd64/sys/amd64_detect_rdfsgsbase.c
new file mode 100644
index 000000000000..7a7f94249c94
--- /dev/null
+++ b/lib/libc/amd64/sys/amd64_detect_rdfsgsbase.c
@@ -0,0 +1,63 @@
+/*-
+ * Copyright (c) 2017 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
+ * under sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#define	IN_RTLD	1
+#include <sys/param.h>
+#undef IN_RTLD
+#include <machine/cpufunc.h>
+#include <machine/specialreg.h>
+#include "amd64_detect_rdfsgsbase.h"
+#include "libc_private.h"
+
+static int state = RDFSGS_UNKNOWN;
+
+int
+amd64_detect_rdfsgsbase(void)
+{
+	u_int p[4];
+
+	if (__predict_true(state != RDFSGS_UNKNOWN))
+		return (state);
+
+	if (__getosreldate() >= P_OSREL_WRFSBASE) {
+		do_cpuid(0x0, p);
+		if (p[0] >= 0x7) {
+			cpuid_count(0x7, 0x0, p);
+			if ((p[1] & CPUID_STDEXT_FSGSBASE) != 0) {
+				state = RDFSGS_SUPPORTED;
+				return (state);
+			}
+		}
+	}
+	state = RDFSGS_UNSUPPORTED;
+	return (state);
+}
diff --git a/lib/libc/amd64/sys/amd64_detect_rdfsgsbase.h b/lib/libc/amd64/sys/amd64_detect_rdfsgsbase.h
new file mode 100644
index 000000000000..ecd9e7d816fc
--- /dev/null
+++ b/lib/libc/amd64/sys/amd64_detect_rdfsgsbase.h
@@ -0,0 +1,43 @@
+/*-
+ * Copyright (c) 2017 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
+ * under sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _AMD64_DETECT_RDFSGSBASE_H_
+#define _AMD64_DETECT_RDFSGSBASE_H_
+
+enum {
+	RDFSGS_UNKNOWN,
+	RDFSGS_SUPPORTED,
+	RDFSGS_UNSUPPORTED,
+};
+
+int amd64_detect_rdfsgsbase(void);
+
+#endif
diff --git a/lib/libc/amd64/sys/amd64_get_fsbase.c b/lib/libc/amd64/sys/amd64_get_fsbase.c
index ff5eb8f305ae..afe1e7a81521 100644
--- a/lib/libc/amd64/sys/amd64_get_fsbase.c
+++ b/lib/libc/amd64/sys/amd64_get_fsbase.c
@@ -1,7 +1,11 @@
 /*-
  * Copyright (c) 2003 Peter Wemm
+ * Copyright (c) 2017 The FreeBSD Foundation
  * All rights reserved.
  *
+ * Portions of this software were developed by Konstantin Belousov
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -27,11 +31,18 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include <sys/types.h>
+#include <machine/cpufunc.h>
 #include <machine/sysarch.h>
+#include "amd64_detect_rdfsgsbase.h"
 
 int
 amd64_get_fsbase(void **addr)
 {
 
+	if (amd64_detect_rdfsgsbase() == RDFSGS_SUPPORTED) {
+		*addr = (void *)rdfsbase();
+		return (0);
+	}
 	return (sysarch(AMD64_GET_FSBASE, addr));
 }
diff --git a/lib/libc/amd64/sys/amd64_get_gsbase.c b/lib/libc/amd64/sys/amd64_get_gsbase.c
index ddbf977f5dd1..2e4fa114b585 100644
--- a/lib/libc/amd64/sys/amd64_get_gsbase.c
+++ b/lib/libc/amd64/sys/amd64_get_gsbase.c
@@ -1,7 +1,11 @@
 /*-
  * Copyright (c) 2003 Peter Wemm
+ * Copyright (c) 2017 The FreeBSD Foundation
  * All rights reserved.
  *
+ * Portions of this software were developed by Konstantin Belousov
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -27,11 +31,18 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include <sys/types.h>
+#include <machine/cpufunc.h>
 #include <machine/sysarch.h>
+#include "amd64_detect_rdfsgsbase.h"
 
 int
 amd64_get_gsbase(void **addr)
 {
 
+	if  (amd64_detect_rdfsgsbase() == RDFSGS_SUPPORTED) {
+		*addr = (void *)rdgsbase();
+		return (0);
+	}
 	return (sysarch(AMD64_GET_GSBASE, addr));
 }
diff --git a/lib/libc/amd64/sys/amd64_set_fsbase.c b/lib/libc/amd64/sys/amd64_set_fsbase.c
index e0914859a1ae..be43b4f895b8 100644
--- a/lib/libc/amd64/sys/amd64_set_fsbase.c
+++ b/lib/libc/amd64/sys/amd64_set_fsbase.c
@@ -1,7 +1,11 @@
 /*-
  * Copyright (c) 2003 Peter Wemm
+ * Copyright (c) 2017 The FreeBSD Foundation
  * All rights reserved.
  *
+ * Portions of this software were developed by Konstantin Belousov
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -27,11 +31,18 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include <sys/types.h>
+#include <machine/cpufunc.h>
 #include <machine/sysarch.h>
+#include "amd64_detect_rdfsgsbase.h"
 
 int
 amd64_set_fsbase(void *addr)
 {
 
+	if (amd64_detect_rdfsgsbase() == RDFSGS_SUPPORTED) {
+		wrfsbase((uintptr_t)addr);
+		return (0);
+	}
 	return (sysarch(AMD64_SET_FSBASE, &addr));
 }
diff --git a/lib/libc/amd64/sys/amd64_set_gsbase.c b/lib/libc/amd64/sys/amd64_set_gsbase.c
index f8963288ac07..da9b830a9691 100644
--- a/lib/libc/amd64/sys/amd64_set_gsbase.c
+++ b/lib/libc/amd64/sys/amd64_set_gsbase.c
@@ -1,7 +1,11 @@
 /*-
  * Copyright (c) 2003 Peter Wemm
+ * Copyright (c) 2017 The FreeBSD Foundation
  * All rights reserved.
  *
+ * Portions of this software were developed by Konstantin Belousov
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -27,11 +31,18 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include <sys/types.h>
+#include <machine/cpufunc.h>
 #include <machine/sysarch.h>
+#include "amd64_detect_rdfsgsbase.h"
 
 int
 amd64_set_gsbase(void *addr)
 {
 
+	if (amd64_detect_rdfsgsbase() == RDFSGS_SUPPORTED) {
+		wrgsbase((uintptr_t)addr);
+		return (0);
+	}
 	return (sysarch(AMD64_SET_GSBASE, &addr));
 }