libc: Fix parsing of hexadecimal numbers in strtol() family.

This had previously been partly fixed in 2571c7f7200f.

MFC after:	1 week
Reviewed by:	imp, allanjude, emaste
Differential Revision:	https://reviews.freebsd.org/D41510

8 files changed, 33 insertions, 9 deletions

diff --git a/lib/libc/iconv/_strtol.h b/lib/libc/iconv/_strtol.h
index 786ae2efbaa9..d183edbe8c3a 100644
--- a/lib/libc/iconv/_strtol.h
+++ b/lib/libc/iconv/_strtol.h
@@ -83,7 +83,10 @@ _FUNCNAME(const char *nptr, char **endptr, int base)
 			c = *s++;
 	}
 	if ((base == 0 || base == 16) &&
-	    c == '0' && (*s == 'x' || *s == 'X')) {
+	    c == '0' && (*s == 'x' || *s == 'X') &&
+	    ((s[1] >= '0' && s[1] <= '9') ||
+	    (s[1] >= 'A' && s[1] <= 'F') ||
+	    (s[1] >= 'a' && s[1] <= 'f'))) {
 		c = s[1];
 		s += 2;
 		base = 16;
diff --git a/lib/libc/iconv/_strtoul.h b/lib/libc/iconv/_strtoul.h
index 2d5c7776b0e3..eade72e9c2e6 100644
--- a/lib/libc/iconv/_strtoul.h
+++ b/lib/libc/iconv/_strtoul.h
@@ -79,7 +79,10 @@ _FUNCNAME(const char *nptr, char **endptr, int base)
 			c = *s++;
 	}
 	if ((base == 0 || base == 16) &&
-	    c == '0' && (*s == 'x' || *s == 'X')) {
+	    c == '0' && (*s == 'x' || *s == 'X') &&
+	    ((s[1] >= '0' && s[1] <= '9') ||
+	    (s[1] >= 'A' && s[1] <= 'F') ||
+	    (s[1] >= 'a' && s[1] <= 'f'))) {
 		c = s[1];
 		s += 2;
 		base = 16;
diff --git a/lib/libc/locale/wcstoimax.c b/lib/libc/locale/wcstoimax.c
index 6447fe6dccfd..259faa2b011c 100644
--- a/lib/libc/locale/wcstoimax.c
+++ b/lib/libc/locale/wcstoimax.c
@@ -78,7 +78,10 @@ wcstoimax_l(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr,
 			c = *s++;
 	}
 	if ((base == 0 || base == 16) &&
-	    c == L'0' && (*s == L'x' || *s == L'X')) {
+	    c == L'0' && (*s == L'x' || *s == L'X') &&
+	    ((s[1] >= L'0' && s[1] <= L'9') ||
+	    (s[1] >= L'A' && s[1] <= L'F') ||
+	    (s[1] >= L'a' && s[1] <= L'f'))) {
 		c = s[1];
 		s += 2;
 		base = 16;
diff --git a/lib/libc/locale/wcstol.c b/lib/libc/locale/wcstol.c
index 183fb00bc507..b0b787384f39 100644
--- a/lib/libc/locale/wcstol.c
+++ b/lib/libc/locale/wcstol.c
@@ -63,7 +63,7 @@ wcstol_l(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr, int
 	do {
 		c = *s++;
 	} while (iswspace_l(c, locale));
-	if (c == '-') {
+	if (c == L'-') {
 		neg = 1;
 		c = *s++;
 	} else {
@@ -72,7 +72,10 @@ wcstol_l(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr, int
 			c = *s++;
 	}
 	if ((base == 0 || base == 16) &&
-	    c == L'0' && (*s == L'x' || *s == L'X')) {
+	    c == L'0' && (*s == L'x' || *s == L'X') &&
+	    ((s[1] >= L'0' && s[1] <= L'9') ||
+	    (s[1] >= L'A' && s[1] <= L'F') ||
+	    (s[1] >= L'a' && s[1] <= L'f'))) {
 		c = s[1];
 		s += 2;
 		base = 16;
diff --git a/lib/libc/locale/wcstoll.c b/lib/libc/locale/wcstoll.c
index 5e42d878e1c8..ac07d6c6adbf 100644
--- a/lib/libc/locale/wcstoll.c
+++ b/lib/libc/locale/wcstoll.c
@@ -78,7 +78,10 @@ wcstoll_l(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr,
 			c = *s++;
 	}
 	if ((base == 0 || base == 16) &&
-	    c == L'0' && (*s == L'x' || *s == L'X')) {
+	    c == L'0' && (*s == L'x' || *s == L'X') &&
+	    ((s[1] >= L'0' && s[1] <= L'9') ||
+	    (s[1] >= L'A' && s[1] <= L'F') ||
+	    (s[1] >= L'a' && s[1] <= L'f'))) {
 		c = s[1];
 		s += 2;
 		base = 16;
diff --git a/lib/libc/locale/wcstoul.c b/lib/libc/locale/wcstoul.c
index d097c20db40a..9f58db799c0e 100644
--- a/lib/libc/locale/wcstoul.c
+++ b/lib/libc/locale/wcstoul.c
@@ -72,7 +72,10 @@ wcstoul_l(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr,
 			c = *s++;
 	}
 	if ((base == 0 || base == 16) &&
-	    c == L'0' && (*s == L'x' || *s == L'X')) {
+	    c == L'0' && (*s == L'x' || *s == L'X') &&
+	    ((s[1] >= L'0' && s[1] <= L'9') ||
+	    (s[1] >= L'A' && s[1] <= L'F') ||
+	    (s[1] >= L'a' && s[1] <= L'f'))) {
 		c = s[1];
 		s += 2;
 		base = 16;
diff --git a/lib/libc/locale/wcstoull.c b/lib/libc/locale/wcstoull.c
index 5f8e8367308b..cbc7253f884d 100644
--- a/lib/libc/locale/wcstoull.c
+++ b/lib/libc/locale/wcstoull.c
@@ -78,7 +78,10 @@ wcstoull_l(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr,
 			c = *s++;
 	}
 	if ((base == 0 || base == 16) &&
-	    c == L'0' && (*s == L'x' || *s == L'X')) {
+	    c == L'0' && (*s == L'x' || *s == L'X') &&
+	    ((s[1] >= L'0' && s[1] <= L'9') ||
+	    (s[1] >= L'A' && s[1] <= L'F') ||
+	    (s[1] >= L'a' && s[1] <= L'f'))) {
 		c = s[1];
 		s += 2;
 		base = 16;
diff --git a/lib/libc/locale/wcstoumax.c b/lib/libc/locale/wcstoumax.c
index ad9e9c334186..4380cccf2424 100644
--- a/lib/libc/locale/wcstoumax.c
+++ b/lib/libc/locale/wcstoumax.c
@@ -78,7 +78,10 @@ wcstoumax_l(const wchar_t * __restrict nptr, wchar_t ** __restrict endptr,
 			c = *s++;
 	}
 	if ((base == 0 || base == 16) &&
-	    c == L'0' && (*s == L'x' || *s == L'X')) {
+	    c == L'0' && (*s == L'x' || *s == L'X') &&
+	    ((s[1] >= L'0' && s[1] <= L'9') ||
+	    (s[1] >= L'A' && s[1] <= L'F') ||
+	    (s[1] >= L'a' && s[1] <= L'f'))) {
 		c = s[1];
 		s += 2;
 		base = 16;