getln: We cannot expand the buffer beyond INT_MAX.

In such cases return ENOMEM. This is a limitation of our
implementation, alternatively you may consider getline(3).

Differential Revision:	https://reviews.freebsd.org/D442 (Partial)
Obtained from:	Apple Inc. (Libc 997.90.3)
Relnotes:	yes

2 files changed, 12 insertions, 2 deletions

diff --git a/lib/libc/stdio/fgetln.3 b/lib/libc/stdio/fgetln.3
index 4b83664d0338..7d4b89a5a06f 100644
--- a/lib/libc/stdio/fgetln.3
+++ b/lib/libc/stdio/fgetln.3
@@ -28,7 +28,7 @@
 .\"     @(#)fgetln.3	8.3 (Berkeley) 4/19/94
 .\" $FreeBSD$
 .\"
-.Dd April 19, 1994
+.Dd February 15, 2016
 .Dt FGETLN 3
 .Os
 .Sh NAME
@@ -97,6 +97,9 @@ These changes are lost as soon as the pointer becomes invalid.
 The argument
 .Fa stream
 is not a stream open for reading.
+.It Bq Er ENOMEM
+The internal line buffer could not be expanded due to lack of available memory,
+or because it would need to expand beyond INT_MAX in size.
 .El
 .Pp
 The
diff --git a/lib/libc/stdio/fgetln.c b/lib/libc/stdio/fgetln.c
index 1779de2ae559..1509bc880a4a 100644
--- a/lib/libc/stdio/fgetln.c
+++ b/lib/libc/stdio/fgetln.c
@@ -37,6 +37,8 @@ static char sccsid[] = "@(#)fgetln.c	8.2 (Berkeley) 1/2/94";
 __FBSDID("$FreeBSD$");
 
 #include "namespace.h"
+#include <errno.h>
+#include <limits.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -61,6 +63,10 @@ __slbexpand(FILE *fp, size_t newsize)
 #endif
 	if (fp->_lb._size >= newsize)
 		return (0);
+	if (newsize > INT_MAX) {
+		errno = ENOMEM;
+		return (-1);
+	}
 	if ((p = realloc(fp->_lb._base, newsize)) == NULL)
 		return (-1);
 	fp->_lb._base = p;
@@ -152,13 +158,14 @@ fgetln(FILE *fp, size_t *lenp)
 	}
 	*lenp = len;
 #ifdef notdef
-	fp->_lb._base[len] = 0;
+	fp->_lb._base[len] = '\0';
 #endif
 	FUNLOCKFILE(fp);
 	return ((char *)fp->_lb._base);
 
 error:
 	*lenp = 0;		/* ??? */
+	fp->_flags |= __SERR;
 	FUNLOCKFILE(fp);
 	return (NULL);		/* ??? */
 }