Update meta mode makefiles

meta2deps - add checks to detect truncated/corrupted filemon data
(only known to happen on Linux hosts), to ensure we do not auto
update dependencies based on incomplete data.

meta.stage.mk adds STAGE_SHLIB_LINKS_FILTER and STAGE_LINK_AS_*

We also allow for hosts where egrep is deprecated for grep -E

Reviewed by: stevek

6 files changed, 119 insertions, 30 deletions

diff --git a/share/mk/dirdeps-options.mk b/share/mk/dirdeps-options.mk
index b31d2033ae98..9a97615bbeb8 100644
--- a/share/mk/dirdeps-options.mk
+++ b/share/mk/dirdeps-options.mk
@@ -1,4 +1,4 @@
-# $Id: dirdeps-options.mk,v 1.20 2022/03/17 20:11:36 sjg Exp $
+# $Id: dirdeps-options.mk,v 1.21 2022/09/06 22:18:45 sjg Exp $
 #
 #	@(#) Copyright (c) 2018-2022, Simon J. Gerraty
 #
@@ -42,6 +42,13 @@
 # so we qualify MK_FOO with .${TARGET_SPEC} and each component
 # TARGET_SPEC_VAR (in reverse order) before using MK_FOO.
 #
+# Because Makefile.depend.options are processed at both level 0 (when
+# computing DIRDEPS to build) and higher (when updating
+# Makefile.depend* after successful build), it is important that 
+# all references to TARGET_SPEC_VARs should use the syntax
+# ${DEP_${TARGET_SPEC_VAR}:U${TARGET_SPEC_VAR}} to ensure correct
+# behavior.
+#
 
 # This should have been set by Makefile.depend.options
 # before including us
diff --git a/share/mk/gendirdeps.mk b/share/mk/gendirdeps.mk
index 6d26b3d308b1..1ff2036237ed 100644
--- a/share/mk/gendirdeps.mk
+++ b/share/mk/gendirdeps.mk
@@ -1,5 +1,4 @@
-# $FreeBSD$
-# $Id: gendirdeps.mk,v 1.46 2020/08/19 17:51:53 sjg Exp $
+# $Id: gendirdeps.mk,v 1.48 2022/09/09 17:44:29 sjg Exp $
 
 # Copyright (c) 2011-2020, Simon J. Gerraty
 # Copyright (c) 2010-2018, Juniper Networks, Inc.
@@ -89,7 +88,7 @@ META_FILES := ${META_FILES:T:O:u}
 # they should all be absolute paths
 SKIP_GENDIRDEPS ?=
 .if !empty(SKIP_GENDIRDEPS)
-_skip_gendirdeps = egrep -v '^(${SKIP_GENDIRDEPS:O:u:ts|})' |
+_skip_gendirdeps = ${EGREP:Uegrep} -v '^(${SKIP_GENDIRDEPS:O:u:ts|})' |
 .else
 _skip_gendirdeps =
 .endif
@@ -340,6 +339,8 @@ CAT_DEPEND ?= .depend
 .PHONY: ${_DEPENDFILE}
 .endif
 
+# set this to 'no' and we will not capture any
+# local depends
 LOCAL_DEPENDS_GUARD ?= _{.MAKE.LEVEL} > 0
 
 # 'cat .depend' should suffice, but if we are mixing build modes
@@ -352,6 +353,7 @@ ${_DEPENDFILE}: .NOMETA ${CAT_DEPEND:M.depend} ${META_FILES:O:u:@m@${exists($m):
 	echo '${DIRDEPS:@d@	$d \\${.newline}@}'; echo; \
 	${_include_src_dirdeps} \
 	echo '.include <dirdeps.mk>'; \
+	[ "${LOCAL_DEPENDS_GUARD:[1]:tl}" != no ] || exit 0; \
 	echo; \
 	echo '.if ${LOCAL_DEPENDS_GUARD}'; \
 	echo '# local dependencies - needed for -jN in clean tree'; \
diff --git a/share/mk/meta.autodep.mk b/share/mk/meta.autodep.mk
index 842554beea45..cd08ac3b3520 100644
--- a/share/mk/meta.autodep.mk
+++ b/share/mk/meta.autodep.mk
@@ -1,5 +1,4 @@
-# $FreeBSD$
-# $Id: meta.autodep.mk,v 1.55 2021/12/13 08:12:01 sjg Exp $
+# $Id: meta.autodep.mk,v 1.56 2022/09/09 17:44:29 sjg Exp $
 
 #
 #	@(#) Copyright (c) 2010, Simon J. Gerraty
@@ -175,7 +174,7 @@ DEPEND_SUFFIXES += .c .h .cpp .hpp .cxx .hxx .cc .hh
 .endif
 .depend: .NOMETA $${.MAKE.META.CREATED} ${_this}
 	@echo "Updating $@: ${.OODATE:T:[1..8]}"
-	@egrep -i '^R .*\.(${DEPEND_SUFFIXES:tl:O:u:S,^.,,:ts|})$$' /dev/null ${.MAKE.META.FILES:T:O:u:${META_FILE_FILTER:ts:}:M*o.meta} | \
+	@${EGREP:Uegrep} -i '^R .*\.(${DEPEND_SUFFIXES:tl:O:u:S,^.,,:ts|})$$' /dev/null ${.MAKE.META.FILES:T:O:u:${META_FILE_FILTER:ts:}:M*o.meta} | \
 	sed -e 's, \./, ,${OBJDIR_REFS:O:u:@d@;s, $d/, ,@};/\//d' \
 		-e 's,^\([^/][^/]*\).meta...[0-9]* ,\1: ,' | \
 	sort -u | \
diff --git a/share/mk/meta.stage.mk b/share/mk/meta.stage.mk
index 9f54f3b1f9c5..168e46d22a82 100644
--- a/share/mk/meta.stage.mk
+++ b/share/mk/meta.stage.mk
@@ -1,5 +1,4 @@
-# $FreeBSD$
-# $Id: meta.stage.mk,v 1.60 2020/08/19 17:51:53 sjg Exp $
+# $Id: meta.stage.mk,v 1.67 2023/04/17 01:22:10 sjg Exp $
 #
 #	@(#) Copyright (c) 2011-2017, Simon J. Gerraty
 #
@@ -31,8 +30,11 @@ _dirdep ?= ${RELDIR}
 CLEANFILES+= .dirdep
 
 # this allows us to trace dependencies back to their src dir
-.dirdep:	.NOPATH
+.dirdep: .NOPATH
+.if !commands(.dirdep)
+.dirdep:
 	@echo '${_dirdep}' > $@
+.endif
 
 .if defined(NO_POSIX_SHELL) || ${type printf:L:sh:Mbuiltin} == ""
 _stage_file_basename = `basename $$f`
@@ -64,7 +66,7 @@ GENDIRDEPS_FILTER += Nnot-empty-is-important \
 LN_CP_SCRIPT = LnCp() { \
   rm -f $$2 2> /dev/null; \
   { [ -z "$$mode" ] && ${LN:Uln} $$1 $$2 2> /dev/null; } || \
-  cp -p $$1 $$2; }
+  cp -p $$1 $$2 2> /dev/null || cp $$1 $$2; }
 
 # a staging conflict should cause an error
 # a warning is handy when bootstapping different options.
@@ -171,7 +173,7 @@ stage_libs:	.dirdep
 .if !defined(NO_SHLIB_LINKS)
 .if !empty(SHLIB_LINKS)
 	@${STAGE_LINKS_SCRIPT}; StageLinks -s ${STAGE_LIBDIR:${STAGE_DIR_FILTER}} \
-	${SHLIB_LINKS:@t@${STAGE_LIBS:T:M$t.*} $t@}
+	${SHLIB_LINKS:@t@${STAGE_LIBS:T:M$t.*:${STAGE_SHLIB_LINKS_FILTER:U}} $t@}
 .elif !empty(SHLIB_LINK) && !empty(SHLIB_NAME)
 	@${STAGE_LINKS_SCRIPT}; StageLinks -s ${STAGE_LIBDIR:${STAGE_DIR_FILTER}} ${SHLIB_NAME} ${SHLIB_LINK}
 .endif
@@ -262,7 +264,8 @@ CLEANFILES += ${STAGE_AS_SETS:@s@stage*$s@}
 # sometimes things need to be renamed as they are staged
 # each ${file} will be staged as ${STAGE_AS_${file:T}}
 # one could achieve the same with SYMLINKS
-# stage_as_and_symlink makes the original name a symlink to the new name
+# stage_as_and_symlink makes the original name (or ${STAGE_LINK_AS_${name}})
+# a symlink to the new name
 # it is the same as using stage_as and stage_symlinks but ensures
 # both operations happen together
 .for s in ${STAGE_AS_SETS:O:u}
@@ -292,7 +295,7 @@ STAGE_AS_AND_SYMLINK.$s ?= ${.ALLSRC:N.dirdep:Nstage_*}
 stage_as_and_symlink:	stage_as_and_symlink.$s
 stage_as_and_symlink.$s:	.dirdep
 	@${STAGE_AS_SCRIPT}; StageAs ${FLAGS.$@} ${STAGE_FILES_DIR.$s:U${STAGE_DIR.$s}:${STAGE_DIR_FILTER}} ${STAGE_AS_AND_SYMLINK.$s:O:@f@$f ${STAGE_AS_${f:tA}:U${STAGE_AS_${f:T}:U${f:T}}}@}
-	@${STAGE_LINKS_SCRIPT}; StageLinks -s ${STAGE_FILES_DIR.$s:U${STAGE_DIR.$s}:${STAGE_DIR_FILTER}} ${STAGE_AS_AND_SYMLINK.$s:O:@f@${STAGE_AS_${f:tA}:U${STAGE_AS_${f:T}:U${f:T}}} $f@}
+	@${STAGE_LINKS_SCRIPT}; StageLinks -s ${STAGE_FILES_DIR.$s:U${STAGE_DIR.$s}:${STAGE_DIR_FILTER}} ${STAGE_AS_AND_SYMLINK.$s:O:@f@${STAGE_AS_${f:tA}:U${STAGE_AS_${f:T}:U${f:T}}} ${STAGE_LINK_AS_${f}:U$f}@}
 	@touch $@
 .endif
 .endif
@@ -304,7 +307,7 @@ CLEANFILES += ${STAGE_TARGETS} stage_incs stage_includes
 
 # this lot also only makes sense the first time...
 .if !target(__${.PARSEFILE}__)
-__${.PARSEFILE}__:
+__${.PARSEFILE}__: .NOTMAIN
 
 # stage_*links usually needs to follow any others.
 # for non-jobs mode the order here matters
@@ -351,7 +354,7 @@ all: stale_staged
 # get a list of paths that we have previously staged to those same dirs
 # anything in the 2nd list but not the first is stale - remove it.
 stale_staged: staging .NOMETA
-	@egrep '^[WL] .*${STAGE_OBJTOP}' /dev/null ${.MAKE.META.FILES:M*stage_*} | \
+	@${EGREP:Uegrep} '^[WL] .*${STAGE_OBJTOP}' /dev/null ${.MAKE.META.FILES:M*stage_*} | \
 	sed "/\.dirdep/d;s,.* '*\(${STAGE_OBJTOP}/[^ '][^ ']*\).*,\1," | \
 	sort > ${.TARGET}.staged1
 	@grep -l '${_dirdep}' /dev/null ${_STAGED_DIRS:M${STAGE_OBJTOP}*:O:u:@d@$d/*.dirdep@} | \
diff --git a/share/mk/meta2deps.py b/share/mk/meta2deps.py
index 1bcf1baedaaa..d7820ec71b5b 100755
--- a/share/mk/meta2deps.py
+++ b/share/mk/meta2deps.py
@@ -37,8 +37,7 @@ We only pay attention to a subset of the information in the
 
 """
 RCSid:
-	$FreeBSD$
-	$Id: meta2deps.py,v 1.40 2021/12/13 19:32:46 sjg Exp $
+	$Id: meta2deps.py,v 1.45 2023/01/18 01:35:24 sjg Exp $
 
 	Copyright (c) 2011-2020, Simon J. Gerraty
 	Copyright (c) 2011-2017, Juniper Networks, Inc.
@@ -67,7 +66,10 @@ RCSid:
 
 """
 
-import os, re, sys
+import os
+import re
+import sys
+import stat
 
 def resolve(path, cwd, last_dir=None, debug=0, debug_out=sys.stderr):
     """
@@ -245,6 +247,7 @@ class MetaFile:
         self.curdir = conf.get('CURDIR')
         self.reldir = conf.get('RELDIR')
         self.dpdeps = conf.get('DPDEPS')
+        self.pids = {}
         self.line = 0
 
         if not self.conf:
@@ -445,12 +448,13 @@ class MetaFile:
         pid_cwd = {}
         pid_last_dir = {}
         last_pid = 0
+        eof_token = False
 
         self.line = 0
         if self.curdir:
             self.seenit(self.curdir)    # we ignore this
 
-        interesting = 'CEFLRV'
+        interesting = '#CEFLRVX'
         for line in f:
             self.line += 1
             # ignore anything we don't care about
@@ -477,6 +481,12 @@ class MetaFile:
                         print("%s: CWD=%s" % (self.name, cwd), file=self.debug_out)
                 continue
 
+            if w[0] == '#':
+                # check the file has not been truncated
+                if line.find('Bye') > 0:
+                    eof_token = True
+                continue
+
             pid = int(w[1])
             if pid != last_pid:
                 if last_pid:
@@ -506,6 +516,13 @@ class MetaFile:
                     print("cwd=", cwd, file=self.debug_out)
                 continue
 
+            if w[0] == 'X':
+                try:
+                    del self.pids[pid]
+                except KeyError:
+                    pass
+                continue
+
             if w[2] in self.seen:
                 if self.debug > 2:
                     print("seen:", w[2], file=self.debug_out)
@@ -519,11 +536,34 @@ class MetaFile:
                 continue
             elif w[0] in 'ERWS':
                 path = w[2]
-                if path == '.':
+                if w[0] == 'E':
+                    self.pids[pid] = path
+                elif path == '.':
                     continue
                 self.parse_path(path, cwd, w[0], w)
 
-        assert(version > 0)
+        if version == 0:
+            raise AssertionError('missing filemon data')
+        if not eof_token:
+            raise AssertionError('truncated filemon data')
+
+        setid_pids = []
+        # self.pids should be empty!
+        for pid,path in self.pids.items():
+            try:
+                # no guarantee that path is still valid
+                if os.stat(path).st_mode & (stat.S_ISUID|stat.S_ISGID):
+                    # we do not expect anything after Exec
+                    setid_pids.append(pid)
+                    continue
+            except:
+                # we do not care why the above fails,
+                # we do not want to miss the ERROR below.
+                pass
+            print("ERROR: missing eXit for {} pid {}".format(path, pid))
+        for pid in setid_pids:
+            del self.pids[pid]
+        assert(len(self.pids) == 0)
         if not file:
             f.close()
 
diff --git a/share/mk/meta2deps.sh b/share/mk/meta2deps.sh
index 4767b83f4343..56367e0105f4 100755
--- a/share/mk/meta2deps.sh
+++ b/share/mk/meta2deps.sh
@@ -49,8 +49,10 @@
 #	The output, is a set of absolute paths with "SB" like:
 #.nf
 #
+#	$SB/obj-i386/bsd/gnu/lib/csu
+#	$SB/obj-i386/bsd/gnu/lib/libgcc
 #	$SB/obj-i386/bsd/include
-#	$SB/obj-i386/bsd/lib/csu/i386
+#	$SB/obj-i386/bsd/lib/csu/i386-elf
 #	$SB/obj-i386/bsd/lib/libc
 #	$SB/src/bsd/include
 #	$SB/src/bsd/sys/i386/include
@@ -75,8 +77,7 @@
 
 
 # RCSid:
-#	$FreeBSD$
-#	$Id: meta2deps.sh,v 1.14 2020/10/02 03:11:17 sjg Exp $
+#	$Id: meta2deps.sh,v 1.20 2023/01/18 01:35:24 sjg Exp $
 
 # Copyright (c) 2010-2013, Juniper Networks, Inc.
 # All rights reserved.
@@ -138,6 +139,13 @@ add_list() {
     eval "$name=\"$list\""
 }
 
+# some Linux systems have deprecated egrep in favor of grep -E
+# but not everyone supports that
+case "`echo bmake | egrep 'a|b' 2>&1`" in
+bmake) ;;
+*) egrep() { grep -E "$@"; }
+esac
+
 _excludes_f() {
     egrep -v "$EXCLUDES"
 }
@@ -240,8 +248,8 @@ meta2deps() {
 	;;
     *) cat /dev/null "$@";;
     esac 2> /dev/null |
-    sed -e 's,^CWD,C C,;/^[CREFLMV] /!d' -e "s,',,g" |
-    $_excludes | ( version=no
+    sed -e 's,^CWD,C C,;/^[#CREFLMVX] /!d' -e "s,',,g" |
+    $_excludes | ( version=no epids= xpids= eof_token=no
     while read op pid path junk
     do
 	: op=$op pid=$pid path=$path
@@ -259,10 +267,15 @@ meta2deps() {
 	    *) ;;
 	    esac
 	    version=0
+	    case "$eof_token" in
+	    no) ;;		# ignore
+	    0) error "truncated filemon data";;
+	    esac
+	    eof_token=0
 	    continue
 	    ;;
 	$pid,$pid) ;;
-	*)
+	[1-9]*)
 	    case "$lpid" in
 	    "") ;;
 	    *) eval ldir_$lpid=$ldir;;
@@ -272,8 +285,9 @@ meta2deps() {
 	    ;;
 	esac
 
+	: op=$op path=$path
 	case "$op,$path" in
-	V,*) version=$path; continue;;
+	V,*) version=$pid; continue;;
 	W,*srcrel|*.dirdep) continue;;
 	C,*)
 	    case "$path" in
@@ -289,7 +303,18 @@ meta2deps() {
 	    eval cwd_$path=$cwd ldir_$path=$ldir
 	    continue
 	    ;;
+	\#,bye) eof_token=1; continue;;
+	\#*) continue;;
 	*)  dir=${path%/*}
+	    case "$op" in
+	    E)	# setid apps get no tracing so we won't see eXit
+		case `'ls' -l $path 2> /dev/null | sed 's, .*,,'` in
+		*s*) ;;
+		*) epids="$epids $pid";;
+		esac
+		;;
+	    X) xpids="$xpids $pid"; continue;;
+	    esac
 	    case "$path" in
 	    $src_re|$obj_re) ;;
 	    /*/stage/*) ;;
@@ -379,9 +404,22 @@ meta2deps() {
 	    echo $dir;;
 	esac
     done > $tf.dirdep
+    : version=$version
     case "$version" in
     0) error "no filemon data";;
-    esac ) || exit 1
+    esac
+    : eof_token=$eof_token
+    case "$eof_token" in
+    0) error "truncated filemon data";;
+    esac
+    for p in $epids
+    do
+	: p=$p
+	case " $xpids " in
+	*" $p "*) ;;
+	*) error "missing eXit for pid $p";;
+	esac
+    done ) || exit 1
     _nl=echo
     for f in $tf.dirdep $tf.qual $tf.srcdep
     do