1# $NetBSD: test-variants.mk,v 1.9 2024/06/25 05:18:38 rillig Exp $
2#
3# Build several variants of make and run the tests on them.
4#
5# The output of this test suite must be inspected manually to see the
6# interesting details.  The main purpose is to list the available build
7# options.
8
9.MAIN: usage
10usage:
11          @echo 'usage: ${MAKE} -f ${MAKEFILE} list'
12          @echo '       ${MAKE} -f ${MAKEFILE} all'
13          @echo '       ${MAKE} -f ${MAKEFILE} <test>...'
14
15
16TESTS+=                       default
17#ENV.default=                 VAR=value...
18#CPPFLAGS.default=  -DMACRO -I...
19#CFLAGS.default=    -O1
20#SKIP.default=                yes
21#SKIP_TESTS.default=          varmod-subst
22
23TESTS+=                       sanitize
24ENV.sanitize=                 MKSANITIZER=yes LIMIT_RESOURCES=:
25CPPFLAGS.sanitize=  -DCLEANUP
26CFLAGS.sanitize=    -O2 -ggdb
27
28# Try a different compiler, with slightly different warnings and error
29# messages.  Clang has a few stricter checks than GCC, concerning enums
30# and non-literal format strings.
31#
32TESTS+=                       llvm
33ENV.llvm=           HAVE_LLVM="yes"
34
35# Use emalloc for memory allocation.
36TESTS+=                       emalloc
37ENV.emalloc=                  TOOLDIR=""
38
39# NetBSD 10 fails with:
40# filemon_dev.c:93:19: error: 'FILEMON_SET_FD' undeclared
41TESTS+=                       filemon-dev
42ENV.filemon-dev=    USE_FILEMON="dev"
43SKIP.filemon-dev=   yes
44
45TESTS+=                       filemon-ktrace
46ENV.filemon-ktrace= USE_FILEMON="ktrace"
47
48TESTS+=                       filemon-none
49ENV.filemon-none=   USE_FILEMON="no"
50# The following tests only fail due to the extra variable in the debug log.
51SKIP_TESTS.filemon-none= \
52                              opt-debug-graph1 \
53                              opt-debug-graph2 \
54                              opt-debug-graph3 \
55                              suff-main-several \
56                              suff-transform-debug
57
58TESTS+=                       no-meta
59ENV.no-meta=                  USE_META="no"
60SKIP_TESTS.no-meta= depsrc-meta meta-cmd-cmp
61
62TESTS+=                       cleanup
63ENV.cleanup=                  MKLINT=yes
64CPPFLAGS.cleanup=   -DCLEANUP
65
66TESTS+=                       debug-refcnt
67CPPFLAGS.debug-refcnt=        -DDEBUG_REFCNT
68
69TESTS+=                       debug-hash
70CPPFLAGS.debug-hash=          -DDEBUG_HASH_LOOKUP
71SKIP_TESTS.debug-hash=        opt-debug-hash      # mixes regular and debug output
72
73TESTS+=                       debug-meta
74CPPFLAGS.debug-meta=          -DDEBUG_META_MODE
75SKIP_TESTS.debug-meta=        depsrc-meta meta-cmd-cmp # generate extra debug output
76
77# Produces lots of debugging output.
78#
79TESTS+=                       debug-src
80CPPFLAGS.debug-src= -DDEBUG_SRC
81SKIP.debug-src=               yes
82
83# NetBSD 8.0 x86_64
84# In file included from arch.c:135:0:
85# /usr/include/sys/param.h:357:0: error: "MAXPATHLEN" redefined [-Werror]
86TESTS+=                       maxpathlen
87CPPFLAGS.maxpathlen=          -DMAXPATHLEN=20
88SKIP.maxpathlen=    yes
89
90# NetBSD 8.0 x86_64 says:
91# In file included from /usr/include/sys/param.h:115:0,
92#                 from arch.c:135:
93# /usr/include/sys/syslimits.h:60:0: error: "PATH_MAX" redefined [-Werror]
94TESTS+=                       path_max
95CPPFLAGS.path_max=  -DPATH_MAX=20
96SKIP.path_max=                yes
97
98# This higher optimization level may trigger additional "may be used
99# uninitialized" errors. Could be combined with other compilers as well.
100#
101TESTS+=                       opt-3
102CFLAGS.opt-3=                 -O3
103
104# When optimizing for small code size, GCC gets confused by the initialization
105# status of local variables in some cases.
106TESTS+=                       opt-size
107CFLAGS.opt-size=    -Os
108
109TESTS+=                       opt-none
110CFLAGS.opt-none=    -O0 -ggdb
111
112# Ensure that every inline function is declared as MAKE_ATTR_UNUSED.
113TESTS+=                       no-inline
114CPPFLAGS.no-inline= -Dinline=
115
116# Is expected to fail with "<stdbool.h> is included in pre-C99 mode" since
117# main.c includes <sys/sysctl.h>, which includes <stdbool.h> even in pre-C99
118# mode.  This would lead to inconsistent definitions of bool and thus
119# differently sized struct CmdOpts and others.
120TESTS+=                       c90-plain
121ENV.c90-plain=                USE_FILEMON=no      # filemon uses designated initializers
122CFLAGS.c90-plain=   -std=c90 -ansi -pedantic -Wno-system-headers
123SKIP.c90-plain=               yes
124
125# The make source code is _intended_ to be compatible with C90, it uses some
126# C99 features though (snprintf).  The workaround with USE_C99_BOOLEAN is
127# necessary on NetBSD 9.99 since main.c includes <sys/sysctl.h>, which
128# includes <stdbool.h> even in pre-C99 mode.
129TESTS+=                       c90-stdbool
130ENV.c90-stdbool=    USE_FILEMON=no      # filemon uses designated initializers
131CFLAGS.c90-stdbool= -std=c90 -ansi -pedantic -Wno-system-headers
132CPPFLAGS.c90-stdbool=         -DUSE_C99_BOOLEAN
133
134# Ensure that there are only side-effect-free conditions in the assert
135# macro, or at least none that affect the outcome of the tests.
136#
137TESTS+=                       no-assert
138CPPFLAGS.no-assert= -DNDEBUG
139
140# Only in native mode, make dares to use a shortcut in Compat_RunCommand
141# that circumvents the shell and instead calls execvp directly.
142# Another effect is that the shell is run with -q, which prevents the
143# -x and -v flags from echoing the commands from profile files.
144TESTS+=                       non-native
145CPPFLAGS.non-native=          -UMAKE_NATIVE
146CPPFLAGS.non-native+=         -DHAVE_STRERROR -DHAVE_SETENV -DHAVE_VSNPRINTF
147
148# Running the code coverage using gcov took a long time on NetBSD < 10, due to
149# https://gnats.netbsd.org/55808.
150#
151# Combining USE_COVERAGE with HAVE_LLVM does not work since ld fails to link
152# with the coverage library.
153#
154# Turning the optimization off is required because gcov does not work on the
155# source code level but on the intermediate code after optimization:
156# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=96622
157#
158TESTS+=                       coverage
159ENV.coverage=                 USE_COVERAGE="yes"
160CFLAGS.coverage=    -O0 -ggdb
161
162TESTS+=                       fort
163ENV.fort=           USE_FORT="yes"
164
165# Ensure that the tests can be specified either as relative filenames or
166# as absolute filenames.
167TESTS+=                       abs
168ENV.abs=            USE_ABSOLUTE_TESTNAMES="yes"
169
170# This test is the result of reading through the GCC 10 "Warning Options"
171# documentation, noting down everything that sounded interesting.
172#
173TESTS+=                       gcc-warn
174CFLAGS.gcc-warn=    -Wmisleading-indentation
175CFLAGS.gcc-warn+=   -Wmissing-attributes
176CFLAGS.gcc-warn+=   -Wmissing-braces
177CFLAGS.gcc-warn+=   -Wextra
178CFLAGS.gcc-warn+=   -Wnonnull
179CFLAGS.gcc-warn+=   -Wnonnull-compare
180CFLAGS.gcc-warn+=   -Wnull-dereference
181CFLAGS.gcc-warn+=   -Wimplicit
182CFLAGS.gcc-warn+=   -Wimplicit-fallthrough=4
183CFLAGS.gcc-warn+=   -Wignored-qualifiers
184CFLAGS.gcc-warn+=   -Wunused
185CFLAGS.gcc-warn+=   -Wunused-but-set-variable
186CFLAGS.gcc-warn+=   -Wunused-parameter
187CFLAGS.gcc-warn+=   -Wduplicated-branches
188CFLAGS.gcc-warn+=   -Wduplicated-cond
189CFLAGS.gcc-warn+=   -Wunused-macros
190CFLAGS.gcc-warn+=   -Wcast-function-type
191CFLAGS.gcc-warn+=   -Wconversion
192CFLAGS.gcc-warn+=   -Wenum-compare
193CFLAGS.gcc-warn+=   -Wmissing-field-initializers
194CFLAGS.gcc-warn+=   -Wredundant-decls
195CFLAGS.gcc-warn+=   -Wno-error=conversion
196CFLAGS.gcc-warn+=   -Wno-error=sign-conversion
197CFLAGS.gcc-warn+=   -Wno-error=unused-macros
198CFLAGS.gcc-warn+=   -Wno-error=unused-parameter
199CFLAGS.gcc-warn+=   -Wno-error=duplicated-branches
200
201.for shell in /usr/pkg/bin/bash /usr/pkg/bin/dash
202.  if exists(${shell})
203TESTS+=                       ${shell:T}
204CPPFLAGS.${shell:T}=          -DDEFSHELL_CUSTOM="\"${shell}\""
205
206.    for name in ${shell:T}-sanitize
207TESTS+=                       ${name}
208ENV.${name}=                  MKSANITIZER=yes LIMIT_RESOURCES=:
209CPPFLAGS.${name}=   -DDEFSHELL_CUSTOM="\"${shell}\"" -DCLEANUP
210CFLAGS.${name}=               -ggdb -O0
211.    endfor
212.  endif
213.endfor
214
215
216.MAKEFLAGS: -k
217.MAKE.DEPENDFILE=   nonexistent
218
219.PHONY: usage list all ${TESTS}
220
221all: ${TESTS:${TESTS:@t@${SKIP.$t:Myes:%=N$t}@:ts:}}
222
223SLOW_TESTS=         dotwait sh-flags
224
225.for test in ${TESTS}
226_ENV.${test}=       PATH="$$PATH"
227_ENV.${test}+=      USETOOLS="no"
228_ENV.${test}+=      MALLOC_OPTIONS="JA"           # for jemalloc 1.0.0
229_ENV.${test}+=      MALLOC_CONF="junk:true"                 # for jemalloc 5.1.0
230_ENV.${test}+=      _MKMSG_COMPILE=":"
231_ENV.${test}+=      _MKMSG_CREATE=":"
232_ENV.${test}+=      _MKMSG_FORMAT=":"
233_ENV.${test}+=      _MKMSG_TEST=":"
234_ENV.${test}+=      ${ENV.${test}}
235_ENV.${test}+=      USER_CPPFLAGS=${CPPFLAGS.${.TARGET}:Q}
236_ENV.${test}+=      USER_CFLAGS=${CFLAGS.${.TARGET}:Q}
237_ENV.${test}+=      BROKEN_TESTS=${${SLOW_TESTS} ${SKIP_TESTS.${.TARGET}}:L:Q}
238.endfor
239
240${TESTS}: run-test
241run-test: .USE
242          @echo "===> Running ${.TARGET}"
243          @echo "env: "${ENV.${.TARGET}:U"(none)"}
244          @echo "cflags: "${CFLAGS.${.TARGET}:U(none):Q}
245          @echo "cppflags: "${CPPFLAGS.${.TARGET}:U(none):Q}
246
247          @env -i ${_ENV.${.TARGET}} sh -ce "ma""ke -s cleandir"
248          @env -i ${_ENV.${.TARGET}} sh -ce "ma""ke -ks -j6 dependall"
249          @size *.o make
250          @env -i ${_ENV.${.TARGET}} sh -ce "ma""ke -s test"
251
252list:
253          @printf '%s%s\n' ${TESTS:O:@t@$t '${SKIP.$t:Myes:%= (skipped)}'@}
254