[Midnightbsd-cvs] src [7461] trunk/tools/tools/tinderbox: sync tinderbox with DES version 2.22 from May 2014
laffer1 at midnightbsd.org
laffer1 at midnightbsd.org
Fri Mar 4 08:41:09 EST 2016
Revision: 7461
http://svnweb.midnightbsd.org/src/?rev=7461
Author: laffer1
Date: 2016-03-04 08:37:08 -0500 (Fri, 04 Mar 2016)
Log Message:
-----------
sync tinderbox with DES version 2.22 from May 2014
Modified Paths:
--------------
trunk/tools/tools/tinderbox/etc/Makefile
trunk/tools/tools/tinderbox/etc/default.rc
trunk/tools/tools/tinderbox/etc/head-build.rc
trunk/tools/tools/tinderbox/etc/head-update.rc
trunk/tools/tools/tinderbox/etc/midnightbsd0.4-build.rc
trunk/tools/tools/tinderbox/etc/midnightbsd0.4-update.rc
trunk/tools/tools/tinderbox/tbmaster.1
trunk/tools/tools/tinderbox/tbmaster.pl
trunk/tools/tools/tinderbox/tinderbox.1
trunk/tools/tools/tinderbox/tinderbox.pl
trunk/tools/tools/tinderbox/www/index.cgi
Added Paths:
-----------
trunk/tools/tools/tinderbox/etc/head-noclang-build.rc
Modified: trunk/tools/tools/tinderbox/etc/Makefile
===================================================================
--- trunk/tools/tools/tinderbox/etc/Makefile 2016-03-04 02:50:34 UTC (rev 7460)
+++ trunk/tools/tools/tinderbox/etc/Makefile 2016-03-04 13:37:08 UTC (rev 7461)
@@ -1,9 +1,10 @@
-# $MidnightBSD: src/tools/tools/tinderbox/etc/Makefile,v 1.2 2008/03/06 20:23:45 laffer1 Exp $
+# $MidnightBSD$
ETCFILES =
ETCFILES += default.rc
ETCFILES += midnightbsd0.4-build.rc midnightbsd0.4-update.rc
ETCFILES += head-build.rc head-update.rc
+ETCFILES += head-noclang-build.rc
ETCDIR ?= ${HOME}/etc
ETCOWN ?= ${USER}
ETCGRP ?= ${USER}
Modified: trunk/tools/tools/tinderbox/etc/default.rc
===================================================================
--- trunk/tools/tools/tinderbox/etc/default.rc 2016-03-04 02:50:34 UTC (rev 7460)
+++ trunk/tools/tools/tinderbox/etc/default.rc 2016-03-04 13:37:08 UTC (rev 7461)
@@ -1,15 +1,14 @@
#
# Tinderbox defaults
#
-# $MidnightBSD: src/tools/tools/tinderbox/etc/default.rc,v 1.3 2008/03/07 04:32:06 laffer1 Exp $
-# $FreeBSD: src/tools/tools/tinderbox/etc/default.rc,v 1.9 2005/04/30 18:26:05 des Exp $
+# $MidnightBSD$
+# $FreeBSD: user/des/tinderbox/etc/default.rc 256000 2013-10-02 18:23:46Z des $
#
-comment = %%branch%% tinderbox
+comment = ${branch} tinderbox
# Paths
-sandbox = %%HOME%%
-logdir = %%HOME%%/logs
+sandbox = ${HOME}
# Tinderbox options
options = --verbose
@@ -18,6 +17,6 @@
# Reporting
sender = MidnightBSD Tinderbox <tinderbox at midnightbsd.org>
-recipient = %%SENDER%%
-subject = [%%COMMENT%%] failure on %%arch%%/%%machine%%
+recipient = ${SENDER}
+subject = [${COMMENT}] failure on ${arch}/${machine}
urlbase = http://tinderbox.midnightbsd.org/
Modified: trunk/tools/tools/tinderbox/etc/head-build.rc
===================================================================
--- trunk/tools/tools/tinderbox/etc/head-build.rc 2016-03-04 02:50:34 UTC (rev 7460)
+++ trunk/tools/tools/tinderbox/etc/head-build.rc 2016-03-04 13:37:08 UTC (rev 7461)
@@ -1,9 +1,11 @@
+#
# HEAD tinderbox setup, shared source - build stage
#
+# $FreeBSD: user/des/tinderbox/etc/head-build.rc 268247 2014-07-04 09:32:22Z des $
# $MidnightBSD$
branches = HEAD
-platforms = amd64, i386, sparc64
+platforms = amd64, i386
targets = precleanobj, version, world, lint, kernels, postcleanobj
-srcdir = %%SANDBOX%%/%%BRANCH%%/src
-recipient += <luke at midnightbsd.org>,%%SENDER%%
+srcdir = ${SANDBOX}/${BRANCH}/src
+recipient += <luke at midnightbsd.org>
Added: trunk/tools/tools/tinderbox/etc/head-noclang-build.rc
===================================================================
--- trunk/tools/tools/tinderbox/etc/head-noclang-build.rc (rev 0)
+++ trunk/tools/tools/tinderbox/etc/head-noclang-build.rc 2016-03-04 13:37:08 UTC (rev 7461)
@@ -0,0 +1,8 @@
+#
+# HEAD tinderbox setup, shared source - build without clang
+#
+# $FreeBSD: user/des/tinderbox/etc/head-noclang-build.rc 257786 2013-11-07 10:32:37Z des $
+# $MidnightBSD$
+
+include head-build
+env += WITHOUT_CLANG=YES, WITHOUT_CLANG_IS_CC=YES, WITH_GCC=YES, WITH_GNUCXX=YES
Modified: trunk/tools/tools/tinderbox/etc/head-update.rc
===================================================================
--- trunk/tools/tools/tinderbox/etc/head-update.rc 2016-03-04 02:50:34 UTC (rev 7460)
+++ trunk/tools/tools/tinderbox/etc/head-update.rc 2016-03-04 13:37:08 UTC (rev 7461)
@@ -1,10 +1,12 @@
+#
# HEAD tinderbox setup, shared source - update stage
#
+# $FreeBSD: user/des/tinderbox/etc/head-update.rc 266156 2014-05-15 16:24:20Z des $
# $MidnightBSD$
branches = HEAD
platforms = none
-srcdir = %%SANDBOX%%/%%BRANCH%%/src
-targets = update, version
+srcdir = ${SANDBOX}/${BRANCH}/src
+targets = revert, update, version
recipient += <luke at midnightbsd.org>
-subject = [%%COMMENT%%] source tree update failure
+subject = [${COMMENT}] source tree update failure
Modified: trunk/tools/tools/tinderbox/etc/midnightbsd0.4-build.rc
===================================================================
--- trunk/tools/tools/tinderbox/etc/midnightbsd0.4-build.rc 2016-03-04 02:50:34 UTC (rev 7460)
+++ trunk/tools/tools/tinderbox/etc/midnightbsd0.4-build.rc 2016-03-04 13:37:08 UTC (rev 7461)
@@ -5,5 +5,5 @@
branches = RELENG_0_4
platforms = amd64, i386, sparc64
targets = precleanobj, version, world, lint, kernels, postcleanobj
-srcdir = %%SANDBOX%%/%%BRANCH%%/src
+srcdir = ${SANDBOX}/${BRANCH}/src
recipient += %%SENDER%%,<luke at midnightbsd.org>
Modified: trunk/tools/tools/tinderbox/etc/midnightbsd0.4-update.rc
===================================================================
--- trunk/tools/tools/tinderbox/etc/midnightbsd0.4-update.rc 2016-03-04 02:50:34 UTC (rev 7460)
+++ trunk/tools/tools/tinderbox/etc/midnightbsd0.4-update.rc 2016-03-04 13:37:08 UTC (rev 7461)
@@ -4,7 +4,7 @@
branches = RELENG_0_4
platforms = none
-srcdir = %%SANDBOX%%/%%BRANCH%%/src
+srcdir = ${SANDBOX}/${BRANCH}/src
targets = update, version
recipient += <luke at midnightbsd.org>
subject = [%%COMMENT%%] source tree update failure
Modified: trunk/tools/tools/tinderbox/tbmaster.1
===================================================================
--- trunk/tools/tools/tinderbox/tbmaster.1 2016-03-04 02:50:34 UTC (rev 7460)
+++ trunk/tools/tools/tinderbox/tbmaster.1 2016-03-04 13:37:08 UTC (rev 7461)
@@ -1,5 +1,5 @@
.\"-
-.\" Copyright (c) 2003-2012 Dag-Erling Smørgrav
+.\" Copyright (c) 2003-2014 Dag-Erling Smørgrav
.\" All rights reserved.
.\"
.\" Redistribution and use in source and binary forms, with or without
@@ -23,10 +23,9 @@
.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
.\" SUCH DAMAGE.
.\"
-.\" $MidnightBSD: src/tools/tools/tinderbox/tbmaster.1,v 1.2 2008/03/07 04:28:13 laffer1 Exp $
-.\" $FreeBSD: projects/tinderbox/tbmaster.1,v 1.25 2008/01/26 19:34:31 des Exp $
+.\" $FreeBSD: user/des/tinderbox/tbmaster.1 263695 2014-03-24 22:34:22Z des $
.\"
-.Dd December 18, 2012
+.Dd March 24, 2014
.Dt TBMASTER 1
.Os
.Sh NAME
@@ -67,17 +66,30 @@
The directory where configuration files are located.
The default value is
.Pa $HOME/etc .
-.It Fl l Ar FILE
+.It Fl h Ar HOSTNAME , Fl -hostname Ns = Ns Ar HOSTNAME
+The name of the host running the tinderbox, used in logs and reports.
+Can be overridden by the
+.Va HOSTNAME
+configuration variable.
+The default value is that reported by
+.Xr uname 1 .
+.It Fl l Ar FILE , Fl -lockfile Ns = Ns Ar FILE
The name of a file to lock upon startup.
If the lock is already held by another process,
.Nm
will terminate immediately rather than block.
-.It Fl n Ar ncpu
-The number of concurrent jobs to run.
-The default value is the number of CPUs in the machine, or 1 if that
+.It Fl n Ar NUMBER , Fl -ncpu Ns = Ns Ar NUMBER
+The maximum number of concurrent builds to run.
+Can be overridden by the
+.Va NCPU
+configuration variable.
+The default value is the number of cores in the machine, or 1 if that
number could not be determined.
+See the
+.Sx Concurrency
+section for additional information.
.El
-.Ss Configuration
+.Ss Configuration files
The
.Nm
script uses named configurations located in individual files named for
@@ -95,33 +107,79 @@
.Pa default.rc
and
.Pa site.rc
-before reading the actual configuration file; thus, these files may be
-used to specify default values shared by multiple configurations.
+before and after the actual configuration file, respectively; thus,
+they can be used to specify default values shared by multiple
+configurations and to override the values set in the individual
+configuration files.
.Pp
-The configuration consists of a list of single- or multiple-value
-variable assignments:
+Each configuration file consists of a list of single- or
+multiple-value variable assignments:
.Bl -tag
-.It Va single_variable = Ar value
-.It Va multi_variable = Ar value1 , Ar value2 , ...
-.It Va multi_variable += Ar value3 , ...
+.It Va variable No = Ar value
+Assigns
+.Ar value
+to the single-value variable
+.Va variable .
+.It Va variable No = Ar value1 Op No , Ar value2 ...
+Assigns
+.Ar value1 ,
+.Ar value2
+etc. to the multi-value variable
+.Va variable .
+.It Va variable No += Ar value3 Op No , Ar value4 ...
+Appends
+.Ar value3 ,
+.Ar value4
+etc. to the multi-value variable
+.Va variable .
+.It Va variable No -= Ar value5 Op No , Ar value6 ...
+Removes
+.Ar value5 ,
+.Ar value6
+etc. from the multi-value variable
+.Va variable .
.El
.Pp
-Whitespace around the equal sign and around the commas separating
-multiple values is optional.
+Whitespace around the assigment operator and around the commas
+separating multiple values is optional.
.Pp
Blank lines are ignored, as is anything following a hash sign
-.Pq Dq # .
+.Pq Sq # .
.Pp
-The following configuration variables are defined:
+Additionally,
+.Cm include
+statements can be used to include one configuration in another:
+.Bl -tag
+.It Cm include Ar otherconfig
+.El
+.Ss Configuration variables
+Below is a list of the configuration variables
+.Nm
+recognizes and their semantics.
+.Pp
+Note that many of these variables are passed on as command-line
+arguments to
+.Xr tinderbox 1 ,
+which may provide its own default values for variables which are left
+undefined by
+.Nm .
+.Pp
+Some variables are read-only and are provided so that other variables
+may include them.
+For instance, a common idiom is to derive
+.Va OBJDIR
+from a combination of
+.Va BRANCH ,
+.Va ARCH
+and
+.Va MACHINE .
.Bl -tag -width 12n
.It ARCH
-.Pq Vt single
+.Pq Vt single, read-only
The architecture currently being built for.
-Read-only.
.It BRANCH
-.Pq Vt single
+.Pq Vt single, read-only
The branch currently being built.
-Read-only.
.It BRANCHES
.Pq Vt multiple
A list of source branches to build.
@@ -153,12 +211,6 @@
.Pq Vt single
A terse comment describing the setup.
No default value.
-.It CVSUP
-.Pq Vt single
-The name of the
-.Xr cvsup 1
-server to use.
-No default value.
.It ENV
.Pq Vt multiple
A list of environment variables to pass to the
@@ -166,46 +218,45 @@
script.
Each value is the name and value of an environment variable, separated
by an equal sign
-.Pq Dq = .
+.Pq Sq = .
No default value.
.It HOME
-.Pq Vt single
+.Pq Vt single, read-only
The current user's home directory, as specified by the
.Ev HOME
environment variable.
Note that it will not be defined unless it passes some simple sanity
checks.
-Read-only.
.It HOSTNAME
.Pq Vt single
The name of the host running the tinderbox.
-This defaults to the name reported by the
-.Fl n
-option of the
-.Xr uname 1
-command, and is only used for cosmetic purposes.
.It JOBS
The maximum number of concurrent
.Xr make 1
-jobs to run.
+processes to run within each build.
No default value.
+See the
+.Sx Concurrency
+section for additional information.
.It LOGDIR
.Pq Vt single
The location of the log directory.
The default value is
-.Pa %%SANDBOX%%/logs .
+.Pa ${SANDBOX}/logs .
.It MACHINE
+.Pq Vt single, read-only
+The machine currently being built for.
+.It NCPU
.Pq Vt single
-The machine currently being built for.
-Read-only.
+The maximum number of concurrent builds to run.
+No default value.
+See the
+.Sx Concurrency
+section for additional information.
.It OBJDIR
.Pq Vt single
The object directory.
-There is no default value; see the
-.Xr tinderbox 1
-script's
-.Fl -objdir
-option for details.
+No default value.
.It OPTIONS
.Pq Vt multiple
A list of additional options to pass to the
@@ -224,7 +275,7 @@
Which architectures and machines to build for.
Each value is the name of an architecture, optionally followed by a
forward slash
-.Pq Dq /
+.Pq Sq /
and a machine name.
If the machine name is not specified, it is assumed to be identical to
the architecture name.
@@ -234,7 +285,7 @@
.Pq Vt multiple
The addresses to which failure reports should be mailed.
The default value is
-.Dq %%SENDER%% .
+.Dq ${SENDER} .
.Pp
To avoid unintentional spamming,
.Nm
@@ -241,12 +292,6 @@
will strip recipients in the
.Li freebsd.org
domain from this list unless the correct magic sauce is used.
-.It REPOSITORY
-.Pq Vt single
-The location of the
-.Xr cvs 1
-repository.
-No default value.
.It SANDBOX
.Pq Vt single
The location of the sandbox directory.
@@ -260,11 +305,7 @@
.It SRCDIR
.Pq Vt single
The source directory.
-There is no default value; see the
-.Xr tinderbox 1
-script's
-.Fl -srcdir
-option for details.
+No default value.
.Pp
Normally, a separate directory within the sandbox will be used for
each build.
@@ -281,10 +322,11 @@
.Pq Vt single
The subject to use on failure reports.
The default value is
-.Dq Tinderbox failure on %%arch%%/%%machine%% .
+.Dq Tinderbox failure on ${arch}/${machine} .
.It SVNBASE
.Pq Vt single
The URL to the base of the Subversion repository.
+No default value.
.It TARGETS
.Pq Vt multiple
A list of targets (commands) to pass to the
@@ -303,7 +345,7 @@
.Xr tinderbox 1
script.
The default value is
-.Dq %%HOME%%/bin/tinderbox .
+.Dq ${HOME}/bin/tinderbox .
.It URLBASE
.Pq Vt single
If defined, a URL constructed by appending the file name of the full
@@ -317,26 +359,32 @@
.Bl -bullet
.It
If a single-value variable contains substrings of the form
-.Va %%VAR%%
+.Va ${VAR}
or
-.Va %%var%% ,
+.Va ${var} ,
those substrings are replaced with the values of the corresponding
variables, after recursive substitution.
The difference between the first and the second form is that the
latter is converted to lower-case before use.
For instance,
-.Dq %%BRANCH%%
+.Dq ${BRANCH}
might expand to
.Dq RELENG_4
while
-.Dq %%branch%%
+.Dq ${branch}
would expand to
.Dq releng_4 .
.It
+If a single-value varaible contains substrings of the form
+.Va $ENV{VAR} ,
+those substrings are replaced with the values of the corresponding
+environment variables.
+Use this with care.
+.It
If an element of a multiple-value variable is of the form
-.Va %%VAR%%
+.Va ${VAR}
or
-.Va %%var%%
+.Va ${var}
and the corresponding variable is a multiple-value variable, recursive
substitution is first performed on that variable, and the resulting
values are included individually in the result.
@@ -344,8 +392,53 @@
Otherwise, elements of multiple-value variables are expanded
individually according to the same rules as single-value variables.
.El
+.Pp
+For backward compatibility with earlier versions, the forms
+.Va %%VAR%%
+and
+.Va %%var%%
+may be used instead of
+.Va ${VAR}
+and
+.Va ${var} .
+.Ss Concurrency
+On multiprocessor machines, performance can generally be improved by
+running multiple builds in parallel, up to a certain limit.
+By default,
+.Nm
+will run one build for each processor core in the system.
+This can be overridden with the
+.Fl -ncpu
+command-line option and the
+.Va NCPU
+configuration variable, the latter taking precedence.
+.Pp
+In addition, each build may run multiple
+.Xr make 1
+processes in parallel, up to the number specified by the
+.Va JOBS
+configuration variable.
+.Pp
+The total number of parallel
+.Xr make 1
+processes will vary, but can be as high as the product of of
+.Va NCPU
+and
+.Va JOBS.
+As a result of processor, memory and filesystem contention, an
+excessively large value can have a significant negative impact on
+performance.
+.Pp
+As a rule of thumb,
+.Va NCPU
+should not exceed one build per gigabyte of physical memory in the
+system, and the
+.Va NCPU
+x
+.Va JOBS
+product should not exceed the number of processor cores in the system
+by a large amount.
.Sh SEE ALSO
-.Xr perl 1 ,
.Xr tinderbox 1
.Sh AUTHORS
.Nm
Modified: trunk/tools/tools/tinderbox/tbmaster.pl
===================================================================
--- trunk/tools/tools/tinderbox/tbmaster.pl 2016-03-04 02:50:34 UTC (rev 7460)
+++ trunk/tools/tools/tinderbox/tbmaster.pl 2016-03-04 13:37:08 UTC (rev 7461)
@@ -1,6 +1,6 @@
#!/usr/bin/perl -Tw
#-
-# Copyright (c) 2003-2012 Dag-Erling Smørgrav
+# Copyright (c) 2003-2014 Dag-Erling Smørgrav
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
@@ -24,9 +24,8 @@
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
-# $FreeBSD: src/tools/tools/tinderbox/tbmaster.pl,v 1.53 2005/03/03 07:18:01 des Exp $
-# $MidnightBSD: src/tools/tools/tinderbox/tbmaster.pl,v 1.3 2008/03/07 04:26:00 laffer1 Exp $
-#
+# $FreeBSD: user/des/tinderbox/tbmaster.pl 266153 2014-05-15 16:17:21Z des $
+# $MidnightBSD$
use v5.10.1;
use strict;
@@ -33,12 +32,13 @@
use Fcntl qw(:DEFAULT :flock);
use POSIX;
use Getopt::Long;
+use Storable qw(dclone);
-my $VERSION = "2.10";
-my $COPYRIGHT = "Copyright (c) 2003-2012 Dag-Erling Smørgrav. " .
+my $VERSION = "2.22";
+my $COPYRIGHT = "Copyright (c) 2003-2014 Dag-Erling Smørgrav. " .
"All rights reserved.";
-my $BACKLOG = 8;
+my $BACKLOG = 20;
my $abbreviate; # Abbreviate path names in log file
my @configs; # Names of requested configations
@@ -46,6 +46,7 @@
my $etcdir; # Configuration directory
my $lockfile; # Lock file name
my $lock; # Lock file descriptor
+my $hostname; # Hostname
my $ncpu; # Number of CPUs
my %platforms; # Specific platforms to build
@@ -54,27 +55,30 @@
'CFLAGS' => '',
'COPTFLAGS' => '',
'COMMENT' => '',
- 'CVSUP' => '',
'ENV' => [ ],
'HOSTNAME' => '',
'JOBS' => '',
- 'LOGDIR' => '%%SANDBOX%%/logs',
+ 'LOGDIR' => '${SANDBOX}/logs',
+ 'NCPU' => '',
'OBJDIR' => '',
'OPTIONS' => [ ],
'PATCH' => '',
'PLATFORMS' => [ 'i386' ],
- 'RECIPIENT' => [ '%%SENDER%%' ],
- 'REPOSITORY'=> '',
+ 'RECIPIENT' => [ '${SENDER}' ],
'SANDBOX' => '/tmp/tinderbox',
'SENDER' => '',
'SRCDIR' => '',
- 'SUBJECT' => 'Tinderbox failure on %%arch%%/%%machine%%',
+ 'SUBJECT' => 'Tinderbox failure on ${arch}/${machine}',
'SVNBASE' => '',
'TARGETS' => [ 'update', 'world' ],
'TIMEOUT' => '',
- 'TINDERBOX' => '%%HOME%%/bin/tinderbox',
+ 'TINDERBOX' => '${HOME}/bin/tinderbox',
'URLBASE' => '',
);
+my %NUMERIC_OPTIONS = map { $_ => 1 } qw(JOBS NCPU TIMEOUT);
+my %PATHNAME_OPTIONS =
+ map { $_ => 1 } qw(LOGDIR OBJDIR PATCH SANDBOX SRCDIR TINDERBOX);
+my %WORD_OPTIONS = map { $_ => 1 } qw(PLATFORMS TARGETS);
my %CONFIG;
#
@@ -122,18 +126,47 @@
if (ref($elem)) {
# prepend to queue for further processing
unshift(@elements, @{$elem});
- } elsif ($elem =~ m/^\%\%(\w+)\%\%$/) {
+ } elsif ($elem =~ m/^\%\%(\w+)\%\%$/ || $elem =~ m/^\%\{(\w+)\}$/) {
# prepend to queue for further processing
# note - can expand to a list
unshift(@elements, expand($1));
} else {
+ $elem =~ s/\$ENV\{(\w+)\}/$ENV{$1}/g;
$elem =~ s/\%\%(\w+)\%\%/expand($1)/eg;
+ $elem =~ s/\$\{(\w+)\}/expand($1)/eg;
push(@expanded, $elem);
}
}
+
+ # Upper / lower case
if ($key !~ m/[A-Z]/) {
@expanded = map { lc($_) } @expanded;
}
+
+ # Validate and untaint expanded value(s)
+ if ($NUMERIC_OPTIONS{uc($key)}) {
+ @expanded = map {
+ m/^(\d+|)$/
+ or die("invalid value for numeric variable $key: $_\n");
+ $1
+ } @expanded;
+ } elsif ($PATHNAME_OPTIONS{uc($key)}) {
+ @expanded = map {
+ m@^((?:/+[\w.-]+)+/*|)$@
+ or die("invalid value for pathname variable $key: $_\n");
+ $1
+ } @expanded;
+ } elsif ($WORD_OPTIONS{uc($key)}) {
+ @expanded = map {
+ # hack - support not only "word" but also "word/word" so
+ # platform designations will pass the test.
+ m@^([\w.-]+(?:/[\w.-]+)?|)$@
+ or die("invalid value for word variable $key: $_\n");
+ $1
+ } @expanded;
+ }
+
+ # Verify single / multiple and return result
if (ref($value)) {
return @expanded;
} elsif (@expanded != 1) {
@@ -148,16 +181,17 @@
#
sub clearconf() {
- %CONFIG = %INITIAL_CONFIG;
+ %CONFIG = %{dclone(\%INITIAL_CONFIG)};
}
#
# Read in a configuration file
#
+sub readconf($);
sub readconf($) {
my $fn = shift;
- open(my $fh, "<", $fn)
+ open(my $fh, '<', $fn)
or return undef;
my $line = "";
my $n = 0;
@@ -167,34 +201,43 @@
s/\s*(\#.*)?$//;
$line .= $_;
if (length($line) && $line !~ s/\\$/ /) {
- die("$fn: syntax error on line $n\n")
- unless ($line =~ m/^(\w+)\s*([+]?=)\s*(.*)$/);
- my ($key, $op, $val) = (uc($1), $2, $3);
- $val = ''
- unless defined($val);
- die("$fn: unknown keyword on line $n\n")
- unless (exists($CONFIG{$key}));
- if (ref($CONFIG{$key})) {
- my @a = split(/\s*,\s*/, $val);
- foreach (@a) {
- s/^\'([^\']*)\'$/$1/;
- }
- if ($op eq '=') {
- $CONFIG{$key} = \@a;
- } elsif ($op eq '+=') {
- push(@{$CONFIG{$key}}, @a);
+ if ($line =~ m/^include\s+([\w-]+)$/) {
+ readconf("$1.rc")
+ or die("$fn: include $1: $!\n");
+ } elsif ($line =~ m/^(\w+)\s*([+-]?=)\s*(.*)$/) {
+ my ($key, $op, $val) = (uc($1), $2, $3);
+ $val = ''
+ unless defined($val);
+ die("$fn: $key is not a known keyword on line $n\n")
+ unless (exists($CONFIG{$key}));
+ if (ref($CONFIG{$key})) {
+ my @a = split(/\s*,\s*/, $val);
+ foreach (@a) {
+ s/^\'([^\']*)\'$/$1/;
+ }
+ if ($op eq '=') {
+ $CONFIG{$key} = \@a;
+ } elsif ($op eq '+=') {
+ push(@{$CONFIG{$key}}, @a);
+ } elsif ($op eq '-=') {
+ my %a = map { $_ => $_ } @a;
+ @{$CONFIG{$key}} =
+ grep { !exists($a{$_}) } @{$CONFIG{$key}};
+ } else {
+ die("can't happen\n");
+ }
} else {
- die("can't happen\n");
+ $val =~ s/^\'([^\']*)\'$/$1/;
+ if ($op eq '=') {
+ $CONFIG{$key} = $val;
+ } elsif ($op eq '+=' || $op eq '-=') {
+ die("$fn: $key is not an array on line $n\n");
+ } else {
+ die("can't happen\n");
+ }
}
} else {
- $val =~ s/^\'([^\']*)\'$/$1/;
- if ($op eq '=') {
- $CONFIG{$key} = $val;
- } elsif ($op eq '+=') {
- die("$fn: invalid operator on line $n\n");
- } else {
- die("can't happen\n");
- }
+ die("$fn: syntax error on line $n\n")
}
$line = "";
}
@@ -221,7 +264,7 @@
$history .= $success ? "OK\n" : "FAIL\n";
my $fn = expand('LOGDIR') . "/history";
- if (open(my $fh, ">>", $fn)) {
+ if (open(my $fh, '>>', $fn)) {
print($fh $history);
close($fh);
} else {
@@ -248,7 +291,7 @@
return;
}
- if (open(my $pipe, "|-", "/usr/sbin/sendmail", "-i", "-t", "-f$sender")) {
+ if (open(my $pipe, '|-', qw(/usr/sbin/sendmail -i -t -f), $sender)) {
print($pipe "Sender: $sender\n");
print($pipe "From: $sender\n");
print($pipe "To: $recipient\n");
@@ -274,7 +317,7 @@
my $config = expand('CONFIG');
my $start = time();
- $0 = "tbmaster: building $branch for $arch/$machine";
+ $0 = "tbmaster [$config]: building $branch for $arch/$machine";
$CONFIG{'BRANCH'} = $branch;
$CONFIG{'ARCH'} = $arch;
@@ -282,10 +325,14 @@
# Open log files: one for the full log and one for the summary
my $logdir = expand('LOGDIR');
- my $logfile = "tinderbox-$config-$branch-$arch-$machine";
+ if (!-d $logdir) {
+ die("nonexistent log directory: $logdir\n");
+ }
+ my $logname = "tinderbox-$config-$branch-$arch-$machine";
+ my $logbase = "$logdir/$logname";
my $full;
- if (!open($full, ">", "$logdir/$logfile.full.$$")) {
- warn("$logdir/$logfile.full.$$: $!\n");
+ if (!open($full, '>', "$logbase.full.$$")) {
+ warn("$logbase.full.$$: $!\n");
return undef;
}
select($full);
@@ -292,8 +339,8 @@
$| = 1;
select(STDOUT);
my $brief;
- if (!open($brief, ">", "$logdir/$logfile.brief.$$")) {
- warn("$logdir/$logfile.brief.$$: $!\n");
+ if (!open($brief, '>', "$logbase.brief.$$")) {
+ warn("$logbase.brief.$$: $!\n");
return undef;
}
select($brief);
@@ -304,9 +351,9 @@
my ($rpipe, $wpipe);
if (!pipe($rpipe, $wpipe)) {
warn("pipe(): $!\n");
- unlink("$logdir/$logfile.brief.$$");
+ unlink("$logbase.brief.$$");
close($brief);
- unlink("$logdir/$logfile.full.$$");
+ unlink("$logbase.full.$$");
close($full);
return undef;
}
@@ -321,8 +368,6 @@
if ($CONFIG{'OBJDIR'});
push(@args, "--arch=$arch");
push(@args, "--machine=$machine");
- push(@args, "--cvsup=" . expand('CVSUP'))
- if ($CONFIG{'CVSUP'});
push(@args, "--repository=" . expand('REPOSITORY'))
if ($CONFIG{'REPOSITORY'});
push(@args, "--branch=$branch");
@@ -343,15 +388,15 @@
my $pid = fork();
if (!defined($pid)) {
warn("fork(): $!\n");
- unlink("$logdir/$logfile.brief.$$");
+ unlink("$logbase.brief.$$");
close($brief);
- unlink("$logdir/$logfile.full.$$");
+ unlink("$logbase.full.$$");
close($full);
return undef;
} elsif ($pid == 0) {
close($rpipe);
- open(STDOUT, ">&", $wpipe);
- open(STDERR, ">&", $wpipe);
+ open(STDOUT, '>&', $wpipe);
+ open(STDERR, '>&', $wpipe);
$| = 1;
exec(expand('TINDERBOX'), @args);
die("exec(): $!\n");
@@ -363,12 +408,12 @@
my $error = 0;
my $summary = "";
my $root = realpath(expand('SANDBOX') . "/$branch/$arch/$machine");
- my $srcdir = realpath(expand('SRCDIR')) || "$root/src";
- my $objdir = realpath(expand('OBJDIR')) || "$root/obj";
+ my $srcdir = realpath(expand('SRCDIR') || "$root/src");
+ my $objdir = realpath(expand('OBJDIR') || "$root/obj");
while (<$rpipe>) {
if ($abbreviate) {
- s/\Q$srcdir\E/\/src/g;
- s/\Q$objdir\E/\/obj/g;
+ s/\Q$srcdir\E/\/src/go;
+ s/\Q$objdir\E/\/obj/go;
}
print($full $_);
if (/^TB ---/ || /^>>> /) {
@@ -425,7 +470,7 @@
my @recipients = expand('RECIPIENT');
if (!$ENV{'MAGIC_SAUCE'} ||
$ENV{'MAGIC_SAUCE'} ne 'MIDNIGHTBSD_TINDERBOX') {
- @recipients = grep { ! m/\@midnightbsd\.org/i } @recipients;
+ @recipients = grep { ! m/\@midnightbsd.org\.org/i } @recipients;
}
# Mail out error reports
@@ -434,13 +479,13 @@
my $recipient = join(', ', @recipients);
my $subject = expand('SUBJECT');
if ($CONFIG{'URLBASE'}) {
- $summary .= "\n\n" . expand('URLBASE') . "$logfile.full";
+ $summary .= "\n\n" . expand('URLBASE') . "$logname.full";
}
report($sender, $recipient, $subject, $summary);
}
- rename("$logdir/$logfile.full.$$", "$logdir/$logfile.full");
- rename("$logdir/$logfile.brief.$$", "$logdir/$logfile.brief");
+ rename("$logbase.full.$$", "$logbase.full");
+ rename("$logbase.brief.$$", "$logbase.brief");
}
#
@@ -489,7 +534,7 @@
sub usage() {
(my $self = $0) =~ s|^.*/||;
- print(STDERR "This is the BSD tinderbox manager, version $VERSION.
+ print(STDERR "This is the MidnightBSD tinderbox manager, version $VERSION.
$COPYRIGHT
Usage:
@@ -517,9 +562,9 @@
clearconf();
readconf('default.rc');
- readconf('site.rc');
readconf("$config.rc")
or die("$config.rc: $!\n");
+ readconf('site.rc');
$CONFIG{'CONFIG'} = $config;
$CONFIG{'ETCDIR'} = $etcdir;
@@ -540,6 +585,7 @@
die("Where is the tinderbox script?\n");
}
+ # Check stop file
my $stopfile = expand('SANDBOX') . "/stop";
my @jobs;
foreach my $branch (expand('BRANCHES')) {
@@ -552,12 +598,15 @@
}
}
- $0 = "tbmaster: supervisor";
+ # Main loop: start as many concurrent jobs as permitted, then keep
+ # starting new jobs as soon as existing jobs terminate, until all
+ # jobs have terminated and there are none left in the queue.
+ $0 = "tbmaster [$config]: supervisor";
my %children;
my $done = 0;
while (@jobs || keys(%children)) {
# start more children if we can
- while (@jobs && keys(%children) < $ncpu) {
+ while (@jobs && keys(%children) < expand('NCPU')) {
my ($branch, $arch, $machine) = @{shift(@jobs)};
if (-e $stopfile || -e "$stopfile.$branch" ||
-e "$stopfile.$arch" || -e "$stopfile.$arch.$machine") {
@@ -575,7 +624,7 @@
}
warn("forked child $child for $branch $arch/$machine\n");
}
- $0 = "tbmaster: supervisor (" .
+ $0 = "tbmaster [$config]: supervisor (" .
keys(%children) . " running, " .
@jobs . " pending, " .
$done . " completed)";
@@ -593,6 +642,21 @@
}
#
+# Read the input from a command
+#
+sub slurp(@) {
+ my @cmdline = @_;
+
+ if (open(my $pipe, '-|', @cmdline)) {
+ local $/;
+ my $input = <$pipe>;
+ close($pipe);
+ return $input;
+ }
+ return undef;
+}
+
+#
# Main
#
MAIN:{
@@ -599,12 +663,6 @@
# Set defaults
$ENV{'TZ'} = "UTC";
$ENV{'PATH'} = "/usr/bin:/usr/sbin:/bin:/sbin";
- $INITIAL_CONFIG{'HOSTNAME'} = `/usr/bin/uname -n`;
- if ($INITIAL_CONFIG{'HOSTNAME'} =~ m/^([0-9a-z-]+(?:\.[0-9a-z-]+)*)$/) {
- $INITIAL_CONFIG{'HOSTNAME'} = $1;
- } else {
- $INITIAL_CONFIG{'HOSTNAME'} = 'unknown';
- }
if ($ENV{'HOME'} =~ m/^((?:\/[\w\.-]+)+)\/*$/) {
$INITIAL_CONFIG{'HOME'} = realpath($1);
$etcdir = "$1/etc";
@@ -611,12 +669,6 @@
$ENV{'PATH'} = "$1/bin:$ENV{'PATH'}"
if (-d "$1/bin");
}
- $ncpu = `/sbin/sysctl -n hw.ncpu`;
- if ($ncpu =~ m/^\s*(\d+)\s*$/) {
- $ncpu = int($1);
- } else {
- $ncpu = 1;
- }
# Get options
{Getopt::Long::Configure("auto_abbrev", "bundling");}
@@ -625,10 +677,12 @@
"c|config=s" => \@configs,
"d|dump" => \$dump,
"e|etcdir=s" => \$etcdir,
+ "h|hostname=s" => \$hostname,
"l|lockfile=s" => \$lockfile,
"n|ncpu=i" => \$ncpu,
) or usage();
+ # Subsequent arguments are platforms to build
foreach (@ARGV) {
if (m/^(\w+(?:\/\w+)?)$/) {
$platforms{$1} = 1;
@@ -637,11 +691,34 @@
}
}
+ # Get / check hostname
+ if (!$hostname) {
+ $hostname = slurp(qw(/usr/bin/uname -n));
+ }
+ if ($hostname &&
+ $hostname =~ m/^\s*([a-z][0-9a-z-]+(?:\.[a-z][0-9a-z-]+)*)\s*$/s) {
+ $hostname = $1;
+ } else {
+ $hostname = 'unknown';
+ }
+ $INITIAL_CONFIG{'HOSTNAME'} = $hostname;
+
+ # Get / check number of CPUs
+ if (!$ncpu) {
+ $ncpu = slurp(qw(/sbin/sysctl -n hw.ncpu));
+ }
+ if ($ncpu && $ncpu =~ m/^\s*(\d+)\s*$/s) {
+ $ncpu = int($1);
+ } else {
+ $ncpu = 1;
+ }
+ $INITIAL_CONFIG{'NCPU'} = $ncpu;
+
# Check options
if (@configs) {
@configs = split(/,/, join(',', @configs));
} else {
- $configs[0] = `/usr/bin/uname -n`;
+ $configs[0] = $hostname;
chomp($configs[0]);
$configs[0] =~ s/^(\w+)(\..*)?/$1/;
}
@@ -665,7 +742,7 @@
die("invalid lockfile\n");
}
$lockfile = $1;
- $lock = open_locked($lockfile, ">", 0600)
+ $lock = open_locked($lockfile, '>', 0600)
or die("unable to acquire lock on $lockfile\n");
# Lock will be released upon termination.
}
Modified: trunk/tools/tools/tinderbox/tinderbox.1
===================================================================
--- trunk/tools/tools/tinderbox/tinderbox.1 2016-03-04 02:50:34 UTC (rev 7460)
+++ trunk/tools/tools/tinderbox/tinderbox.1 2016-03-04 13:37:08 UTC (rev 7461)
@@ -1,5 +1,5 @@
.\"-
-.\" Copyright (c) 2003-2012 Dag-Erling Smørgrav
+.\" Copyright (c) 2003-2014 Dag-Erling Smørgrav
.\" All rights reserved.
.\"
.\" Redistribution and use in source and binary forms, with or without
@@ -23,10 +23,10 @@
.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
.\" SUCH DAMAGE.
.\"
-.\" $MidnightBSD: src/tools/tools/tinderbox/tinderbox.1,v 1.2 2008/03/07 04:28:13 laffer1 Exp $
-.\" $FreeBSD: projects/tinderbox/tinderbox.1,v 1.18 2008/01/26 19:34:31 des Exp $
+.\" $FreeBSD: user/des/tinderbox/tinderbox.1 268719 2014-07-15 22:34:54Z des $
+.\" $MidnightBSD$
.\"
-.Dd March 29, 2013
+.Dd May 24, 2014
.Dt TINDERBOX 1
.Os
.Sh NAME
@@ -41,7 +41,7 @@
The
.Nm
script tests the
-.Fx
+.Mx
build and release system by performing a cross-build (and optionally a
cross-release) of an arbitrary branch of the source tree for an
arbitrary target platform.
@@ -49,20 +49,18 @@
The following options are recognized:
.Bl -tag -width 12n
.It Fl -arch Ns = Ns Ar ARCH
-Specifies the target architecture.
+Specifies the target architecture
+.Pq Va $arch .
The default value is whatever the host system's
.Xr uname 1
reports.
.It Fl -branch Ns = Ns Ar BRANCH
The branch to check out from
-.Xr svn 1 ,
-.Xr cvs 1
-or
-.Xr cvsup 1
-when updating the source tree.
+.Xr svn 1
+when updating the source tree
+.Pq Va $branch .
The default is to check out HEAD.
-When using
-.Xr svn 1 ,
+For historical compatibility,
.Xr cvs 1 Ns -style
branch names are translated to
.Xr svn 1
@@ -74,15 +72,6 @@
.Dv RELENG_9_1
becomes
.Pa releng/9.1 .
-.It Fl -cvsup Ns = Ns Ar NAME
-The name of the
-.Xr cvsup 1
-server from which to update the source tree
-.Cm update
-command is specified.
-The default is to use
-.Xr cvs 1
-instead.
.It Fl -destdir Ns = Ns Ar DIR
The directory (aka
.Dv DESTDIR )
@@ -99,7 +88,9 @@
.Xr uname 1
command, and is only used for cosmetic purposes.
.It Fl -jobs Ns = Ns Ar NUM
-The maximum number of paralell jobs, as specified to
+The maximum number of paralell jobs
+.Pq Va $jobs ,
+as specified to
.Xr make 1
using the
.Fl j
@@ -112,7 +103,8 @@
The default is to redirect all output to
.Pa /dev/stdout .
.It Fl -machine Ns = Ns Ar MACHINE
-Specifies the target machine.
+Specifies the target machine
+.Pq Va $machine .
The default value is whatever the host system's
.Xr uname 1
reports, unless a target architecture was specified with the
@@ -119,7 +111,8 @@
.Fl -arch
option, in which case the target machine is set to the same value.
.It Fl -objdir Ns = Ns Ar DIR
-Specifies the object directory prefix.
+Specifies the object directory prefix
+.Pq Va $objdir .
.It Fl -patch Ns = Ns Ar PATCH
The file name of a patch to apply to the source tree before building
if the
@@ -135,20 +128,13 @@
.Cm patch
command was specified.
The default is to not apply any patches.
-.It Fl -repository Ns = Ns Ar DIR
-The location of the
-.Xr cvs 1
-repository from which to update the source tree if the
-.Cm update
-command is specified.
-The default is
-.Pa /home/ncvs .
.It Fl -sandbox Ns = Ns Ar DIR
The location of the sandbox in which the builds are to take place.
This directory should reside on a reasonably fast disk with at least
1.5 GB available (3 GB if building a release).
.It Fl -srcdir Ns = Ns Ar DIR
-Specifies the source directory.
+Specifies the source directory
+.Pq Va $srcdir .
Normally, the
.Nm
script creates a separate source directory within the sandbox for each
@@ -167,6 +153,9 @@
commands.
.It Fl -svnbase Ns = Ns Ar URL
The URL to the base of the Subversion repository.
+The default is
+.\" There does not seem to be an mdoc macro for URLs; use Pa instead.
+.Pa svn://svn.freebsd.org/base .
.It Fl -timeout Ns = Ns Ar NUM
The maximum wall-time duration of the run, in seconds.
The default is to continue until all targets are completed.
@@ -213,12 +202,11 @@
Delete the installation tree at the start of each job.
.It Cm precleanroot
Delete the release chroot tree at the start of each job.
+.It Cm revert
+Revert the source tree to a clean state.
.It Cm update
-Update the sources using
-.Xr cvs 1 .
-This is highly recommended, for obvious reasons, if the
-.Cm clean
-command is specified.
+Update the source tree using
+.Xr svn 1 .
.It Cm patch
Apply the patch specified with the
.Fl -patch
@@ -226,6 +214,9 @@
If the specified patch file does not exist, the
.Cm patch
command will fail gracefully.
+.It Cm version
+After updating and patching the source tree but before doing anything
+else, log information about the current state of the source tree.
.It Cm world
Build the world.
.It Cm lint
@@ -289,9 +280,6 @@
As
.Cm precleanroot ,
but at the end of each job.
-.It Cm version
-After updating and patching the source tree but before doing anything
-else, log information about the current state of the source tree.
.El
.Pp
The commands are executed in the order in which they are listed above,
@@ -322,35 +310,41 @@
The following variables are set for all builds:
.Bl -tag -width 18n
.It PATH
-.Va /usr/bin:/usr/sbin:/bin:/sbin
+.Ar /usr/bin:/usr/sbin:/bin:/sbin
.It TZ
-.Va UTC
+.Ar UTC
.It __MAKE_CONF
-.Va /dev/null
+.Ar /dev/null
+.It SRCCONF
+.Ar /dev/null
+.It MAKEOBJDIRPREFIX
+.Ar $objdir
+.It TARGET
+.Ar $machine
+.It TARGET_ARCH
+.Ar $arch
+.It CROSS_BUILD_TESTING
+.Ar YES
.El
.Pp
The following additional variables are set for release builds:
.Bl -tag -width 18n
.It CHROOTDIR
-.Va $SANDBOX/root
-.It CVSROOT
-As specified with the
-.Fl -repository
-option.
+.Ar $SANDBOX/root
.It RELEASETAG
-.Va -rBRANCH
-if
-.Fl -branch Ns = Ns Ar BRANCH
-was specified, or
-.Va -A
+.Ar -r$branch
+if specified using the
+.Fl -branch
+option, or
+.Ar -A
otherwise.
.It WORLD_FLAGS , KERNEL_FLAGS
Both of these are set to
-.Va -jN
-if
-.Fl -jobs Ns = Ns Ar N
-was specified, or
-.Va -B
+.Ar -j$jobs
+if specified using the
+.Fl -jobs
+option, or
+.Ar -B
otherwise.
.It LOCAL_PATCHES
Set to the path of the patch that was specified with the
@@ -358,21 +352,21 @@
option, if any.
.It PATCH_FLAGS
Set to
-.Va -fs
+.Ar -fs
if a patch was specified.
.It NOCDROM
-.Va YES
+.Ar YES
.It NODOC
-.Va YES
+.Ar YES
.It NOPORTS
-.Va YES
+.Ar YES
.El
.Pp
None of these variables may be overridden by command-line arguments.
.Sh SEE ALSO
-.Xr cvs 1 ,
.Xr make 1 ,
.Xr patch 1 ,
+.Xr svn 1 ,
.Xr tbmaster 1 ,
.Xr build 7 ,
.Xr release 7
Modified: trunk/tools/tools/tinderbox/tinderbox.pl
===================================================================
--- trunk/tools/tools/tinderbox/tinderbox.pl 2016-03-04 02:50:34 UTC (rev 7460)
+++ trunk/tools/tools/tinderbox/tinderbox.pl 2016-03-04 13:37:08 UTC (rev 7461)
@@ -1,6 +1,6 @@
#!/usr/bin/perl -Tw
#-
-# Copyright (c) 2003-2012 Dag-Erling Smørgrav
+# Copyright (c) 2003-2014 Dag-Erling Smørgrav
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
@@ -24,8 +24,8 @@
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
-# $FreeBSD: src/tools/tools/tinderbox/tinderbox.pl,v 1.42 2005/04/30 18:22:12 des Exp $
-# $MidnightBSD: src/tools/tools/tinderbox/tinderbox.pl,v 1.3 2008/03/07 04:20:26 laffer1 Exp $
+# $FreeBSD: user/des/tinderbox/tinderbox.pl 268719 2014-07-15 22:34:54Z des $
+# $MidnightBSD$
#
use v5.10.1;
@@ -33,14 +33,14 @@
use Fcntl qw(:DEFAULT :flock);
use POSIX;
use Getopt::Long;
+use Scalar::Util qw(tainted);
-my $VERSION = "2.10";
-my $COPYRIGHT = "Copyright (c) 2003-2012 Dag-Erling Smørgrav. " .
+my $VERSION = "2.22";
+my $COPYRIGHT = "Copyright (c) 2003-2014 Dag-Erling Smørgrav. " .
"All rights reserved.";
my $arch; # Target architecture
my $branch; # CVS branch to check out
-my $cvsup; # Name of CVSup server
my $destdir; # Destination directory
my $jobs; # Number of paralell jobs
my $hostname; # Name of the host running the tinderbox
@@ -47,8 +47,9 @@
my $logfile; # Path to log file
my $machine; # Target machine
my $objdir; # Location of object tree
+my $objpath; # Full path to object tree
+my $obj32path; # Full path to 32-bit object tree
my $patch; # Patch to apply before building
-my $repository; # Location of CVS repository
my $sandbox; # Location of sandbox
my $srcdir; # Location of source tree
my $svnbase; # Subversion base URL
@@ -65,6 +66,7 @@
'cleanobj' => 0, 'precleanobj' => 0, 'postcleanobj' => 0,
'cleaninst' => 0, 'precleaninst' => 0, 'postcleaninst' => 0,
'cleanobj' => 0, 'precleanobj' => 0, 'postcleanobj' => 0,
+ 'revert' => 0,
'update' => 0,
'patch' => 0,
'world' => 0,
@@ -81,17 +83,6 @@
my $unamecmd = '/usr/bin/uname';
-my @cvscmds = (
- '/usr/bin/cvs',
- '/usr/local/bin/cvs',
-);
-
-my @cvsupcmds = (
- '/usr/bin/csup',
- '/usr/local/bin/csup',
- '/usr/local/bin/cvsup'
-);
-
my @svncmds = (
'/usr/bin/svn',
'/usr/local/bin/svn',
@@ -184,6 +175,34 @@
}
#
+# Expand a path
+#
+sub realpath($;$);
+sub realpath($;$) {
+ my $path = shift;
+ my $base = shift || "";
+
+ my $realpath = ($path =~ m|^/|) ? "" : $base;
+ foreach my $part (split('/', $path)) {
+ if ($part eq '' || $part eq '.') {
+ # nothing
+ } elsif ($part eq '..') {
+ $realpath =~ s|/[^/]+$||
+ or die("'$path' is not a valid path relative to '$base'\n");
+ } elsif (-l "$realpath/$part") {
+ my $target = readlink("$realpath/$part")
+ or die("unable to resolve symlink '$realpath/$part': $!\n");
+ $realpath = realpath($target, $realpath);
+ } else {
+ $part =~ m/^([\w.-]+)$/
+ or die("unsafe path '$realpath/$part'\n");
+ $realpath .= "/$1";
+ }
+ }
+ return $realpath;
+}
+
+#
# Open and lock a file reliably
#
sub open_locked($;$$) {
@@ -294,6 +313,14 @@
my @args = @_; # Arguments
message($cmd, @args);
+ # Check command and arguments for taint. The build will die
+ # anyway, but at least we'll have a starting point for debugging.
+ warning("command name is tainted\n")
+ if tainted($cmd);
+ for (my $i = 0; $i < @args; ++$i) {
+ warning("argv\[$i\] is tainted\n")
+ if tainted($args[$i]);
+ }
my $pid = fork();
if (!defined($pid)) {
return warning("fork(): $!");
@@ -376,7 +403,7 @@
sub usage() {
- print(STDERR "This is the BSD tinderbox script, version $VERSION.
+ print(STDERR "This is the MidnightBSD tinderbox script, version $VERSION.
$COPYRIGHT
Usage:
@@ -387,8 +414,7 @@
Parameters:
--arch=ARCH Target architecture (e.g. i386)
- --branch=BRANCH CVS branch to check out
- --cvsup CVSup server
+ --branch=BRANCH Source branch to check out
--destdir=DIR Destination directory when installing
--jobs=NUM Maximum number of paralell jobs
--hostname=NAME Name of the host running the tinderbox
@@ -395,7 +421,6 @@
--logfile=FILE Path to log file
--machine=MACHINE Target machine (e.g. pc98)
--patch=PATCH Patch to apply before building
- --repository=DIR Location of CVS repository
--sandbox=DIR Location of sandbox
--svnbase=URL Subversion base URL
--timeout=SECONDS Maximum allowed build time
@@ -432,8 +457,8 @@
chomp($hostname);
$branch = "HEAD";
$jobs = 0;
- $repository = "/home/cvs";
$sandbox = "/tmp/tinderbox";
+ $svnbase = "svn://svn.midnightbsd.org/svn/src/";
$timeout = 0;
# Get options
@@ -440,7 +465,6 @@
GetOptions(
"arch=s" => \$arch,
"branch=s" => \$branch,
- "cvsup=s" => \$cvsup,
"destdir=s" => \$destdir,
"jobs=i" => \$jobs,
"hostname=s" => \$hostname,
@@ -448,7 +472,6 @@
"machine=s" => \$machine,
"objdir=s" => \$objdir,
"patch=s" => \$patch,
- "repository=s" => \$repository,
"sandbox=s" => \$sandbox,
"srcdir=s" => \$srcdir,
"svnbase=s" => \$svnbase,
@@ -456,16 +479,18 @@
"verbose+" => \$verbose,
) or usage();
- if ($jobs < 0) {
+ if ($jobs !~ m/^(\d+)$/) {
error("invalid number of jobs");
}
- if ($timeout < 0) {
+ $jobs = $1;
+ if ($timeout !~ m/^(\d+)$/) {
error("invalid timeout");
}
+ $timeout = $1;
if ($branch !~ m|^(\w+)$|) {
error("invalid source branch");
}
- $branch = ($1 eq 'CURRENT') ? 'HEAD' : $1;
+ $branch = ($1 eq 'CURRENT') ? 'trunk' : $1;
if (!defined($arch)) {
$arch = `$unamecmd -p`;
chomp($arch);
@@ -488,6 +513,11 @@
if (!defined($destdir)) {
$destdir = "$sandbox/inst";
}
+ if ($svnbase &&
+ $svnbase !~ m@^((?:svn(?:\+ssh)?://(?:[a-z][0-9a-z-]*)(?:\.[a-z][0-9a-z-]*)*(?::\d+)?|file://)/[\w./-]*)@) {
+ error("invalid SVN base URL");
+ }
+ $svnbase = $1;
if (!@ARGV) {
usage();
@@ -501,7 +531,7 @@
# Find out what we're expected to do
foreach my $cmd (@ARGV) {
- if ($cmd =~ m/^([0-9A-Z_]+)=(.*)\s*$/) {
+ if ($cmd =~ m/^(\w+)=(.*)\s*$/) {
$userenv{$1} = $2;
next;
}
@@ -556,6 +586,7 @@
truncate($lockfile, 0);
print($lockfile "$$\n");
+ # Validate source directory
if (defined($srcdir)) {
if ($srcdir !~ m|^(/[\w./-]+)$|) {
error("invalid source directory");
@@ -564,7 +595,9 @@
} else {
$srcdir = "$sandbox/src";
}
+ $srcdir = realpath($srcdir);
+ # Validate object directory
if (defined($objdir)) {
if ($objdir !~ m|^(/[\w./-]+)$|) {
error("invalid object directory");
@@ -573,96 +606,88 @@
} else {
$objdir = "$sandbox/obj";
}
+ $objdir = realpath($objdir);
+ # Construct full path to object directory
+ $objpath = "$objdir/$machine.$arch$srcdir";
+ $obj32path = "$objdir/$machine.$arch/lib32$srcdir";
+
# Clean up remains from old runs
do_clean(); # no prefix for backward compatibility
do_clean('pre');
+ # Locate svn
+ my $svncmd = '/usr/bin/false';
+ if ($cmds{'revert'} || $cmds{'version'} || $cmds{'update'}) {
+ $svncmd = [grep({ -x } @svncmds)]->[0]
+ or error("unable to locate svn binary");
+ }
+
+ # Upgrade and unlock the working copy
+ if (($cmds{'revert'} || $cmds{'update'}) && -d "$srcdir/.svn") {
+ spawn($svncmd, "upgrade", $srcdir);
+ spawn($svncmd, "cleanup", $srcdir);
+ }
+
+ # Revert sources
+ if ($cmds{'revert'} && -d "$srcdir/.svn") {
+ my @svnargs;
+ push(@svnargs, "--quiet")
+ unless ($verbose);
+ logstage("reverting $srcdir");
+ spawn($svncmd, @svnargs, "revert", "-R", $srcdir)
+ or error("unable to revert the source tree");
+ # remove leftovers... ugly!
+ open(my $pipe, '-|', $svncmd, "stat", "--no-ignore", $srcdir)
+ or error("unable to stat source tree");
+ while (<$pipe>) {
+ m/^[I?]\s+(\S.*)$/ or next;
+ if (-d $1) {
+ remove_dir($1)
+ or error("unable to remove $1");
+ } elsif (-f $1 || -l $1) {
+ unlink($1)
+ or error("unable to remove $1");
+ } else {
+ warning("ignoring $1");
+ }
+ }
+ close($pipe);
+ }
+
# Check out new source tree
if ($cmds{'update'}) {
- if (defined($svnbase)) {
- my @svnargs;
- push(@svnargs, "--quiet")
- unless ($verbose);
- # ugly-bugly magic required because CVS to SVN conversion
- # smashed branch names
- $svnbase =~ s/\/$//;
- if ($branch eq 'HEAD') {
- $svnbase .= '/head';
- } elsif ($branch =~ m/^RELENG_(\d+)_(\d+)$/) {
- $svnbase .= "/releng/$1.$2";
- } elsif ($branch =~ m/^RELENG_(\d+)$/) {
- $svnbase .= "/stable/$1";
+ error("no svn base URL defined")
+ unless defined($svnbase);
+ my @svnargs;
+ push(@svnargs, "--quiet")
+ unless ($verbose);
+ # ugly-bugly magic required because CVS to SVN conversion
+ # smashed branch names
+ $svnbase =~ s/\/$//;
+ if ($branch eq 'HEAD') {
+ $svnbase .= '/head';
+ } elsif ($branch =~ m/^RELENG_(\d+)_(\d+)$/) {
+ $svnbase .= "/releng/$1.$2";
+ } elsif ($branch =~ m/^RELENG_(\d+)$/) {
+ $svnbase .= "/stable/$1";
+ } else {
+ error("unrecognized branch: $branch");
+ }
+ logstage("checking out $srcdir from $svnbase");
+ cd("$sandbox");
+ for (0..$svnattempts) {
+ if (-d "$srcdir/.svn") {
+ last if spawn($svncmd, @svnargs, "update", $srcdir);
} else {
- error("unrecognized branch: $branch");
+ last if spawn($svncmd, @svnargs, "checkout", $svnbase, $srcdir);
}
- logstage("checking out $srcdir from $svnbase");
- my $svncmd = [grep({ -x } @svncmds)]->[0]
- or error("unable to locate svn binary");
- cd("$sandbox");
- if (-d $srcdir) {
- spawn($svncmd, "cleanup", $srcdir);
- push(@svnargs, "update", $srcdir);
- } else {
- push(@svnargs, "checkout", $svnbase, $srcdir);
- }
- for (0..$svnattempts) {
- last if spawn($svncmd, @svnargs);
- error("unable to check out the source tree")
- if ($_ == $svnattempts);
- my $delay = 30 * ($_ + 1);
- warning("sleeping $delay s and retrying...");
- sleep($delay);
- }
- } elsif (defined($cvsup)) {
- logstage("cvsupping the source tree");
- open(my $fh, ">", "$sandbox/supfile")
- or error("$sandbox/supfile: $!");
- print($fh "*default base=$sandbox\n");
- print($fh "*default prefix=$sandbox\n");
- print($fh "*default delete use-rel-suffix\n");
- print($fh "src-all release=cvs");
- if ($branch eq 'HEAD') {
- print($fh " tag=.");
- } else {
- print($fh " tag=$branch");
- }
- print($fh "\n");
- close($fh);
- my @cvsupargs = (
- "-z",
- "-r 3",
- "-g",
- "-L", ($verbose ? 1 : 0),
- "-h",
- split(' ', $cvsup),
- "$sandbox/supfile"
- );
- my $cvsupcmd = [grep({ -x } @cvsupcmds)]->[0]
- or error("unable to locate cvsup / csup binary");
- spawn($cvsupcmd, @cvsupargs)
- or error("unable to cvsup the source tree");
- } else {
- logstage("checking out the source tree from $repository");
- cd("$sandbox");
- my @cvsargs = (
- "-f",
- "-R",
- $verbose ? "-q" : "-Q",
- "-d$repository",
- );
- if (-d $srcdir) {
- push(@cvsargs, "update", "-Pd");
- } else {
- push(@cvsargs, "checkout", "-P");
- };
- push(@cvsargs, ($branch eq 'HEAD') ? "-A" : "-r$branch")
- if defined($branch);
- push(@cvsargs, "src");
- my $cvscmd = [grep({ -x } @cvscmds)]->[0]
- or error("unable to locate cvs binary");
- spawn($cvscmd, @cvsargs)
- or error("unable to check out the source tree");
+ error("unable to check out the source tree")
+ if ($_ == $svnattempts);
+ my $delay = 30 * ($_ + 1);
+ warning("sleeping $delay s and retrying...");
+ sleep($delay);
+ spawn($svncmd, "cleanup", $srcdir);
}
}
@@ -691,12 +716,10 @@
# Print source tree version information
if ($cmds{'version'}) {
if (defined($svnbase)) {
- my $svncmd = [grep({ -x } @svncmds)]->[0]
- or error("unable to locate svn binary");
my $svnversioncmd = [grep({ -x } @svnversioncmds)]->[0]
or error("unable to locate svnversion binary");
if ($verbose) {
- spawn($svncmd, "stat", $srcdir)
+ spawn($svncmd, "stat", "--no-ignore", $srcdir)
or error("unable to stat source tree");
}
my $svnversion = `$svnversioncmd $srcdir`; # XXX
@@ -804,7 +827,10 @@
# Check that the config is appropriate for this target.
cd("$srcdir/sys/$machine/conf");
local *PIPE;
- my @cmdline = ("/usr/sbin/config", "-m", "$kernel");
+ # ugh, we really shouldn't need to know that.
+ my $cmd = "$objpath/tmp/legacy/usr/sbin/config";
+ $cmd = "/usr/sbin/config" unless -x $cmd;
+ my @cmdline = ($cmd, "-m", $kernel);
message(@cmdline);
if (open(PIPE, "-|", @cmdline)) {
local $/;
@@ -849,7 +875,6 @@
# Build a release if requested
if ($cmds{'release'}) {
$ENV{'CHROOTDIR'} = "$sandbox/root";
- $ENV{'CVSROOT'} = $repository;
$ENV{'RELEASETAG'} = $branch
if $branch ne 'HEAD';
$ENV{'WORLD_FLAGS'} = $ENV{'KERNEL_FLAGS'} =
Modified: trunk/tools/tools/tinderbox/www/index.cgi
===================================================================
--- trunk/tools/tools/tinderbox/www/index.cgi 2016-03-04 02:50:34 UTC (rev 7460)
+++ trunk/tools/tools/tinderbox/www/index.cgi 2016-03-04 13:37:08 UTC (rev 7461)
@@ -24,8 +24,7 @@
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
-# $MidnightBSD$
-# $FreeBSD: projects/tinderbox/www/index.cgi,v 1.32 2008/02/10 17:04:26 des Exp $
+# $FreeBSD: user/des/tinderbox/www/index.cgi 256311 2013-10-11 08:53:27Z des $
#
use v5.10.1;
@@ -33,7 +32,7 @@
use POSIX qw(strftime);
use Sys::Hostname;
-my %CONFIGS;
+my %BRANCHES;
my %ARCHES;
my $DIR = ".";
@@ -80,15 +79,16 @@
return branch_rank($b) cmp branch_rank($a);
}
-sub do_config($) {
- my ($config) = @_;
+sub do_branch($) {
+ my ($branch) = @_;
- my %branches = %{$CONFIGS{$config}};
+ my $prettybranch = $branch;
+ $prettybranch =~ s@^HEAD$@head@;
+ $prettybranch =~ s@^RELENG_(\d+)_(\d+)$@releng/$1.$2@;
+ $prettybranch =~ s@^RELENG_(\d+)$@stable/$1@;
- my $prettyconfig = $config;
- $prettyconfig =~ s/^(.*?)-build$/$1/;
print " <tr class='header'>
- <th>$prettyconfig</th>
+ <th> </th>
";
foreach my $arch (sort(keys(%ARCHES))) {
foreach my $machine (sort(keys(%{$ARCHES{$arch}}))) {
@@ -103,13 +103,11 @@
my $now = time();
- foreach my $branch (sort(inverse_branch_sort keys(%branches))) {
- my $prettybranch = $branch;
- $prettybranch =~ s@^HEAD$@/head@;
- $prettybranch =~ s@^RELENG_(\d+)_(\d+)$@/releng/$1.$2@;
- $prettybranch =~ s@^RELENG_(\d+)$@/stable/$1@;
+ foreach my $config (sort(keys(%{$BRANCHES{$branch}}))) {
+ $config =~ m/^(\w+)((?:-\w+)*?)(-build)?$/;
+ my $variant = $2 =~ s/^-//r;
print " <tr>
- <th>$prettybranch</th>
+ <th>$prettybranch" . ($variant ? "<br/>($variant)" : "") . "</th>
";
foreach my $arch (sort(keys(%ARCHES))) {
foreach my $machine (sort(keys(%{$ARCHES{$arch}}))) {
@@ -148,16 +146,16 @@
$| = 1;
if ($ENV{'GATEWAY_INTERFACE'}) {
print "Content-Type: text/html; charset=utf-8\n\n";
- $realthing = ($ENV{'SERVER_NAME'} eq 'tinderbox.midnightbsd.org');
+ $realthing = ($ENV{'SERVER_NAME'} eq 'tinderbox.freebsd.org');
} else {
my $host = hostname();
- $realthing = ($host eq 'defiant.midnightbsd.org');
+ $realthing = ($host eq 'dma.des.no');
}
if ($realthing) {
- $greeting = "<a href='http://tinderbox.midnightbsd.org/'>tinderbox.midnightbsd.org</a>";
+ $greeting = "<a href='http://tinderbox.freebsd.org/'>tinderbox.freebsd.org</a>";
} else {
- $greeting = "For official Tinderbox logs, see <a href='http://tinderbox.midnightbsd.org/'>here</a>";
+ $greeting = "For official Tinderbox logs, see <a href='http://tinderbox.freebsd.org/'>here</a>";
}
local *DIR;
@@ -165,7 +163,7 @@
or die("$DIR: $!\n");
foreach (readdir(DIR)) {
next unless m/^tinderbox-([\w-]+)-(\w+)-(\w+)-(\w+)\.(brief|full)$/;
- $CONFIGS{$1}->{$2} = $ARCHES{$3}->{$4} = 1;
+ $BRANCHES{$2}->{$1} = $ARCHES{$3}->{$4} = 1;
}
closedir(DIR);
@@ -175,13 +173,14 @@
'http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd'>
<html xmlns='http://www.w3.org/1999/xhtml' xml:lang='en' lang='en'>
<head>
- <title>MidnightBSD tinderbox logs</title>
+ <title>FreeBSD tinderbox logs</title>
<meta name='robots' content='nofollow' />
<meta http-equiv='refresh' content='600' />
<link rel='stylesheet' type='text/css' media='screen' href='tb.css' />
+ <link rel='shortcut icon' type='image/x-icon' href='favicon.ico' />
</head>
<body>
- <!-- h1>MidnightBSD tinderbox logs</h1 -->
+ <!-- h1>FreeBSD tinderbox logs</h1 -->
<table border='1'>
";
@@ -197,9 +196,8 @@
}
# Generate rows
- foreach my $config (sort(inverse_branch_sort keys(%CONFIGS))) {
- next if $config =~ m/^update_/;
- do_config($config);
+ foreach my $branch (sort(inverse_branch_sort keys(%BRANCHES))) {
+ do_branch($branch);
}
print "
@@ -210,6 +208,14 @@
</td>
</tr>
</table>
+ <!-- p>
+ <a href='http://validator.w3.org/check/referer'><img
+ src='valid-xhtml10.png'
+ alt='Valid XHTML 1.0!' height='31' width='88' /></a>
+ <a href='http://jigsaw.w3.org/css-validator/check/referer'><img
+ src='valid-css.png'
+ alt='Valid CSS!' height='31' width='88' /></a>
+ </p -->
</body>
</html>
";
More information about the Midnightbsd-cvs
mailing list