|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [Xen-devel] [PATCH v3 5/8] osstest: introduce a FreeBSD build script
In order to generate the FreeBSD installer image and the install
media.
The install sets are the vanilla ones generated by the 'ftp' release
target. The installer image is handcrafted based on the filesystem
created by the 'bootonly' target, which is then populated with the ssh
host keys, and setup in order to use the serial console. The other
difference from upstream FreeBSD installer images is that the one
built by osstest uses a ramdisk instead of relying on the installer
media to be somehow attached, either on a CD or USB drive. This is
required in order to boot the image from pxelinux (where no CD or USB
is actually attached to the host, and everything is fetched from
tftp).
Due to the nature of the FreeBSD build, the outputs are different from
what osstest expects from a buildjob, more specifically
path_freebsddist points to a folder that contains the several outputs
form this buildjob.
The following new helpers are also introduced, that are only used by
the FreeBSD build script: buildcmd_stamped_logged_root and
target_cmd_build_root, they behave exactly the same as the non-root
counterparts.
Signed-off-by: Roger Pau Monné <roger.pau@xxxxxxxxxx>
---
Changes since v2:
- Increase build target timeouts.
- Use sysrc instead echo to set rc.conf options.
Changes since v1:
- Remove the ts-build-check FreeBSD hack.
- Use pkg-static instead of pkg.
- Introduce buildcmd_stamped_logged_root and target_cmd_build_root.
- Use target_cmd_build_root and buildcmd_stamped_logged_root in the
ts-freebsd-build script.
- Fix the script snippets to use <<END.<<'END' in order to avoid
escaping the shell variables.
- Set path_freebsddist runvar to point to the folder where the build
files are stashed.
- Add a comment at the top of the file describing what runvars are
consumed/produced by the build script.
---
Osstest/BuildSupport.pm | 26 +++++-
Osstest/TestSupport.pm | 28 +++++-
ts-freebsd-build | 240 ++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 286 insertions(+), 8 deletions(-)
create mode 100755 ts-freebsd-build
diff --git a/Osstest/BuildSupport.pm b/Osstest/BuildSupport.pm
index 4c2b658b..b96ce766 100644
--- a/Osstest/BuildSupport.pm
+++ b/Osstest/BuildSupport.pm
@@ -36,7 +36,7 @@ BEGIN {
$whhost $ho
builddirsprops
- buildcmd_stamped_logged
+ buildcmd_stamped_logged buildcmd_stamped_logged_root
$builddir $makeflags
$buildcmd_global_prefix
@@ -60,9 +60,9 @@ our ($xendist);
our $buildcmd_global_prefix //= '';
-sub buildcmd_stamped_logged ($$$$$$) {
- my ($timeout, $component, $stampname, $prefix, $cmd, $suffix) = @_;
- target_cmd_build($ho, $timeout, $builddir, <<END);
+sub buildcmd_stamped_logged_script ($$$$$) {
+ my ($component, $stampname, $prefix, $cmd, $suffix) = @_;
+ my $script = <<END;
cd $component
$buildcmd_global_prefix
$prefix
@@ -73,6 +73,24 @@ sub buildcmd_stamped_logged ($$$$$$) {
echo ok.
END
#/;
+
+ return $script;
+}
+
+sub buildcmd_stamped_logged_root ($$$$$$) {
+ my ($timeout, $component, $stampname, $prefix, $cmd, $suffix) = @_;
+ my $script = buildcmd_stamped_logged_script($component, $stampname,
+ $prefix, $cmd, $suffix);
+
+ target_cmd_build_root($ho, $timeout, $builddir, $script);
+}
+
+sub buildcmd_stamped_logged ($$$$$$) {
+ my ($timeout, $component, $stampname, $prefix, $cmd, $suffix) = @_;
+ my $script = buildcmd_stamped_logged_script($component, $stampname,
+ $prefix, $cmd, $suffix);
+
+ target_cmd_build($ho, $timeout, $builddir, $script);
}
sub selectbuildhost {
diff --git a/Osstest/TestSupport.pm b/Osstest/TestSupport.pm
index ba1d6cc9..3f136777 100644
--- a/Osstest/TestSupport.pm
+++ b/Osstest/TestSupport.pm
@@ -53,6 +53,7 @@ BEGIN {
stashfilecontents
target_cmd_root target_cmd target_cmd_build
+ target_cmd_build_root
target_cmd_output_root target_cmd_output
target_cmd_inputfh_root sshuho
target_getfile target_getfile_root
@@ -635,8 +636,8 @@ sub target_editfile ($$$;$$) {
teditfileex('osstest',@_); }
# my ($ho,$rfile, $lleaf,$rdest) = @_;
# ^^^^^^^^^^^^^ optional
-sub target_cmd_build ($$$$) {
- my ($ho,$timeout,$builddir,$script) = @_;
+sub target_cmd_build_script ($$$) {
+ my ($ho, $builddir, $script) = @_;
my $distcc_hosts = get_host_property($ho,'DistccHosts',undef);
my $distcc = defined($distcc_hosts) ? <<END : "";
@@ -648,8 +649,8 @@ END
my $httpproxy = http_proxy_envsettings($ho);
- target_cmd($ho, <<END.$distcc.<<END.$script, $timeout);
- set -xe
+ my $cmd = <<END.$distcc.<<END.$script;
+ set -xe
LC_ALL=C; export LC_ALL
PATH=/usr/lib/ccache:\$PATH:/usr/lib/git-core
$httpproxy
@@ -658,8 +659,27 @@ END
cd $builddir
rm -f build-ok-stamp
END
+
+ return $cmd;
+}
+
+sub target_cmd_build ($$$$) {
+ my ($ho,$timeout,$builddir,$script) = @_;
+
+ my $cmd = target_cmd_build_script($ho, $builddir, $script);
+
+ target_cmd($ho, $cmd, $timeout);
+}
+
+sub target_cmd_build_root ($$$$) {
+ my ($ho,$timeout,$builddir,$script) = @_;
+
+ my $cmd = target_cmd_build_script($ho, $builddir, $script);
+
+ target_cmd_root($ho, $cmd, $timeout);
}
+
sub target_ping_check_core {
my ($ho, $exp) = @_;
my $out= `ping -c 5 $ho->{Ip} 2>&1`;
diff --git a/ts-freebsd-build b/ts-freebsd-build
new file mode 100755
index 00000000..6c10eece
--- /dev/null
+++ b/ts-freebsd-build
@@ -0,0 +1,240 @@
+#!/usr/bin/perl -w
+# This is part of "osstest", an automated testing framework for Xen.
+# Copyright (C) 2017 Citrix Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Consumes the following input runvars:
+#
+# revision_freebsd: FreeBSD commit ID to generate the install media from.
+# tree_freebsd: FreeBSD git tree to fetch the source code from.
+#
+# Produces the following output:
+#
+# Compressed install sets: kernel.txz, base.txz
+# Compressed debug symbols for the kernel: kernel-dbg.txz
+# Manifest file (checkums): MANIFEST
+# Installer image: install.img
+#
+# Sets the following runvar:
+#
+# path_freebsddist: points to the folder where the above files are stored.
+# freebsd_buildversion: version of FreeBSD built.
+
+use strict qw(vars);
+use DBI;
+use POSIX;
+
+unshift @INC, qw(.);
+use Osstest;
+use Osstest::TestSupport;
+use Osstest::BuildSupport;
+
+tsreadconfig();
+
+selectbuildhost(\@ARGV);
+builddirsprops();
+
+sub install_deps () {
+ target_cmd_root($ho, 'pkg-static install git', 300);
+}
+
+sub checkout () {
+ prepbuilddirs();
+
+ # Remove the directory as root, there might be files owned by root
+ target_cmd_build_root($ho, 300, $builddir, <<'END');
+# chflags will return error if the directory doesn't exist
+chflags -fR noschg freebsd || true
+rm -rf freebsd
+END
+
+ logm("Checkout the FreeBSD source tree");
+ build_clone($ho, 'freebsd', $builddir, 'freebsd', );
+}
+
+sub build () {
+ my $authkeys = authorized_keys();
+ my $target = "bootonly";
+ my $prefix = <<END;
+export MAKEOBJDIRPREFIX=$builddir/obj
+export TARGET=$r{arch}
+END
+
+ # Build process as documented in the handbook:
+ # https://www.freebsd.org/doc/handbook/updating-src.html
+
+ logm("Cleaning up previous builds");
+ buildcmd_stamped_logged(300, 'freebsd', 'cleanworld',
+ $prefix, 'make cleanworld', '');
+
+ logm("Building world");
+ buildcmd_stamped_logged(25200, 'freebsd', 'buildworld',
+ $prefix, <<END, '');
+make $makeflags buildworld
+END
+
+ logm("Building kernel");
+ buildcmd_stamped_logged(3600, 'freebsd', 'buildkernel',
+ $prefix, <<END, '');
+make $makeflags buildkernel
+END
+
+ logm("Creating the install sets");
+ # NB: the steps below need to be done as root or the permissions
+ # of the files won't be properly set (and the target will fail).
+ buildcmd_stamped_logged_root(3600, 'freebsd', 'release-ftp',
+ $prefix, <<END, '');
+make -C release ftp
+END
+
+ logm("Populating the installer image");
+ buildcmd_stamped_logged_root(3600, 'freebsd', "release-$target",
+ $prefix, <<END, '');
+make -C release $target
+END
+
+ logm("Placing ssh host keys");
+ foreach my $file (<$c{OverlayLocal}/etc/ssh/ssh_host_*_key*>) {
+ target_putfile_root($ho, 30, $file,
+ "$builddir/freebsd/release/$target/etc/ssh/");
+ }
+
+ logm("Configuring the installer image");
+ target_cmd_build_root($ho, 30, $builddir, <<END.<<'END');
+authkeys="$authkeys"
+bauds="$c{Baud}"
+cd freebsd/release/$target
+END
+# Enable sshd by default
+sysrc -f etc/rc.conf sshd_enable=YES
+
+# Allow root login and copy the keys
+echo 'PermitRootLogin yes' >> etc/ssh/sshd_config
+mkdir -p root/.ssh
+cat << ENDKEYS > root/.ssh/authorized_keys
+$authkeys
+ENDKEYS
+
+# Set host keys permissions
+chown root:wheel etc/ssh/ssh_host_*_key*
+chmod 0600 etc/ssh/ssh_host_*_key
+chmod 0644 etc/ssh/ssh_host_*_key.pub
+
+# Setup serial console output for stage1
+printf "%s" "-h -S$bauds" >> boot.config
+cat << ENDBOOT >> boot/loader.conf
+# Serial console configuration
+boot_serial="YES"
+comconsole_speed="$bauds"
+console="comconsole"
+boot_verbose="YES"
+beastie_disable="YES"
+
+# mfs boot parameters
+mfs_load="YES"
+mfs_type="mfs_root"
+mfs_name="/mfsroot"
+vfs.root.mountfrom="ufs:/dev/ufs/FreeBSD_Install"
+ENDBOOT
+
+# Enable DHCP on all network interfaces
+sysrc -f etc/rc.conf ifconfig_DEFAULT=DHCP
+
+# Remove the local script that launches the installer by default
+rm -rf etc/rc.local
+
+# Create a temporary fstab with the root dir
+echo '/dev/ufs/FreeBSD_Install / ufs rw 1 1' > etc/fstab
+
+# Remove the linked resolv.conf
+rm -rf etc/resolv.conf
+END
+
+ logm("Create the installer");
+ target_cmd_build_root($ho, 900, $builddir, <<END.<<'END');
+target="freebsd/release/$target"
+output="install.img"
+END
+mkdir -p $output.tmp
+
+# Do some pruning
+rm -rf $target/usr/share/man
+rm -rf $target/usr/share/examples
+rm -rf $target/usr/share/doc
+rm -rf $target/usr/share/dtrace
+
+# Create a mfs root image
+makefs -b 10% -B little -o label=FreeBSD_Install $output.tmp/mfsroot $target
+# Compress image
+gzip $output.tmp/mfsroot
+
+# Copy boot to the staging dir
+cp -r $target/boot $output.tmp/
+cp $target/boot.config $output.tmp/
+
+# The loader doesn't need any modules in order to boot into the mfsroot.
+# The rest of the modules can be loaded from the mfs root itself.
+rm -f $output.tmp/boot/kernel/*.ko
+
+# Compress the kernel
+gzip $output.tmp/boot/kernel/kernel
+
+makefs -B little $output.part $output.tmp
+
+# Make the image bootable
+mkimg -s gpt -b $target/boot/pmbr -p efi:=$target/boot/boot1.efifat \
+ -p freebsd-boot:=$target/boot/gptboot -p freebsd-ufs:=$output.part \
+ -p freebsd-swap::1M -o $output
+
+rm $output.part
+rm -rf $output.tmp
+END
+}
+
+sub stash () {
+ my @sets = qw(MANIFEST base.txz kernel.txz kernel-dbg.txz);
+ my @symbols = qw(kernel-dbg.txz);
+
+
+ logm("Stashing FreeBSD build output");
+ foreach my $set (@sets) {
+ built_stash_file($ho, $builddir, $set,
+ "freebsd/release/ftp/$set", 0);
+ }
+ foreach my $symbol (@symbols) {
+ built_stash_debugfile($ho, $builddir, $symbol,
+ "freebsd/release/ftp/$symbol", 0);
+ }
+ built_stash_file($ho, $builddir, "install.img", "install.img", 0);
+
+
+ my $srcversion = target_cmd_output_root($ho, <<END, 30);
+awk '/^\\\#define[[:space:]]*__FreeBSD_version/ { print \$3 }' \\
+ $builddir/freebsd/sys/sys/param.h | cut -c1-2
+END
+ store_runvar("freebsd_buildversion", "$srcversion");
+
+ # Set path_freebsddist to point to the build output folder
+ # in order to make ts-build-check happy.
+ store_runvar("path_freebsddist", "build/");
+}
+
+install_deps();
+checkout();
+build();
+stash();
+
+logm("FreeBSD build successful");
+
--
2.11.0 (Apple Git-81)
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
https://lists.xen.org/xen-devel
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |