Re: [Xen-devel] [stage1-xen PATCH v1 04/10] build/fedora: Add `run` and `components/*` scripts

[Stefano Stabellini <sstabellini@xxxxxxxxxx>]

On Sun, 27 Aug 2017, Rajiv Ranganath wrote:
> From: Rajiv M Ranganath <rajiv.ranganath@xxxxxxxxxxx>
> 
> Signed-off-by: Rajiv Ranganath <rajiv.ranganath@xxxxxxxxxxx>
> ---
>  build/fedora/components/qemu |   50 ++++++++++++++++++++++++++++++++++++
>  build/fedora/components/rkt  |   58 
> ++++++++++++++++++++++++++++++++++++++++++
>  build/fedora/components/xen  |   46 +++++++++++++++++++++++++++++++++
>  build/fedora/run             |   56 +++++++++++++++++++++++++++++++++++++++++
>  4 files changed, 210 insertions(+)
>  create mode 100755 build/fedora/components/qemu
>  create mode 100755 build/fedora/components/rkt
>  create mode 100755 build/fedora/components/xen
>  create mode 100755 build/fedora/run
> 
> diff --git a/build/fedora/components/qemu b/build/fedora/components/qemu
> new file mode 100755
> index 0000000..6c89e2c
> --- /dev/null
> +++ b/build/fedora/components/qemu
> @@ -0,0 +1,50 @@
> +#!/usr/bin/python2
> +
> +import shlex
> +import subprocess
> +import sys
> +import os
> +
> +# Modify this if you would like to install Qemu elsewhere on your filesystem 
> or
> +# a different version of Qemu
> +QEMU_PREFIX = '/opt/qemu-unstable'
> +QEMU_BRANCH = 'master'

I am not sure we want to checkout always the latest QEMU. It is a
running target. It makes sense to use one of the latest releases
instead, such as v2.10.0?


> +# This should correspond to your Xen install prefix
> +XEN_PREFIX = '/opt/xen-unstable'
> +
> +
> +# helper function to capture stdout from a long running process
> +def subprocess_stdout(cmd, cwd, env):
> +    p = subprocess.Popen(
> +        shlex.split(cmd), cwd=cwd, env=env, stdout=subprocess.PIPE)
> +    while p.poll() is None:
> +        l = p.stdout.readline()
> +        sys.stdout.write(l)
> +    if p.returncode != 0:
> +        sys.exit(1)

Is this the same as
  #!/bin/bash
  set -e
?

Please add a few words in the commit message about the benefit of this
approach of writing scripts.


> +env = os.environ.copy()
> +
> +# build and install qemu
> +print "Cloning qemu..."
> +cmd = "git clone --branch %(branch)s git://git.qemu.org/qemu.git" % {
> +    'branch': QEMU_BRANCH
> +}
> +subprocess.check_output(shlex.split(cmd), cwd='/root')
> +
> +steps = [
> +    "./configure --prefix=%(qemu_prefix)s --enable-xen 
> --target-list=i386-softmmu --extra-cflags=\"-I%(xen_prefix)s/include\" 
> --extra-ldflags=\"-L%(xen_prefix)s/lib -Wl,-rpath,%(xen_prefix)s/lib\" 
> --disable-kvm --enable-virtfs --enable-linux-aio"
> +    % {
> +        'qemu_prefix': QEMU_PREFIX,
> +        'xen_prefix': XEN_PREFIX
> +    }, 'make', 'make install'
> +]
> +for cmd in steps:
> +    cwd = '/root/qemu'
> +    subprocess_stdout(cmd, cwd, env)
> +
> +cmd = "cp i386-softmmu/qemu-system-i386 
> %(xen_prefix)s/lib/xen/bin/qemu-system-i386" % {
> +    'xen_prefix': XEN_PREFIX
> +}
> +subprocess.check_output(shlex.split(cmd), cwd='/root/qemu')
> diff --git a/build/fedora/components/rkt b/build/fedora/components/rkt
> new file mode 100755
> index 0000000..edfdd1c
> --- /dev/null
> +++ b/build/fedora/components/rkt
> @@ -0,0 +1,58 @@
> +#!/usr/bin/python2
> +
> +import shlex
> +import subprocess
> +import sys
> +import os
> +
> +# `rkt` is installed in the same prefix as `stage1-xen`. Modify this if you
> +# would like to install rkt elsewhere on your filesystem.
> +STAGE1_XEN_PREFIX = '/opt/stage1-xen'
> +RKT_PREFIX = STAGE1_XEN_PREFIX
> +RKT_BRANCH = 'master'
> +
> +# Adjust this according to what RKT_BRANCH generates
> +RKT_BUILD_VER = 'rkt-1.28.1+git'

I think it would be best if git-checked out the tag (v1.28.1) so that we
are sure there are no version mismatches. In fact, I would remove
RKT_BRANCH and just use a single variable to specify the version to
clone and build.


> +# helper function to capture stdout from a long running process
> +def subprocess_stdout(cmd, cwd, env):
> +    p = subprocess.Popen(
> +        shlex.split(cmd), cwd=cwd, env=env, stdout=subprocess.PIPE)
> +    while p.poll() is None:
> +        l = p.stdout.readline()
> +        sys.stdout.write(l)
> +    if p.returncode != 0:
> +        sys.exi(1)
> +
> +
> +env = os.environ.copy()
> +
> +# build rkt
> +print "Cloning rkt..."
> +cmd = "git clone --branch %(branch)s https://github.com/rkt/rkt.git"; % {
> +    'branch': RKT_BRANCH
> +}
> +subprocess.check_output(shlex.split(cmd), cwd='/root')
> +
> +steps = [
> +    './autogen.sh', './configure --disable-tpm --with-stage1-flavors=coreos',
> +    'make'
> +]
> +for cmd in steps:
> +    cwd = '/root/rkt'
> +    subprocess_stdout(cmd, cwd, env)
> +
> +# install rkt build artifacts to RKT_PREFIX
> +steps = [
> +    "mkdir -p %(prefix)s/bin" % {
> +        'prefix': RKT_PREFIX
> +    },
> +    "cp /root/rkt/build-%(build_ver)s/target/bin/rkt %(prefix)s/bin/rkt" % {
> +        'build_ver': RKT_BUILD_VER,
> +        'prefix': RKT_PREFIX
> +    }
> +]
> +for cmd in steps:
> +    cwd = '/root/rkt'
> +    subprocess_stdout(cmd, cwd, env)
> diff --git a/build/fedora/components/xen b/build/fedora/components/xen
> new file mode 100755
> index 0000000..95da9a6
> --- /dev/null
> +++ b/build/fedora/components/xen
> @@ -0,0 +1,46 @@
> +#!/usr/bin/python2
> +
> +import shlex
> +import subprocess
> +import sys
> +import os
> +
> +# Modify this if you would like to install Xen elsewhere on your filesystem 
> or
> +# a different version of Xen
> +XEN_PREFIX = '/opt/xen-unstable'
> +XEN_BRANCH = 'master'

Same here, I think that for a stage1-xen CI-loop it's best to build and
test again a stable target. Thus, instead of master, I would use a tag
like RELEASE-4.9.0.


> +# helper function to capture stdout from a long running process
> +def subprocess_stdout(cmd, cwd, env):
> +    p = subprocess.Popen(
> +        shlex.split(cmd), cwd=cwd, env=env, stdout=subprocess.PIPE)
> +    while p.poll() is None:
> +        l = p.stdout.readline()
> +        sys.stdout.write(l)
> +    if p.returncode != 0:
> +        sys.exit(1)
> +
> +
> +env = os.environ.copy()
> +
> +# build and install xen
> +print "Cloning xen..."
> +cmd = "git clone --branch %(branch)s git://xenbits.xen.org/xen.git" % {
> +    'branch': XEN_BRANCH
> +}
> +subprocess.check_output(shlex.split(cmd), cwd='/root')
> +
> +steps = [
> +    "./configure --prefix=%(prefix)s 
> --with-system-qemu=%(prefix)s/lib/xen/bin/qemu-system-i386 --disable-stubdom 
> --disable-qemu-traditional --disable-rombios --sysconfdir=%(prefix)s/etc 
> --enable-rpath --disable-systemd"
> +    % {
> +        'prefix': XEN_PREFIX
> +    }, 'make',
> +    "make install BOOT_DIR=%(prefix)s/boot DEBUG_DIR=%(prefix)s/lib/debug 
> EFI_DIR=%(prefix)s/boot/efi/EFI/xen"
> +    % {
> +        'prefix': XEN_PREFIX
> +    }
> +]
> +for cmd in steps:
> +    cwd = '/root/xen'
> +    subprocess_stdout(cmd, cwd, env)
> diff --git a/build/fedora/run b/build/fedora/run
> new file mode 100755
> index 0000000..6cb6417
> --- /dev/null
> +++ b/build/fedora/run
> @@ -0,0 +1,56 @@
> +#!/usr/bin/python2
> +
> +import shlex
> +import subprocess
> +import sys
> +import os
> +
> +# This scripts calls out to `xen`, `qemu` and `rkt` scripts in the
> +# `components/` directory within a container. It is expected that components
> +# directory is present at the same directory level as run script.
> +STAGE1_XEN_COMPONENTS = ['xen', 'qemu', 'rkt']
> +
> +
> +# helper function to capture stdout from a long running process
> +def subprocess_stdout(cmd, cwd, env):
> +    p = subprocess.Popen(
> +        shlex.split(cmd), cwd=cwd, env=env, stdout=subprocess.PIPE)
> +    while p.poll() is None:
> +        l = p.stdout.readline()
> +        sys.stdout.write(l)
> +    if p.returncode != 0:
> +        sys.exit(1)
> +
> +
> +env = os.environ.copy()
> +
> +dirname = os.path.dirname(os.path.realpath(__file__))
> +steps = [os.path.join(dirname, 'components', x) for x in 
> STAGE1_XEN_COMPONENTS]
> +for cmd in steps:
> +    cwd = '/root'
> +    subprocess_stdout(cmd, cwd, env)
> +
> +# build stage1-xen
> +env['GOPATH'] = '/root/gopath'
> +cwd = '/root/gopath/src/github.com/rkt/stage1-xen'
> +cmd = 'bash build.sh'
> +subprocess_stdout(cmd, cwd, env)
> +
> +# install build artifacts to `/opt/stage1-xen/aci` and create a tarball
> +steps = [
> +    'mkdir -p /opt/stage1-xen/aci',
> +    'cp /root/gopath/src/github.com/rkt/stage1-xen/stage1-xen.aci 
> /opt/stage1-xen/aci/stage1-xen.aci',
> +    'cp 
> /root/gopath/src/github.com/rkt/stage1-xen/build/fedora/source_path.sh 
> /opt/stage1-xen/bin/source_path.sh',
> +    'cp -r 
> /root/gopath/src/github.com/rkt/stage1-xen/build/fedora/xen-unstable-runit 
> /opt/xen-unstable-runit'
> +]
> +for cmd in steps:
> +    cwd = '/root'
> +    subprocess_stdout(cmd, cwd, env)
> +
> +cwd = '/opt'
> +cmd = 'tar zcvf /root/stage1-xen-build.tar.gz qemu-unstable/ stage1-xen/ 
> xen-unstable/ xen-unstable-runit/'
> +subprocess_stdout(cmd, cwd, env)
> +
> +cwd = '/root'
> +cmd = 'mv /root/stage1-xen-build.tar.gz /tmp'
> +subprocess_stdout(cmd, cwd, env)
> 

_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
https://lists.xen.org/xen-devel