| Message ID | 20190510104633.9428-11-kraxel@redhat.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | tests/vm: serial console autoinstall, misc fixes. | expand |
Hi Gerd, On 5/10/19 12:46 PM, Gerd Hoffmann wrote: > Instead of fetching the prebuilt image from patchew download the install > iso and prepare the image locally. Install to disk, using the serial > console. Create qemu user, configure ssh login. Install packages > needed for qemu builds. I'm impressed how charmly this works :) 3 comments so far. 1/ We could record (in tests/vm/freebsd header?) roughly how many local storage will be used (or display in 'make vm-help'?). FYI this image takes ~3.1GiB. 2/ "Autoboot in 9 seconds, hit [Enter] to boot or any other key to stop" 3/ I am a bit annoyed it overwrote my previous ~/.cache/qemu-vm/images/freebsd.img VM. Not sure what's the best hash to use, maybe "git log -n 1 --pretty=format:%H -- tests/vm/freebsd"? (Similarly for other images). > Note that freebsd package downloads are delivered as non-cachable > content, so I had to configure squid with "ignore-no-store > ignore-private ignore-reload" for pkgmir.geo.freebsd.org to make the > caching actually work. > > Signed-off-by: Gerd Hoffmann <kraxel@redhat.com> Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com> Tested-by: Philippe Mathieu-Daudé <philmd@redhat.com> > --- > tests/vm/freebsd | 175 ++++++++++++++++++++++++++++++++++++++++++++--- > 1 file changed, 165 insertions(+), 10 deletions(-) > > diff --git a/tests/vm/freebsd b/tests/vm/freebsd > index b0066017a617..57e5c97f3b26 100755 > --- a/tests/vm/freebsd > +++ b/tests/vm/freebsd > @@ -2,43 +2,198 @@ > # > # FreeBSD VM image > # > -# Copyright 2017 Red Hat Inc. > +# Copyright 2017-2019 Red Hat Inc. > # > # Authors: > # Fam Zheng <famz@redhat.com> > +# Gerd Hoffmann <kraxel@redhat.com> > # > # This code is licensed under the GPL version 2 or later. See > # the COPYING file in the top-level directory. > # > > import os > +import re > import sys > +import time > +import socket > import subprocess > import basevm > > class FreeBSDVM(basevm.BaseVM): > name = "freebsd" > arch = "x86_64" > + > + link = "https://download.freebsd.org/ftp/releases/ISO-IMAGES/12.0/FreeBSD-12.0-RELEASE-amd64-disc1.iso.xz" > + csum = "1d40015bea89d05b8bd13e2ed80c40b522a9ec1abd8e7c8b80954fb485fb99db" > + size = "20G" > + pkgs = [ > + # build tools > + "git", > + "pkgconf", > + "bzip2", > + > + # gnu tools > + "bash", > + "gmake", > + "gsed", > + "flex", "bison", > + > + # libs: crypto > + "gnutls", > + > + # libs: images > + "jpeg-turbo", > + "png", > + > + # libs: ui > + "sdl2", > + "gtk3", > + "libxkbcommon", > + > + # libs: opengl > + "libepoxy", > + "mesa-libs", > + ] > + > BUILD_SCRIPT = """ > set -e; > - rm -rf /var/tmp/qemu-test.* > - cd $(mktemp -d /var/tmp/qemu-test.XXXXXX); > + rm -rf /home/qemu/qemu-test.* > + cd $(mktemp -d /home/qemu/qemu-test.XXXXXX); > + mkdir src build; cd src; > tar -xf /dev/vtbd1; > - ./configure {configure_opts}; > + cd ../build > + ../src/configure --python=python3.6 {configure_opts}; > gmake --output-sync -j{jobs} {target} {verbose}; > """ > > + def console_boot_serial(self): > + self.console_wait_send("Autoboot", "3") > + self.console_wait_send("OK", "set console=comconsole\n") > + self.console_wait_send("OK", "boot\n") > + > def build_image(self, img): > - cimg = self._download_with_cache("http://download.patchew.org/freebsd-11.1-amd64.img.xz", > - sha256sum='adcb771549b37bc63826c501f05121a206ed3d9f55f49145908f7e1432d65891') > - img_tmp_xz = img + ".tmp.xz" > + self.print_step("Downloading install iso") > + cimg = self._download_with_cache(self.link, sha256sum=self.csum) > img_tmp = img + ".tmp" > - sys.stderr.write("Extracting the image...\n") > - subprocess.check_call(["cp", "-f", cimg, img_tmp_xz]) > - subprocess.check_call(["xz", "-dvf", img_tmp_xz]) > + iso = img + ".install.iso" > + iso_xz = iso + ".xz" > + > + self.print_step("Preparing iso and disk image") > + subprocess.check_call(["cp", "-f", cimg, iso_xz]) > + subprocess.check_call(["xz", "-dvf", iso_xz]) > + subprocess.check_call(["qemu-img", "create", "-f", "qcow2", > + img_tmp, self.size]) > + > + self.print_step("Booting installer") > + self.boot(img_tmp, extra_args = [ > + "-machine", "graphics=off", > + "-cdrom", iso > + ]) > + self.console_init() > + self.console_boot_serial() > + self.console_wait_send("Console type", "xterm\n") > + > + # pre-install configuration > + self.console_wait_send("Welcome", "\n") > + self.console_wait_send("Keymap Selection", "\n") > + self.console_wait_send("Set Hostname", "freebsd\n") > + self.console_wait_send("Distribution Select", "\n") > + self.console_wait_send("Partitioning", "\n") > + self.console_wait_send("Partition", "\n") > + self.console_wait_send("Scheme", "\n") > + self.console_wait_send("Editor", "f") > + self.console_wait_send("Confirmation", "c") > + > + self.print_step("Installation started now, this will take a while") > + > + # post-install configuration > + self.console_wait("New Password:") > + self.console_send("%s\n" % self.ROOT_PASS) > + self.console_wait("Retype New Password:") > + self.console_send("%s\n" % self.ROOT_PASS) > + > + self.console_wait_send("Network Configuration", "\n") > + self.console_wait_send("IPv4", "y") > + self.console_wait_send("DHCP", "y") > + self.console_wait_send("IPv6", "n") > + self.console_wait_send("Resolver", "\n") > + > + self.console_wait_send("Time Zone Selector", "a\n") > + self.console_wait_send("Confirmation", "y") > + self.console_wait_send("Time & Date", "\n") > + self.console_wait_send("Time & Date", "\n") > + > + self.console_wait_send("System Configuration", "\n") > + self.console_wait_send("System Hardening", "\n") > + > + # qemu user > + self.console_wait_send("Add User Accounts", "y") > + self.console_wait("Username") > + self.console_send("%s\n" % self.GUEST_USER) > + self.console_wait("Full name") > + self.console_send("%s\n" % self.GUEST_USER) > + self.console_wait_send("Uid", "\n") > + self.console_wait_send("Login group", "\n") > + self.console_wait_send("Login group", "\n") > + self.console_wait_send("Login class", "\n") > + self.console_wait_send("Shell", "\n") > + self.console_wait_send("Home directory", "\n") > + self.console_wait_send("Home directory perm", "\n") > + self.console_wait_send("Use password", "\n") > + self.console_wait_send("Use an empty password", "\n") > + self.console_wait_send("Use a random password", "\n") > + self.console_wait("Enter password:") > + self.console_send("%s\n" % self.GUEST_PASS) > + self.console_wait("Enter password again:") > + self.console_send("%s\n" % self.GUEST_PASS) > + self.console_wait_send("Lock out", "\n") > + self.console_wait_send("OK", "yes\n") > + self.console_wait_send("Add another user", "no\n") > + > + self.console_wait_send("Final Configuration", "\n") > + self.console_wait_send("Manual Configuration", "\n") > + self.console_wait_send("Complete", "\n") > + > + self.print_step("Installation finished, rebooting") > + self.console_boot_serial() > + > + # setup qemu user > + prompt = "$" > + self.console_ssh_init(prompt, self.GUEST_USER, self.GUEST_PASS) > + self.console_wait_send(prompt, "exit\n") > + > + # setup root user > + prompt = "root@freebsd:~ #" > + self.console_ssh_init(prompt, "root", self.ROOT_PASS) > + self.console_sshd_config(prompt) > + > + # setup serial console > + self.console_wait(prompt) > + self.console_send("echo 'console=comconsole' >> /boot/loader.conf\n") > + > + # setup virtio-blk #1 (tarfile) > + self.console_wait(prompt) > + self.console_send("echo 'chmod 666 /dev/vtbd1' >> /etc/rc.local\n") > + > + self.print_step("Configuration finished, rebooting") > + self.console_wait_send(prompt, "reboot\n") > + self.console_wait("login:") > + self.wait_ssh() > + > + self.print_step("Installing packages") > + self.ssh_root_check("pkg install -y %s\n" % " ".join(self.pkgs)) > + > + # shutdown > + self.ssh_root(self.poweroff) > + self.console_wait("Uptime:") > + self.wait() > + > if os.path.exists(img): > os.remove(img) > os.rename(img_tmp, img) > + os.remove(iso) > + self.print_step("All done") > > if __name__ == "__main__": > sys.exit(basevm.main(FreeBSDVM)) >
On Sun, May 19, 2019 at 12:55:09AM +0200, Philippe Mathieu-Daudé wrote: > Hi Gerd, > > On 5/10/19 12:46 PM, Gerd Hoffmann wrote: > > Instead of fetching the prebuilt image from patchew download the install > > iso and prepare the image locally. Install to disk, using the serial > > console. Create qemu user, configure ssh login. Install packages > > needed for qemu builds. > > I'm impressed how charmly this works :) > > 3 comments so far. > > 1/ We could record (in tests/vm/freebsd header?) roughly how many local > storage will be used (or display in 'make vm-help'?). FYI this image > takes ~3.1GiB. vm-help would be more useful I think. > 2/ "Autoboot in 9 seconds, hit [Enter] to boot or any other key to stop" Probably tweakable via loader.conf, I'll check. > 3/ I am a bit annoyed it overwrote my previous > ~/.cache/qemu-vm/images/freebsd.img VM. Not sure what's the best hash to > use, maybe "git log -n 1 --pretty=format:%H -- tests/vm/freebsd"? > (Similarly for other images). Then use ~/.cache/qemu-vm/images/freebsd-${hash}.img ? Which will fill ~/.cache/qemu-vm/images with stale images over time, which isn't great either ... Or do you have something else in mind? cheers, Gerd
On 20/05/2019 07.26, Gerd Hoffmann wrote: > On Sun, May 19, 2019 at 12:55:09AM +0200, Philippe Mathieu-Daudé wrote: >> Hi Gerd, >> >> On 5/10/19 12:46 PM, Gerd Hoffmann wrote: >>> Instead of fetching the prebuilt image from patchew download the install >>> iso and prepare the image locally. Install to disk, using the serial >>> console. Create qemu user, configure ssh login. Install packages >>> needed for qemu builds. >> >> I'm impressed how charmly this works :) >> >> 3 comments so far. >> >> 1/ We could record (in tests/vm/freebsd header?) roughly how many local >> storage will be used (or display in 'make vm-help'?). FYI this image >> takes ~3.1GiB. > > vm-help would be more useful I think. > >> 2/ "Autoboot in 9 seconds, hit [Enter] to boot or any other key to stop" > > Probably tweakable via loader.conf, I'll check. > >> 3/ I am a bit annoyed it overwrote my previous >> ~/.cache/qemu-vm/images/freebsd.img VM. Not sure what's the best hash to >> use, maybe "git log -n 1 --pretty=format:%H -- tests/vm/freebsd"? >> (Similarly for other images). > > Then use ~/.cache/qemu-vm/images/freebsd-${hash}.img ? > > Which will fill ~/.cache/qemu-vm/images with stale images over time, > which isn't great either ... > > Or do you have something else in mind? I'd vote to keep the current approach to overwrite "freebsd.img". It's a "cache" directory - if you want to have something persisten, Philippe, I think you should simply copy your image to a different location instead. Thomas
On 5/20/19 9:14 AM, Thomas Huth wrote: > On 20/05/2019 07.26, Gerd Hoffmann wrote: >> On Sun, May 19, 2019 at 12:55:09AM +0200, Philippe Mathieu-Daudé wrote: >>> Hi Gerd, >>> >>> On 5/10/19 12:46 PM, Gerd Hoffmann wrote: >>>> Instead of fetching the prebuilt image from patchew download the install >>>> iso and prepare the image locally. Install to disk, using the serial >>>> console. Create qemu user, configure ssh login. Install packages >>>> needed for qemu builds. >>> >>> I'm impressed how charmly this works :) >>> >>> 3 comments so far. >>> >>> 1/ We could record (in tests/vm/freebsd header?) roughly how many local >>> storage will be used (or display in 'make vm-help'?). FYI this image >>> takes ~3.1GiB. >> >> vm-help would be more useful I think. >> >>> 2/ "Autoboot in 9 seconds, hit [Enter] to boot or any other key to stop" >> >> Probably tweakable via loader.conf, I'll check. >> >>> 3/ I am a bit annoyed it overwrote my previous >>> ~/.cache/qemu-vm/images/freebsd.img VM. Not sure what's the best hash to >>> use, maybe "git log -n 1 --pretty=format:%H -- tests/vm/freebsd"? >>> (Similarly for other images). >> >> Then use ~/.cache/qemu-vm/images/freebsd-${hash}.img ? >> >> Which will fill ~/.cache/qemu-vm/images with stale images over time, >> which isn't great either ... >> >> Or do you have something else in mind? Maybe we can simply use a ${VM_IMG_VERSION} and manually increase it when there are significant changes. We could add a "make vm-clean-cache-older-than-%" parsing the "20days/3months/2y" formats? > I'd vote to keep the current approach to overwrite "freebsd.img". It's a > "cache" directory - if you want to have something persisten, Philippe, I > think you should simply copy your image to a different location instead. My concern came when thinking about bisecting: $ git log --oneline \ $(git log --pretty=format:%H \ --diff-filter=A -- tests/vm/freebsd)..origin/master | wc -l 13016 But then I only used "make check-vm" while bisecting twice, so I can stop worrying about testing old VMs. Regards, Phil.
Hi, > > I'd vote to keep the current approach to overwrite "freebsd.img". It's a > > "cache" directory - if you want to have something persisten, Philippe, I > > think you should simply copy your image to a different location instead. > > My concern came when thinking about bisecting: /me tricked make with "touch ~/.cache/qemu-vm/images/${name}.img" to avoid image rebuilds while bisecting. A bit hacky, but I think better than building fragile infrastructure for rare usecases ... cheers, Gerd
diff --git a/tests/vm/freebsd b/tests/vm/freebsd index b0066017a617..57e5c97f3b26 100755 --- a/tests/vm/freebsd +++ b/tests/vm/freebsd @@ -2,43 +2,198 @@ # # FreeBSD VM image # -# Copyright 2017 Red Hat Inc. +# Copyright 2017-2019 Red Hat Inc. # # Authors: # Fam Zheng <famz@redhat.com> +# Gerd Hoffmann <kraxel@redhat.com> # # This code is licensed under the GPL version 2 or later. See # the COPYING file in the top-level directory. # import os +import re import sys +import time +import socket import subprocess import basevm class FreeBSDVM(basevm.BaseVM): name = "freebsd" arch = "x86_64" + + link = "https://download.freebsd.org/ftp/releases/ISO-IMAGES/12.0/FreeBSD-12.0-RELEASE-amd64-disc1.iso.xz" + csum = "1d40015bea89d05b8bd13e2ed80c40b522a9ec1abd8e7c8b80954fb485fb99db" + size = "20G" + pkgs = [ + # build tools + "git", + "pkgconf", + "bzip2", + + # gnu tools + "bash", + "gmake", + "gsed", + "flex", "bison", + + # libs: crypto + "gnutls", + + # libs: images + "jpeg-turbo", + "png", + + # libs: ui + "sdl2", + "gtk3", + "libxkbcommon", + + # libs: opengl + "libepoxy", + "mesa-libs", + ] + BUILD_SCRIPT = """ set -e; - rm -rf /var/tmp/qemu-test.* - cd $(mktemp -d /var/tmp/qemu-test.XXXXXX); + rm -rf /home/qemu/qemu-test.* + cd $(mktemp -d /home/qemu/qemu-test.XXXXXX); + mkdir src build; cd src; tar -xf /dev/vtbd1; - ./configure {configure_opts}; + cd ../build + ../src/configure --python=python3.6 {configure_opts}; gmake --output-sync -j{jobs} {target} {verbose}; """ + def console_boot_serial(self): + self.console_wait_send("Autoboot", "3") + self.console_wait_send("OK", "set console=comconsole\n") + self.console_wait_send("OK", "boot\n") + def build_image(self, img): - cimg = self._download_with_cache("http://download.patchew.org/freebsd-11.1-amd64.img.xz", - sha256sum='adcb771549b37bc63826c501f05121a206ed3d9f55f49145908f7e1432d65891') - img_tmp_xz = img + ".tmp.xz" + self.print_step("Downloading install iso") + cimg = self._download_with_cache(self.link, sha256sum=self.csum) img_tmp = img + ".tmp" - sys.stderr.write("Extracting the image...\n") - subprocess.check_call(["cp", "-f", cimg, img_tmp_xz]) - subprocess.check_call(["xz", "-dvf", img_tmp_xz]) + iso = img + ".install.iso" + iso_xz = iso + ".xz" + + self.print_step("Preparing iso and disk image") + subprocess.check_call(["cp", "-f", cimg, iso_xz]) + subprocess.check_call(["xz", "-dvf", iso_xz]) + subprocess.check_call(["qemu-img", "create", "-f", "qcow2", + img_tmp, self.size]) + + self.print_step("Booting installer") + self.boot(img_tmp, extra_args = [ + "-machine", "graphics=off", + "-cdrom", iso + ]) + self.console_init() + self.console_boot_serial() + self.console_wait_send("Console type", "xterm\n") + + # pre-install configuration + self.console_wait_send("Welcome", "\n") + self.console_wait_send("Keymap Selection", "\n") + self.console_wait_send("Set Hostname", "freebsd\n") + self.console_wait_send("Distribution Select", "\n") + self.console_wait_send("Partitioning", "\n") + self.console_wait_send("Partition", "\n") + self.console_wait_send("Scheme", "\n") + self.console_wait_send("Editor", "f") + self.console_wait_send("Confirmation", "c") + + self.print_step("Installation started now, this will take a while") + + # post-install configuration + self.console_wait("New Password:") + self.console_send("%s\n" % self.ROOT_PASS) + self.console_wait("Retype New Password:") + self.console_send("%s\n" % self.ROOT_PASS) + + self.console_wait_send("Network Configuration", "\n") + self.console_wait_send("IPv4", "y") + self.console_wait_send("DHCP", "y") + self.console_wait_send("IPv6", "n") + self.console_wait_send("Resolver", "\n") + + self.console_wait_send("Time Zone Selector", "a\n") + self.console_wait_send("Confirmation", "y") + self.console_wait_send("Time & Date", "\n") + self.console_wait_send("Time & Date", "\n") + + self.console_wait_send("System Configuration", "\n") + self.console_wait_send("System Hardening", "\n") + + # qemu user + self.console_wait_send("Add User Accounts", "y") + self.console_wait("Username") + self.console_send("%s\n" % self.GUEST_USER) + self.console_wait("Full name") + self.console_send("%s\n" % self.GUEST_USER) + self.console_wait_send("Uid", "\n") + self.console_wait_send("Login group", "\n") + self.console_wait_send("Login group", "\n") + self.console_wait_send("Login class", "\n") + self.console_wait_send("Shell", "\n") + self.console_wait_send("Home directory", "\n") + self.console_wait_send("Home directory perm", "\n") + self.console_wait_send("Use password", "\n") + self.console_wait_send("Use an empty password", "\n") + self.console_wait_send("Use a random password", "\n") + self.console_wait("Enter password:") + self.console_send("%s\n" % self.GUEST_PASS) + self.console_wait("Enter password again:") + self.console_send("%s\n" % self.GUEST_PASS) + self.console_wait_send("Lock out", "\n") + self.console_wait_send("OK", "yes\n") + self.console_wait_send("Add another user", "no\n") + + self.console_wait_send("Final Configuration", "\n") + self.console_wait_send("Manual Configuration", "\n") + self.console_wait_send("Complete", "\n") + + self.print_step("Installation finished, rebooting") + self.console_boot_serial() + + # setup qemu user + prompt = "$" + self.console_ssh_init(prompt, self.GUEST_USER, self.GUEST_PASS) + self.console_wait_send(prompt, "exit\n") + + # setup root user + prompt = "root@freebsd:~ #" + self.console_ssh_init(prompt, "root", self.ROOT_PASS) + self.console_sshd_config(prompt) + + # setup serial console + self.console_wait(prompt) + self.console_send("echo 'console=comconsole' >> /boot/loader.conf\n") + + # setup virtio-blk #1 (tarfile) + self.console_wait(prompt) + self.console_send("echo 'chmod 666 /dev/vtbd1' >> /etc/rc.local\n") + + self.print_step("Configuration finished, rebooting") + self.console_wait_send(prompt, "reboot\n") + self.console_wait("login:") + self.wait_ssh() + + self.print_step("Installing packages") + self.ssh_root_check("pkg install -y %s\n" % " ".join(self.pkgs)) + + # shutdown + self.ssh_root(self.poweroff) + self.console_wait("Uptime:") + self.wait() + if os.path.exists(img): os.remove(img) os.rename(img_tmp, img) + os.remove(iso) + self.print_step("All done") if __name__ == "__main__": sys.exit(basevm.main(FreeBSDVM))
Instead of fetching the prebuilt image from patchew download the install iso and prepare the image locally. Install to disk, using the serial console. Create qemu user, configure ssh login. Install packages needed for qemu builds. Note that freebsd package downloads are delivered as non-cachable content, so I had to configure squid with "ignore-no-store ignore-private ignore-reload" for pkgmir.geo.freebsd.org to make the caching actually work. Signed-off-by: Gerd Hoffmann <kraxel@redhat.com> --- tests/vm/freebsd | 175 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 165 insertions(+), 10 deletions(-)