| Message ID | 20190508085645.11595-12-kraxel@redhat.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | tests/vm: serial console autoinstall, misc fixes. | expand |
On 08.05.2019 10:56, Gerd Hoffmann wrote: > Instead of fetching the prebuilt image from patchew download the install > iso and prepare the image locally. Install to disk, using the serial > console. Create qemu user, configure ssh login. Install packages > needed for qemu builds. > I recommend to add one extra step into generated image: echo security.pax.mprotect.enabled=0 >> /etc/sysctl.conf Alternatively (and preferably) enhance qemu to handle RWX allocation for JIT on NetBSD. Example in libffi. https://github.com/libffi/libffi/commit/2bfcd29955c02b67fa10a68cc4200f6838181e0f > Signed-off-by: Gerd Hoffmann <kraxel@redhat.com> > --- > tests/vm/netbsd | 178 +++++++++++++++++++++++++++++++++++++++++++++--- > 1 file changed, 169 insertions(+), 9 deletions(-) > > diff --git a/tests/vm/netbsd b/tests/vm/netbsd > index 4c6624ea5ed5..eaf0ae21db42 100755 > --- a/tests/vm/netbsd > +++ b/tests/vm/netbsd > @@ -13,32 +13,192 @@ > > import os > import sys > +import time > import subprocess > import basevm > > class NetBSDVM(basevm.BaseVM): > name = "netbsd" > arch = "x86_64" > + > + link = "https://cdn.netbsd.org/pub/NetBSD/NetBSD-8.0/images/NetBSD-8.0-amd64.iso" > + size = "20G" > + pkgs = [ > + # tools > + "git", > + "pkgconf", > + "bzip2", "xz", > + > + # gnu tools > + "bash", > + "gmake", > + "gsed", > + "flex", "bison", > + > + # libs: crypto > + "gnutls", > + > + # libs: images > + "jpeg", > + "png", > + > + # libs: ui > + "SDL2", > + "gtk3+", > + "libxkbcommon", > + ] > + > BUILD_SCRIPT = """ > set -e; > - rm -rf /var/tmp/qemu-test.* > - cd $(mktemp -d /var/tmp/qemu-test.XXXXXX); > + rm -rf /home/qemu/qemu-test.* > + cd $(mktemp -d /home/qemu/qemu-test.XXXXXX); > + mkdir src build; cd src; > tar -xf /dev/rld1a; > - ./configure --python=python2.7 {configure_opts}; > + cd ../build > + ../src/configure --python=python2.7 --disable-opengl {configure_opts}; > gmake --output-sync -j{jobs} {target} {verbose}; > """ > > def build_image(self, img): > - cimg = self._download_with_cache("http://download.patchew.org/netbsd-7.1-amd64.img.xz", > - sha256sum='b633d565b0eac3d02015cd0c81440bd8a7a8df8512615ac1ee05d318be015732') > - img_tmp_xz = img + ".tmp.xz" > + cimg = self._download_with_cache(self.link) > img_tmp = img + ".tmp" > - sys.stderr.write("Extracting the image...\n") > - subprocess.check_call(["cp", "-f", cimg, img_tmp_xz]) > - subprocess.check_call(["xz", "-dvf", img_tmp_xz]) > + iso = img + ".install.iso" > + > + self.print_step("Preparing iso and disk image") > + subprocess.check_call(["cp", "-f", cimg, iso]) > + subprocess.check_call(["qemu-img", "create", "-f", "qcow2", > + img_tmp, self.size]) > + > + self.print_step("Booting installer") > + self.boot(img_tmp, extra_args = [ > + "-device", "VGA", > + "-machine", "graphics=off", > + "-cdrom", iso > + ]) > + self.console_init() > + self.console_wait("Primary Bootstrap") > + > + # serial console boot menu output doesn't work for some > + # reason, so we have to fly blind ... > + for char in list("5consdev com0\n"): > + time.sleep(0.2) > + self.console_send(char) > + self.console_wait("") > + self.console_wait_send("> ", "boot\n") > + > + self.console_wait_send("Terminal type", "xterm\n") > + self.console_wait_send("a: Installation messages", "a\n") > + self.console_wait_send("b: US-English", "b\n") > + self.console_wait_send("a: Install NetBSD", "a\n") > + self.console_wait("Shall we continue?") > + self.console_wait_send("b: Yes", "b\n") > + > + self.console_wait_send("a: ld0", "a\n") > + self.console_wait_send("a: This is the correct", "a\n") > + self.console_wait_send("b: Use the entire disk", "b\n") > + self.console_wait("NetBSD bootcode") > + self.console_wait_send("a: Yes", "a\n") > + self.console_wait_send("b: Use existing part", "b\n") > + self.console_wait_send("x: Partition sizes ok", "x\n") > + self.console_wait_send("for your NetBSD disk", "\n") > + self.console_wait("Shall we continue?") > + self.console_wait_send("b: Yes", "b\n") > + > + self.console_wait_send("b: Use serial port com0", "b\n") > + self.console_wait_send("f: Set serial baud rate", "f\n") > + self.console_wait_send("a: 9600", "a\n") > + self.console_wait_send("x: Exit", "x\n") > + > + self.console_wait_send("a: Full installation", "a\n") > + self.console_wait_send("a: CD-ROM", "a\n") > + > + self.print_step("Installation started now, this will take a while") > + self.console_wait_send("Hit enter to continue", "\n") > + > + self.console_wait_send("d: Change root password", "d\n") > + self.console_wait_send("a: Yes", "a\n") > + self.console_wait("New password:") > + self.console_send("%s\n" % self.ROOT_PASS) > + self.console_wait("New password:") > + self.console_send("%s\n" % self.ROOT_PASS) > + self.console_wait("Retype new password:") > + self.console_send("%s\n" % self.ROOT_PASS) > + > + self.console_wait_send("o: Add a user", "o\n") > + self.console_wait("username") > + self.console_send("%s\n" % self.GUEST_USER) > + self.console_wait("to group wheel") > + self.console_wait_send("a: Yes", "a\n") > + self.console_wait_send("a: /bin/sh", "a\n") > + self.console_wait("New password:") > + self.console_send("%s\n" % self.GUEST_PASS) > + self.console_wait("New password:") > + self.console_send("%s\n" % self.GUEST_PASS) > + self.console_wait("Retype new password:") > + self.console_send("%s\n" % self.GUEST_PASS) > + > + self.console_wait_send("a: Configure network", "a\n") > + self.console_wait_send("a: vioif0", "a\n") > + self.console_wait_send("Network media type", "\n") > + self.console_wait("autoconfiguration") > + self.console_wait_send("a: Yes", "a\n") > + self.console_wait_send("DNS domain", "localnet\n") > + self.console_wait("Are they OK?") > + self.console_wait_send("a: Yes", "a\n") > + self.console_wait("installed in /etc") > + self.console_wait_send("a: Yes", "a\n") > + > + self.console_wait_send("e: Enable install", "e\n") > + proxy = os.environ.get("http_proxy") > + if not proxy is None: > + self.console_wait_send("f: Proxy", "f\n") > + self.console_wait("Proxy") > + self.console_send("%s\n" % proxy) > + self.console_wait_send("x: Install pkgin", "x\n") > + self.console_init(1200) > + self.console_wait_send("Hit enter to continue", "\n") > + self.console_init() > + > + self.console_wait_send("g: Enable sshd", "g\n") > + self.console_wait_send("x: Finished conf", "x\n") > + self.console_wait_send("Hit enter to continue", "\n") > + > + self.print_step("Installation finished, rebooting") > + self.console_wait_send("d: Reboot the computer", "d\n") > + > + # setup qemu user > + prompt = "localhost$" > + self.console_ssh_init(prompt, self.GUEST_USER, self.GUEST_PASS) > + self.console_wait_send(prompt, "exit\n") > + > + # setup root user > + prompt = "localhost#" > + self.console_ssh_init(prompt, "root", self.ROOT_PASS) > + self.console_sshd_config(prompt) > + > + # setup virtio-blk #1 (tarfile) > + self.console_wait(prompt) > + self.console_send("echo 'chmod 666 /dev/rld1a' >> /etc/rc.local\n") > + > + self.print_step("Configuration finished, rebooting") > + self.console_wait_send(prompt, "reboot\n") > + self.console_wait("login:") > + self.wait_ssh() > + > + self.print_step("Installing packages") > + self.ssh_root_check("pkgin update\n") > + self.ssh_root_check("pkgin -y install %s\n" % " ".join(self.pkgs)) > + > + # shutdown > + self.ssh_root("/sbin/poweroff") > + self.console_wait("entering state S5") > + self.wait() > + > if os.path.exists(img): > os.remove(img) > os.rename(img_tmp, img) > + os.remove(iso) > + self.print_step("All done") > > if __name__ == "__main__": > sys.exit(basevm.main(NetBSDVM)) >
Hi, > I recommend to add one extra step into generated image: > > echo security.pax.mprotect.enabled=0 >> /etc/sysctl.conf Done. > Alternatively (and preferably) enhance qemu to handle RWX allocation for > JIT on NetBSD. > > Example in libffi. > > https://github.com/libffi/libffi/commit/2bfcd29955c02b67fa10a68cc4200f6838181e0f So, from looking at the patch it seems you need two mappings of the same page, one writable and one executable. Or, maybe it is also possible with one mapping which is writable first when you fill it with code, then gets flipped over to executable when you are done with the initialization and want use it. Is that correct? I suspect supporting that in tcg isn't exactly trivial. Does OpenBSD work the same way btw? cheers, Gerd
On 5/8/19 11:47 PM, Gerd Hoffmann wrote: > So, from looking at the patch it seems you need two mappings of the same > page, one writable and one executable. > > Or, maybe it is also possible with one mapping which is writable first > when you fill it with code, then gets flipped over to executable when > you are done with the initialization and want use it. > > Is that correct? That's certainly the way I read that patch. > I suspect supporting that in tcg isn't exactly trivial. It shouldn't be too hard, if required. All of the writing to the buffer is isolated to a couple of inline functions. I do wonder if using paxctl -m as a part of the build process for affected *BSD isn't just as easy? r~
On 09.05.2019 08:47, Gerd Hoffmann wrote: > Hi, > >> I recommend to add one extra step into generated image: >> >> echo security.pax.mprotect.enabled=0 >> /etc/sysctl.conf > > Done. > Thanks! Once there will be PaX MPROTECT support in qemu, we can and should drop it. >> Alternatively (and preferably) enhance qemu to handle RWX allocation for >> JIT on NetBSD. >> >> Example in libffi. >> >> https://github.com/libffi/libffi/commit/2bfcd29955c02b67fa10a68cc4200f6838181e0f > > So, from looking at the patch it seems you need two mappings of the same > page, one writable and one executable. > > Or, maybe it is also possible with one mapping which is writable first > when you fill it with code, then gets flipped over to executable when > you are done with the initialization and want use it. > > Is that correct? > If we need RWX in the same time we need double mapping (RW and RX). This has been done in libffi. If we can switch between RW and RX it's sufficient to use a single mapping, however there is need to allocate a memory region with mmap(2) using the PROT_MPROTECT() macro. Example from the LLVM code: https://github.com/llvm-mirror/llvm/blob/90dd07f5c5946a3d9d6861effe3291620c88c06f/lib/Support/Unix/Memory.inc#L99 > I suspect supporting that in tcg isn't exactly trivial. > Does OpenBSD work the same way btw? > No. OpenBSD does not implement escape API. There is need to pass "-z wxneeded" to the linker flags, but I'm not the right person to test this or write a patch. https://man.openbsd.org/ld FreeBSD does not support W^X. > cheers, > Gerd >
On 09.05.2019 18:39, Richard Henderson wrote: > On 5/8/19 11:47 PM, Gerd Hoffmann wrote: >> So, from looking at the patch it seems you need two mappings of the same >> page, one writable and one executable. >> >> Or, maybe it is also possible with one mapping which is writable first >> when you fill it with code, then gets flipped over to executable when >> you are done with the initialization and want use it. >> >> Is that correct? > > That's certainly the way I read that patch. > >> I suspect supporting that in tcg isn't exactly trivial. > > It shouldn't be too hard, if required. All of the writing to the buffer is > isolated to a couple of inline functions. > > I do wonder if using paxctl -m as a part of the build process for affected *BSD > isn't just as easy? > paxctl(8) is available only for NetBSD. paxctl(8) is the last resort solution and shall be avoided due to a security risk. Only few exceptions shall be allowed to switch mapping protection or request RWX mappings (mainly JIT and a process running under a debugger). > > r~ >
diff --git a/tests/vm/netbsd b/tests/vm/netbsd index 4c6624ea5ed5..eaf0ae21db42 100755 --- a/tests/vm/netbsd +++ b/tests/vm/netbsd @@ -13,32 +13,192 @@ import os import sys +import time import subprocess import basevm class NetBSDVM(basevm.BaseVM): name = "netbsd" arch = "x86_64" + + link = "https://cdn.netbsd.org/pub/NetBSD/NetBSD-8.0/images/NetBSD-8.0-amd64.iso" + size = "20G" + pkgs = [ + # tools + "git", + "pkgconf", + "bzip2", "xz", + + # gnu tools + "bash", + "gmake", + "gsed", + "flex", "bison", + + # libs: crypto + "gnutls", + + # libs: images + "jpeg", + "png", + + # libs: ui + "SDL2", + "gtk3+", + "libxkbcommon", + ] + BUILD_SCRIPT = """ set -e; - rm -rf /var/tmp/qemu-test.* - cd $(mktemp -d /var/tmp/qemu-test.XXXXXX); + rm -rf /home/qemu/qemu-test.* + cd $(mktemp -d /home/qemu/qemu-test.XXXXXX); + mkdir src build; cd src; tar -xf /dev/rld1a; - ./configure --python=python2.7 {configure_opts}; + cd ../build + ../src/configure --python=python2.7 --disable-opengl {configure_opts}; gmake --output-sync -j{jobs} {target} {verbose}; """ def build_image(self, img): - cimg = self._download_with_cache("http://download.patchew.org/netbsd-7.1-amd64.img.xz", - sha256sum='b633d565b0eac3d02015cd0c81440bd8a7a8df8512615ac1ee05d318be015732') - img_tmp_xz = img + ".tmp.xz" + cimg = self._download_with_cache(self.link) img_tmp = img + ".tmp" - sys.stderr.write("Extracting the image...\n") - subprocess.check_call(["cp", "-f", cimg, img_tmp_xz]) - subprocess.check_call(["xz", "-dvf", img_tmp_xz]) + iso = img + ".install.iso" + + self.print_step("Preparing iso and disk image") + subprocess.check_call(["cp", "-f", cimg, iso]) + subprocess.check_call(["qemu-img", "create", "-f", "qcow2", + img_tmp, self.size]) + + self.print_step("Booting installer") + self.boot(img_tmp, extra_args = [ + "-device", "VGA", + "-machine", "graphics=off", + "-cdrom", iso + ]) + self.console_init() + self.console_wait("Primary Bootstrap") + + # serial console boot menu output doesn't work for some + # reason, so we have to fly blind ... + for char in list("5consdev com0\n"): + time.sleep(0.2) + self.console_send(char) + self.console_wait("") + self.console_wait_send("> ", "boot\n") + + self.console_wait_send("Terminal type", "xterm\n") + self.console_wait_send("a: Installation messages", "a\n") + self.console_wait_send("b: US-English", "b\n") + self.console_wait_send("a: Install NetBSD", "a\n") + self.console_wait("Shall we continue?") + self.console_wait_send("b: Yes", "b\n") + + self.console_wait_send("a: ld0", "a\n") + self.console_wait_send("a: This is the correct", "a\n") + self.console_wait_send("b: Use the entire disk", "b\n") + self.console_wait("NetBSD bootcode") + self.console_wait_send("a: Yes", "a\n") + self.console_wait_send("b: Use existing part", "b\n") + self.console_wait_send("x: Partition sizes ok", "x\n") + self.console_wait_send("for your NetBSD disk", "\n") + self.console_wait("Shall we continue?") + self.console_wait_send("b: Yes", "b\n") + + self.console_wait_send("b: Use serial port com0", "b\n") + self.console_wait_send("f: Set serial baud rate", "f\n") + self.console_wait_send("a: 9600", "a\n") + self.console_wait_send("x: Exit", "x\n") + + self.console_wait_send("a: Full installation", "a\n") + self.console_wait_send("a: CD-ROM", "a\n") + + self.print_step("Installation started now, this will take a while") + self.console_wait_send("Hit enter to continue", "\n") + + self.console_wait_send("d: Change root password", "d\n") + self.console_wait_send("a: Yes", "a\n") + self.console_wait("New password:") + self.console_send("%s\n" % self.ROOT_PASS) + self.console_wait("New password:") + self.console_send("%s\n" % self.ROOT_PASS) + self.console_wait("Retype new password:") + self.console_send("%s\n" % self.ROOT_PASS) + + self.console_wait_send("o: Add a user", "o\n") + self.console_wait("username") + self.console_send("%s\n" % self.GUEST_USER) + self.console_wait("to group wheel") + self.console_wait_send("a: Yes", "a\n") + self.console_wait_send("a: /bin/sh", "a\n") + self.console_wait("New password:") + self.console_send("%s\n" % self.GUEST_PASS) + self.console_wait("New password:") + self.console_send("%s\n" % self.GUEST_PASS) + self.console_wait("Retype new password:") + self.console_send("%s\n" % self.GUEST_PASS) + + self.console_wait_send("a: Configure network", "a\n") + self.console_wait_send("a: vioif0", "a\n") + self.console_wait_send("Network media type", "\n") + self.console_wait("autoconfiguration") + self.console_wait_send("a: Yes", "a\n") + self.console_wait_send("DNS domain", "localnet\n") + self.console_wait("Are they OK?") + self.console_wait_send("a: Yes", "a\n") + self.console_wait("installed in /etc") + self.console_wait_send("a: Yes", "a\n") + + self.console_wait_send("e: Enable install", "e\n") + proxy = os.environ.get("http_proxy") + if not proxy is None: + self.console_wait_send("f: Proxy", "f\n") + self.console_wait("Proxy") + self.console_send("%s\n" % proxy) + self.console_wait_send("x: Install pkgin", "x\n") + self.console_init(1200) + self.console_wait_send("Hit enter to continue", "\n") + self.console_init() + + self.console_wait_send("g: Enable sshd", "g\n") + self.console_wait_send("x: Finished conf", "x\n") + self.console_wait_send("Hit enter to continue", "\n") + + self.print_step("Installation finished, rebooting") + self.console_wait_send("d: Reboot the computer", "d\n") + + # setup qemu user + prompt = "localhost$" + self.console_ssh_init(prompt, self.GUEST_USER, self.GUEST_PASS) + self.console_wait_send(prompt, "exit\n") + + # setup root user + prompt = "localhost#" + self.console_ssh_init(prompt, "root", self.ROOT_PASS) + self.console_sshd_config(prompt) + + # setup virtio-blk #1 (tarfile) + self.console_wait(prompt) + self.console_send("echo 'chmod 666 /dev/rld1a' >> /etc/rc.local\n") + + self.print_step("Configuration finished, rebooting") + self.console_wait_send(prompt, "reboot\n") + self.console_wait("login:") + self.wait_ssh() + + self.print_step("Installing packages") + self.ssh_root_check("pkgin update\n") + self.ssh_root_check("pkgin -y install %s\n" % " ".join(self.pkgs)) + + # shutdown + self.ssh_root("/sbin/poweroff") + self.console_wait("entering state S5") + self.wait() + if os.path.exists(img): os.remove(img) os.rename(img_tmp, img) + os.remove(iso) + self.print_step("All done") if __name__ == "__main__": sys.exit(basevm.main(NetBSDVM))
Instead of fetching the prebuilt image from patchew download the install iso and prepare the image locally. Install to disk, using the serial console. Create qemu user, configure ssh login. Install packages needed for qemu builds. Signed-off-by: Gerd Hoffmann <kraxel@redhat.com> --- tests/vm/netbsd | 178 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 169 insertions(+), 9 deletions(-)