| Message ID | 20180824012126.22721-1-famz@redhat.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | tests: vm: auto_install OpenBSD | expand |
On Fri, Aug 24, 2018 at 09:21:26AM +0800, Fam Zheng wrote: > Upgrade OpenBSD to 6.3 using auto_install. Especially, drop SDL1, > include SDL2. > > One limitation of this patch is that we need a temporary HTTP server on > host 80 port for auto_install, because slirp cannot do guest forward on > "host addr". That's a pretty big limitation, as port 80 requires root privileges, and that's a pretty strict no for automated testing IMHO. Why does it have to be port 80 in particular - is it not possible to use 8080 or even better, detect a random free port ? > > Signed-off-by: Fam Zheng <famz@redhat.com> > --- > tests/vm/basevm.py | 28 ++++++++++++++-- > tests/vm/openbsd | 81 ++++++++++++++++++++++++++++++++++++++++------ > 2 files changed, 96 insertions(+), 13 deletions(-) > > diff --git a/tests/vm/basevm.py b/tests/vm/basevm.py > index d7149dea7d..4080039e66 100755 > --- a/tests/vm/basevm.py > +++ b/tests/vm/basevm.py > @@ -65,8 +65,6 @@ class BaseVM(object): > self._stdout = self._devnull > self._args = [ \ > "-nodefaults", "-m", "4G", > - "-netdev", "user,id=vnet,hostfwd=:127.0.0.1:0-:22", > - "-device", "virtio-net-pci,netdev=vnet", > "-vnc", "127.0.0.1:0,to=20", > "-serial", "file:%s" % os.path.join(self._tmpdir, "serial.out")] > if vcpus: > @@ -145,8 +143,10 @@ class BaseVM(object): > "-device", > "virtio-blk,drive=%s,serial=%s,bootindex=1" % (name, name)] > > - def boot(self, img, extra_args=[]): > + def boot(self, img, extra_args=[], extra_usernet_args=""): > args = self._args + [ > + "-netdev", "user,id=vnet,hostfwd=:127.0.0.1:0-:22" + extra_usernet_args, > + "-device", "virtio-net-pci,netdev=vnet", > "-device", "VGA", > "-drive", "file=%s,if=none,id=drive0,cache=writeback" % img, > "-device", "virtio-blk,drive=drive0,bootindex=0"] > @@ -196,6 +196,28 @@ class BaseVM(object): > def qmp(self, *args, **kwargs): > return self._guest.qmp(*args, **kwargs) > > + def start_http_server(self, workdir, ports=range(8010, 8020), sudo=False): > + p = None > + token = "%d-%d" % (os.getpid(), time.time()) > + with open(os.path.join(workdir, token), "w") as f: > + f.write("# QEMU VM testing HTTP server token file #") > + f.flush() > + for port in ports: > + cmd = ["python3", "-m", "http.server", str(port)] > + if sudo: > + cmd = ["sudo", "-n"] + cmd > + p = subprocess.Popen(cmd, cwd=workdir) > + for retry in range(5): > + if p.poll() != None: > + break > + if subprocess.call("curl http://127.0.0.1:%d/%s &>/dev/null" % \ > + (port, token), > + shell=True) == 0: > + atexit.register(p.terminate) > + return port > + time.sleep(0.3) > + raise IOError("Failed to start HTTP server") > + > def parse_args(vm_name): > parser = optparse.OptionParser( > description="VM test utility. Exit codes: " > diff --git a/tests/vm/openbsd b/tests/vm/openbsd > index 52500ee52b..7e118572a9 100755 > --- a/tests/vm/openbsd > +++ b/tests/vm/openbsd > @@ -14,6 +14,9 @@ > import os > import sys > import subprocess > +import time > +import atexit > +import tempfile > import basevm > > class OpenBSDVM(basevm.BaseVM): > @@ -23,22 +26,80 @@ class OpenBSDVM(basevm.BaseVM): > rm -rf /var/tmp/qemu-test.* > cd $(mktemp -d /var/tmp/qemu-test.XXXXXX); > tar -xf /dev/rsd1c; > - ./configure --cc=x86_64-unknown-openbsd6.1-gcc-4.9.4 --python=python2.7 {configure_opts}; > + ./configure {configure_opts}; > gmake --output-sync -j{jobs} {verbose}; > # XXX: "gmake check" seems to always hang or fail > #gmake --output-sync -j{jobs} check {verbose}; > """ > > + def _install_os(self, img): > + tmpdir = tempfile.mkdtemp() > + pxeboot = self._download_with_cache("https://fastly.cdn.openbsd.org/pub/OpenBSD/6.3/amd64/pxeboot", > + sha256sum="60029919798f48ea40ecb123adfed6217f099d5ed9cd1a6c7de5b544d7b7b0f6") > + bsd_rd = self._download_with_cache("https://fastly.cdn.openbsd.org/pub/OpenBSD/6.3/amd64/bsd.rd", > + sha256sum="1c0adb43a02ae3aee512bcf0829dac0ccb2e4d614b161049af7ce530e5da2dfc") > + install = self._download_with_cache("https://fastly.cdn.openbsd.org/pub/OpenBSD/6.3/amd64/install63.iso", > + sha256sum='ee775405dd7926975befbc3fef23de8c4b5a726c3b5075e4848fcd3a2a712ea8') > + subprocess.check_call(["qemu-img", "create", img, "32G"]) > + subprocess.check_call(["cp", pxeboot, os.path.join(tmpdir, "auto_install")]) > + subprocess.check_call(["cp", bsd_rd, os.path.join(tmpdir, "bsd")]) > + > + self._gen_install_conf(tmpdir) > + try: > + self.start_http_server(tmpdir, ports=[80], sudo=True) > + except Exception: > + sys.stdout.write("Cannot open HTTP server on port 80. Maybe use sudo?\n") > + sys.exit(1) > + # BOOTP filename being auto_install makes sure OpenBSD installer > + # not prompt for "auto install mode" > + tftp_args = ",tftp=%s,bootfile=/auto_install" % tmpdir > + self.boot(img, > + extra_args=["-boot", "once=n", "-no-reboot", > + "-cdrom", install], > + extra_usernet_args=tftp_args) > + self.wait() > + > + def _gen_install_conf(self, tmpdir): > + contents = """ > +System hostname = qemu-openbsd > +Password for root = qemupass > +Public ssh key for root = {pub_key} > +Allow root ssh login = yes > +Network interfaces = vio0 > +IPv4 address for vio0 = dhcp > +Setup a user = qemu > +Password for user = qemupass > +Public ssh key for user = {pub_key} > +What timezone are you in = US/Eastern > +Server = fastly.cdn.openbsd.org > +Use http = yes > +Default IPv4 route = 10.0.2.2 > +Location of sets = cd0 > +Set name(s) = all > +Continue without verification = yes > +""".format(pub_key=basevm.SSH_PUB_KEY) > + with open(os.path.join(tmpdir, "install.conf"), "w") as f: > + f.write(contents) > + > def build_image(self, img): > - cimg = self._download_with_cache("http://download.patchew.org/openbsd-6.1-amd64.img.xz", > - sha256sum='8c6cedc483e602cfee5e04f0406c64eb99138495e8ca580bc0293bcf0640c1bf') > - img_tmp_xz = img + ".tmp.xz" > - img_tmp = img + ".tmp" > - subprocess.check_call(["cp", "-f", cimg, img_tmp_xz]) > - subprocess.check_call(["xz", "-df", img_tmp_xz]) > - if os.path.exists(img): > - os.remove(img) > - os.rename(img_tmp, img) > + > + self._install_os(img + ".tmp") > + > + self.boot(img + ".tmp") > + self.wait_ssh() > + > + self.ssh_root("usermod -G operator qemu") > + self.ssh_root("echo https://fastly.cdn.openbsd.org/pub/OpenBSD > /etc/installurl") > + for pkg in ["git", "gmake", "glib2", "bison", "sdl2"]: > + self.ssh_root("pkg_add " + pkg) > + self.ssh_root("ln -sf /usr/local/bin/python2.7 /usr/local/bin/python") > + self.ssh_root("ln -sf /usr/local/bin/python2.7-2to3 /usr/local/bin/2to3") > + self.ssh_root("ln -sf /usr/local/bin/python2.7-config /usr/local/bin/python-config") > + self.ssh_root("ln -sf /usr/local/bin/pydoc2.7 /usr/local/bin/pydoc") > + self.ssh_root("shutdown -p now") > + self.wait() > + > + subprocess.check_call(["mv", img + ".tmp", img]) > > if __name__ == "__main__": > sys.exit(basevm.main(OpenBSDVM)) > -- > 2.17.1 > Regards, Daniel
On Fri, 08/24 09:22, Daniel P. Berrangé wrote: > On Fri, Aug 24, 2018 at 09:21:26AM +0800, Fam Zheng wrote: > > Upgrade OpenBSD to 6.3 using auto_install. Especially, drop SDL1, > > include SDL2. > > > > One limitation of this patch is that we need a temporary HTTP server on > > host 80 port for auto_install, because slirp cannot do guest forward on > > "host addr". > > That's a pretty big limitation, as port 80 requires root privileges, > and that's a pretty strict no for automated testing IMHO. > > Why does it have to be port 80 in particular - is it not possible > to use 8080 or even better, detect a random free port ? OpenBSD autoinstall[1] is hardcoded to fetch from port 80. Maybe we can instead persuade it into fetching from a different http server than 10.0.2.2. To do that we'd need to implement a new slirp option for either "option tftp-server-name" or "next-server" according to the manpage. I'll have to experiment with it to see if it will work. Or, do you think there are better options to configure the VM network to serve the file? (Another possibility is to inject the install.conf file into bsd.rd (ramdisk embedded in ELF), but the tools seem OpenBSD specific.) [1] https://man.openbsd.org/autoinstall > > > > > Signed-off-by: Fam Zheng <famz@redhat.com> > > --- > > tests/vm/basevm.py | 28 ++++++++++++++-- > > tests/vm/openbsd | 81 ++++++++++++++++++++++++++++++++++++++++------ > > 2 files changed, 96 insertions(+), 13 deletions(-) > > > > diff --git a/tests/vm/basevm.py b/tests/vm/basevm.py > > index d7149dea7d..4080039e66 100755 > > --- a/tests/vm/basevm.py > > +++ b/tests/vm/basevm.py > > @@ -65,8 +65,6 @@ class BaseVM(object): > > self._stdout = self._devnull > > self._args = [ \ > > "-nodefaults", "-m", "4G", > > - "-netdev", "user,id=vnet,hostfwd=:127.0.0.1:0-:22", > > - "-device", "virtio-net-pci,netdev=vnet", > > "-vnc", "127.0.0.1:0,to=20", > > "-serial", "file:%s" % os.path.join(self._tmpdir, "serial.out")] > > if vcpus: > > @@ -145,8 +143,10 @@ class BaseVM(object): > > "-device", > > "virtio-blk,drive=%s,serial=%s,bootindex=1" % (name, name)] > > > > - def boot(self, img, extra_args=[]): > > + def boot(self, img, extra_args=[], extra_usernet_args=""): > > args = self._args + [ > > + "-netdev", "user,id=vnet,hostfwd=:127.0.0.1:0-:22" + extra_usernet_args, > > + "-device", "virtio-net-pci,netdev=vnet", > > "-device", "VGA", > > "-drive", "file=%s,if=none,id=drive0,cache=writeback" % img, > > "-device", "virtio-blk,drive=drive0,bootindex=0"] > > @@ -196,6 +196,28 @@ class BaseVM(object): > > def qmp(self, *args, **kwargs): > > return self._guest.qmp(*args, **kwargs) > > > > + def start_http_server(self, workdir, ports=range(8010, 8020), sudo=False): > > + p = None > > + token = "%d-%d" % (os.getpid(), time.time()) > > + with open(os.path.join(workdir, token), "w") as f: > > + f.write("# QEMU VM testing HTTP server token file #") > > + f.flush() > > + for port in ports: > > + cmd = ["python3", "-m", "http.server", str(port)] > > + if sudo: > > + cmd = ["sudo", "-n"] + cmd > > + p = subprocess.Popen(cmd, cwd=workdir) > > + for retry in range(5): > > + if p.poll() != None: > > + break > > + if subprocess.call("curl http://127.0.0.1:%d/%s &>/dev/null" % \ > > + (port, token), > > + shell=True) == 0: > > + atexit.register(p.terminate) > > + return port > > + time.sleep(0.3) > > + raise IOError("Failed to start HTTP server") > > + > > def parse_args(vm_name): > > parser = optparse.OptionParser( > > description="VM test utility. Exit codes: " > > diff --git a/tests/vm/openbsd b/tests/vm/openbsd > > index 52500ee52b..7e118572a9 100755 > > --- a/tests/vm/openbsd > > +++ b/tests/vm/openbsd > > @@ -14,6 +14,9 @@ > > import os > > import sys > > import subprocess > > +import time > > +import atexit > > +import tempfile > > import basevm > > > > class OpenBSDVM(basevm.BaseVM): > > @@ -23,22 +26,80 @@ class OpenBSDVM(basevm.BaseVM): > > rm -rf /var/tmp/qemu-test.* > > cd $(mktemp -d /var/tmp/qemu-test.XXXXXX); > > tar -xf /dev/rsd1c; > > - ./configure --cc=x86_64-unknown-openbsd6.1-gcc-4.9.4 --python=python2.7 {configure_opts}; > > + ./configure {configure_opts}; > > gmake --output-sync -j{jobs} {verbose}; > > # XXX: "gmake check" seems to always hang or fail > > #gmake --output-sync -j{jobs} check {verbose}; > > """ > > > > + def _install_os(self, img): > > + tmpdir = tempfile.mkdtemp() > > + pxeboot = self._download_with_cache("https://fastly.cdn.openbsd.org/pub/OpenBSD/6.3/amd64/pxeboot", > > + sha256sum="60029919798f48ea40ecb123adfed6217f099d5ed9cd1a6c7de5b544d7b7b0f6") > > + bsd_rd = self._download_with_cache("https://fastly.cdn.openbsd.org/pub/OpenBSD/6.3/amd64/bsd.rd", > > + sha256sum="1c0adb43a02ae3aee512bcf0829dac0ccb2e4d614b161049af7ce530e5da2dfc") > > + install = self._download_with_cache("https://fastly.cdn.openbsd.org/pub/OpenBSD/6.3/amd64/install63.iso", > > + sha256sum='ee775405dd7926975befbc3fef23de8c4b5a726c3b5075e4848fcd3a2a712ea8') > > + subprocess.check_call(["qemu-img", "create", img, "32G"]) > > + subprocess.check_call(["cp", pxeboot, os.path.join(tmpdir, "auto_install")]) > > + subprocess.check_call(["cp", bsd_rd, os.path.join(tmpdir, "bsd")]) > > + > > + self._gen_install_conf(tmpdir) > > + try: > > + self.start_http_server(tmpdir, ports=[80], sudo=True) > > + except Exception: > > + sys.stdout.write("Cannot open HTTP server on port 80. Maybe use sudo?\n") > > + sys.exit(1) > > + # BOOTP filename being auto_install makes sure OpenBSD installer > > + # not prompt for "auto install mode" > > + tftp_args = ",tftp=%s,bootfile=/auto_install" % tmpdir > > + self.boot(img, > > + extra_args=["-boot", "once=n", "-no-reboot", > > + "-cdrom", install], > > + extra_usernet_args=tftp_args) > > + self.wait() > > + > > + def _gen_install_conf(self, tmpdir): > > + contents = """ > > +System hostname = qemu-openbsd > > +Password for root = qemupass > > +Public ssh key for root = {pub_key} > > +Allow root ssh login = yes > > +Network interfaces = vio0 > > +IPv4 address for vio0 = dhcp > > +Setup a user = qemu > > +Password for user = qemupass > > +Public ssh key for user = {pub_key} > > +What timezone are you in = US/Eastern > > +Server = fastly.cdn.openbsd.org > > +Use http = yes > > +Default IPv4 route = 10.0.2.2 > > +Location of sets = cd0 > > +Set name(s) = all > > +Continue without verification = yes > > +""".format(pub_key=basevm.SSH_PUB_KEY) > > + with open(os.path.join(tmpdir, "install.conf"), "w") as f: > > + f.write(contents) > > + > > def build_image(self, img): > > - cimg = self._download_with_cache("http://download.patchew.org/openbsd-6.1-amd64.img.xz", > > - sha256sum='8c6cedc483e602cfee5e04f0406c64eb99138495e8ca580bc0293bcf0640c1bf') > > - img_tmp_xz = img + ".tmp.xz" > > - img_tmp = img + ".tmp" > > - subprocess.check_call(["cp", "-f", cimg, img_tmp_xz]) > > - subprocess.check_call(["xz", "-df", img_tmp_xz]) > > - if os.path.exists(img): > > - os.remove(img) > > - os.rename(img_tmp, img) > > + > > + self._install_os(img + ".tmp") > > + > > + self.boot(img + ".tmp") > > + self.wait_ssh() > > + > > + self.ssh_root("usermod -G operator qemu") > > + self.ssh_root("echo https://fastly.cdn.openbsd.org/pub/OpenBSD > /etc/installurl") > > + for pkg in ["git", "gmake", "glib2", "bison", "sdl2"]: > > + self.ssh_root("pkg_add " + pkg) > > + self.ssh_root("ln -sf /usr/local/bin/python2.7 /usr/local/bin/python") > > + self.ssh_root("ln -sf /usr/local/bin/python2.7-2to3 /usr/local/bin/2to3") > > + self.ssh_root("ln -sf /usr/local/bin/python2.7-config /usr/local/bin/python-config") > > + self.ssh_root("ln -sf /usr/local/bin/pydoc2.7 /usr/local/bin/pydoc") > > + self.ssh_root("shutdown -p now") > > + self.wait() > > + > > + subprocess.check_call(["mv", img + ".tmp", img]) > > > > if __name__ == "__main__": > > sys.exit(basevm.main(OpenBSDVM)) > > -- > > 2.17.1 > > > > Regards, > Daniel > -- > |: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| > |: https://libvirt.org -o- https://fstop138.berrange.com :| > |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
On 24 August 2018 at 09:52, Fam Zheng <famz@redhat.com> wrote: > On Fri, 08/24 09:22, Daniel P. Berrangé wrote: >> On Fri, Aug 24, 2018 at 09:21:26AM +0800, Fam Zheng wrote: >> > Upgrade OpenBSD to 6.3 using auto_install. Especially, drop SDL1, >> > include SDL2. >> > >> > One limitation of this patch is that we need a temporary HTTP server on >> > host 80 port for auto_install, because slirp cannot do guest forward on >> > "host addr". >> >> That's a pretty big limitation, as port 80 requires root privileges, >> and that's a pretty strict no for automated testing IMHO. >> >> Why does it have to be port 80 in particular - is it not possible >> to use 8080 or even better, detect a random free port ? > > OpenBSD autoinstall[1] is hardcoded to fetch from port 80. Maybe we can instead > persuade it into fetching from a different http server than 10.0.2.2. To do that > we'd need to implement a new slirp option for either "option tftp-server-name" > or "next-server" according to the manpage. I'll have to experiment with it to > see if it will work. Can you use the usermode networking "guestfwd" option to make guest tcp connections to wherever:80 go to host:8080 or something instead? (watch out for https://bugs.launchpad.net/qemu/+bug/1628971 if it is still relevant, I guess). thanks -- PMM
On Fri, Aug 24, 2018 at 04:52:20PM +0800, Fam Zheng wrote: > On Fri, 08/24 09:22, Daniel P. Berrangé wrote: > > On Fri, Aug 24, 2018 at 09:21:26AM +0800, Fam Zheng wrote: > > > Upgrade OpenBSD to 6.3 using auto_install. Especially, drop SDL1, > > > include SDL2. > > > > > > One limitation of this patch is that we need a temporary HTTP server on > > > host 80 port for auto_install, because slirp cannot do guest forward on > > > "host addr". > > > > That's a pretty big limitation, as port 80 requires root privileges, > > and that's a pretty strict no for automated testing IMHO. > > > > Why does it have to be port 80 in particular - is it not possible > > to use 8080 or even better, detect a random free port ? > > OpenBSD autoinstall[1] is hardcoded to fetch from port 80. Maybe we can instead > persuade it into fetching from a different http server than 10.0.2.2. To do that > we'd need to implement a new slirp option for either "option tftp-server-name" > or "next-server" according to the manpage. I'll have to experiment with it to > see if it will work. > > Or, do you think there are better options to configure the VM network to serve > the file? Why not use guestfwd eg guestfwd:tcp:10.0.2.1:80-tcp:127.0.0.1:8080 would redirect the guest IP address port 80, to the loopback interface on the host port 8080. Now you can tell autoinstall to use 10.0.2.1 ? Regards, Daniel
On Fri, 08/24 10:06, Peter Maydell wrote: > On 24 August 2018 at 09:52, Fam Zheng <famz@redhat.com> wrote: > > On Fri, 08/24 09:22, Daniel P. Berrangé wrote: > >> On Fri, Aug 24, 2018 at 09:21:26AM +0800, Fam Zheng wrote: > >> > Upgrade OpenBSD to 6.3 using auto_install. Especially, drop SDL1, > >> > include SDL2. > >> > > >> > One limitation of this patch is that we need a temporary HTTP server on > >> > host 80 port for auto_install, because slirp cannot do guest forward on > >> > "host addr". > >> > >> That's a pretty big limitation, as port 80 requires root privileges, > >> and that's a pretty strict no for automated testing IMHO. > >> > >> Why does it have to be port 80 in particular - is it not possible > >> to use 8080 or even better, detect a random free port ? > > > > OpenBSD autoinstall[1] is hardcoded to fetch from port 80. Maybe we can instead > > persuade it into fetching from a different http server than 10.0.2.2. To do that > > we'd need to implement a new slirp option for either "option tftp-server-name" > > or "next-server" according to the manpage. I'll have to experiment with it to > > see if it will work. > > Can you use the usermode networking "guestfwd" option to make guest > tcp connections to wherever:80 go to host:8080 or something instead? > (watch out for https://bugs.launchpad.net/qemu/+bug/1628971 if it > is still relevant, I guess). Yes, that is what I mean above. Will post v2. Fam
I very much appreciate the effort to bump up to 6.3 as I was going
to suggest doing that at some point. But bumping up to 6.3 at the
moment will fail with the configure script. We've switched from GCC 4.2
to Clang. The TLS check will fail with Clang's emulated TLS. We've
had a local patch for awhile to fix the test but I don't think it is
appropriate to upstream as is.
Index: configure
--- configure.orig
+++ configure
@@ -1876,7 +1876,7 @@ static __thread int tls_var;
int main(void) { return tls_var; }
EOF
-if ! compile_prog "-Werror" "" ; then
+if ! compile_prog "-Werror" "-pthread" ; then
error_exit "Your compiler does not support the __thread specifier for " \
"Thread-Local Storage (TLS). Please upgrade to a version that does."
fi
On 8/23/2018 9:21 PM, Fam Zheng wrote:
> Upgrade OpenBSD to 6.3 using auto_install. Especially, drop SDL1,
> include SDL2.
>
> One limitation of this patch is that we need a temporary HTTP server on
> host 80 port for auto_install, because slirp cannot do guest forward on
> "host addr".
On Fri, Aug 24, 2018 at 10:36:30AM -0400, Brad Smith wrote: > I very much appreciate the effort to bump up to 6.3 as I was going > to suggest doing that at some point. But bumping up to 6.3 at the > moment will fail with the configure script. We've switched from GCC 4.2 > to Clang. The TLS check will fail with Clang's emulated TLS. We've > had a local patch for awhile to fix the test but I don't think it is > appropriate to upstream as is. > > Index: configure > --- configure.orig > +++ configure > @@ -1876,7 +1876,7 @@ static __thread int tls_var; > int main(void) { return tls_var; } > EOF > -if ! compile_prog "-Werror" "" ; then > +if ! compile_prog "-Werror" "-pthread" ; then > error_exit "Your compiler does not support the __thread specifier for " \ > "Thread-Local Storage (TLS). Please upgrade to a version that does." > fi Later on in the configure script there's a check for pthreads that sets $PTHREAD_LIB to the desired arg. Best is probably to move that check higher up, and then use $PTHREAD_LIB as the argument to the compile_prog call you show. Regards, Daniel
On 24 August 2018 at 15:43, Daniel P. Berrangé <berrange@redhat.com> wrote: > On Fri, Aug 24, 2018 at 10:36:30AM -0400, Brad Smith wrote: >> I very much appreciate the effort to bump up to 6.3 as I was going >> to suggest doing that at some point. But bumping up to 6.3 at the >> moment will fail with the configure script. We've switched from GCC 4.2 >> to Clang. The TLS check will fail with Clang's emulated TLS. We've >> had a local patch for awhile to fix the test but I don't think it is >> appropriate to upstream as is. >> >> Index: configure >> --- configure.orig >> +++ configure >> @@ -1876,7 +1876,7 @@ static __thread int tls_var; >> int main(void) { return tls_var; } >> EOF >> -if ! compile_prog "-Werror" "" ; then >> +if ! compile_prog "-Werror" "-pthread" ; then >> error_exit "Your compiler does not support the __thread specifier for " \ >> "Thread-Local Storage (TLS). Please upgrade to a version that does." >> fi > > Later on in the configure script there's a check for pthreads that > sets $PTHREAD_LIB to the desired arg. > > Best is probably to move that check higher up, and then use $PTHREAD_LIB > as the argument to the compile_prog call you show. Is emulated TLS a sufficiently complete/performant TLS implementation for our purposes? (I think it would be better if OpenBSD just implemented real TLS like every other host OS we support.) thanks -- PMM
On Fri, Aug 24, 2018 at 03:46:31PM +0100, Peter Maydell wrote: > On 24 August 2018 at 15:43, Daniel P. Berrangé <berrange@redhat.com> wrote: > > On Fri, Aug 24, 2018 at 10:36:30AM -0400, Brad Smith wrote: > >> I very much appreciate the effort to bump up to 6.3 as I was going > >> to suggest doing that at some point. But bumping up to 6.3 at the > >> moment will fail with the configure script. We've switched from GCC 4.2 > >> to Clang. The TLS check will fail with Clang's emulated TLS. We've > >> had a local patch for awhile to fix the test but I don't think it is > >> appropriate to upstream as is. > >> > >> Index: configure > >> --- configure.orig > >> +++ configure > >> @@ -1876,7 +1876,7 @@ static __thread int tls_var; > >> int main(void) { return tls_var; } > >> EOF > >> -if ! compile_prog "-Werror" "" ; then > >> +if ! compile_prog "-Werror" "-pthread" ; then > >> error_exit "Your compiler does not support the __thread specifier for " \ > >> "Thread-Local Storage (TLS). Please upgrade to a version that does." > >> fi > > > > Later on in the configure script there's a check for pthreads that > > sets $PTHREAD_LIB to the desired arg. > > > > Best is probably to move that check higher up, and then use $PTHREAD_LIB > > as the argument to the compile_prog call you show. > > Is emulated TLS a sufficiently complete/performant TLS > implementation for our purposes? > > (I think it would be better if OpenBSD just implemented real TLS > like every other host OS we support.) My reading of the situation is that OpenBSD *does* support real TLS, but our configure script failed to detect that it supported it, due to missing "-pthread" arg. Regards, Daniel
On 24 August 2018 at 15:48, Daniel P. Berrangé <berrange@redhat.com> wrote: > On Fri, Aug 24, 2018 at 03:46:31PM +0100, Peter Maydell wrote: >> Is emulated TLS a sufficiently complete/performant TLS >> implementation for our purposes? >> >> (I think it would be better if OpenBSD just implemented real TLS >> like every other host OS we support.) > > My reading of the situation is that OpenBSD *does* support real TLS, > but our configure script failed to detect that it supported it, > due to missing "-pthread" arg. Real TLS doesn't require that you have to call out to pthread_* functions or link against a library to get it. It's directly implemented by the C compiler/linker/runtime. thanks -- PMM
On 8/24/2018 10:46 AM, Peter Maydell wrote: > On 24 August 2018 at 15:43, Daniel P. Berrangé <berrange@redhat.com> wrote: >> On Fri, Aug 24, 2018 at 10:36:30AM -0400, Brad Smith wrote: >>> I very much appreciate the effort to bump up to 6.3 as I was going >>> to suggest doing that at some point. But bumping up to 6.3 at the >>> moment will fail with the configure script. We've switched from GCC 4.2 >>> to Clang. The TLS check will fail with Clang's emulated TLS. We've >>> had a local patch for awhile to fix the test but I don't think it is >>> appropriate to upstream as is. >>> >>> Index: configure >>> --- configure.orig >>> +++ configure >>> @@ -1876,7 +1876,7 @@ static __thread int tls_var; >>> int main(void) { return tls_var; } >>> EOF >>> -if ! compile_prog "-Werror" "" ; then >>> +if ! compile_prog "-Werror" "-pthread" ; then >>> error_exit "Your compiler does not support the __thread specifier for " \ >>> "Thread-Local Storage (TLS). Please upgrade to a version that does." >>> fi >> Later on in the configure script there's a check for pthreads that >> sets $PTHREAD_LIB to the desired arg. >> >> Best is probably to move that check higher up, and then use $PTHREAD_LIB >> as the argument to the compile_prog call you show. > Is emulated TLS a sufficiently complete/performant TLS > implementation for our purposes? I won't make any claims for performance but it's been used for 2 1/2 years now with QEMU with GCC 4.9 and recently with Clang. The various libraries/apps and so forth that we had issues in the past with TLS all seem to work Ok. It's used on Android too. Both newer versions of GCC and Clang implement this. > (I think it would be better if OpenBSD just implemented real TLS > like every other host OS we support.) There is very much a desire to do so and it will be completed at some point. I know some work has gone into our threads library and linker towards implementing TLS but it's hasn't been completed yet. I have no idea how long it will be or what is currently holding things up but I know it won't be anytime soon.
On Fri, Aug 24, 2018 at 10:36:30AM -0400, Brad Smith wrote: > I very much appreciate the effort to bump up to 6.3 as I was going > to suggest doing that at some point. But bumping up to 6.3 at the > moment will fail with the configure script. We've switched from GCC 4.2 > to Clang. The TLS check will fail with Clang's emulated TLS. We've > had a local patch for awhile to fix the test but I don't think it is > appropriate to upstream as is. Hmm, /me looks a bit surprised. Seems to work fine for me ... openbsd kraxel ~/build/qemu# uname -a OpenBSD openbsd.sirius.kraxel.org 6.3 GENERIC.MP#107 amd64 openbsd kraxel ~/build/qemu# cc --version OpenBSD clang version 5.0.1 (tags/RELEASE_501/final) (based on LLVM 5.0.1) Target: amd64-unknown-openbsd6.3 Thread model: posix InstalledDir: /usr/bin cheers, Gerd
On Fri, 08/24 10:36, Brad Smith wrote: > I very much appreciate the effort to bump up to 6.3 as I was going > to suggest doing that at some point. But bumping up to 6.3 at the > moment will fail with the configure script. We've switched from GCC 4.2 > to Clang. The TLS check will fail with Clang's emulated TLS. We've > had a local patch for awhile to fix the test but I don't think it is > appropriate to upstream as is. > > Index: configure > --- configure.orig > +++ configure > @@ -1876,7 +1876,7 @@ static __thread int tls_var; > int main(void) { return tls_var; } > EOF > -if ! compile_prog "-Werror" "" ; then > +if ! compile_prog "-Werror" "-pthread" ; then > error_exit "Your compiler does not support the __thread specifier for " \ > "Thread-Local Storage (TLS). Please upgrade to a version that does." > fi Are you suggesting we use 6.2 for now? Fam > > On 8/23/2018 9:21 PM, Fam Zheng wrote: > > Upgrade OpenBSD to 6.3 using auto_install. Especially, drop SDL1, > > include SDL2. > > > > One limitation of this patch is that we need a temporary HTTP server on > > host 80 port for auto_install, because slirp cannot do guest forward on > > "host addr".
On 8/27/2018 5:18 AM, Fam Zheng wrote: > On Fri, 08/24 10:36, Brad Smith wrote: >> I very much appreciate the effort to bump up to 6.3 as I was going >> to suggest doing that at some point. But bumping up to 6.3 at the >> moment will fail with the configure script. We've switched from GCC 4.2 >> to Clang. The TLS check will fail with Clang's emulated TLS. We've >> had a local patch for awhile to fix the test but I don't think it is >> appropriate to upstream as is. >> >> Index: configure >> --- configure.orig >> +++ configure >> @@ -1876,7 +1876,7 @@ static __thread int tls_var; >> int main(void) { return tls_var; } >> EOF >> -if ! compile_prog "-Werror" "" ; then >> +if ! compile_prog "-Werror" "-pthread" ; then >> error_exit "Your compiler does not support the __thread specifier for " \ >> "Thread-Local Storage (TLS). Please upgrade to a version that does." >> fi > Are you suggesting we use 6.2 for now? Until the configure script has been fixed you will have to stick with 6.1 for now.
On 8/27/2018 5:05 AM, Gerd Hoffmann wrote: > On Fri, Aug 24, 2018 at 10:36:30AM -0400, Brad Smith wrote: >> I very much appreciate the effort to bump up to 6.3 as I was going >> to suggest doing that at some point. But bumping up to 6.3 at the >> moment will fail with the configure script. We've switched from GCC 4.2 >> to Clang. The TLS check will fail with Clang's emulated TLS. We've >> had a local patch for awhile to fix the test but I don't think it is >> appropriate to upstream as is. > Hmm, /me looks a bit surprised. > Seems to work fine for me ... I am quite surprised as well. Something has changed since we first brought in Clang and this patch no longer seems to be necessary. I just removed it from our ports tree.
On 8/27/2018 5:18 AM, Fam Zheng wrote: > On Fri, 08/24 10:36, Brad Smith wrote: >> I very much appreciate the effort to bump up to 6.3 as I was going >> to suggest doing that at some point. But bumping up to 6.3 at the >> moment will fail with the configure script. We've switched from GCC 4.2 >> to Clang. The TLS check will fail with Clang's emulated TLS. We've >> had a local patch for awhile to fix the test but I don't think it is >> appropriate to upstream as is. >> >> Index: configure >> --- configure.orig >> +++ configure >> @@ -1876,7 +1876,7 @@ static __thread int tls_var; >> int main(void) { return tls_var; } >> EOF >> -if ! compile_prog "-Werror" "" ; then >> +if ! compile_prog "-Werror" "-pthread" ; then >> error_exit "Your compiler does not support the __thread specifier for " \ >> "Thread-Local Storage (TLS). Please upgrade to a version that does." >> fi > Are you suggesting we use 6.2 for now? > > Fam Disregard what I said earlier. It seems something has changed along the way since we first brought in Clang and the patch is no longer necessary. Please go ahead with 6.3 as is.
On Wed, Sep 05, 2018 at 08:54:39AM -0400, Brad Smith wrote: > On 8/27/2018 5:05 AM, Gerd Hoffmann wrote: > > > On Fri, Aug 24, 2018 at 10:36:30AM -0400, Brad Smith wrote: > > > I very much appreciate the effort to bump up to 6.3 as I was going > > > to suggest doing that at some point. But bumping up to 6.3 at the > > > moment will fail with the configure script. We've switched from GCC 4.2 > > > to Clang. The TLS check will fail with Clang's emulated TLS. We've > > > had a local patch for awhile to fix the test but I don't think it is > > > appropriate to upstream as is. > > Hmm, /me looks a bit surprised. > > Seems to work fine for me ... > > I am quite surprised as well. Something has changed since we first brought > in Clang > and this patch no longer seems to be necessary. I just removed it from our > ports tree. Cool. Fam, what is the status here? There was a slirp patch on the list, and I guess you also have a updated auto-install patch using that? cheers, Gerd
On Wed, 09/05 16:24, Gerd Hoffmann wrote: > On Wed, Sep 05, 2018 at 08:54:39AM -0400, Brad Smith wrote: > > On 8/27/2018 5:05 AM, Gerd Hoffmann wrote: > > > > > On Fri, Aug 24, 2018 at 10:36:30AM -0400, Brad Smith wrote: > > > > I very much appreciate the effort to bump up to 6.3 as I was going > > > > to suggest doing that at some point. But bumping up to 6.3 at the > > > > moment will fail with the configure script. We've switched from GCC 4.2 > > > > to Clang. The TLS check will fail with Clang's emulated TLS. We've > > > > had a local patch for awhile to fix the test but I don't think it is > > > > appropriate to upstream as is. > > > Hmm, /me looks a bit surprised. > > > Seems to work fine for me ... > > > > I am quite surprised as well. Something has changed since we first brought > > in Clang > > and this patch no longer seems to be necessary. I just removed it from our > > ports tree. > > Cool. Fam, what is the status here? There was a slirp patch on the > list, and I guess you also have a updated auto-install patch using that? Yes, it seems pretty close to me. But backing to 6.3 is the news of the day. I guess I can wrap things up tomorrow and look to sending another version. Fam
diff --git a/tests/vm/basevm.py b/tests/vm/basevm.py index d7149dea7d..4080039e66 100755 --- a/tests/vm/basevm.py +++ b/tests/vm/basevm.py @@ -65,8 +65,6 @@ class BaseVM(object): self._stdout = self._devnull self._args = [ \ "-nodefaults", "-m", "4G", - "-netdev", "user,id=vnet,hostfwd=:127.0.0.1:0-:22", - "-device", "virtio-net-pci,netdev=vnet", "-vnc", "127.0.0.1:0,to=20", "-serial", "file:%s" % os.path.join(self._tmpdir, "serial.out")] if vcpus: @@ -145,8 +143,10 @@ class BaseVM(object): "-device", "virtio-blk,drive=%s,serial=%s,bootindex=1" % (name, name)] - def boot(self, img, extra_args=[]): + def boot(self, img, extra_args=[], extra_usernet_args=""): args = self._args + [ + "-netdev", "user,id=vnet,hostfwd=:127.0.0.1:0-:22" + extra_usernet_args, + "-device", "virtio-net-pci,netdev=vnet", "-device", "VGA", "-drive", "file=%s,if=none,id=drive0,cache=writeback" % img, "-device", "virtio-blk,drive=drive0,bootindex=0"] @@ -196,6 +196,28 @@ class BaseVM(object): def qmp(self, *args, **kwargs): return self._guest.qmp(*args, **kwargs) + def start_http_server(self, workdir, ports=range(8010, 8020), sudo=False): + p = None + token = "%d-%d" % (os.getpid(), time.time()) + with open(os.path.join(workdir, token), "w") as f: + f.write("# QEMU VM testing HTTP server token file #") + f.flush() + for port in ports: + cmd = ["python3", "-m", "http.server", str(port)] + if sudo: + cmd = ["sudo", "-n"] + cmd + p = subprocess.Popen(cmd, cwd=workdir) + for retry in range(5): + if p.poll() != None: + break + if subprocess.call("curl http://127.0.0.1:%d/%s &>/dev/null" % \ + (port, token), + shell=True) == 0: + atexit.register(p.terminate) + return port + time.sleep(0.3) + raise IOError("Failed to start HTTP server") + def parse_args(vm_name): parser = optparse.OptionParser( description="VM test utility. Exit codes: " diff --git a/tests/vm/openbsd b/tests/vm/openbsd index 52500ee52b..7e118572a9 100755 --- a/tests/vm/openbsd +++ b/tests/vm/openbsd @@ -14,6 +14,9 @@ import os import sys import subprocess +import time +import atexit +import tempfile import basevm class OpenBSDVM(basevm.BaseVM): @@ -23,22 +26,80 @@ class OpenBSDVM(basevm.BaseVM): rm -rf /var/tmp/qemu-test.* cd $(mktemp -d /var/tmp/qemu-test.XXXXXX); tar -xf /dev/rsd1c; - ./configure --cc=x86_64-unknown-openbsd6.1-gcc-4.9.4 --python=python2.7 {configure_opts}; + ./configure {configure_opts}; gmake --output-sync -j{jobs} {verbose}; # XXX: "gmake check" seems to always hang or fail #gmake --output-sync -j{jobs} check {verbose}; """ + def _install_os(self, img): + tmpdir = tempfile.mkdtemp() + pxeboot = self._download_with_cache("https://fastly.cdn.openbsd.org/pub/OpenBSD/6.3/amd64/pxeboot", + sha256sum="60029919798f48ea40ecb123adfed6217f099d5ed9cd1a6c7de5b544d7b7b0f6") + bsd_rd = self._download_with_cache("https://fastly.cdn.openbsd.org/pub/OpenBSD/6.3/amd64/bsd.rd", + sha256sum="1c0adb43a02ae3aee512bcf0829dac0ccb2e4d614b161049af7ce530e5da2dfc") + install = self._download_with_cache("https://fastly.cdn.openbsd.org/pub/OpenBSD/6.3/amd64/install63.iso", + sha256sum='ee775405dd7926975befbc3fef23de8c4b5a726c3b5075e4848fcd3a2a712ea8') + subprocess.check_call(["qemu-img", "create", img, "32G"]) + subprocess.check_call(["cp", pxeboot, os.path.join(tmpdir, "auto_install")]) + subprocess.check_call(["cp", bsd_rd, os.path.join(tmpdir, "bsd")]) + + self._gen_install_conf(tmpdir) + try: + self.start_http_server(tmpdir, ports=[80], sudo=True) + except Exception: + sys.stdout.write("Cannot open HTTP server on port 80. Maybe use sudo?\n") + sys.exit(1) + # BOOTP filename being auto_install makes sure OpenBSD installer + # not prompt for "auto install mode" + tftp_args = ",tftp=%s,bootfile=/auto_install" % tmpdir + self.boot(img, + extra_args=["-boot", "once=n", "-no-reboot", + "-cdrom", install], + extra_usernet_args=tftp_args) + self.wait() + + def _gen_install_conf(self, tmpdir): + contents = """ +System hostname = qemu-openbsd +Password for root = qemupass +Public ssh key for root = {pub_key} +Allow root ssh login = yes +Network interfaces = vio0 +IPv4 address for vio0 = dhcp +Setup a user = qemu +Password for user = qemupass +Public ssh key for user = {pub_key} +What timezone are you in = US/Eastern +Server = fastly.cdn.openbsd.org +Use http = yes +Default IPv4 route = 10.0.2.2 +Location of sets = cd0 +Set name(s) = all +Continue without verification = yes +""".format(pub_key=basevm.SSH_PUB_KEY) + with open(os.path.join(tmpdir, "install.conf"), "w") as f: + f.write(contents) + def build_image(self, img): - cimg = self._download_with_cache("http://download.patchew.org/openbsd-6.1-amd64.img.xz", - sha256sum='8c6cedc483e602cfee5e04f0406c64eb99138495e8ca580bc0293bcf0640c1bf') - img_tmp_xz = img + ".tmp.xz" - img_tmp = img + ".tmp" - subprocess.check_call(["cp", "-f", cimg, img_tmp_xz]) - subprocess.check_call(["xz", "-df", img_tmp_xz]) - if os.path.exists(img): - os.remove(img) - os.rename(img_tmp, img) + + self._install_os(img + ".tmp") + + self.boot(img + ".tmp") + self.wait_ssh() + + self.ssh_root("usermod -G operator qemu") + self.ssh_root("echo https://fastly.cdn.openbsd.org/pub/OpenBSD > /etc/installurl") + for pkg in ["git", "gmake", "glib2", "bison", "sdl2"]: + self.ssh_root("pkg_add " + pkg) + self.ssh_root("ln -sf /usr/local/bin/python2.7 /usr/local/bin/python") + self.ssh_root("ln -sf /usr/local/bin/python2.7-2to3 /usr/local/bin/2to3") + self.ssh_root("ln -sf /usr/local/bin/python2.7-config /usr/local/bin/python-config") + self.ssh_root("ln -sf /usr/local/bin/pydoc2.7 /usr/local/bin/pydoc") + self.ssh_root("shutdown -p now") + self.wait() + + subprocess.check_call(["mv", img + ".tmp", img]) if __name__ == "__main__": sys.exit(basevm.main(OpenBSDVM))
Upgrade OpenBSD to 6.3 using auto_install. Especially, drop SDL1, include SDL2. One limitation of this patch is that we need a temporary HTTP server on host 80 port for auto_install, because slirp cannot do guest forward on "host addr". Signed-off-by: Fam Zheng <famz@redhat.com> --- tests/vm/basevm.py | 28 ++++++++++++++-- tests/vm/openbsd | 81 ++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 96 insertions(+), 13 deletions(-)