[3/7] scripts/nsis.py: Automatically package required DLLs of QEMU executables

Commit Message

Bin Meng Sept. 8, 2022, 1:28 p.m. UTC

From: Bin Meng <[e-mail]>

At present packaging the required DLLs of QEMU executables is a
manual process, and error prone.

Actually build/config-host.mak contains a GLIB_BINDIR variable
which is the directory where glib and other DLLs reside. This
works for both Windows native build and cross-build on Linux.
We can use it as the search directory for DLLs and automate
the whole DLL packaging process.

Signed-off-by: Bin Meng <[e-mail]>
---

 meson.build     |  1 +
 scripts/nsis.py | 46 ++++++++++++++++++++++++++++++++++++++++++----
 2 files changed, 43 insertions(+), 4 deletions(-)

Comments

Marc-André Lureau Sept. 8, 2022, 1:56 p.m. UTC | #1

Hi

(adding Stefan Weil in CC, who currently provides Windows installer)

On Thu, Sep 8, 2022 at 5:34 PM Bin Meng <[e-mail]> wrote:

> From: Bin Meng <[e-mail]>
>
> At present packaging the required DLLs of QEMU executables is a
> manual process, and error prone.
>
> Actually build/config-host.mak contains a GLIB_BINDIR variable
> which is the directory where glib and other DLLs reside. This
> works for both Windows native build and cross-build on Linux.
> We can use it as the search directory for DLLs and automate
> the whole DLL packaging process.
>
> Signed-off-by: Bin Meng <[e-mail]>
>

That seems reasonable to me, although packaging dependencies is not just
about linked DLLs.. There are dynamic stuff, executables, data, legal docs
etc etc. I have no clear picture how is everything really packaged in the
installer tbh (I would recommend msys2 qemu installation at this point)

anyhow, for the patch, as far as I am concerned:
Reviewed-by: Marc-André Lureau <[e-mail]>


---
>
>  meson.build     |  1 +
>  scripts/nsis.py | 46 ++++++++++++++++++++++++++++++++++++++++++----
>  2 files changed, 43 insertions(+), 4 deletions(-)
>
> diff --git a/meson.build b/meson.build
> index c2adb7caf4..4c03850f9f 100644
> --- a/meson.build
> +++ b/meson.build
> @@ -3657,6 +3657,7 @@ if host_machine.system() == 'windows'
>      '@OUTPUT@',
>      get_option('prefix'),
>      meson.current_source_dir(),
> +    config_host['GLIB_BINDIR'],
>      host_machine.cpu(),
>      '--',
>      '-DDISPLAYVERSION=' + meson.project_version(),
> diff --git a/scripts/nsis.py b/scripts/nsis.py
> index baa6ef9594..03ed7608a2 100644
> --- a/scripts/nsis.py
> +++ b/scripts/nsis.py
> @@ -18,12 +18,36 @@ def signcode(path):
>          return
>      subprocess.run([cmd, path])
>
> +def find_deps(exe_or_dll, search_path, analyzed_deps):
> +    deps = [exe_or_dll]
> +    output = subprocess.check_output(["objdump", "-p", exe_or_dll],
> text=True)
> +    output = output.split("\n")
> +    for line in output:
> +        if not line.startswith("\tDLL Name: "):
> +            continue
> +
> +        dep = line.split("DLL Name: ")[1].strip()
> +        if dep in analyzed_deps:
> +            continue
> +
> +        dll = os.path.join(search_path, dep)
> +        if not os.path.exists(dll):
> +            # assume it's a Windows provided dll, skip it
> +            continue
> +
> +        analyzed_deps.add(dep)
> +        # locate the dll dependencies recursively
> +        rdeps = find_deps(dll, search_path, analyzed_deps)
> +        deps.extend(rdeps)
> +
> +    return deps
>
>  def main():
>      parser = argparse.ArgumentParser(description="QEMU NSIS build
> helper.")
>      parser.add_argument("outfile")
>      parser.add_argument("prefix")
>      parser.add_argument("srcdir")
> +    parser.add_argument("dlldir")
>      parser.add_argument("cpu")
>      parser.add_argument("nsisargs", nargs="*")
>      args = parser.parse_args()
> @@ -63,9 +87,26 @@ def main():
>                  !insertmacro MUI_DESCRIPTION_TEXT ${{Section_{0}}} "{1}"
>                  """.format(arch, desc))
>
> +        search_path = args.dlldir
> +        print("Searching '%s' for the dependent dlls ..." % search_path)
> +        dlldir = os.path.join(destdir + prefix, "dll")
> +        os.mkdir(dlldir)
> +
>          for exe in glob.glob(os.path.join(destdir + prefix, "*.exe")):
>              signcode(exe)
>
> +            # find all dll dependencies
> +            deps = set(find_deps(exe, search_path, set()))
> +            deps.remove(exe)
> +
> +            # copy all dlls to the DLLDIR
> +            for dep in deps:
> +                dllfile = os.path.join(dlldir, os.path.basename(dep))
> +                if (os.path.exists(dllfile)):
> +                    continue
> +                print("Copying '%s' to '%s'" % (dep, dllfile))
> +                shutil.copy(dep, dllfile)
> +
>          makensis = [
>              "makensis",
>              "-V2",
> @@ -73,12 +114,9 @@ def main():
>              "-DSRCDIR=" + args.srcdir,
>              "-DBINDIR=" + destdir + prefix,
>          ]
> -        dlldir = "w32"
>          if args.cpu == "x86_64":
> -            dlldir = "w64"
>              makensis += ["-DW64"]
> -        if os.path.exists(os.path.join(args.srcdir, "dll")):
> -            makensis += ["-DDLLDIR={0}/dll/{1}".format(args.srcdir,
> dlldir)]
> +        makensis += ["-DDLLDIR=" + dlldir]
>
>          makensis += ["-DOUTFILE=" + args.outfile] + args.nsisargs
>          subprocess.run(makensis)
> --
> 2.34.1
>
>
>

Mark Cave-Ayland Sept. 9, 2022, 4:49 p.m. UTC | #2

On 08/09/2022 14:28, Bin Meng wrote:

> From: Bin Meng <[e-mail]>
> 
> At present packaging the required DLLs of QEMU executables is a
> manual process, and error prone.
> 
> Actually build/config-host.mak contains a GLIB_BINDIR variable
> which is the directory where glib and other DLLs reside. This
> works for both Windows native build and cross-build on Linux.
> We can use it as the search directory for DLLs and automate
> the whole DLL packaging process.
> 
> Signed-off-by: Bin Meng <[e-mail]>
> ---
> 
>   meson.build     |  1 +
>   scripts/nsis.py | 46 ++++++++++++++++++++++++++++++++++++++++++----
>   2 files changed, 43 insertions(+), 4 deletions(-)
> 
> diff --git a/meson.build b/meson.build
> index c2adb7caf4..4c03850f9f 100644
> --- a/meson.build
> +++ b/meson.build
> @@ -3657,6 +3657,7 @@ if host_machine.system() == 'windows'
>       '@OUTPUT@',
>       get_option('prefix'),
>       meson.current_source_dir(),
> +    config_host['GLIB_BINDIR'],
>       host_machine.cpu(),
>       '--',
>       '-DDISPLAYVERSION=' + meson.project_version(),
> diff --git a/scripts/nsis.py b/scripts/nsis.py
> index baa6ef9594..03ed7608a2 100644
> --- a/scripts/nsis.py
> +++ b/scripts/nsis.py
> @@ -18,12 +18,36 @@ def signcode(path):
>           return
>       subprocess.run([cmd, path])
>   
> +def find_deps(exe_or_dll, search_path, analyzed_deps):
> +    deps = [exe_or_dll]
> +    output = subprocess.check_output(["objdump", "-p", exe_or_dll], text=True)
> +    output = output.split("\n")
> +    for line in output:
> +        if not line.startswith("\tDLL Name: "):
> +            continue
> +
> +        dep = line.split("DLL Name: ")[1].strip()
> +        if dep in analyzed_deps:
> +            continue
> +
> +        dll = os.path.join(search_path, dep)
> +        if not os.path.exists(dll):
> +            # assume it's a Windows provided dll, skip it
> +            continue
> +
> +        analyzed_deps.add(dep)
> +        # locate the dll dependencies recursively
> +        rdeps = find_deps(dll, search_path, analyzed_deps)
> +        deps.extend(rdeps)
> +
> +    return deps
>   
>   def main():
>       parser = argparse.ArgumentParser(description="QEMU NSIS build helper.")
>       parser.add_argument("outfile")
>       parser.add_argument("prefix")
>       parser.add_argument("srcdir")
> +    parser.add_argument("dlldir")
>       parser.add_argument("cpu")
>       parser.add_argument("nsisargs", nargs="*")
>       args = parser.parse_args()
> @@ -63,9 +87,26 @@ def main():
>                   !insertmacro MUI_DESCRIPTION_TEXT ${{Section_{0}}} "{1}"
>                   """.format(arch, desc))
>   
> +        search_path = args.dlldir
> +        print("Searching '%s' for the dependent dlls ..." % search_path)
> +        dlldir = os.path.join(destdir + prefix, "dll")
> +        os.mkdir(dlldir)
> +
>           for exe in glob.glob(os.path.join(destdir + prefix, "*.exe")):
>               signcode(exe)
>   
> +            # find all dll dependencies
> +            deps = set(find_deps(exe, search_path, set()))
> +            deps.remove(exe)
> +
> +            # copy all dlls to the DLLDIR
> +            for dep in deps:
> +                dllfile = os.path.join(dlldir, os.path.basename(dep))
> +                if (os.path.exists(dllfile)):
> +                    continue
> +                print("Copying '%s' to '%s'" % (dep, dllfile))
> +                shutil.copy(dep, dllfile)
> +
>           makensis = [
>               "makensis",
>               "-V2",
> @@ -73,12 +114,9 @@ def main():
>               "-DSRCDIR=" + args.srcdir,
>               "-DBINDIR=" + destdir + prefix,
>           ]
> -        dlldir = "w32"
>           if args.cpu == "x86_64":
> -            dlldir = "w64"
>               makensis += ["-DW64"]
> -        if os.path.exists(os.path.join(args.srcdir, "dll")):
> -            makensis += ["-DDLLDIR={0}/dll/{1}".format(args.srcdir, dlldir)]
> +        makensis += ["-DDLLDIR=" + dlldir]
>   
>           makensis += ["-DOUTFILE=" + args.outfile] + args.nsisargs
>           subprocess.run(makensis)

FWIW I wrote a similar script a while back to help package a custom Windows build for 
a client, however I used ldd instead of objdump since it provided the full paths for 
DLLs installed in the msys2/mingw-w64 environment via pacman which were outside the 
QEMU build tree.

Once the complete list of DLLs was obtained, it was simple matter of filtering out 
those DLLs that started with the %WINDIR% prefix before copying them to the final 
distribution directory.


ATB,

Mark.

Bin Meng Sept. 10, 2022, 12:37 a.m. UTC | #3

On Sat, Sep 10, 2022 at 12:49 AM Mark Cave-Ayland
<[e-mail]> wrote:
>
> On 08/09/2022 14:28, Bin Meng wrote:
>
> > From: Bin Meng <[e-mail]>
> >
> > At present packaging the required DLLs of QEMU executables is a
> > manual process, and error prone.
> >
> > Actually build/config-host.mak contains a GLIB_BINDIR variable
> > which is the directory where glib and other DLLs reside. This
> > works for both Windows native build and cross-build on Linux.
> > We can use it as the search directory for DLLs and automate
> > the whole DLL packaging process.
> >
> > Signed-off-by: Bin Meng <[e-mail]>
> > ---
> >
> >   meson.build     |  1 +
> >   scripts/nsis.py | 46 ++++++++++++++++++++++++++++++++++++++++++----
> >   2 files changed, 43 insertions(+), 4 deletions(-)
> >
> > diff --git a/meson.build b/meson.build
> > index c2adb7caf4..4c03850f9f 100644
> > --- a/meson.build
> > +++ b/meson.build
> > @@ -3657,6 +3657,7 @@ if host_machine.system() == 'windows'
> >       '@OUTPUT@',
> >       get_option('prefix'),
> >       meson.current_source_dir(),
> > +    config_host['GLIB_BINDIR'],
> >       host_machine.cpu(),
> >       '--',
> >       '-DDISPLAYVERSION=' + meson.project_version(),
> > diff --git a/scripts/nsis.py b/scripts/nsis.py
> > index baa6ef9594..03ed7608a2 100644
> > --- a/scripts/nsis.py
> > +++ b/scripts/nsis.py
> > @@ -18,12 +18,36 @@ def signcode(path):
> >           return
> >       subprocess.run([cmd, path])
> >
> > +def find_deps(exe_or_dll, search_path, analyzed_deps):
> > +    deps = [exe_or_dll]
> > +    output = subprocess.check_output(["objdump", "-p", exe_or_dll], text=True)
> > +    output = output.split("\n")
> > +    for line in output:
> > +        if not line.startswith("\tDLL Name: "):
> > +            continue
> > +
> > +        dep = line.split("DLL Name: ")[1].strip()
> > +        if dep in analyzed_deps:
> > +            continue
> > +
> > +        dll = os.path.join(search_path, dep)
> > +        if not os.path.exists(dll):
> > +            # assume it's a Windows provided dll, skip it
> > +            continue
> > +
> > +        analyzed_deps.add(dep)
> > +        # locate the dll dependencies recursively
> > +        rdeps = find_deps(dll, search_path, analyzed_deps)
> > +        deps.extend(rdeps)
> > +
> > +    return deps
> >
> >   def main():
> >       parser = argparse.ArgumentParser(description="QEMU NSIS build helper.")
> >       parser.add_argument("outfile")
> >       parser.add_argument("prefix")
> >       parser.add_argument("srcdir")
> > +    parser.add_argument("dlldir")
> >       parser.add_argument("cpu")
> >       parser.add_argument("nsisargs", nargs="*")
> >       args = parser.parse_args()
> > @@ -63,9 +87,26 @@ def main():
> >                   !insertmacro MUI_DESCRIPTION_TEXT ${{Section_{0}}} "{1}"
> >                   """.format(arch, desc))
> >
> > +        search_path = args.dlldir
> > +        print("Searching '%s' for the dependent dlls ..." % search_path)
> > +        dlldir = os.path.join(destdir + prefix, "dll")
> > +        os.mkdir(dlldir)
> > +
> >           for exe in glob.glob(os.path.join(destdir + prefix, "*.exe")):
> >               signcode(exe)
> >
> > +            # find all dll dependencies
> > +            deps = set(find_deps(exe, search_path, set()))
> > +            deps.remove(exe)
> > +
> > +            # copy all dlls to the DLLDIR
> > +            for dep in deps:
> > +                dllfile = os.path.join(dlldir, os.path.basename(dep))
> > +                if (os.path.exists(dllfile)):
> > +                    continue
> > +                print("Copying '%s' to '%s'" % (dep, dllfile))
> > +                shutil.copy(dep, dllfile)
> > +
> >           makensis = [
> >               "makensis",
> >               "-V2",
> > @@ -73,12 +114,9 @@ def main():
> >               "-DSRCDIR=" + args.srcdir,
> >               "-DBINDIR=" + destdir + prefix,
> >           ]
> > -        dlldir = "w32"
> >           if args.cpu == "x86_64":
> > -            dlldir = "w64"
> >               makensis += ["-DW64"]
> > -        if os.path.exists(os.path.join(args.srcdir, "dll")):
> > -            makensis += ["-DDLLDIR={0}/dll/{1}".format(args.srcdir, dlldir)]
> > +        makensis += ["-DDLLDIR=" + dlldir]
> >
> >           makensis += ["-DOUTFILE=" + args.outfile] + args.nsisargs
> >           subprocess.run(makensis)
>
> FWIW I wrote a similar script a while back to help package a custom Windows build for
> a client, however I used ldd instead of objdump since it provided the full paths for
> DLLs installed in the msys2/mingw-w64 environment via pacman which were outside the
> QEMU build tree.
>

Yep, ldd also works, but only on Windows native build. objdump can
work on both Windows native and Linux cross builds.

> Once the complete list of DLLs was obtained, it was simple matter of filtering out
> those DLLs that started with the %WINDIR% prefix before copying them to the final
> distribution directory.
>

Regards,
Bin

Stefan Weil Oct. 29, 2022, 9:04 a.m. UTC | #4

Am 08.09.22 um 15:28 schrieb Bin Meng:
> From: Bin Meng <[e-mail]>
> 
> At present packaging the required DLLs of QEMU executables is a
> manual process, and error prone.
> 
> Actually build/config-host.mak contains a GLIB_BINDIR variable
> which is the directory where glib and other DLLs reside. This
> works for both Windows native build and cross-build on Linux.
> We can use it as the search directory for DLLs and automate
> the whole DLL packaging process.
> 
> Signed-off-by: Bin Meng <[e-mail]>
> ---


Thank you, that is really helpful to avoid broken installers because of 
missing DLL files.

Tested-by: Stefan Weil <[e-mail]>

Stefan Weil Feb. 25, 2024, 5:37 p.m. UTC | #5

Am 10.09.22 um 02:37 schrieb Bin Meng:
> On Sat, Sep 10, 2022 at 12:49 AM Mark Cave-Ayland
> <[e-mail]> wrote:
>>
>> On 08/09/2022 14:28, Bin Meng wrote:
>>
>>> From: Bin Meng <[e-mail]>
>>>
>>> At present packaging the required DLLs of QEMU executables is a
>>> manual process, and error prone.
>>>
>>> Actually build/config-host.mak contains a GLIB_BINDIR variable
>>> which is the directory where glib and other DLLs reside. This
>>> works for both Windows native build and cross-build on Linux.
>>> We can use it as the search directory for DLLs and automate
>>> the whole DLL packaging process.
>>>
>>> Signed-off-by: Bin Meng <[e-mail]>
>>> ---
>>>
>>>    meson.build     |  1 +
>>>    scripts/nsis.py | 46 ++++++++++++++++++++++++++++++++++++++++++----
>>>    2 files changed, 43 insertions(+), 4 deletions(-)
>>>
[...]>>> diff --git a/scripts/nsis.py b/scripts/nsis.py
>>> index baa6ef9594..03ed7608a2 100644
>>> --- a/scripts/nsis.py
>>> +++ b/scripts/nsis.py
>>> @@ -18,12 +18,36 @@ def signcode(path):
>>>            return
>>>        subprocess.run([cmd, path])
>>>
>>> +def find_deps(exe_or_dll, search_path, analyzed_deps):
>>> +    deps = [exe_or_dll]
>>> +    output = subprocess.check_output(["objdump", "-p", exe_or_dll], text=True)

This fails on non x86 hosts where objdump does not know how to handle a 
Windows x86_64 exe file.

>>> +    output = output.split("\n")
>>> +    for line in output:
>>> +        if not line.startswith("\tDLL Name: "):
>>> +            continue
>>> +
>>> +        dep = line.split("DLL Name: ")[1].strip()
>>> +        if dep in analyzed_deps:
>>> +            continue
>>> +
>>> +        dll = os.path.join(search_path, dep)
>>> +        if not os.path.exists(dll):
>>> +            # assume it's a Windows provided dll, skip it
>>> +            continue
>>> +
>>> +        analyzed_deps.add(dep)
>>> +        # locate the dll dependencies recursively
>>> +        rdeps = find_deps(dll, search_path, analyzed_deps)
>>> +        deps.extend(rdeps)
>>> +
>>> +    return deps
[...]
>>
>> FWIW I wrote a similar script a while back to help package a custom Windows build for
>> a client, however I used ldd instead of objdump since it provided the full paths for
>> DLLs installed in the msys2/mingw-w64 environment via pacman which were outside the
>> QEMU build tree.
>>
> 
> Yep, ldd also works, but only on Windows native build. objdump can
> work on both Windows native and Linux cross builds.


objdump fails on Linux cross builds on any non x86 host, because objdump 
typically only supports the native host architecture.

Therefore I get an error on an ARM64 host (podman on Mac M1):

         objdump: /tmp/tmpvae5u0qm/qemu/qemu-system-aarch64.exe: file 
format not recognized

I could use x86_64-w64-mingw32-objdump to fix the cross builds, but then 
native builds might fail because they don't have x86_64-w64-mingw32-objdump.

Is there a simple way how we can get the information here whether 
objdump requires a cross build prefix?

Stefan

Bin Meng Feb. 26, 2024, 4:35 a.m. UTC | #6

On Mon, Feb 26, 2024 at 1:37 AM Stefan Weil <[e-mail]> wrote:
>
> Am 10.09.22 um 02:37 schrieb Bin Meng:
> > On Sat, Sep 10, 2022 at 12:49 AM Mark Cave-Ayland
> > <[e-mail]> wrote:
> >>
> >> On 08/09/2022 14:28, Bin Meng wrote:
> >>
> >>> From: Bin Meng <[e-mail]>
> >>>
> >>> At present packaging the required DLLs of QEMU executables is a
> >>> manual process, and error prone.
> >>>
> >>> Actually build/config-host.mak contains a GLIB_BINDIR variable
> >>> which is the directory where glib and other DLLs reside. This
> >>> works for both Windows native build and cross-build on Linux.
> >>> We can use it as the search directory for DLLs and automate
> >>> the whole DLL packaging process.
> >>>
> >>> Signed-off-by: Bin Meng <[e-mail]>
> >>> ---
> >>>
> >>>    meson.build     |  1 +
> >>>    scripts/nsis.py | 46 ++++++++++++++++++++++++++++++++++++++++++----
> >>>    2 files changed, 43 insertions(+), 4 deletions(-)
> >>>
> [...]>>> diff --git a/scripts/nsis.py b/scripts/nsis.py
> >>> index baa6ef9594..03ed7608a2 100644
> >>> --- a/scripts/nsis.py
> >>> +++ b/scripts/nsis.py
> >>> @@ -18,12 +18,36 @@ def signcode(path):
> >>>            return
> >>>        subprocess.run([cmd, path])
> >>>
> >>> +def find_deps(exe_or_dll, search_path, analyzed_deps):
> >>> +    deps = [exe_or_dll]
> >>> +    output = subprocess.check_output(["objdump", "-p", exe_or_dll], text=True)
>
> This fails on non x86 hosts where objdump does not know how to handle a
> Windows x86_64 exe file.

Does this command work in the MSYS2 environment on Windows Arm?

>
> >>> +    output = output.split("\n")
> >>> +    for line in output:
> >>> +        if not line.startswith("\tDLL Name: "):
> >>> +            continue
> >>> +
> >>> +        dep = line.split("DLL Name: ")[1].strip()
> >>> +        if dep in analyzed_deps:
> >>> +            continue
> >>> +
> >>> +        dll = os.path.join(search_path, dep)
> >>> +        if not os.path.exists(dll):
> >>> +            # assume it's a Windows provided dll, skip it
> >>> +            continue
> >>> +
> >>> +        analyzed_deps.add(dep)
> >>> +        # locate the dll dependencies recursively
> >>> +        rdeps = find_deps(dll, search_path, analyzed_deps)
> >>> +        deps.extend(rdeps)
> >>> +
> >>> +    return deps
> [...]
> >>
> >> FWIW I wrote a similar script a while back to help package a custom Windows build for
> >> a client, however I used ldd instead of objdump since it provided the full paths for
> >> DLLs installed in the msys2/mingw-w64 environment via pacman which were outside the
> >> QEMU build tree.
> >>
> >
> > Yep, ldd also works, but only on Windows native build. objdump can
> > work on both Windows native and Linux cross builds.
>
>
> objdump fails on Linux cross builds on any non x86 host, because objdump
> typically only supports the native host architecture.
>
> Therefore I get an error on an ARM64 host (podman on Mac M1):
>
>          objdump: /tmp/tmpvae5u0qm/qemu/qemu-system-aarch64.exe: file
> format not recognized
>
> I could use x86_64-w64-mingw32-objdump to fix the cross builds, but then
> native builds might fail because they don't have x86_64-w64-mingw32-objdump.
>
> Is there a simple way how we can get the information here whether
> objdump requires a cross build prefix?

For QEMU Windows, I believe the only supported architecture is x86_64,
correct? Do we want to support (cross) building QEMU for Windows Arm?

Based on your observation, objdump on Linux cross builds on any x86
host works, but not on non x86 host.
Maybe it's a hint to ask for binutils guys to include the PE format
support for objdump Arm by default.

Regards,
Bin

Stefan Weil Feb. 26, 2024, 6:30 a.m. UTC | #7

Am 26.02.24 um 05:35 schrieb Bin Meng:

> On Mon, Feb 26, 2024 at 1:37 AM Stefan Weil<[e-mail]>  wrote:
>> Am 10.09.22 um 02:37 schrieb Bin Meng:
>>> On Sat, Sep 10, 2022 at 12:49 AM Mark Cave-Ayland
>>> <[e-mail]>  wrote:
>>>> On 08/09/2022 14:28, Bin Meng wrote:
>>>>
>>>>> From: Bin Meng<[e-mail]>
>>>>>
>>>>> At present packaging the required DLLs of QEMU executables is a
>>>>> manual process, and error prone.
>>>>>
>>>>> Actually build/config-host.mak contains a GLIB_BINDIR variable
>>>>> which is the directory where glib and other DLLs reside. This
>>>>> works for both Windows native build and cross-build on Linux.
>>>>> We can use it as the search directory for DLLs and automate
>>>>> the whole DLL packaging process.
>>>>>
>>>>> Signed-off-by: Bin Meng<[e-mail]>
>>>>> ---
>>>>>
>>>>>     meson.build     |  1 +
>>>>>     scripts/nsis.py | 46 ++++++++++++++++++++++++++++++++++++++++++----
>>>>>     2 files changed, 43 insertions(+), 4 deletions(-)
>>>>>
>> [...]>>> diff --git a/scripts/nsis.py b/scripts/nsis.py
>>>>> index baa6ef9594..03ed7608a2 100644
>>>>> --- a/scripts/nsis.py
>>>>> +++ b/scripts/nsis.py
>>>>> @@ -18,12 +18,36 @@ def signcode(path):
>>>>>             return
>>>>>         subprocess.run([cmd, path])
>>>>>
>>>>> +def find_deps(exe_or_dll, search_path, analyzed_deps):
>>>>> +    deps = [exe_or_dll]
>>>>> +    output = subprocess.check_output(["objdump", "-p", exe_or_dll], text=True)
>> This fails on non x86 hosts where objdump does not know how to handle a
>> Windows x86_64 exe file.
> Does this command work in the MSYS2 environment on Windows Arm?


I don't know and cannot test that, because I don't run Windows on ARM.

>>>>> +    output = output.split("\n")
>>>>> +    for line in output:
>>>>> +        if not line.startswith("\tDLL Name: "):
>>>>> +            continue
>>>>> +
>>>>> +        dep = line.split("DLL Name: ")[1].strip()
>>>>> +        if dep in analyzed_deps:
>>>>> +            continue
>>>>> +
>>>>> +        dll = os.path.join(search_path, dep)
>>>>> +        if not os.path.exists(dll):
>>>>> +            # assume it's a Windows provided dll, skip it
>>>>> +            continue
>>>>> +
>>>>> +        analyzed_deps.add(dep)
>>>>> +        # locate the dll dependencies recursively
>>>>> +        rdeps = find_deps(dll, search_path, analyzed_deps)
>>>>> +        deps.extend(rdeps)
>>>>> +
>>>>> +    return deps
>> [...]
>>>> FWIW I wrote a similar script a while back to help package a custom Windows build for
>>>> a client, however I used ldd instead of objdump since it provided the full paths for
>>>> DLLs installed in the msys2/mingw-w64 environment via pacman which were outside the
>>>> QEMU build tree.
>>>>
>>> Yep, ldd also works, but only on Windows native build. objdump can
>>> work on both Windows native and Linux cross builds.
>>
>> objdump fails on Linux cross builds on any non x86 host, because objdump
>> typically only supports the native host architecture.
>>
>> Therefore I get an error on an ARM64 host (podman on Mac M1):
>>
>>           objdump: /tmp/tmpvae5u0qm/qemu/qemu-system-aarch64.exe: file
>> format not recognized
>>
>> I could use x86_64-w64-mingw32-objdump to fix the cross builds, but then
>> native builds might fail because they don't have x86_64-w64-mingw32-objdump.
>>
>> Is there a simple way how we can get the information here whether
>> objdump requires a cross build prefix?
> For QEMU Windows, I believe the only supported architecture is x86_64,
> correct? Do we want to support (cross) building QEMU for Windows Arm?


Yes, I think we only support QEMU on Windows x86_64. I also don't know 
anyone who has tried building for Windows ARM. And up to now I also was 
never asked for that, so obviously there is still no need for it.


> Based on your observation, objdump on Linux cross builds on any x86
> host works, but not on non x86 host.
> Maybe it's a hint to ask for binutils guys to include the PE format
> support for objdump Arm by default.


I am afraid that we'll have to find a solution on the QEMU side, not 
wait until all binutils support the PE format for x86_64 (which would 
make the binaries or the library much larger).

Stefan

Mark Cave-Ayland March 10, 2024, 8:02 a.m. UTC | #8

On 26/02/2024 06:30, Stefan Weil via wrote:

> Am 26.02.24 um 05:35 schrieb Bin Meng:
> 
>> On Mon, Feb 26, 2024 at 1:37 AM Stefan Weil<[e-mail]>  wrote:
>>> Am 10.09.22 um 02:37 schrieb Bin Meng:
>>>> On Sat, Sep 10, 2022 at 12:49 AM Mark Cave-Ayland
>>>> <[e-mail]>  wrote:
>>>>> On 08/09/2022 14:28, Bin Meng wrote:
>>>>>
>>>>>> From: Bin Meng<[e-mail]>
>>>>>>
>>>>>> At present packaging the required DLLs of QEMU executables is a
>>>>>> manual process, and error prone.
>>>>>>
>>>>>> Actually build/config-host.mak contains a GLIB_BINDIR variable
>>>>>> which is the directory where glib and other DLLs reside. This
>>>>>> works for both Windows native build and cross-build on Linux.
>>>>>> We can use it as the search directory for DLLs and automate
>>>>>> the whole DLL packaging process.
>>>>>>
>>>>>> Signed-off-by: Bin Meng<[e-mail]>
>>>>>> ---
>>>>>>
>>>>>>     meson.build     |  1 +
>>>>>>     scripts/nsis.py | 46 ++++++++++++++++++++++++++++++++++++++++++----
>>>>>>     2 files changed, 43 insertions(+), 4 deletions(-)
>>>>>>
>>> [...]>>> diff --git a/scripts/nsis.py b/scripts/nsis.py
>>>>>> index baa6ef9594..03ed7608a2 100644
>>>>>> --- a/scripts/nsis.py
>>>>>> +++ b/scripts/nsis.py
>>>>>> @@ -18,12 +18,36 @@ def signcode(path):
>>>>>>             return
>>>>>>         subprocess.run([cmd, path])
>>>>>>
>>>>>> +def find_deps(exe_or_dll, search_path, analyzed_deps):
>>>>>> +    deps = [exe_or_dll]
>>>>>> +    output = subprocess.check_output(["objdump", "-p", exe_or_dll], text=True)
>>> This fails on non x86 hosts where objdump does not know how to handle a
>>> Windows x86_64 exe file.
>> Does this command work in the MSYS2 environment on Windows Arm?
> 
> 
> I don't know and cannot test that, because I don't run Windows on ARM.
> 
>>>>>> +    output = output.split("\n")
>>>>>> +    for line in output:
>>>>>> +        if not line.startswith("\tDLL Name: "):
>>>>>> +            continue
>>>>>> +
>>>>>> +        dep = line.split("DLL Name: ")[1].strip()
>>>>>> +        if dep in analyzed_deps:
>>>>>> +            continue
>>>>>> +
>>>>>> +        dll = os.path.join(search_path, dep)
>>>>>> +        if not os.path.exists(dll):
>>>>>> +            # assume it's a Windows provided dll, skip it
>>>>>> +            continue
>>>>>> +
>>>>>> +        analyzed_deps.add(dep)
>>>>>> +        # locate the dll dependencies recursively
>>>>>> +        rdeps = find_deps(dll, search_path, analyzed_deps)
>>>>>> +        deps.extend(rdeps)
>>>>>> +
>>>>>> +    return deps
>>> [...]
>>>>> FWIW I wrote a similar script a while back to help package a custom Windows build for
>>>>> a client, however I used ldd instead of objdump since it provided the full paths for
>>>>> DLLs installed in the msys2/mingw-w64 environment via pacman which were outside the
>>>>> QEMU build tree.
>>>>>
>>>> Yep, ldd also works, but only on Windows native build. objdump can
>>>> work on both Windows native and Linux cross builds.
>>> objdump fails on Linux cross builds on any non x86 host, because objdump
>>> typically only supports the native host architecture.
>>>
>>> Therefore I get an error on an ARM64 host (podman on Mac M1):
>>>
>>>           objdump: /tmp/tmpvae5u0qm/qemu/qemu-system-aarch64.exe: file
>>> format not recognized
>>>
>>> I could use x86_64-w64-mingw32-objdump to fix the cross builds, but then
>>> native builds might fail because they don't have x86_64-w64-mingw32-objdump.
>>>
>>> Is there a simple way how we can get the information here whether
>>> objdump requires a cross build prefix?
>> For QEMU Windows, I believe the only supported architecture is x86_64,
>> correct? Do we want to support (cross) building QEMU for Windows Arm?
> 
> 
> Yes, I think we only support QEMU on Windows x86_64. I also don't know anyone who has 
> tried building for Windows ARM. And up to now I also was never asked for that, so 
> obviously there is still no need for it.
> 
> 
>> Based on your observation, objdump on Linux cross builds on any x86
>> host works, but not on non x86 host.
>> Maybe it's a hint to ask for binutils guys to include the PE format
>> support for objdump Arm by default.
> 
> 
> I am afraid that we'll have to find a solution on the QEMU side, not wait until all 
> binutils support the PE format for x86_64 (which would make the binaries or the 
> library much larger).

Another thought here: now that python is required to build QEMU, is it possible to 
use a python-based PE parser to extract the dependency information for both Windows 
native and cross builds? For example using something like 
https://github.com/erocarrera/pefile?


ATB,

Mark.

Patch

diff --git a/meson.build b/meson.build
index c2adb7caf4..4c03850f9f 100644
--- a/meson.build
+++ b/meson.build
@@ -3657,6 +3657,7 @@  if host_machine.system() == 'windows'
     '@OUTPUT@',
     get_option('prefix'),
     meson.current_source_dir(),
+    config_host['GLIB_BINDIR'],
     host_machine.cpu(),
     '--',
     '-DDISPLAYVERSION=' + meson.project_version(),
diff --git a/scripts/nsis.py b/scripts/nsis.py
index baa6ef9594..03ed7608a2 100644
--- a/scripts/nsis.py
+++ b/scripts/nsis.py
@@ -18,12 +18,36 @@  def signcode(path):
         return
     subprocess.run([cmd, path])
 
+def find_deps(exe_or_dll, search_path, analyzed_deps):
+    deps = [exe_or_dll]
+    output = subprocess.check_output(["objdump", "-p", exe_or_dll], text=True)
+    output = output.split("\n")
+    for line in output:
+        if not line.startswith("\tDLL Name: "):
+            continue
+
+        dep = line.split("DLL Name: ")[1].strip()
+        if dep in analyzed_deps:
+            continue
+
+        dll = os.path.join(search_path, dep)
+        if not os.path.exists(dll):
+            # assume it's a Windows provided dll, skip it
+            continue
+
+        analyzed_deps.add(dep)
+        # locate the dll dependencies recursively
+        rdeps = find_deps(dll, search_path, analyzed_deps)
+        deps.extend(rdeps)
+
+    return deps
 
 def main():
     parser = argparse.ArgumentParser(description="QEMU NSIS build helper.")
     parser.add_argument("outfile")
     parser.add_argument("prefix")
     parser.add_argument("srcdir")
+    parser.add_argument("dlldir")
     parser.add_argument("cpu")
     parser.add_argument("nsisargs", nargs="*")
     args = parser.parse_args()
@@ -63,9 +87,26 @@  def main():
                 !insertmacro MUI_DESCRIPTION_TEXT ${{Section_{0}}} "{1}"
                 """.format(arch, desc))
 
+        search_path = args.dlldir
+        print("Searching '%s' for the dependent dlls ..." % search_path)
+        dlldir = os.path.join(destdir + prefix, "dll")
+        os.mkdir(dlldir)
+
         for exe in glob.glob(os.path.join(destdir + prefix, "*.exe")):
             signcode(exe)
 
+            # find all dll dependencies
+            deps = set(find_deps(exe, search_path, set()))
+            deps.remove(exe)
+
+            # copy all dlls to the DLLDIR
+            for dep in deps:
+                dllfile = os.path.join(dlldir, os.path.basename(dep))
+                if (os.path.exists(dllfile)):
+                    continue
+                print("Copying '%s' to '%s'" % (dep, dllfile))
+                shutil.copy(dep, dllfile)
+
         makensis = [
             "makensis",
             "-V2",
@@ -73,12 +114,9 @@  def main():
             "-DSRCDIR=" + args.srcdir,
             "-DBINDIR=" + destdir + prefix,
         ]
-        dlldir = "w32"
         if args.cpu == "x86_64":
-            dlldir = "w64"
             makensis += ["-DW64"]
-        if os.path.exists(os.path.join(args.srcdir, "dll")):
-            makensis += ["-DDLLDIR={0}/dll/{1}".format(args.srcdir, dlldir)]
+        makensis += ["-DDLLDIR=" + dlldir]
 
         makensis += ["-DOUTFILE=" + args.outfile] + args.nsisargs
         subprocess.run(makensis)