diff mbox series

iotests: Dump QCOW2 dirty bitmaps metadata

Message ID 1590504866-679474-1-git-send-email-andrey.shinkevich@virtuozzo.com (mailing list archive)
State New, archived
Headers show
Series iotests: Dump QCOW2 dirty bitmaps metadata | expand

Commit Message

Andrey Shinkevich May 26, 2020, 2:54 p.m. UTC
Add dirty bitmap information to QCOW2 metadata dump in qcow2.py script.
The sample output:

Header extension (Bitmaps):
magic                     0x23852875
length                    24
nb_bitmaps                2
reserved32                0
bitmap_directory_size     0x40
bitmap_directory_offset   0x100000

Bitmap name               bitmap-1
flag                      "auto"
bitmap_table_offset       0x90000
bitmap_table_size         8
flags                     2
type                      1
granularity_bits          15
name_size                 8
extra_data_size           0

Bitmap table
   0     serialized, offset 0xa0000
   1     all-zeroes, offset 0x0
   2     all-zeroes, offset 0x0
   3     all-zeroes, offset 0x0
   4     all-zeroes, offset 0x0
   5     all-zeroes, offset 0x0
   6     all-zeroes, offset 0x0
   7     all-zeroes, offset 0x0

Signed-off-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
---
 tests/qemu-iotests/qcow2.py | 149 +++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 141 insertions(+), 8 deletions(-)

Comments

no-reply@patchew.org May 26, 2020, 6:16 p.m. UTC | #1
Patchew URL: https://patchew.org/QEMU/1590504866-679474-1-git-send-email-andrey.shinkevich@virtuozzo.com/



Hi,

This series failed the docker-quick@centos7 build test. Please find the testing commands and
their output below. If you have Docker installed, you can probably reproduce it
locally.

=== TEST SCRIPT BEGIN ===
#!/bin/bash
make docker-image-centos7 V=1 NETWORK=1
time make docker-test-quick@centos7 SHOW_ENV=1 J=14 NETWORK=1
=== TEST SCRIPT END ===

Not run: 259
Failures: 031 036 061
Failed 3 of 119 iotests
make: *** [check-tests/check-block.sh] Error 1
make: *** Waiting for unfinished jobs....
  TEST    check-qtest-aarch64: tests/qtest/test-hmp
  TEST    check-qtest-aarch64: tests/qtest/qos-test
---
    raise CalledProcessError(retcode, cmd)
subprocess.CalledProcessError: Command '['sudo', '-n', 'docker', 'run', '--label', 'com.qemu.instance.uuid=350f72f6732d405b861f0e9334ef155a', '-u', '1001', '--security-opt', 'seccomp=unconfined', '--rm', '-e', 'TARGET_LIST=', '-e', 'EXTRA_CONFIGURE_OPTS=', '-e', 'V=', '-e', 'J=14', '-e', 'DEBUG=', '-e', 'SHOW_ENV=1', '-e', 'CCACHE_DIR=/var/tmp/ccache', '-v', '/home/patchew/.cache/qemu-docker-ccache:/var/tmp/ccache:z', '-v', '/var/tmp/patchew-tester-tmp-kwr2oe7u/src/docker-src.2020-05-26-14.02.04.28988:/var/tmp/qemu:z,ro', 'qemu:centos7', '/var/tmp/qemu/run', 'test-quick']' returned non-zero exit status 2.
filter=--filter=label=com.qemu.instance.uuid=350f72f6732d405b861f0e9334ef155a
make[1]: *** [docker-run] Error 1
make[1]: Leaving directory `/var/tmp/patchew-tester-tmp-kwr2oe7u/src'
make: *** [docker-run-test-quick@centos7] Error 2

real    14m37.383s
user    0m8.950s


The full log is available at
http://patchew.org/logs/1590504866-679474-1-git-send-email-andrey.shinkevich@virtuozzo.com/testing.docker-quick@centos7/?type=message.
---
Email generated automatically by Patchew [https://patchew.org/].
Please send your feedback to patchew-devel@redhat.com
Eric Blake May 26, 2020, 6:58 p.m. UTC | #2
On 5/26/20 9:54 AM, Andrey Shinkevich wrote:
> Add dirty bitmap information to QCOW2 metadata dump in qcow2.py script.
> The sample output:
> 
> Header extension (Bitmaps):

This change to the output is independently useful.  However, per 
patchew, it does cause 'make check' to fail:

https://patchew.org/logs/1590504866-679474-1-git-send-email-andrey.shinkevich@virtuozzo.com/testing.docker-quick@centos7/?type=message

...
--- /tmp/qemu-test/src/tests/qemu-iotests/031.out	2020-05-26 
14:44:51.000000000 +0000
+++ /tmp/qemu-test/build/tests/qemu-iotests/031.out.bad	2020-05-26 
18:07:11.753556518 +0000
@@ -24,7 +24,7 @@
  refcount_order            4
  header_length             72

-Header extension:
+Header extension (Unknown):
...
Failures: 031 036 061

I think it would be wise to split this into two patches, one that makes 
_just_ the following change:


> @@ -143,30 +267,39 @@ class QcowHeader:
>               print("%-25s" % f[2], value_str)
>           print("")
>   
> -    def dump_extensions(self):
> +    def dump_extensions(self, fd):
>           for ex in self.extensions:
>   
> +            print("Header extension (%s):" % self.extension_name(ex.magic))
> +            print("%-25s %#x" % ("magic", ex.magic))
> +            print("%-25s %d" % ("length", ex.length))

and whatever is needed to support that, plus the changes necessary to 
the iotests output to keep them passing (hopefully, the 3 tests 
identified by 'make check' covers all of the existing tests already 
using qcow2.py), then the second patch adding the rest of this that then 
gives details about the bitmap contents.
diff mbox series

Patch

diff --git a/tests/qemu-iotests/qcow2.py b/tests/qemu-iotests/qcow2.py
index 91e4420..41aa030 100755
--- a/tests/qemu-iotests/qcow2.py
+++ b/tests/qemu-iotests/qcow2.py
@@ -5,6 +5,122 @@  import sys
 import struct
 import string
 
+
+class Qcow2BitmapDirEntry:
+
+    name = ''
+    BME_FLAG_IN_USE = 1
+    BME_FLAG_AUTO = 1 << 1
+
+    uint8_t = 'B'
+    uint16_t = 'H'
+    uint32_t = 'I'
+    uint64_t = 'Q'
+
+    fields = [
+        [uint64_t, '%#x', 'bitmap_table_offset'],
+        [uint32_t, '%d',  'bitmap_table_size'],
+        [uint32_t, '%d',  'flags'],
+        [uint8_t,  '%d',  'type'],
+        [uint8_t,  '%d',  'granularity_bits'],
+        [uint16_t, '%d',  'name_size'],
+        [uint32_t, '%d',  'extra_data_size']
+    ]
+
+    fmt = '>' + ''.join(field[0] for field in fields)
+
+    def __init__(self, data):
+
+        entry = struct.unpack(Qcow2BitmapDirEntry.fmt, data)
+        self.__dict__ = dict((field[2], entry[i])
+                             for i, field in enumerate(
+                                 Qcow2BitmapDirEntry.fields))
+
+        self.bitmap_table_size = self.bitmap_table_size \
+            * struct.calcsize(self.uint64_t)
+
+    def bitmap_dir_entry_size(self):
+        size = struct.calcsize(self.fmt) + self.name_size + \
+            self.extra_data_size
+        return (size + 7) & ~7
+
+    def dump_bitmap_dir_entry(self):
+        print("%-25s" % 'Bitmap name', self.name)
+        if (self.flags & self.BME_FLAG_IN_USE) != 0:
+            print("%-25s" % 'flag', '"in-use"')
+        if (self.flags & self.BME_FLAG_AUTO) != 0:
+            print("%-25s" % 'flag', '"auto"')
+        for f in Qcow2BitmapDirEntry.fields:
+            value = self.__dict__[f[2]]
+            value_str = f[1] % value
+
+            print("%-25s" % f[2], value_str)
+        print("")
+
+    def dump_bitmap_table(self, fd):
+        fd.seek(self.bitmap_table_offset)
+        table_size = self.bitmap_table_size * struct.calcsize(self.uint64_t)
+        bitmap_table = [e[0] for e in struct.iter_unpack('>Q',
+                                                         fd.read(table_size))]
+        BME_TABLE_ENTRY_OFFSET_MASK = 0x00fffffffffffe00
+        BME_TABLE_ENTRY_FLAG_ALL_ONES = 1
+        bmt_type = ['all-zeroes', 'all-ones', 'serialized']
+        items = enumerate(bitmap_table)
+        print("Bitmap table")
+        for i, entry in items:
+            offset = entry & BME_TABLE_ENTRY_OFFSET_MASK
+            if offset != 0:
+                index = 2
+            else:
+                index = entry & BME_TABLE_ENTRY_FLAG_ALL_ONES
+            print("   %-4d  %s, offset %#x" % (i, bmt_type[index], offset))
+        print("")
+
+
+class Qcow2BitmapExt:
+
+    uint32_t = 'I'
+    uint64_t = 'Q'
+
+    fields = [
+        [uint32_t, '%d',  'nb_bitmaps'],
+        [uint32_t, '%d',  'reserved32'],
+        [uint64_t, '%#x', 'bitmap_directory_size'],
+        [uint64_t, '%#x', 'bitmap_directory_offset']
+    ]
+
+    fmt = '>' + ''.join(field[0] for field in fields)
+
+    def __init__(self, data):
+
+        extension = struct.unpack(Qcow2BitmapExt.fmt, data)
+        self.__dict__ = dict((field[2], extension[i])
+                             for i, field in enumerate(Qcow2BitmapExt.fields))
+
+    def dump_bitmap_ext(self):
+        for f in Qcow2BitmapExt.fields:
+            value = self.__dict__[f[2]]
+            value_str = f[1] % value
+
+            print("%-25s" % f[2], value_str)
+        print("")
+
+    def bitmap_directory(self, fd):
+        offset = self.bitmap_directory_offset
+        buf_size = struct.calcsize(Qcow2BitmapDirEntry.fmt)
+
+        for n in range(self.nb_bitmaps):
+            fd.seek(offset)
+            buf = fd.read(buf_size)
+            dir_entry = Qcow2BitmapDirEntry(buf)
+            fd.seek(dir_entry.extra_data_size, 1)
+            bitmap_name = fd.read(dir_entry.name_size)
+            dir_entry.name = bitmap_name.decode('ascii')
+            dir_entry.dump_bitmap_dir_entry()
+            dir_entry.dump_bitmap_table(fd)
+            offset += dir_entry.bitmap_dir_entry_size()
+
+
 class QcowHeaderExtension:
 
     def __init__(self, magic, length, data):
@@ -22,6 +138,8 @@  class QcowHeaderExtension:
 
 class QcowHeader:
 
+    QCOW2_EXT_MAGIC_FEATURE_TABLE = 0x6803f857
+    QCOW2_EXT_MAGIC_BITMAPS = 0x23852875
     uint32_t = 'I'
     uint64_t = 'Q'
 
@@ -128,6 +246,12 @@  class QcowHeader:
         buf = buf[0:header_bytes-1]
         fd.write(buf)
 
+    def extension_name(self, magic):
+       return {
+            self.QCOW2_EXT_MAGIC_FEATURE_TABLE: 'Feature table',
+            self.QCOW2_EXT_MAGIC_BITMAPS: 'Bitmaps',
+        }.get(magic, 'Unknown')
+
     def dump(self):
         for f in QcowHeader.fields:
             value = self.__dict__[f[2]]
@@ -143,30 +267,39 @@  class QcowHeader:
             print("%-25s" % f[2], value_str)
         print("")
 
-    def dump_extensions(self):
+    def dump_extensions(self, fd):
         for ex in self.extensions:
 
+            print("Header extension (%s):" % self.extension_name(ex.magic))
+            print("%-25s %#x" % ("magic", ex.magic))
+            print("%-25s %d" % ("length", ex.length))
+
             data = ex.data[:ex.length]
             if all(c in string.printable.encode('ascii') for c in data):
                 data = "'%s'" % data.decode('ascii')
+                print("%-25s %s" % ("data", data))
             else:
-                data = "<binary>"
+                self.dump_extension_data(fd, ex)
 
-            print("Header extension:")
-            print("%-25s %#x" % ("magic", ex.magic))
-            print("%-25s %d" % ("length", ex.length))
-            print("%-25s %s" % ("data", data))
             print("")
 
+    def dump_extension_data(self, fd, ext):
+        if ext.magic == self.QCOW2_EXT_MAGIC_BITMAPS:
+            b_ext = Qcow2BitmapExt(ext.data)
+            b_ext.dump_bitmap_ext()
+            b_ext.bitmap_directory(fd)
+        else:
+            print("%-25s %s" % ("data", "<binary>"))
+
 
 def cmd_dump_header(fd):
     h = QcowHeader(fd)
     h.dump()
-    h.dump_extensions()
+    h.dump_extensions(fd)
 
 def cmd_dump_header_exts(fd):
     h = QcowHeader(fd)
-    h.dump_extensions()
+    h.dump_extensions(fd)
 
 def cmd_set_header(fd, name, value):
     try: