diff mbox series

[2/3] blktests: add python scripts for parsing fio json output

Message ID 20181204174733.7423-3-josef@toxicpanda.com (mailing list archive)
State New, archived
Headers show
Series io.latency test for blktests | expand

Commit Message

Josef Bacik Dec. 4, 2018, 5:47 p.m. UTC 
I wrote these scripts for xfstests, just copying them over to blktests
as well.  The terse output fio support that blktests doesn't get all of
the various fio performance things that we may want to look at, this
gives us a lot of flexibility around getting specific values out of the
fio results data.

Signed-off-by: Josef Bacik <josef@toxicpanda.com>
---
 src/FioResultDecoder.py | 64 +++++++++++++++++++++++++++++++++++++++++++++++++
 src/fio-key-value.py    | 28 ++++++++++++++++++++++
 2 files changed, 92 insertions(+)
 create mode 100644 src/FioResultDecoder.py
 create mode 100644 src/fio-key-value.py

Comments

Federico Motta Dec. 4, 2018, 10:28 p.m. UTC | #1 
On 12/4/18 6:47 PM, Josef Bacik wrote:
> I wrote these scripts for xfstests, just copying them over to blktests
> as well.  The terse output fio support that blktests doesn't get all of
> the various fio performance things that we may want to look at, this
> gives us a lot of flexibility around getting specific values out of the
> fio results data.
> 
Hi,
some suggestions below :)

> Signed-off-by: Josef Bacik <josef@toxicpanda.com>
> ---
>  src/FioResultDecoder.py | 64 +++++++++++++++++++++++++++++++++++++++++++++++++
>  src/fio-key-value.py    | 28 ++++++++++++++++++++++
>  2 files changed, 92 insertions(+)
>  create mode 100644 src/FioResultDecoder.py
>  create mode 100644 src/fio-key-value.py
> 
> diff --git a/src/FioResultDecoder.py b/src/FioResultDecoder.py
> new file mode 100644
> index 000000000000..d004140c0fdf
> --- /dev/null
> +++ b/src/FioResultDecoder.py
> @@ -0,0 +1,64 @@
I think a shebang could clarify this runs under python2.

> +# SPDX-License-Identifier: GPL-2.0
> +
> +import json
> +
> +class FioResultDecoder(json.JSONDecoder):
> +    """Decoder for decoding fio result json to an object for our database
> +
> +    This decodes the json output from fio into an object that can be directly
> +    inserted into our database.  This just strips out the fields we don't care
> +    about and collapses the read/write/trim classes into a flat value structure
> +    inside of the jobs object.
> +
> +    For example
> +        "write" : {
> +            "io_bytes" : 313360384,
> +            "bw" : 1016,
> +        }
> +
> +    Get's collapsed to
> +
> +        "write_io_bytes" : 313360384,
> +        "write_bw": 1016,
> +
> +    Currently any dict under 'jobs' get's dropped, with the exception of 'read',
> +    'write', and 'trim'.  For those sub sections we drop any dict's under those.
> +
> +    Attempt to keep this as generic as possible, we don't want to break every
> +    time fio changes it's json output format.
> +    """
> +    _ignore_types = ['dict', 'list']
> +    _override_keys = ['lat_ns', 'lat']
> +    _io_ops = ['read', 'write', 'trim']
> +
> +    _transform_keys = { 'lat': 'lat_ns' }
> +
> +    def decode(self, json_string):
> +        """This does the dirty work of converting everything"""
> +        default_obj = super(FioResultDecoder, self).decode(json_string)
> +        obj = {}
> +        obj['global'] = {}
> +        obj['global']['time'] = default_obj['time']
> +        obj['jobs'] = []
> +        for job in default_obj['jobs']:
> +            new_job = {}
> +            for key,value in job.iteritems():
> +                if key not in self._io_ops:
> +                    if value.__class__.__name__ in self._ignore_types:
> +                        continue
> +                    new_job[key] = value
> +                    continue
> +                for k,v in value.iteritems():
> +                    if k in self._override_keys:
> +                        if k in self._transform_keys:
> +                            k = self._transform_keys[k]
> +                        for subk,subv in v.iteritems():
> +                            collapsed_key = "{}_{}_{}".format(key, k, subk)
> +                            new_job[collapsed_key] = subv
> +                        continue
> +                    if v.__class__.__name__ in self._ignore_types:
> +                        continue
> +                    collapsed_key = "{}_{}".format(key, k)
> +                    new_job[collapsed_key] = v
> +            obj['jobs'].append(new_job)
> +        return obj
> diff --git a/src/fio-key-value.py b/src/fio-key-value.py
> new file mode 100644
> index 000000000000..208e9a453a19
> --- /dev/null
> +++ b/src/fio-key-value.py
> @@ -0,0 +1,28 @@
Also here a shebang could be fine.

> +# SPDX-License-Identifier: GPL-2.0
> +
> +import FioResultDecoder
> +import json
> +import argparse
> +import sys
> +import platform
> +
> +parser = argparse.ArgumentParser()
> +parser.add_argument('-j', '--jobname', type=str,
> +                    help="The jobname we want our key from.",
> +                    required=True)
> +parser.add_argument('-k', '--key', type=str,
> +                    help="The key we want the value of", required=True)
> +parser.add_argument('result', type=str,
If here you set type=open              ^

> +                    help="The result file read")
> +args = parser.parse_args()
> +
> +json_data = open(args.result)
you could avoid this line ^

> +data = json.load(json_data, cls=FioResultDecoder.FioResultDecoder)
and substitute json_data with args.result which would return a file object.

In short the above piece of patch would become something like:

parser.add_argument('result', type=open, help="The result file read")
args = parser.parse_args()
data = json.load(args.result, cls=FioResultDecoder.FioResultDecoder)

Which still raises IOError if bad arguments are passed.

> +
> +for job in data['jobs']:
> +    if job['jobname'] == args.jobname:
> +        if args.key not in job:
> +            print('')
> +        else:
> +            print("{}".format(job[args.key]))
> +        break
> 


Have a nice day,
Federico
Johannes Thumshirn Dec. 5, 2018, 9:20 a.m. UTC | #2 
On 04/12/2018 18:47, Josef Bacik wrote:
> +    For example
> +        "write" : {
> +            "io_bytes" : 313360384,
> +            "bw" : 1016,
> +        }
> +
> +    Get's collapsed to
> +
> +        "write_io_bytes" : 313360384,
> +        "write_bw": 1016,

I'd definitively prefer to not pull in python as a dependency. The above
example can be done in shell (with the jq utility) as follows:

jthumshirn@linux-x5ow:~$ cat test.json |\
   jq '. | {write_io_bytes: .write.io_bytes, write_bw: .write.bw}'
{
  "write_io_bytes": 313360384,
  "write_bw": 1016
}

And on a plus side jq is small as well:

jthumshirn@linux-x5ow:~$ zypper info jq
Loading repository data...
Reading installed packages...


Information for package jq:
---------------------------
Repository     : openSUSE:Leap:15.0
Name           : jq
Version        : 1.5-lp150.1.10
Arch           : x86_64
Vendor         : openSUSE
Installed Size : 93.4 KiB
Installed      : Yes
[...]
diff mbox series

Patch

diff --git a/src/FioResultDecoder.py b/src/FioResultDecoder.py
new file mode 100644
index 000000000000..d004140c0fdf
--- /dev/null
+++ b/src/FioResultDecoder.py
@@ -0,0 +1,64 @@ 
+# SPDX-License-Identifier: GPL-2.0
+
+import json
+
+class FioResultDecoder(json.JSONDecoder):
+    """Decoder for decoding fio result json to an object for our database
+
+    This decodes the json output from fio into an object that can be directly
+    inserted into our database.  This just strips out the fields we don't care
+    about and collapses the read/write/trim classes into a flat value structure
+    inside of the jobs object.
+
+    For example
+        "write" : {
+            "io_bytes" : 313360384,
+            "bw" : 1016,
+        }
+
+    Get's collapsed to
+
+        "write_io_bytes" : 313360384,
+        "write_bw": 1016,
+
+    Currently any dict under 'jobs' get's dropped, with the exception of 'read',
+    'write', and 'trim'.  For those sub sections we drop any dict's under those.
+
+    Attempt to keep this as generic as possible, we don't want to break every
+    time fio changes it's json output format.
+    """
+    _ignore_types = ['dict', 'list']
+    _override_keys = ['lat_ns', 'lat']
+    _io_ops = ['read', 'write', 'trim']
+
+    _transform_keys = { 'lat': 'lat_ns' }
+
+    def decode(self, json_string):
+        """This does the dirty work of converting everything"""
+        default_obj = super(FioResultDecoder, self).decode(json_string)
+        obj = {}
+        obj['global'] = {}
+        obj['global']['time'] = default_obj['time']
+        obj['jobs'] = []
+        for job in default_obj['jobs']:
+            new_job = {}
+            for key,value in job.iteritems():
+                if key not in self._io_ops:
+                    if value.__class__.__name__ in self._ignore_types:
+                        continue
+                    new_job[key] = value
+                    continue
+                for k,v in value.iteritems():
+                    if k in self._override_keys:
+                        if k in self._transform_keys:
+                            k = self._transform_keys[k]
+                        for subk,subv in v.iteritems():
+                            collapsed_key = "{}_{}_{}".format(key, k, subk)
+                            new_job[collapsed_key] = subv
+                        continue
+                    if v.__class__.__name__ in self._ignore_types:
+                        continue
+                    collapsed_key = "{}_{}".format(key, k)
+                    new_job[collapsed_key] = v
+            obj['jobs'].append(new_job)
+        return obj
diff --git a/src/fio-key-value.py b/src/fio-key-value.py
new file mode 100644
index 000000000000..208e9a453a19
--- /dev/null
+++ b/src/fio-key-value.py
@@ -0,0 +1,28 @@ 
+# SPDX-License-Identifier: GPL-2.0
+
+import FioResultDecoder
+import json
+import argparse
+import sys
+import platform
+
+parser = argparse.ArgumentParser()
+parser.add_argument('-j', '--jobname', type=str,
+                    help="The jobname we want our key from.",
+                    required=True)
+parser.add_argument('-k', '--key', type=str,
+                    help="The key we want the value of", required=True)
+parser.add_argument('result', type=str,
+                    help="The result file read")
+args = parser.parse_args()
+
+json_data = open(args.result)
+data = json.load(json_data, cls=FioResultDecoder.FioResultDecoder)
+
+for job in data['jobs']:
+    if job['jobname'] == args.jobname:
+        if args.key not in job:
+            print('')
+        else:
+            print("{}".format(job[args.key]))
+        break