@@ -153,26 +153,36 @@ callbacks per mdev parent device, per mdev type, or any other categorization.
Vendor drivers are expected to be fully asynchronous in this respect or
provide their own internal resource protection.)
-The callbacks in the mdev_parent_ops structure are as follows:
+A driver should use the mdev_parent_ops structure in the function call
+to register itself with the mdev core driver::
-* open: open callback of mediated device
-* close: close callback of mediated device
-* ioctl: ioctl callback of mediated device
+ extern int mdev_vfio_register_device(struct device *dev,
+ const struct mdev_parent_ops *ops);
+
+However, the mdev_parent_ops structure is not required in the function call
+that a driver should use to unregister itself with the mdev core driver::
+
+ extern void mdev_vfio_unregister_device(struct device *dev);
+
+The VFIO specific callbacks is abstracted in mdev_vfio_ops structure
+are as follows:
+
+* open: open callback of VFIO mediated device
+* close: close callback of VFIO mediated device
+* ioctl: ioctl callback of VFIO mediated device
* read : read emulation callback
* write: write emulation callback
* mmap: mmap emulation callback
-A driver should use the mdev_parent_ops structure and bus type in the
-function call to register itself with the mdev core driver::
+During the creation of VFIO mediated device, mdev_vfio_ops need to be
+specified::
- extern int mdev_register_device(struct device *dev,
- const struct mdev_parent_ops *ops,
- struct bus_type *bus);
+ void mdev_vfio_set_ops(struct mdev_device *mdev,
+ const struct mdev_vfio_ops *ops);
-However, the mdev_parent_ops structure is not required in the function call
-that a driver should use to unregister itself with the mdev core driver::
+Those callbacks could be fetched by drivers through::
- extern void mdev_unregister_device(struct device *dev);
+ const struct mdev_vfio_ops *mdev_vfio_get_ops(struct mdev_device *mdev);
Mediated Device Management Interface Through sysfs
@@ -643,6 +643,8 @@ static void kvmgt_put_vfio_device(void *vgpu)
vfio_device_put(((struct intel_vgpu *)vgpu)->vdev.vfio_device);
}
+static const struct mdev_vfio_ops intel_mdev_vfio_ops;
+
static int intel_vgpu_create(struct kobject *kobj, struct mdev_device *mdev)
{
struct intel_vgpu *vgpu = NULL;
@@ -678,6 +680,7 @@ static int intel_vgpu_create(struct kobject *kobj, struct mdev_device *mdev)
dev_name(mdev_dev(mdev)));
ret = 0;
+ mdev_vfio_set_ops(mdev, &intel_mdev_vfio_ops);
out:
return ret;
}
@@ -1581,20 +1584,21 @@ static const struct attribute_group *intel_vgpu_groups[] = {
NULL,
};
-static struct mdev_parent_ops intel_vgpu_ops = {
- .mdev_attr_groups = intel_vgpu_groups,
- .create = intel_vgpu_create,
- .remove = intel_vgpu_remove,
-
+static const struct mdev_vfio_ops intel_mdev_vfio_ops = {
.open = intel_vgpu_open,
.release = intel_vgpu_release,
-
.read = intel_vgpu_read,
.write = intel_vgpu_write,
.mmap = intel_vgpu_mmap,
.ioctl = intel_vgpu_ioctl,
};
+static struct mdev_parent_ops intel_vgpu_ops = {
+ .mdev_attr_groups = intel_vgpu_groups,
+ .create = intel_vgpu_create,
+ .remove = intel_vgpu_remove,
+};
+
static int kvmgt_host_init(struct device *dev, void *gvt, const void *ops)
{
struct attribute **kvm_type_attrs;
@@ -110,6 +110,8 @@ static struct attribute_group *mdev_type_groups[] = {
NULL,
};
+static const struct mdev_vfio_ops vfio_ccw_mdev_vfio_ops;
+
static int vfio_ccw_mdev_create(struct kobject *kobj, struct mdev_device *mdev)
{
struct vfio_ccw_private *private =
@@ -129,6 +131,8 @@ static int vfio_ccw_mdev_create(struct kobject *kobj, struct mdev_device *mdev)
private->sch->schid.ssid,
private->sch->schid.sch_no);
+ mdev_vfio_set_ops(mdev, &vfio_ccw_mdev_vfio_ops);
+
return 0;
}
@@ -574,16 +578,19 @@ static ssize_t vfio_ccw_mdev_ioctl(struct mdev_device *mdev,
}
}
-static const struct mdev_parent_ops vfio_ccw_mdev_ops = {
- .owner = THIS_MODULE,
- .supported_type_groups = mdev_type_groups,
- .create = vfio_ccw_mdev_create,
- .remove = vfio_ccw_mdev_remove,
+static const struct mdev_vfio_ops vfio_ccw_mdev_vfio_ops = {
.open = vfio_ccw_mdev_open,
.release = vfio_ccw_mdev_release,
.read = vfio_ccw_mdev_read,
.write = vfio_ccw_mdev_write,
.ioctl = vfio_ccw_mdev_ioctl,
+}
+
+static const struct mdev_parent_ops vfio_ccw_mdev_ops = {
+ .owner = THIS_MODULE,
+ .supported_type_groups = mdev_type_groups,
+ .create = vfio_ccw_mdev_create,
+ .remove = vfio_ccw_mdev_remove,
};
int vfio_ccw_mdev_reg(struct subchannel *sch)
@@ -321,6 +321,8 @@ static void vfio_ap_matrix_init(struct ap_config_info *info,
matrix->adm_max = info->apxa ? info->Nd : 15;
}
+static const struct mdev_vfio_ops vfio_ap_matrix_mdev_ops;
+
static int vfio_ap_mdev_create(struct kobject *kobj, struct mdev_device *mdev)
{
struct ap_matrix_mdev *matrix_mdev;
@@ -343,6 +345,8 @@ static int vfio_ap_mdev_create(struct kobject *kobj, struct mdev_device *mdev)
list_add(&matrix_mdev->node, &matrix_dev->mdev_list);
mutex_unlock(&matrix_dev->lock);
+ mdev_vfio_set_ops(mdev, &vfio_ap_matrix_mdev_ops);
+
return 0;
}
@@ -1280,15 +1284,18 @@ static ssize_t vfio_ap_mdev_ioctl(struct mdev_device *mdev,
return ret;
}
+static const struct mdev_vfio_ops vfio_ap_matrix_mdev_ops = {
+ .open = vfio_ap_mdev_open,
+ .release = vfio_ap_mdev_release,
+ .ioctl = vfio_ap_mdev_ioctl,
+}
+
static const struct mdev_parent_ops vfio_ap_matrix_ops = {
.owner = THIS_MODULE,
.supported_type_groups = vfio_ap_mdev_type_groups,
.mdev_attr_groups = vfio_ap_mdev_attr_groups,
.create = vfio_ap_mdev_create,
.remove = vfio_ap_mdev_remove,
- .open = vfio_ap_mdev_open,
- .release = vfio_ap_mdev_release,
- .ioctl = vfio_ap_mdev_ioctl,
};
int vfio_ap_mdev_register(void)
@@ -179,7 +179,7 @@ static struct class_compat *mdev_alloc_class_compat(struct bus_type *bus)
* Returns a negative value on error, otherwise 0.
*/
int mdev_register_device(struct device *dev, const struct mdev_parent_ops *ops,
- struct bus_type *bus)
+ struct bus_type *bus, size_t dev_size)
{
int ret;
struct mdev_parent *parent;
@@ -217,6 +217,7 @@ int mdev_register_device(struct device *dev, const struct mdev_parent_ops *ops,
parent->dev = dev;
parent->ops = ops;
parent->bus = bus;
+ parent->dev_size = dev_size;
mutex_lock(&compat_list_lock);
class_compat = mdev_alloc_class_compat(bus);
@@ -339,7 +340,7 @@ int mdev_device_create(struct kobject *kobj,
}
}
- mdev = kzalloc(sizeof(*mdev), GFP_KERNEL);
+ mdev = kzalloc(parent->dev_size, GFP_KERNEL);
if (!mdev) {
mutex_unlock(&mdev_list_lock);
ret = -ENOMEM;
@@ -20,6 +20,7 @@ struct mdev_parent {
struct list_head type_list;
/* Synchronize device creation/removal with parent unregistration */
struct rw_semaphore unreg_sem;
+ size_t dev_size;
};
struct mdev_device {
@@ -66,4 +67,8 @@ int mdev_device_create(struct kobject *kobj,
struct device *dev, const guid_t *uuid);
int mdev_device_remove(struct device *dev);
+int mdev_register_device(struct device *dev, const struct mdev_parent_ops *ops,
+ struct bus_type *bus, size_t dev_size);
+void mdev_unregister_device(struct device *dev);
+
#endif /* MDEV_PRIVATE_H */
@@ -4,6 +4,8 @@
#include <linux/device.h>
#include <linux/mdev_vfio.h>
+#include "mdev_private.h"
+
#define DRIVER_VERSION "0.1"
#define DRIVER_AUTHOR "Jason Wang"
#define DRIVER_DESC "Mediated VFIO bus"
@@ -15,6 +17,31 @@ struct bus_type mdev_vfio_bus_type = {
};
EXPORT_SYMBOL(mdev_vfio_bus_type);
+#define to_vfio_mdev_device(mdev) container_of(mdev, \
+ struct mdev_vfio_device, mdev)
+
+struct mdev_vfio_device {
+ struct mdev_device mdev;
+ const struct mdev_vfio_ops *ops;
+};
+
+void mdev_vfio_set_ops(struct mdev_device *mdev,
+ const struct mdev_vfio_ops *ops)
+{
+ struct mdev_vfio_device *mdev_vfio = to_vfio_mdev_device(mdev);
+
+ mdev_vfio->ops = ops;
+}
+EXPORT_SYMBOL(mdev_vfio_set_ops);
+
+const struct mdev_vfio_ops *mdev_vfio_get_ops(struct mdev_device *mdev)
+{
+ struct mdev_vfio_device *mdev_vfio = to_vfio_mdev_device(mdev);
+
+ return mdev_vfio->ops;
+}
+EXPORT_SYMBOL(mdev_vfio_get_ops);
+
static int __init mdev_init(void)
{
return mdev_register_bus(&mdev_vfio_bus_type);
@@ -28,7 +55,8 @@ static void __exit mdev_exit(void)
int mdev_vfio_register_device(struct device *dev,
const struct mdev_parent_ops *ops)
{
- return mdev_register_device(dev, ops, &mdev_vfio_bus_type);
+ return mdev_register_device(dev, ops, &mdev_vfio_bus_type,
+ sizeof(struct mdev_vfio_device));
}
EXPORT_SYMBOL(mdev_vfio_register_device);
@@ -24,16 +24,16 @@
static int vfio_mdev_open(void *device_data)
{
struct mdev_device *mdev = device_data;
- struct mdev_parent *parent = mdev->parent;
+ const struct mdev_vfio_ops *ops = mdev_vfio_get_ops(mdev);
int ret;
- if (unlikely(!parent->ops->open))
+ if (unlikely(!ops->open))
return -EINVAL;
if (!try_module_get(THIS_MODULE))
return -ENODEV;
- ret = parent->ops->open(mdev);
+ ret = ops->open(mdev);
if (ret)
module_put(THIS_MODULE);
@@ -43,10 +43,10 @@ static int vfio_mdev_open(void *device_data)
static void vfio_mdev_release(void *device_data)
{
struct mdev_device *mdev = device_data;
- struct mdev_parent *parent = mdev->parent;
+ const struct mdev_vfio_ops *ops = mdev_vfio_get_ops(mdev);
- if (likely(parent->ops->release))
- parent->ops->release(mdev);
+ if (likely(ops->release))
+ ops->release(mdev);
module_put(THIS_MODULE);
}
@@ -55,47 +55,47 @@ static long vfio_mdev_unlocked_ioctl(void *device_data,
unsigned int cmd, unsigned long arg)
{
struct mdev_device *mdev = device_data;
- struct mdev_parent *parent = mdev->parent;
+ const struct mdev_vfio_ops *ops = mdev_vfio_get_ops(mdev);
- if (unlikely(!parent->ops->ioctl))
+ if (unlikely(!ops->ioctl))
return -EINVAL;
- return parent->ops->ioctl(mdev, cmd, arg);
+ return ops->ioctl(mdev, cmd, arg);
}
static ssize_t vfio_mdev_read(void *device_data, char __user *buf,
size_t count, loff_t *ppos)
{
struct mdev_device *mdev = device_data;
- struct mdev_parent *parent = mdev->parent;
+ const struct mdev_vfio_ops *ops = mdev_vfio_get_ops(mdev);
- if (unlikely(!parent->ops->read))
+ if (unlikely(!ops->read))
return -EINVAL;
- return parent->ops->read(mdev, buf, count, ppos);
+ return ops->read(mdev, buf, count, ppos);
}
static ssize_t vfio_mdev_write(void *device_data, const char __user *buf,
size_t count, loff_t *ppos)
{
struct mdev_device *mdev = device_data;
- struct mdev_parent *parent = mdev->parent;
+ const struct mdev_vfio_ops *ops = mdev_vfio_get_ops(mdev);
- if (unlikely(!parent->ops->write))
+ if (unlikely(!ops->write))
return -EINVAL;
- return parent->ops->write(mdev, buf, count, ppos);
+ return ops->write(mdev, buf, count, ppos);
}
static int vfio_mdev_mmap(void *device_data, struct vm_area_struct *vma)
{
struct mdev_device *mdev = device_data;
- struct mdev_parent *parent = mdev->parent;
+ const struct mdev_vfio_ops *ops = mdev_vfio_get_ops(mdev);
- if (unlikely(!parent->ops->mmap))
+ if (unlikely(!ops->mmap))
return -EINVAL;
- return parent->ops->mmap(mdev, vma);
+ return ops->mmap(mdev, vma);
}
static const struct vfio_device_ops vfio_mdev_dev_ops = {
@@ -110,7 +110,7 @@ static const struct vfio_device_ops vfio_mdev_dev_ops = {
static int vfio_mdev_probe(struct device *dev)
{
- struct mdev_device *mdev = to_mdev_device(dev);
+ struct mdev_device *mdev = mdev_from_dev(dev, &mdev_vfio_bus_type);
return vfio_add_group_dev(dev, &vfio_mdev_dev_ops, mdev);
}
@@ -48,30 +48,6 @@ struct device *mdev_get_iommu_device(struct device *dev);
* @mdev: mdev_device device structure which is being
* destroyed
* Returns integer: success (0) or error (< 0)
- * @open: Open mediated device.
- * @mdev: mediated device.
- * Returns integer: success (0) or error (< 0)
- * @release: release mediated device
- * @mdev: mediated device.
- * @read: Read emulation callback
- * @mdev: mediated device structure
- * @buf: read buffer
- * @count: number of bytes to read
- * @ppos: address.
- * Retuns number on bytes read on success or error.
- * @write: Write emulation callback
- * @mdev: mediated device structure
- * @buf: write buffer
- * @count: number of bytes to be written
- * @ppos: address.
- * Retuns number on bytes written on success or error.
- * @ioctl: IOCTL callback
- * @mdev: mediated device structure
- * @cmd: ioctl command
- * @arg: arguments to ioctl
- * @mmap: mmap callback
- * @mdev: mediated device structure
- * @vma: vma structure
* Parent device that support mediated device should be registered with mdev
* module with mdev_parent_ops structure.
**/
@@ -83,15 +59,6 @@ struct mdev_parent_ops {
int (*create)(struct kobject *kobj, struct mdev_device *mdev);
int (*remove)(struct mdev_device *mdev);
- int (*open)(struct mdev_device *mdev);
- void (*release)(struct mdev_device *mdev);
- ssize_t (*read)(struct mdev_device *mdev, char __user *buf,
- size_t count, loff_t *ppos);
- ssize_t (*write)(struct mdev_device *mdev, const char __user *buf,
- size_t count, loff_t *ppos);
- long (*ioctl)(struct mdev_device *mdev, unsigned int cmd,
- unsigned long arg);
- int (*mmap)(struct mdev_device *mdev, struct vm_area_struct *vma);
};
/* interface for exporting mdev supported type attributes */
@@ -133,10 +100,6 @@ void *mdev_get_drvdata(struct mdev_device *mdev);
void mdev_set_drvdata(struct mdev_device *mdev, void *data);
const guid_t *mdev_uuid(struct mdev_device *mdev);
-int mdev_register_device(struct device *dev, const struct mdev_parent_ops *ops,
- struct bus_type *bus);
-void mdev_unregister_device(struct device *dev);
-
int mdev_register_driver(struct mdev_driver *drv, struct module *owner,
struct bus_type *bus);
void mdev_unregister_driver(struct mdev_driver *drv);
@@ -13,6 +13,45 @@
extern struct bus_type mdev_vfio_bus_type;
+/* VFIO mdev ops
+ *
+ * @open: Open mediated device.
+ * @mdev: mediated device.
+ * Returns integer: success (0) or error (< 0)
+ * @release: release mediated device
+ * @mdev: mediated device.
+ * @read: Read emulation callback
+ * @mdev: mediated device structure
+ * @buf: read buffer
+ * @count: number of bytes to read
+ * @ppos: address.
+ * Retuns number on bytes read on success or error.
+ * @write: Write emulation callback
+ * @mdev: mediated device structure
+ * @buf: write buffer
+ * @count: number of bytes to be written
+ * @ppos: address.
+ * Retuns number on bytes written on success or error.
+ * @ioctl: IOCTL callback
+ * @mdev: mediated device structure
+ * @cmd: ioctl command
+ * @arg: arguments to ioctl
+ * @mmap: mmap callback
+ * @mdev: mediated device structure
+ * @vma: vma structure
+ */
+struct mdev_vfio_ops {
+ int (*open)(struct mdev_device *mdev);
+ void (*release)(struct mdev_device *mdev);
+ ssize_t (*read)(struct mdev_device *mdev, char __user *buf,
+ size_t count, loff_t *ppos);
+ ssize_t (*write)(struct mdev_device *mdev, const char __user *buf,
+ size_t count, loff_t *ppos);
+ long (*ioctl)(struct mdev_device *mdev, unsigned int cmd,
+ unsigned long arg);
+ int (*mmap)(struct mdev_device *mdev, struct vm_area_struct *vma);
+};
+
int mdev_vfio_register_device(struct device *dev,
const struct mdev_parent_ops *ops);
void mdev_vfio_unregister_device(struct device *dev);
@@ -22,4 +61,8 @@ static inline struct mdev_device *vfio_mdev_from_dev(struct device *dev)
return mdev_from_dev(dev, &mdev_vfio_bus_type);
}
+void mdev_vfio_set_ops(struct mdev_device *mdev,
+ const struct mdev_vfio_ops *ops);
+const struct mdev_vfio_ops *mdev_vfio_get_ops(struct mdev_device *mdev);
+
#endif
@@ -516,6 +516,8 @@ static int mbochs_reset(struct mdev_device *mdev)
return 0;
}
+static const struct mdev_vfio_ops mdev_ops;
+
static int mbochs_create(struct kobject *kobj, struct mdev_device *mdev)
{
const struct mbochs_type *type = mbochs_find_type(kobj);
@@ -561,6 +563,7 @@ static int mbochs_create(struct kobject *kobj, struct mdev_device *mdev)
mbochs_reset(mdev);
mbochs_used_mbytes += type->mbytes;
+ mdev_vfio_set_ops(mdev, &mdev_ops);
return 0;
err_mem:
@@ -1418,12 +1421,7 @@ static struct attribute_group *mdev_type_groups[] = {
NULL,
};
-static const struct mdev_parent_ops mdev_fops = {
- .owner = THIS_MODULE,
- .mdev_attr_groups = mdev_dev_groups,
- .supported_type_groups = mdev_type_groups,
- .create = mbochs_create,
- .remove = mbochs_remove,
+static const struct mdev_vfio_ops mdev_ops = {
.open = mbochs_open,
.release = mbochs_close,
.read = mbochs_read,
@@ -1432,6 +1430,14 @@ static const struct mdev_parent_ops mdev_fops = {
.mmap = mbochs_mmap,
};
+static const struct mdev_parent_ops mdev_fops = {
+ .owner = THIS_MODULE,
+ .mdev_attr_groups = mdev_dev_groups,
+ .supported_type_groups = mdev_type_groups,
+ .create = mbochs_create,
+ .remove = mbochs_remove,
+};
+
static const struct file_operations vd_fops = {
.owner = THIS_MODULE,
};
@@ -226,6 +226,8 @@ static int mdpy_reset(struct mdev_device *mdev)
return 0;
}
+static const struct mdev_vfio_ops mdev_ops;
+
static int mdpy_create(struct kobject *kobj, struct mdev_device *mdev)
{
const struct mdpy_type *type = mdpy_find_type(kobj);
@@ -269,6 +271,8 @@ static int mdpy_create(struct kobject *kobj, struct mdev_device *mdev)
mdpy_reset(mdev);
mdpy_count++;
+
+ mdev_vfio_set_ops(mdev, &mdev_ops);
return 0;
}
@@ -725,12 +729,7 @@ static struct attribute_group *mdev_type_groups[] = {
NULL,
};
-static const struct mdev_parent_ops mdev_fops = {
- .owner = THIS_MODULE,
- .mdev_attr_groups = mdev_dev_groups,
- .supported_type_groups = mdev_type_groups,
- .create = mdpy_create,
- .remove = mdpy_remove,
+static const struct mdev_vfio_ops mdev_ops = {
.open = mdpy_open,
.release = mdpy_close,
.read = mdpy_read,
@@ -739,6 +738,14 @@ static const struct mdev_parent_ops mdev_fops = {
.mmap = mdpy_mmap,
};
+static const struct mdev_parent_ops mdev_fops = {
+ .owner = THIS_MODULE,
+ .mdev_attr_groups = mdev_dev_groups,
+ .supported_type_groups = mdev_type_groups,
+ .create = mdpy_create,
+ .remove = mdpy_remove,
+};
+
static const struct file_operations vd_fops = {
.owner = THIS_MODULE,
};
@@ -708,6 +708,8 @@ static ssize_t mdev_access(struct mdev_device *mdev, u8 *buf, size_t count,
return ret;
}
+static const struct mdev_vfio_ops mdev_ops;
+
static int mtty_create(struct kobject *kobj, struct mdev_device *mdev)
{
struct mdev_state *mdev_state;
@@ -755,6 +757,7 @@ static int mtty_create(struct kobject *kobj, struct mdev_device *mdev)
list_add(&mdev_state->next, &mdev_devices_list);
mutex_unlock(&mdev_list_lock);
+ mdev_vfio_set_ops(mdev, &mdev_ops);
return 0;
}
@@ -1387,6 +1390,14 @@ static struct attribute_group *mdev_type_groups[] = {
NULL,
};
+static const struct mdev_vfio_ops mdev_ops = {
+ .open = mtty_open,
+ .release = mtty_close,
+ .read = mtty_read,
+ .write = mtty_write,
+ .ioctl = mtty_ioctl,
+};
+
static const struct mdev_parent_ops mdev_fops = {
.owner = THIS_MODULE,
.dev_attr_groups = mtty_dev_groups,
@@ -1394,11 +1405,6 @@ static const struct mdev_parent_ops mdev_fops = {
.supported_type_groups = mdev_type_groups,
.create = mtty_create,
.remove = mtty_remove,
- .open = mtty_open,
- .release = mtty_close,
- .read = mtty_read,
- .write = mtty_write,
- .ioctl = mtty_ioctl,
};
static void mtty_device_release(struct device *dev)
The only thing left for generalizing mdev is the VFIO specific parent ops. This is basically the open/release/read/write/ioctl/mmap. To support this, mdev core is extend to support a specific size of structure during create, this will allow to compose mdev structure into mdev vfio structure and place the VFIO specific callbacks there like: struct mdev_vfio { struct mdev_device mdev; const struct mdev_vfio_ops *ops; }; Helpers for setting and getting the ops were introduced to support mdev vfio device to set ops and vfio mdev driver to use the ops. Signed-off-by: Jason Wang <jasowang@redhat.com> --- .../driver-api/vfio-mediated-device.rst | 34 +++++++++------ drivers/gpu/drm/i915/gvt/kvmgt.c | 16 ++++--- drivers/s390/cio/vfio_ccw_ops.c | 17 +++++--- drivers/s390/crypto/vfio_ap_ops.c | 13 ++++-- drivers/vfio/mdev/mdev_core.c | 5 ++- drivers/vfio/mdev/mdev_private.h | 5 +++ drivers/vfio/mdev/mdev_vfio.c | 30 ++++++++++++- drivers/vfio/mdev/vfio_mdev.c | 38 ++++++++-------- include/linux/mdev.h | 37 ---------------- include/linux/mdev_vfio.h | 43 +++++++++++++++++++ samples/vfio-mdev/mbochs.c | 18 +++++--- samples/vfio-mdev/mdpy.c | 19 +++++--- samples/vfio-mdev/mtty.c | 16 ++++--- 13 files changed, 189 insertions(+), 102 deletions(-)