| Message ID | 1688002673-28493-2-git-send-email-shengjiu.wang@nxp.com (mailing list archive) |
|---|---|
| State | Superseded |
| Headers | show |
| Series | Add audio support in v4l2 framework | expand |
Hi Shengjiu, On Thu, Jun 29, 2023 at 09:37:48AM +0800, Shengjiu Wang wrote: > Audio signal processing has the requirement for memory to > memory similar as Video. > > This patch is to add this support in v4l2 framework, defined > new buffer type V4L2_BUF_TYPE_AUDIO_CAPTURE and > V4L2_BUF_TYPE_AUDIO_OUTPUT, defined new format v4l2_audio_format > for audio case usage. Why are you proposing to add this to V4L2 framework instead of doing this within ALSA? Also cc Hans and Jacopo. > > The created audio device is named "/dev/audioX". > > Signed-off-by: Shengjiu Wang <shengjiu.wang@nxp.com> > --- > .../media/common/videobuf2/videobuf2-v4l2.c | 4 ++ > drivers/media/v4l2-core/v4l2-dev.c | 17 ++++++ > drivers/media/v4l2-core/v4l2-ioctl.c | 52 +++++++++++++++++++ > include/media/v4l2-dev.h | 2 + > include/media/v4l2-ioctl.h | 34 ++++++++++++ > include/uapi/linux/videodev2.h | 19 +++++++ > 6 files changed, 128 insertions(+) > > diff --git a/drivers/media/common/videobuf2/videobuf2-v4l2.c b/drivers/media/common/videobuf2/videobuf2-v4l2.c > index c7a54d82a55e..12f2be2773a2 100644 > --- a/drivers/media/common/videobuf2/videobuf2-v4l2.c > +++ b/drivers/media/common/videobuf2/videobuf2-v4l2.c > @@ -785,6 +785,10 @@ int vb2_create_bufs(struct vb2_queue *q, struct v4l2_create_buffers *create) > case V4L2_BUF_TYPE_META_OUTPUT: > requested_sizes[0] = f->fmt.meta.buffersize; > break; > + case V4L2_BUF_TYPE_AUDIO_CAPTURE: > + case V4L2_BUF_TYPE_AUDIO_OUTPUT: > + requested_sizes[0] = f->fmt.audio.buffersize; > + break; > default: > return -EINVAL; > } > diff --git a/drivers/media/v4l2-core/v4l2-dev.c b/drivers/media/v4l2-core/v4l2-dev.c > index f81279492682..67484f4c6eaf 100644 > --- a/drivers/media/v4l2-core/v4l2-dev.c > +++ b/drivers/media/v4l2-core/v4l2-dev.c > @@ -553,6 +553,7 @@ static void determine_valid_ioctls(struct video_device *vdev) > bool is_tch = vdev->vfl_type == VFL_TYPE_TOUCH; > bool is_meta = vdev->vfl_type == VFL_TYPE_VIDEO && > (vdev->device_caps & meta_caps); > + bool is_audio = vdev->vfl_type == VFL_TYPE_AUDIO; > bool is_rx = vdev->vfl_dir != VFL_DIR_TX; > bool is_tx = vdev->vfl_dir != VFL_DIR_RX; > bool is_io_mc = vdev->device_caps & V4L2_CAP_IO_MC; > @@ -664,6 +665,19 @@ static void determine_valid_ioctls(struct video_device *vdev) > SET_VALID_IOCTL(ops, VIDIOC_S_FMT, vidioc_s_fmt_meta_out); > SET_VALID_IOCTL(ops, VIDIOC_TRY_FMT, vidioc_try_fmt_meta_out); > } > + if (is_audio && is_rx) { > + /* audio capture specific ioctls */ > + SET_VALID_IOCTL(ops, VIDIOC_ENUM_FMT, vidioc_enum_fmt_audio_cap); > + SET_VALID_IOCTL(ops, VIDIOC_G_FMT, vidioc_g_fmt_audio_cap); > + SET_VALID_IOCTL(ops, VIDIOC_S_FMT, vidioc_s_fmt_audio_cap); > + SET_VALID_IOCTL(ops, VIDIOC_TRY_FMT, vidioc_try_fmt_audio_cap); > + } else if (is_audio && is_tx) { > + /* audio output specific ioctls */ > + SET_VALID_IOCTL(ops, VIDIOC_ENUM_FMT, vidioc_enum_fmt_audio_out); > + SET_VALID_IOCTL(ops, VIDIOC_G_FMT, vidioc_g_fmt_audio_out); > + SET_VALID_IOCTL(ops, VIDIOC_S_FMT, vidioc_s_fmt_audio_out); > + SET_VALID_IOCTL(ops, VIDIOC_TRY_FMT, vidioc_try_fmt_audio_out); > + } > if (is_vbi) { > /* vbi specific ioctls */ > if ((is_rx && (ops->vidioc_g_fmt_vbi_cap || > @@ -927,6 +941,9 @@ int __video_register_device(struct video_device *vdev, > case VFL_TYPE_TOUCH: > name_base = "v4l-touch"; > break; > + case VFL_TYPE_AUDIO: > + name_base = "audio"; > + break; > default: > pr_err("%s called with unknown type: %d\n", > __func__, type); > diff --git a/drivers/media/v4l2-core/v4l2-ioctl.c b/drivers/media/v4l2-core/v4l2-ioctl.c > index a858acea6547..26bc4b0d8ef0 100644 > --- a/drivers/media/v4l2-core/v4l2-ioctl.c > +++ b/drivers/media/v4l2-core/v4l2-ioctl.c > @@ -188,6 +188,8 @@ const char *v4l2_type_names[] = { > [V4L2_BUF_TYPE_SDR_OUTPUT] = "sdr-out", > [V4L2_BUF_TYPE_META_CAPTURE] = "meta-cap", > [V4L2_BUF_TYPE_META_OUTPUT] = "meta-out", > + [V4L2_BUF_TYPE_AUDIO_CAPTURE] = "audio-cap", > + [V4L2_BUF_TYPE_AUDIO_OUTPUT] = "audio-out", > }; > EXPORT_SYMBOL(v4l2_type_names); > > @@ -276,6 +278,7 @@ static void v4l_print_format(const void *arg, bool write_only) > const struct v4l2_sliced_vbi_format *sliced; > const struct v4l2_window *win; > const struct v4l2_meta_format *meta; > + const struct v4l2_audio_format *audio; > u32 pixelformat; > u32 planes; > unsigned i; > @@ -346,6 +349,12 @@ static void v4l_print_format(const void *arg, bool write_only) > pr_cont(", dataformat=%p4cc, buffersize=%u\n", > &pixelformat, meta->buffersize); > break; > + case V4L2_BUF_TYPE_AUDIO_CAPTURE: > + case V4L2_BUF_TYPE_AUDIO_OUTPUT: > + audio = &p->fmt.audio; > + pr_cont(", rate=%u, format=%u, channels=%u, buffersize=%u\n", > + audio->rate, audio->format, audio->channels, audio->buffersize); > + break; > } > } > > @@ -927,6 +936,7 @@ static int check_fmt(struct file *file, enum v4l2_buf_type type) > bool is_tch = vfd->vfl_type == VFL_TYPE_TOUCH; > bool is_meta = vfd->vfl_type == VFL_TYPE_VIDEO && > (vfd->device_caps & meta_caps); > + bool is_audio = vfd->vfl_type == VFL_TYPE_AUDIO; > bool is_rx = vfd->vfl_dir != VFL_DIR_TX; > bool is_tx = vfd->vfl_dir != VFL_DIR_RX; > > @@ -992,6 +1002,14 @@ static int check_fmt(struct file *file, enum v4l2_buf_type type) > if (is_meta && is_tx && ops->vidioc_g_fmt_meta_out) > return 0; > break; > + case V4L2_BUF_TYPE_AUDIO_CAPTURE: > + if (is_audio && is_rx && ops->vidioc_g_fmt_audio_cap) > + return 0; > + break; > + case V4L2_BUF_TYPE_AUDIO_OUTPUT: > + if (is_audio && is_tx && ops->vidioc_g_fmt_audio_out) > + return 0; > + break; > default: > break; > } > @@ -1592,6 +1610,16 @@ static int v4l_enum_fmt(const struct v4l2_ioctl_ops *ops, > break; > ret = ops->vidioc_enum_fmt_meta_out(file, fh, arg); > break; > + case V4L2_BUF_TYPE_AUDIO_CAPTURE: > + if (unlikely(!ops->vidioc_enum_fmt_audio_cap)) > + break; > + ret = ops->vidioc_enum_fmt_audio_cap(file, fh, arg); > + break; > + case V4L2_BUF_TYPE_AUDIO_OUTPUT: > + if (unlikely(!ops->vidioc_enum_fmt_audio_out)) > + break; > + ret = ops->vidioc_enum_fmt_audio_out(file, fh, arg); > + break; > } > if (ret == 0) > v4l_fill_fmtdesc(p); > @@ -1668,6 +1696,10 @@ static int v4l_g_fmt(const struct v4l2_ioctl_ops *ops, > return ops->vidioc_g_fmt_meta_cap(file, fh, arg); > case V4L2_BUF_TYPE_META_OUTPUT: > return ops->vidioc_g_fmt_meta_out(file, fh, arg); > + case V4L2_BUF_TYPE_AUDIO_CAPTURE: > + return ops->vidioc_g_fmt_audio_cap(file, fh, arg); > + case V4L2_BUF_TYPE_AUDIO_OUTPUT: > + return ops->vidioc_g_fmt_audio_out(file, fh, arg); > } > return -EINVAL; > } > @@ -1779,6 +1811,16 @@ static int v4l_s_fmt(const struct v4l2_ioctl_ops *ops, > break; > memset_after(p, 0, fmt.meta); > return ops->vidioc_s_fmt_meta_out(file, fh, arg); > + case V4L2_BUF_TYPE_AUDIO_CAPTURE: > + if (unlikely(!ops->vidioc_s_fmt_audio_cap)) > + break; > + memset_after(p, 0, fmt.audio); > + return ops->vidioc_s_fmt_audio_cap(file, fh, arg); > + case V4L2_BUF_TYPE_AUDIO_OUTPUT: > + if (unlikely(!ops->vidioc_s_fmt_audio_out)) > + break; > + memset_after(p, 0, fmt.audio); > + return ops->vidioc_s_fmt_audio_out(file, fh, arg); > } > return -EINVAL; > } > @@ -1887,6 +1929,16 @@ static int v4l_try_fmt(const struct v4l2_ioctl_ops *ops, > break; > memset_after(p, 0, fmt.meta); > return ops->vidioc_try_fmt_meta_out(file, fh, arg); > + case V4L2_BUF_TYPE_AUDIO_CAPTURE: > + if (unlikely(!ops->vidioc_try_fmt_audio_cap)) > + break; > + memset_after(p, 0, fmt.audio); > + return ops->vidioc_try_fmt_audio_cap(file, fh, arg); > + case V4L2_BUF_TYPE_AUDIO_OUTPUT: > + if (unlikely(!ops->vidioc_try_fmt_audio_out)) > + break; > + memset_after(p, 0, fmt.audio); > + return ops->vidioc_try_fmt_audio_out(file, fh, arg); > } > return -EINVAL; > } > diff --git a/include/media/v4l2-dev.h b/include/media/v4l2-dev.h > index e0a13505f88d..0924e6d1dab1 100644 > --- a/include/media/v4l2-dev.h > +++ b/include/media/v4l2-dev.h > @@ -30,6 +30,7 @@ > * @VFL_TYPE_SUBDEV: for V4L2 subdevices > * @VFL_TYPE_SDR: for Software Defined Radio tuners > * @VFL_TYPE_TOUCH: for touch sensors > + * @VFL_TYPE_AUDIO: for audio input/output devices > * @VFL_TYPE_MAX: number of VFL types, must always be last in the enum > */ > enum vfl_devnode_type { > @@ -39,6 +40,7 @@ enum vfl_devnode_type { > VFL_TYPE_SUBDEV, > VFL_TYPE_SDR, > VFL_TYPE_TOUCH, > + VFL_TYPE_AUDIO, > VFL_TYPE_MAX /* Shall be the last one */ > }; > > diff --git a/include/media/v4l2-ioctl.h b/include/media/v4l2-ioctl.h > index edb733f21604..f840cf740ce1 100644 > --- a/include/media/v4l2-ioctl.h > +++ b/include/media/v4l2-ioctl.h > @@ -45,6 +45,12 @@ struct v4l2_fh; > * @vidioc_enum_fmt_meta_out: pointer to the function that implements > * :ref:`VIDIOC_ENUM_FMT <vidioc_enum_fmt>` ioctl logic > * for metadata output > + * @vidioc_enum_fmt_audio_cap: pointer to the function that implements > + * :ref:`VIDIOC_ENUM_FMT <vidioc_enum_fmt>` ioctl logic > + * for audio capture > + * @vidioc_enum_fmt_audio_out: pointer to the function that implements > + * :ref:`VIDIOC_ENUM_FMT <vidioc_enum_fmt>` ioctl logic > + * for audio output > * @vidioc_g_fmt_vid_cap: pointer to the function that implements > * :ref:`VIDIOC_G_FMT <vidioc_g_fmt>` ioctl logic for video capture > * in single plane mode > @@ -79,6 +85,10 @@ struct v4l2_fh; > * :ref:`VIDIOC_G_FMT <vidioc_g_fmt>` ioctl logic for metadata capture > * @vidioc_g_fmt_meta_out: pointer to the function that implements > * :ref:`VIDIOC_G_FMT <vidioc_g_fmt>` ioctl logic for metadata output > + * @vidioc_g_fmt_audio_cap: pointer to the function that implements > + * :ref:`VIDIOC_G_FMT <vidioc_g_fmt>` ioctl logic for audio capture > + * @vidioc_g_fmt_audio_out: pointer to the function that implements > + * :ref:`VIDIOC_G_FMT <vidioc_g_fmt>` ioctl logic for audio output > * @vidioc_s_fmt_vid_cap: pointer to the function that implements > * :ref:`VIDIOC_S_FMT <vidioc_g_fmt>` ioctl logic for video capture > * in single plane mode > @@ -113,6 +123,10 @@ struct v4l2_fh; > * :ref:`VIDIOC_S_FMT <vidioc_g_fmt>` ioctl logic for metadata capture > * @vidioc_s_fmt_meta_out: pointer to the function that implements > * :ref:`VIDIOC_S_FMT <vidioc_g_fmt>` ioctl logic for metadata output > + * @vidioc_s_fmt_audio_cap: pointer to the function that implements > + * :ref:`VIDIOC_S_FMT <vidioc_g_fmt>` ioctl logic for audio capture > + * @vidioc_s_fmt_audio_out: pointer to the function that implements > + * :ref:`VIDIOC_S_FMT <vidioc_g_fmt>` ioctl logic for audio output > * @vidioc_try_fmt_vid_cap: pointer to the function that implements > * :ref:`VIDIOC_TRY_FMT <vidioc_g_fmt>` ioctl logic for video capture > * in single plane mode > @@ -149,6 +163,10 @@ struct v4l2_fh; > * :ref:`VIDIOC_TRY_FMT <vidioc_g_fmt>` ioctl logic for metadata capture > * @vidioc_try_fmt_meta_out: pointer to the function that implements > * :ref:`VIDIOC_TRY_FMT <vidioc_g_fmt>` ioctl logic for metadata output > + * @vidioc_try_fmt_audio_cap: pointer to the function that implements > + * :ref:`VIDIOC_TRY_FMT <vidioc_g_fmt>` ioctl logic for audio capture > + * @vidioc_try_fmt_audio_out: pointer to the function that implements > + * :ref:`VIDIOC_TRY_FMT <vidioc_g_fmt>` ioctl logic for audio output > * @vidioc_reqbufs: pointer to the function that implements > * :ref:`VIDIOC_REQBUFS <vidioc_reqbufs>` ioctl > * @vidioc_querybuf: pointer to the function that implements > @@ -315,6 +333,10 @@ struct v4l2_ioctl_ops { > struct v4l2_fmtdesc *f); > int (*vidioc_enum_fmt_meta_out)(struct file *file, void *fh, > struct v4l2_fmtdesc *f); > + int (*vidioc_enum_fmt_audio_cap)(struct file *file, void *fh, > + struct v4l2_fmtdesc *f); > + int (*vidioc_enum_fmt_audio_out)(struct file *file, void *fh, > + struct v4l2_fmtdesc *f); > > /* VIDIOC_G_FMT handlers */ > int (*vidioc_g_fmt_vid_cap)(struct file *file, void *fh, > @@ -345,6 +367,10 @@ struct v4l2_ioctl_ops { > struct v4l2_format *f); > int (*vidioc_g_fmt_meta_out)(struct file *file, void *fh, > struct v4l2_format *f); > + int (*vidioc_g_fmt_audio_cap)(struct file *file, void *fh, > + struct v4l2_format *f); > + int (*vidioc_g_fmt_audio_out)(struct file *file, void *fh, > + struct v4l2_format *f); > > /* VIDIOC_S_FMT handlers */ > int (*vidioc_s_fmt_vid_cap)(struct file *file, void *fh, > @@ -375,6 +401,10 @@ struct v4l2_ioctl_ops { > struct v4l2_format *f); > int (*vidioc_s_fmt_meta_out)(struct file *file, void *fh, > struct v4l2_format *f); > + int (*vidioc_s_fmt_audio_cap)(struct file *file, void *fh, > + struct v4l2_format *f); > + int (*vidioc_s_fmt_audio_out)(struct file *file, void *fh, > + struct v4l2_format *f); > > /* VIDIOC_TRY_FMT handlers */ > int (*vidioc_try_fmt_vid_cap)(struct file *file, void *fh, > @@ -405,6 +435,10 @@ struct v4l2_ioctl_ops { > struct v4l2_format *f); > int (*vidioc_try_fmt_meta_out)(struct file *file, void *fh, > struct v4l2_format *f); > + int (*vidioc_try_fmt_audio_cap)(struct file *file, void *fh, > + struct v4l2_format *f); > + int (*vidioc_try_fmt_audio_out)(struct file *file, void *fh, > + struct v4l2_format *f); > > /* Buffer handlers */ > int (*vidioc_reqbufs)(struct file *file, void *fh, > diff --git a/include/uapi/linux/videodev2.h b/include/uapi/linux/videodev2.h > index aee75eb9e686..a7af28f4c8c3 100644 > --- a/include/uapi/linux/videodev2.h > +++ b/include/uapi/linux/videodev2.h > @@ -153,6 +153,8 @@ enum v4l2_buf_type { > V4L2_BUF_TYPE_SDR_OUTPUT = 12, > V4L2_BUF_TYPE_META_CAPTURE = 13, > V4L2_BUF_TYPE_META_OUTPUT = 14, > + V4L2_BUF_TYPE_AUDIO_CAPTURE = 15, > + V4L2_BUF_TYPE_AUDIO_OUTPUT = 16, > /* Deprecated, do not use */ > V4L2_BUF_TYPE_PRIVATE = 0x80, > }; > @@ -169,6 +171,7 @@ enum v4l2_buf_type { > || (type) == V4L2_BUF_TYPE_VBI_OUTPUT \ > || (type) == V4L2_BUF_TYPE_SLICED_VBI_OUTPUT \ > || (type) == V4L2_BUF_TYPE_SDR_OUTPUT \ > + || (type) == V4L2_BUF_TYPE_AUDIO_OUTPUT \ > || (type) == V4L2_BUF_TYPE_META_OUTPUT) > > #define V4L2_TYPE_IS_CAPTURE(type) (!V4L2_TYPE_IS_OUTPUT(type)) > @@ -2404,6 +2407,20 @@ struct v4l2_meta_format { > __u32 buffersize; > } __attribute__ ((packed)); > > +/** > + * struct v4l2_audio_format - audio data format definition > + * @rate: sample rate > + * @format: sample format > + * @channels: channel numbers > + * @buffersize: maximum size in bytes required for data > + */ > +struct v4l2_audio_format { > + __u32 rate; > + __u32 format; > + __u32 channels; > + __u32 buffersize; > +} __attribute__ ((packed)); > + > /** > * struct v4l2_format - stream data format > * @type: enum v4l2_buf_type; type of the data stream > @@ -2412,6 +2429,7 @@ struct v4l2_meta_format { > * @win: definition of an overlaid image > * @vbi: raw VBI capture or output parameters > * @sliced: sliced VBI capture or output parameters > + * @audio: definition of an audio format > * @raw_data: placeholder for future extensions and custom formats > * @fmt: union of @pix, @pix_mp, @win, @vbi, @sliced, @sdr, @meta > * and @raw_data > @@ -2426,6 +2444,7 @@ struct v4l2_format { > struct v4l2_sliced_vbi_format sliced; /* V4L2_BUF_TYPE_SLICED_VBI_CAPTURE */ > struct v4l2_sdr_format sdr; /* V4L2_BUF_TYPE_SDR_CAPTURE */ > struct v4l2_meta_format meta; /* V4L2_BUF_TYPE_META_CAPTURE */ > + struct v4l2_audio_format audio; /* V4L2_BUF_TYPE_AUDIO_CAPTURE */ > __u8 raw_data[200]; /* user-defined */ > } fmt; > }; > -- > 2.34.1 >
Hi Sakari On Fri, Jun 30, 2023 at 6:05 PM Sakari Ailus <sakari.ailus@iki.fi> wrote: > Hi Shengjiu, > > On Thu, Jun 29, 2023 at 09:37:48AM +0800, Shengjiu Wang wrote: > > Audio signal processing has the requirement for memory to > > memory similar as Video. > > > > This patch is to add this support in v4l2 framework, defined > > new buffer type V4L2_BUF_TYPE_AUDIO_CAPTURE and > > V4L2_BUF_TYPE_AUDIO_OUTPUT, defined new format v4l2_audio_format > > for audio case usage. > > Why are you proposing to add this to V4L2 framework instead of doing this > within ALSA? > > Also cc Hans and Jacopo. There is no such memory to memory interface defined in ALSA. Seems ALSA is not designed for M2M cases. V4L2 is designed for video, radio, image, sdr, meta..., so I think audio can be naturally added to the support scope. Thanks. Best regards Shengjiu Wang > > > > > The created audio device is named "/dev/audioX". > > > > Signed-off-by: Shengjiu Wang <shengjiu.wang@nxp.com> > > --- > > .../media/common/videobuf2/videobuf2-v4l2.c | 4 ++ > > drivers/media/v4l2-core/v4l2-dev.c | 17 ++++++ > > drivers/media/v4l2-core/v4l2-ioctl.c | 52 +++++++++++++++++++ > > include/media/v4l2-dev.h | 2 + > > include/media/v4l2-ioctl.h | 34 ++++++++++++ > > include/uapi/linux/videodev2.h | 19 +++++++ > > 6 files changed, 128 insertions(+) > > > > diff --git a/drivers/media/common/videobuf2/videobuf2-v4l2.c > b/drivers/media/common/videobuf2/videobuf2-v4l2.c > > index c7a54d82a55e..12f2be2773a2 100644 > > --- a/drivers/media/common/videobuf2/videobuf2-v4l2.c > > +++ b/drivers/media/common/videobuf2/videobuf2-v4l2.c > > @@ -785,6 +785,10 @@ int vb2_create_bufs(struct vb2_queue *q, struct > v4l2_create_buffers *create) > > case V4L2_BUF_TYPE_META_OUTPUT: > > requested_sizes[0] = f->fmt.meta.buffersize; > > break; > > + case V4L2_BUF_TYPE_AUDIO_CAPTURE: > > + case V4L2_BUF_TYPE_AUDIO_OUTPUT: > > + requested_sizes[0] = f->fmt.audio.buffersize; > > + break; > > default: > > return -EINVAL; > > } > > diff --git a/drivers/media/v4l2-core/v4l2-dev.c > b/drivers/media/v4l2-core/v4l2-dev.c > > index f81279492682..67484f4c6eaf 100644 > > --- a/drivers/media/v4l2-core/v4l2-dev.c > > +++ b/drivers/media/v4l2-core/v4l2-dev.c > > @@ -553,6 +553,7 @@ static void determine_valid_ioctls(struct > video_device *vdev) > > bool is_tch = vdev->vfl_type == VFL_TYPE_TOUCH; > > bool is_meta = vdev->vfl_type == VFL_TYPE_VIDEO && > > (vdev->device_caps & meta_caps); > > + bool is_audio = vdev->vfl_type == VFL_TYPE_AUDIO; > > bool is_rx = vdev->vfl_dir != VFL_DIR_TX; > > bool is_tx = vdev->vfl_dir != VFL_DIR_RX; > > bool is_io_mc = vdev->device_caps & V4L2_CAP_IO_MC; > > @@ -664,6 +665,19 @@ static void determine_valid_ioctls(struct > video_device *vdev) > > SET_VALID_IOCTL(ops, VIDIOC_S_FMT, vidioc_s_fmt_meta_out); > > SET_VALID_IOCTL(ops, VIDIOC_TRY_FMT, > vidioc_try_fmt_meta_out); > > } > > + if (is_audio && is_rx) { > > + /* audio capture specific ioctls */ > > + SET_VALID_IOCTL(ops, VIDIOC_ENUM_FMT, > vidioc_enum_fmt_audio_cap); > > + SET_VALID_IOCTL(ops, VIDIOC_G_FMT, vidioc_g_fmt_audio_cap); > > + SET_VALID_IOCTL(ops, VIDIOC_S_FMT, vidioc_s_fmt_audio_cap); > > + SET_VALID_IOCTL(ops, VIDIOC_TRY_FMT, > vidioc_try_fmt_audio_cap); > > + } else if (is_audio && is_tx) { > > + /* audio output specific ioctls */ > > + SET_VALID_IOCTL(ops, VIDIOC_ENUM_FMT, > vidioc_enum_fmt_audio_out); > > + SET_VALID_IOCTL(ops, VIDIOC_G_FMT, vidioc_g_fmt_audio_out); > > + SET_VALID_IOCTL(ops, VIDIOC_S_FMT, vidioc_s_fmt_audio_out); > > + SET_VALID_IOCTL(ops, VIDIOC_TRY_FMT, > vidioc_try_fmt_audio_out); > > + } > > if (is_vbi) { > > /* vbi specific ioctls */ > > if ((is_rx && (ops->vidioc_g_fmt_vbi_cap || > > @@ -927,6 +941,9 @@ int __video_register_device(struct video_device > *vdev, > > case VFL_TYPE_TOUCH: > > name_base = "v4l-touch"; > > break; > > + case VFL_TYPE_AUDIO: > > + name_base = "audio"; > > + break; > > default: > > pr_err("%s called with unknown type: %d\n", > > __func__, type); > > diff --git a/drivers/media/v4l2-core/v4l2-ioctl.c > b/drivers/media/v4l2-core/v4l2-ioctl.c > > index a858acea6547..26bc4b0d8ef0 100644 > > --- a/drivers/media/v4l2-core/v4l2-ioctl.c > > +++ b/drivers/media/v4l2-core/v4l2-ioctl.c > > @@ -188,6 +188,8 @@ const char *v4l2_type_names[] = { > > [V4L2_BUF_TYPE_SDR_OUTPUT] = "sdr-out", > > [V4L2_BUF_TYPE_META_CAPTURE] = "meta-cap", > > [V4L2_BUF_TYPE_META_OUTPUT] = "meta-out", > > + [V4L2_BUF_TYPE_AUDIO_CAPTURE] = "audio-cap", > > + [V4L2_BUF_TYPE_AUDIO_OUTPUT] = "audio-out", > > }; > > EXPORT_SYMBOL(v4l2_type_names); > > > > @@ -276,6 +278,7 @@ static void v4l_print_format(const void *arg, bool > write_only) > > const struct v4l2_sliced_vbi_format *sliced; > > const struct v4l2_window *win; > > const struct v4l2_meta_format *meta; > > + const struct v4l2_audio_format *audio; > > u32 pixelformat; > > u32 planes; > > unsigned i; > > @@ -346,6 +349,12 @@ static void v4l_print_format(const void *arg, bool > write_only) > > pr_cont(", dataformat=%p4cc, buffersize=%u\n", > > &pixelformat, meta->buffersize); > > break; > > + case V4L2_BUF_TYPE_AUDIO_CAPTURE: > > + case V4L2_BUF_TYPE_AUDIO_OUTPUT: > > + audio = &p->fmt.audio; > > + pr_cont(", rate=%u, format=%u, channels=%u, > buffersize=%u\n", > > + audio->rate, audio->format, audio->channels, > audio->buffersize); > > + break; > > } > > } > > > > @@ -927,6 +936,7 @@ static int check_fmt(struct file *file, enum > v4l2_buf_type type) > > bool is_tch = vfd->vfl_type == VFL_TYPE_TOUCH; > > bool is_meta = vfd->vfl_type == VFL_TYPE_VIDEO && > > (vfd->device_caps & meta_caps); > > + bool is_audio = vfd->vfl_type == VFL_TYPE_AUDIO; > > bool is_rx = vfd->vfl_dir != VFL_DIR_TX; > > bool is_tx = vfd->vfl_dir != VFL_DIR_RX; > > > > @@ -992,6 +1002,14 @@ static int check_fmt(struct file *file, enum > v4l2_buf_type type) > > if (is_meta && is_tx && ops->vidioc_g_fmt_meta_out) > > return 0; > > break; > > + case V4L2_BUF_TYPE_AUDIO_CAPTURE: > > + if (is_audio && is_rx && ops->vidioc_g_fmt_audio_cap) > > + return 0; > > + break; > > + case V4L2_BUF_TYPE_AUDIO_OUTPUT: > > + if (is_audio && is_tx && ops->vidioc_g_fmt_audio_out) > > + return 0; > > + break; > > default: > > break; > > } > > @@ -1592,6 +1610,16 @@ static int v4l_enum_fmt(const struct > v4l2_ioctl_ops *ops, > > break; > > ret = ops->vidioc_enum_fmt_meta_out(file, fh, arg); > > break; > > + case V4L2_BUF_TYPE_AUDIO_CAPTURE: > > + if (unlikely(!ops->vidioc_enum_fmt_audio_cap)) > > + break; > > + ret = ops->vidioc_enum_fmt_audio_cap(file, fh, arg); > > + break; > > + case V4L2_BUF_TYPE_AUDIO_OUTPUT: > > + if (unlikely(!ops->vidioc_enum_fmt_audio_out)) > > + break; > > + ret = ops->vidioc_enum_fmt_audio_out(file, fh, arg); > > + break; > > } > > if (ret == 0) > > v4l_fill_fmtdesc(p); > > @@ -1668,6 +1696,10 @@ static int v4l_g_fmt(const struct v4l2_ioctl_ops > *ops, > > return ops->vidioc_g_fmt_meta_cap(file, fh, arg); > > case V4L2_BUF_TYPE_META_OUTPUT: > > return ops->vidioc_g_fmt_meta_out(file, fh, arg); > > + case V4L2_BUF_TYPE_AUDIO_CAPTURE: > > + return ops->vidioc_g_fmt_audio_cap(file, fh, arg); > > + case V4L2_BUF_TYPE_AUDIO_OUTPUT: > > + return ops->vidioc_g_fmt_audio_out(file, fh, arg); > > } > > return -EINVAL; > > } > > @@ -1779,6 +1811,16 @@ static int v4l_s_fmt(const struct v4l2_ioctl_ops > *ops, > > break; > > memset_after(p, 0, fmt.meta); > > return ops->vidioc_s_fmt_meta_out(file, fh, arg); > > + case V4L2_BUF_TYPE_AUDIO_CAPTURE: > > + if (unlikely(!ops->vidioc_s_fmt_audio_cap)) > > + break; > > + memset_after(p, 0, fmt.audio); > > + return ops->vidioc_s_fmt_audio_cap(file, fh, arg); > > + case V4L2_BUF_TYPE_AUDIO_OUTPUT: > > + if (unlikely(!ops->vidioc_s_fmt_audio_out)) > > + break; > > + memset_after(p, 0, fmt.audio); > > + return ops->vidioc_s_fmt_audio_out(file, fh, arg); > > } > > return -EINVAL; > > } > > @@ -1887,6 +1929,16 @@ static int v4l_try_fmt(const struct > v4l2_ioctl_ops *ops, > > break; > > memset_after(p, 0, fmt.meta); > > return ops->vidioc_try_fmt_meta_out(file, fh, arg); > > + case V4L2_BUF_TYPE_AUDIO_CAPTURE: > > + if (unlikely(!ops->vidioc_try_fmt_audio_cap)) > > + break; > > + memset_after(p, 0, fmt.audio); > > + return ops->vidioc_try_fmt_audio_cap(file, fh, arg); > > + case V4L2_BUF_TYPE_AUDIO_OUTPUT: > > + if (unlikely(!ops->vidioc_try_fmt_audio_out)) > > + break; > > + memset_after(p, 0, fmt.audio); > > + return ops->vidioc_try_fmt_audio_out(file, fh, arg); > > } > > return -EINVAL; > > } > > diff --git a/include/media/v4l2-dev.h b/include/media/v4l2-dev.h > > index e0a13505f88d..0924e6d1dab1 100644 > > --- a/include/media/v4l2-dev.h > > +++ b/include/media/v4l2-dev.h > > @@ -30,6 +30,7 @@ > > * @VFL_TYPE_SUBDEV: for V4L2 subdevices > > * @VFL_TYPE_SDR: for Software Defined Radio tuners > > * @VFL_TYPE_TOUCH: for touch sensors > > + * @VFL_TYPE_AUDIO: for audio input/output devices > > * @VFL_TYPE_MAX: number of VFL types, must always be last in the > enum > > */ > > enum vfl_devnode_type { > > @@ -39,6 +40,7 @@ enum vfl_devnode_type { > > VFL_TYPE_SUBDEV, > > VFL_TYPE_SDR, > > VFL_TYPE_TOUCH, > > + VFL_TYPE_AUDIO, > > VFL_TYPE_MAX /* Shall be the last one */ > > }; > > > > diff --git a/include/media/v4l2-ioctl.h b/include/media/v4l2-ioctl.h > > index edb733f21604..f840cf740ce1 100644 > > --- a/include/media/v4l2-ioctl.h > > +++ b/include/media/v4l2-ioctl.h > > @@ -45,6 +45,12 @@ struct v4l2_fh; > > * @vidioc_enum_fmt_meta_out: pointer to the function that implements > > * :ref:`VIDIOC_ENUM_FMT <vidioc_enum_fmt>` ioctl logic > > * for metadata output > > + * @vidioc_enum_fmt_audio_cap: pointer to the function that implements > > + * :ref:`VIDIOC_ENUM_FMT <vidioc_enum_fmt>` ioctl logic > > + * for audio capture > > + * @vidioc_enum_fmt_audio_out: pointer to the function that implements > > + * :ref:`VIDIOC_ENUM_FMT <vidioc_enum_fmt>` ioctl logic > > + * for audio output > > * @vidioc_g_fmt_vid_cap: pointer to the function that implements > > * :ref:`VIDIOC_G_FMT <vidioc_g_fmt>` ioctl logic for video capture > > * in single plane mode > > @@ -79,6 +85,10 @@ struct v4l2_fh; > > * :ref:`VIDIOC_G_FMT <vidioc_g_fmt>` ioctl logic for metadata capture > > * @vidioc_g_fmt_meta_out: pointer to the function that implements > > * :ref:`VIDIOC_G_FMT <vidioc_g_fmt>` ioctl logic for metadata output > > + * @vidioc_g_fmt_audio_cap: pointer to the function that implements > > + * :ref:`VIDIOC_G_FMT <vidioc_g_fmt>` ioctl logic for audio capture > > + * @vidioc_g_fmt_audio_out: pointer to the function that implements > > + * :ref:`VIDIOC_G_FMT <vidioc_g_fmt>` ioctl logic for audio output > > * @vidioc_s_fmt_vid_cap: pointer to the function that implements > > * :ref:`VIDIOC_S_FMT <vidioc_g_fmt>` ioctl logic for video capture > > * in single plane mode > > @@ -113,6 +123,10 @@ struct v4l2_fh; > > * :ref:`VIDIOC_S_FMT <vidioc_g_fmt>` ioctl logic for metadata capture > > * @vidioc_s_fmt_meta_out: pointer to the function that implements > > * :ref:`VIDIOC_S_FMT <vidioc_g_fmt>` ioctl logic for metadata output > > + * @vidioc_s_fmt_audio_cap: pointer to the function that implements > > + * :ref:`VIDIOC_S_FMT <vidioc_g_fmt>` ioctl logic for audio capture > > + * @vidioc_s_fmt_audio_out: pointer to the function that implements > > + * :ref:`VIDIOC_S_FMT <vidioc_g_fmt>` ioctl logic for audio output > > * @vidioc_try_fmt_vid_cap: pointer to the function that implements > > * :ref:`VIDIOC_TRY_FMT <vidioc_g_fmt>` ioctl logic for video capture > > * in single plane mode > > @@ -149,6 +163,10 @@ struct v4l2_fh; > > * :ref:`VIDIOC_TRY_FMT <vidioc_g_fmt>` ioctl logic for metadata > capture > > * @vidioc_try_fmt_meta_out: pointer to the function that implements > > * :ref:`VIDIOC_TRY_FMT <vidioc_g_fmt>` ioctl logic for metadata > output > > + * @vidioc_try_fmt_audio_cap: pointer to the function that implements > > + * :ref:`VIDIOC_TRY_FMT <vidioc_g_fmt>` ioctl logic for audio capture > > + * @vidioc_try_fmt_audio_out: pointer to the function that implements > > + * :ref:`VIDIOC_TRY_FMT <vidioc_g_fmt>` ioctl logic for audio output > > * @vidioc_reqbufs: pointer to the function that implements > > * :ref:`VIDIOC_REQBUFS <vidioc_reqbufs>` ioctl > > * @vidioc_querybuf: pointer to the function that implements > > @@ -315,6 +333,10 @@ struct v4l2_ioctl_ops { > > struct v4l2_fmtdesc *f); > > int (*vidioc_enum_fmt_meta_out)(struct file *file, void *fh, > > struct v4l2_fmtdesc *f); > > + int (*vidioc_enum_fmt_audio_cap)(struct file *file, void *fh, > > + struct v4l2_fmtdesc *f); > > + int (*vidioc_enum_fmt_audio_out)(struct file *file, void *fh, > > + struct v4l2_fmtdesc *f); > > > > /* VIDIOC_G_FMT handlers */ > > int (*vidioc_g_fmt_vid_cap)(struct file *file, void *fh, > > @@ -345,6 +367,10 @@ struct v4l2_ioctl_ops { > > struct v4l2_format *f); > > int (*vidioc_g_fmt_meta_out)(struct file *file, void *fh, > > struct v4l2_format *f); > > + int (*vidioc_g_fmt_audio_cap)(struct file *file, void *fh, > > + struct v4l2_format *f); > > + int (*vidioc_g_fmt_audio_out)(struct file *file, void *fh, > > + struct v4l2_format *f); > > > > /* VIDIOC_S_FMT handlers */ > > int (*vidioc_s_fmt_vid_cap)(struct file *file, void *fh, > > @@ -375,6 +401,10 @@ struct v4l2_ioctl_ops { > > struct v4l2_format *f); > > int (*vidioc_s_fmt_meta_out)(struct file *file, void *fh, > > struct v4l2_format *f); > > + int (*vidioc_s_fmt_audio_cap)(struct file *file, void *fh, > > + struct v4l2_format *f); > > + int (*vidioc_s_fmt_audio_out)(struct file *file, void *fh, > > + struct v4l2_format *f); > > > > /* VIDIOC_TRY_FMT handlers */ > > int (*vidioc_try_fmt_vid_cap)(struct file *file, void *fh, > > @@ -405,6 +435,10 @@ struct v4l2_ioctl_ops { > > struct v4l2_format *f); > > int (*vidioc_try_fmt_meta_out)(struct file *file, void *fh, > > struct v4l2_format *f); > > + int (*vidioc_try_fmt_audio_cap)(struct file *file, void *fh, > > + struct v4l2_format *f); > > + int (*vidioc_try_fmt_audio_out)(struct file *file, void *fh, > > + struct v4l2_format *f); > > > > /* Buffer handlers */ > > int (*vidioc_reqbufs)(struct file *file, void *fh, > > diff --git a/include/uapi/linux/videodev2.h > b/include/uapi/linux/videodev2.h > > index aee75eb9e686..a7af28f4c8c3 100644 > > --- a/include/uapi/linux/videodev2.h > > +++ b/include/uapi/linux/videodev2.h > > @@ -153,6 +153,8 @@ enum v4l2_buf_type { > > V4L2_BUF_TYPE_SDR_OUTPUT = 12, > > V4L2_BUF_TYPE_META_CAPTURE = 13, > > V4L2_BUF_TYPE_META_OUTPUT = 14, > > + V4L2_BUF_TYPE_AUDIO_CAPTURE = 15, > > + V4L2_BUF_TYPE_AUDIO_OUTPUT = 16, > > /* Deprecated, do not use */ > > V4L2_BUF_TYPE_PRIVATE = 0x80, > > }; > > @@ -169,6 +171,7 @@ enum v4l2_buf_type { > > || (type) == V4L2_BUF_TYPE_VBI_OUTPUT \ > > || (type) == V4L2_BUF_TYPE_SLICED_VBI_OUTPUT \ > > || (type) == V4L2_BUF_TYPE_SDR_OUTPUT \ > > + || (type) == V4L2_BUF_TYPE_AUDIO_OUTPUT \ > > || (type) == V4L2_BUF_TYPE_META_OUTPUT) > > > > #define V4L2_TYPE_IS_CAPTURE(type) (!V4L2_TYPE_IS_OUTPUT(type)) > > @@ -2404,6 +2407,20 @@ struct v4l2_meta_format { > > __u32 buffersize; > > } __attribute__ ((packed)); > > > > +/** > > + * struct v4l2_audio_format - audio data format definition > > + * @rate: sample rate > > + * @format: sample format > > + * @channels: channel numbers > > + * @buffersize: maximum size in bytes required for data > > + */ > > +struct v4l2_audio_format { > > + __u32 rate; > > + __u32 format; > > + __u32 channels; > > + __u32 buffersize; > > +} __attribute__ ((packed)); > > + > > /** > > * struct v4l2_format - stream data format > > * @type: enum v4l2_buf_type; type of the data stream > > @@ -2412,6 +2429,7 @@ struct v4l2_meta_format { > > * @win: definition of an overlaid image > > * @vbi: raw VBI capture or output parameters > > * @sliced: sliced VBI capture or output parameters > > + * @audio: definition of an audio format > > * @raw_data: placeholder for future extensions and custom > formats > > * @fmt: union of @pix, @pix_mp, @win, @vbi, @sliced, @sdr, @meta > > * and @raw_data > > @@ -2426,6 +2444,7 @@ struct v4l2_format { > > struct v4l2_sliced_vbi_format sliced; /* > V4L2_BUF_TYPE_SLICED_VBI_CAPTURE */ > > struct v4l2_sdr_format sdr; /* > V4L2_BUF_TYPE_SDR_CAPTURE */ > > struct v4l2_meta_format meta; /* > V4L2_BUF_TYPE_META_CAPTURE */ > > + struct v4l2_audio_format audio; /* > V4L2_BUF_TYPE_AUDIO_CAPTURE */ > > __u8 raw_data[200]; /* user-defined */ > > } fmt; > > }; > > -- > > 2.34.1 > > > > -- > Sakari Ailus >
On 03/07/2023 11:54, Shengjiu Wang wrote: > Hi Sakari > > On Fri, Jun 30, 2023 at 6:05 PM Sakari Ailus <sakari.ailus@iki.fi <mailto:sakari.ailus@iki.fi>> wrote: > > Hi Shengjiu, > > On Thu, Jun 29, 2023 at 09:37:48AM +0800, Shengjiu Wang wrote: > > Audio signal processing has the requirement for memory to > > memory similar as Video. > > > > This patch is to add this support in v4l2 framework, defined > > new buffer type V4L2_BUF_TYPE_AUDIO_CAPTURE and > > V4L2_BUF_TYPE_AUDIO_OUTPUT, defined new format v4l2_audio_format > > for audio case usage. > > Why are you proposing to add this to V4L2 framework instead of doing this > within ALSA? > > Also cc Hans and Jacopo. > > > There is no such memory to memory interface defined in ALSA. Seems > ALSA is not designed for M2M cases. > > V4L2 is designed for video, radio, image, sdr, meta..., so I think audio can be > naturally added to the support scope. While I do not have an objection as such supporting this as part of V4L2, I do want to know if the ALSA maintainers think it is OK as well before I am going to spend time on this. In principle the V4L2 mem2mem framework doesn't really care what type of data is processed, it is just a matter of adding audio types (or reusing them from ALSA, which is presumably the intention here). Regards, Hans > > Thanks. > > Best regards > Shengjiu Wang > > > > > > > > The created audio device is named "/dev/audioX". > > > > Signed-off-by: Shengjiu Wang <shengjiu.wang@nxp.com <mailto:shengjiu.wang@nxp.com>> > > --- > > .../media/common/videobuf2/videobuf2-v4l2.c | 4 ++ > > drivers/media/v4l2-core/v4l2-dev.c | 17 ++++++ > > drivers/media/v4l2-core/v4l2-ioctl.c | 52 +++++++++++++++++++ > > include/media/v4l2-dev.h | 2 + > > include/media/v4l2-ioctl.h | 34 ++++++++++++ > > include/uapi/linux/videodev2.h | 19 +++++++ > > 6 files changed, 128 insertions(+) > > > > diff --git a/drivers/media/common/videobuf2/videobuf2-v4l2.c b/drivers/media/common/videobuf2/videobuf2-v4l2.c > > index c7a54d82a55e..12f2be2773a2 100644 > > --- a/drivers/media/common/videobuf2/videobuf2-v4l2.c > > +++ b/drivers/media/common/videobuf2/videobuf2-v4l2.c > > @@ -785,6 +785,10 @@ int vb2_create_bufs(struct vb2_queue *q, struct v4l2_create_buffers *create) > > case V4L2_BUF_TYPE_META_OUTPUT: > > requested_sizes[0] = f->fmt.meta.buffersize; > > break; > > + case V4L2_BUF_TYPE_AUDIO_CAPTURE: > > + case V4L2_BUF_TYPE_AUDIO_OUTPUT: > > + requested_sizes[0] = f->fmt.audio.buffersize; > > + break; > > default: > > return -EINVAL; > > } > > diff --git a/drivers/media/v4l2-core/v4l2-dev.c b/drivers/media/v4l2-core/v4l2-dev.c > > index f81279492682..67484f4c6eaf 100644 > > --- a/drivers/media/v4l2-core/v4l2-dev.c > > +++ b/drivers/media/v4l2-core/v4l2-dev.c > > @@ -553,6 +553,7 @@ static void determine_valid_ioctls(struct video_device *vdev) > > bool is_tch = vdev->vfl_type == VFL_TYPE_TOUCH; > > bool is_meta = vdev->vfl_type == VFL_TYPE_VIDEO && > > (vdev->device_caps & meta_caps); > > + bool is_audio = vdev->vfl_type == VFL_TYPE_AUDIO; > > bool is_rx = vdev->vfl_dir != VFL_DIR_TX; > > bool is_tx = vdev->vfl_dir != VFL_DIR_RX; > > bool is_io_mc = vdev->device_caps & V4L2_CAP_IO_MC; > > @@ -664,6 +665,19 @@ static void determine_valid_ioctls(struct video_device *vdev) > > SET_VALID_IOCTL(ops, VIDIOC_S_FMT, vidioc_s_fmt_meta_out); > > SET_VALID_IOCTL(ops, VIDIOC_TRY_FMT, vidioc_try_fmt_meta_out); > > } > > + if (is_audio && is_rx) { > > + /* audio capture specific ioctls */ > > + SET_VALID_IOCTL(ops, VIDIOC_ENUM_FMT, vidioc_enum_fmt_audio_cap); > > + SET_VALID_IOCTL(ops, VIDIOC_G_FMT, vidioc_g_fmt_audio_cap); > > + SET_VALID_IOCTL(ops, VIDIOC_S_FMT, vidioc_s_fmt_audio_cap); > > + SET_VALID_IOCTL(ops, VIDIOC_TRY_FMT, vidioc_try_fmt_audio_cap); > > + } else if (is_audio && is_tx) { > > + /* audio output specific ioctls */ > > + SET_VALID_IOCTL(ops, VIDIOC_ENUM_FMT, vidioc_enum_fmt_audio_out); > > + SET_VALID_IOCTL(ops, VIDIOC_G_FMT, vidioc_g_fmt_audio_out); > > + SET_VALID_IOCTL(ops, VIDIOC_S_FMT, vidioc_s_fmt_audio_out); > > + SET_VALID_IOCTL(ops, VIDIOC_TRY_FMT, vidioc_try_fmt_audio_out); > > + } > > if (is_vbi) { > > /* vbi specific ioctls */ > > if ((is_rx && (ops->vidioc_g_fmt_vbi_cap || > > @@ -927,6 +941,9 @@ int __video_register_device(struct video_device *vdev, > > case VFL_TYPE_TOUCH: > > name_base = "v4l-touch"; > > break; > > + case VFL_TYPE_AUDIO: > > + name_base = "audio"; > > + break; > > default: > > pr_err("%s called with unknown type: %d\n", > > __func__, type); > > diff --git a/drivers/media/v4l2-core/v4l2-ioctl.c b/drivers/media/v4l2-core/v4l2-ioctl.c > > index a858acea6547..26bc4b0d8ef0 100644 > > --- a/drivers/media/v4l2-core/v4l2-ioctl.c > > +++ b/drivers/media/v4l2-core/v4l2-ioctl.c > > @@ -188,6 +188,8 @@ const char *v4l2_type_names[] = { > > [V4L2_BUF_TYPE_SDR_OUTPUT] = "sdr-out", > > [V4L2_BUF_TYPE_META_CAPTURE] = "meta-cap", > > [V4L2_BUF_TYPE_META_OUTPUT] = "meta-out", > > + [V4L2_BUF_TYPE_AUDIO_CAPTURE] = "audio-cap", > > + [V4L2_BUF_TYPE_AUDIO_OUTPUT] = "audio-out", > > }; > > EXPORT_SYMBOL(v4l2_type_names); > > > > @@ -276,6 +278,7 @@ static void v4l_print_format(const void *arg, bool write_only) > > const struct v4l2_sliced_vbi_format *sliced; > > const struct v4l2_window *win; > > const struct v4l2_meta_format *meta; > > + const struct v4l2_audio_format *audio; > > u32 pixelformat; > > u32 planes; > > unsigned i; > > @@ -346,6 +349,12 @@ static void v4l_print_format(const void *arg, bool write_only) > > pr_cont(", dataformat=%p4cc, buffersize=%u\n", > > &pixelformat, meta->buffersize); > > break; > > + case V4L2_BUF_TYPE_AUDIO_CAPTURE: > > + case V4L2_BUF_TYPE_AUDIO_OUTPUT: > > + audio = &p->fmt.audio; > > + pr_cont(", rate=%u, format=%u, channels=%u, buffersize=%u\n", > > + audio->rate, audio->format, audio->channels, audio->buffersize); > > + break; > > } > > } > > > > @@ -927,6 +936,7 @@ static int check_fmt(struct file *file, enum v4l2_buf_type type) > > bool is_tch = vfd->vfl_type == VFL_TYPE_TOUCH; > > bool is_meta = vfd->vfl_type == VFL_TYPE_VIDEO && > > (vfd->device_caps & meta_caps); > > + bool is_audio = vfd->vfl_type == VFL_TYPE_AUDIO; > > bool is_rx = vfd->vfl_dir != VFL_DIR_TX; > > bool is_tx = vfd->vfl_dir != VFL_DIR_RX; > > > > @@ -992,6 +1002,14 @@ static int check_fmt(struct file *file, enum v4l2_buf_type type) > > if (is_meta && is_tx && ops->vidioc_g_fmt_meta_out) > > return 0; > > break; > > + case V4L2_BUF_TYPE_AUDIO_CAPTURE: > > + if (is_audio && is_rx && ops->vidioc_g_fmt_audio_cap) > > + return 0; > > + break; > > + case V4L2_BUF_TYPE_AUDIO_OUTPUT: > > + if (is_audio && is_tx && ops->vidioc_g_fmt_audio_out) > > + return 0; > > + break; > > default: > > break; > > } > > @@ -1592,6 +1610,16 @@ static int v4l_enum_fmt(const struct v4l2_ioctl_ops *ops, > > break; > > ret = ops->vidioc_enum_fmt_meta_out(file, fh, arg); > > break; > > + case V4L2_BUF_TYPE_AUDIO_CAPTURE: > > + if (unlikely(!ops->vidioc_enum_fmt_audio_cap)) > > + break; > > + ret = ops->vidioc_enum_fmt_audio_cap(file, fh, arg); > > + break; > > + case V4L2_BUF_TYPE_AUDIO_OUTPUT: > > + if (unlikely(!ops->vidioc_enum_fmt_audio_out)) > > + break; > > + ret = ops->vidioc_enum_fmt_audio_out(file, fh, arg); > > + break; > > } > > if (ret == 0) > > v4l_fill_fmtdesc(p); > > @@ -1668,6 +1696,10 @@ static int v4l_g_fmt(const struct v4l2_ioctl_ops *ops, > > return ops->vidioc_g_fmt_meta_cap(file, fh, arg); > > case V4L2_BUF_TYPE_META_OUTPUT: > > return ops->vidioc_g_fmt_meta_out(file, fh, arg); > > + case V4L2_BUF_TYPE_AUDIO_CAPTURE: > > + return ops->vidioc_g_fmt_audio_cap(file, fh, arg); > > + case V4L2_BUF_TYPE_AUDIO_OUTPUT: > > + return ops->vidioc_g_fmt_audio_out(file, fh, arg); > > } > > return -EINVAL; > > } > > @@ -1779,6 +1811,16 @@ static int v4l_s_fmt(const struct v4l2_ioctl_ops *ops, > > break; > > memset_after(p, 0, fmt.meta); > > return ops->vidioc_s_fmt_meta_out(file, fh, arg); > > + case V4L2_BUF_TYPE_AUDIO_CAPTURE: > > + if (unlikely(!ops->vidioc_s_fmt_audio_cap)) > > + break; > > + memset_after(p, 0, fmt.audio); > > + return ops->vidioc_s_fmt_audio_cap(file, fh, arg); > > + case V4L2_BUF_TYPE_AUDIO_OUTPUT: > > + if (unlikely(!ops->vidioc_s_fmt_audio_out)) > > + break; > > + memset_after(p, 0, fmt.audio); > > + return ops->vidioc_s_fmt_audio_out(file, fh, arg); > > } > > return -EINVAL; > > } > > @@ -1887,6 +1929,16 @@ static int v4l_try_fmt(const struct v4l2_ioctl_ops *ops, > > break; > > memset_after(p, 0, fmt.meta); > > return ops->vidioc_try_fmt_meta_out(file, fh, arg); > > + case V4L2_BUF_TYPE_AUDIO_CAPTURE: > > + if (unlikely(!ops->vidioc_try_fmt_audio_cap)) > > + break; > > + memset_after(p, 0, fmt.audio); > > + return ops->vidioc_try_fmt_audio_cap(file, fh, arg); > > + case V4L2_BUF_TYPE_AUDIO_OUTPUT: > > + if (unlikely(!ops->vidioc_try_fmt_audio_out)) > > + break; > > + memset_after(p, 0, fmt.audio); > > + return ops->vidioc_try_fmt_audio_out(file, fh, arg); > > } > > return -EINVAL; > > } > > diff --git a/include/media/v4l2-dev.h b/include/media/v4l2-dev.h > > index e0a13505f88d..0924e6d1dab1 100644 > > --- a/include/media/v4l2-dev.h > > +++ b/include/media/v4l2-dev.h > > @@ -30,6 +30,7 @@ > > * @VFL_TYPE_SUBDEV: for V4L2 subdevices > > * @VFL_TYPE_SDR: for Software Defined Radio tuners > > * @VFL_TYPE_TOUCH: for touch sensors > > + * @VFL_TYPE_AUDIO: for audio input/output devices > > * @VFL_TYPE_MAX: number of VFL types, must always be last in the enum > > */ > > enum vfl_devnode_type { > > @@ -39,6 +40,7 @@ enum vfl_devnode_type { > > VFL_TYPE_SUBDEV, > > VFL_TYPE_SDR, > > VFL_TYPE_TOUCH, > > + VFL_TYPE_AUDIO, > > VFL_TYPE_MAX /* Shall be the last one */ > > }; > > > > diff --git a/include/media/v4l2-ioctl.h b/include/media/v4l2-ioctl.h > > index edb733f21604..f840cf740ce1 100644 > > --- a/include/media/v4l2-ioctl.h > > +++ b/include/media/v4l2-ioctl.h > > @@ -45,6 +45,12 @@ struct v4l2_fh; > > * @vidioc_enum_fmt_meta_out: pointer to the function that implements > > * :ref:`VIDIOC_ENUM_FMT <vidioc_enum_fmt>` ioctl logic > > * for metadata output > > + * @vidioc_enum_fmt_audio_cap: pointer to the function that implements > > + * :ref:`VIDIOC_ENUM_FMT <vidioc_enum_fmt>` ioctl logic > > + * for audio capture > > + * @vidioc_enum_fmt_audio_out: pointer to the function that implements > > + * :ref:`VIDIOC_ENUM_FMT <vidioc_enum_fmt>` ioctl logic > > + * for audio output > > * @vidioc_g_fmt_vid_cap: pointer to the function that implements > > * :ref:`VIDIOC_G_FMT <vidioc_g_fmt>` ioctl logic for video capture > > * in single plane mode > > @@ -79,6 +85,10 @@ struct v4l2_fh; > > * :ref:`VIDIOC_G_FMT <vidioc_g_fmt>` ioctl logic for metadata capture > > * @vidioc_g_fmt_meta_out: pointer to the function that implements > > * :ref:`VIDIOC_G_FMT <vidioc_g_fmt>` ioctl logic for metadata output > > + * @vidioc_g_fmt_audio_cap: pointer to the function that implements > > + * :ref:`VIDIOC_G_FMT <vidioc_g_fmt>` ioctl logic for audio capture > > + * @vidioc_g_fmt_audio_out: pointer to the function that implements > > + * :ref:`VIDIOC_G_FMT <vidioc_g_fmt>` ioctl logic for audio output > > * @vidioc_s_fmt_vid_cap: pointer to the function that implements > > * :ref:`VIDIOC_S_FMT <vidioc_g_fmt>` ioctl logic for video capture > > * in single plane mode > > @@ -113,6 +123,10 @@ struct v4l2_fh; > > * :ref:`VIDIOC_S_FMT <vidioc_g_fmt>` ioctl logic for metadata capture > > * @vidioc_s_fmt_meta_out: pointer to the function that implements > > * :ref:`VIDIOC_S_FMT <vidioc_g_fmt>` ioctl logic for metadata output > > + * @vidioc_s_fmt_audio_cap: pointer to the function that implements > > + * :ref:`VIDIOC_S_FMT <vidioc_g_fmt>` ioctl logic for audio capture > > + * @vidioc_s_fmt_audio_out: pointer to the function that implements > > + * :ref:`VIDIOC_S_FMT <vidioc_g_fmt>` ioctl logic for audio output > > * @vidioc_try_fmt_vid_cap: pointer to the function that implements > > * :ref:`VIDIOC_TRY_FMT <vidioc_g_fmt>` ioctl logic for video capture > > * in single plane mode > > @@ -149,6 +163,10 @@ struct v4l2_fh; > > * :ref:`VIDIOC_TRY_FMT <vidioc_g_fmt>` ioctl logic for metadata capture > > * @vidioc_try_fmt_meta_out: pointer to the function that implements > > * :ref:`VIDIOC_TRY_FMT <vidioc_g_fmt>` ioctl logic for metadata output > > + * @vidioc_try_fmt_audio_cap: pointer to the function that implements > > + * :ref:`VIDIOC_TRY_FMT <vidioc_g_fmt>` ioctl logic for audio capture > > + * @vidioc_try_fmt_audio_out: pointer to the function that implements > > + * :ref:`VIDIOC_TRY_FMT <vidioc_g_fmt>` ioctl logic for audio output > > * @vidioc_reqbufs: pointer to the function that implements > > * :ref:`VIDIOC_REQBUFS <vidioc_reqbufs>` ioctl > > * @vidioc_querybuf: pointer to the function that implements > > @@ -315,6 +333,10 @@ struct v4l2_ioctl_ops { > > struct v4l2_fmtdesc *f); > > int (*vidioc_enum_fmt_meta_out)(struct file *file, void *fh, > > struct v4l2_fmtdesc *f); > > + int (*vidioc_enum_fmt_audio_cap)(struct file *file, void *fh, > > + struct v4l2_fmtdesc *f); > > + int (*vidioc_enum_fmt_audio_out)(struct file *file, void *fh, > > + struct v4l2_fmtdesc *f); > > > > /* VIDIOC_G_FMT handlers */ > > int (*vidioc_g_fmt_vid_cap)(struct file *file, void *fh, > > @@ -345,6 +367,10 @@ struct v4l2_ioctl_ops { > > struct v4l2_format *f); > > int (*vidioc_g_fmt_meta_out)(struct file *file, void *fh, > > struct v4l2_format *f); > > + int (*vidioc_g_fmt_audio_cap)(struct file *file, void *fh, > > + struct v4l2_format *f); > > + int (*vidioc_g_fmt_audio_out)(struct file *file, void *fh, > > + struct v4l2_format *f); > > > > /* VIDIOC_S_FMT handlers */ > > int (*vidioc_s_fmt_vid_cap)(struct file *file, void *fh, > > @@ -375,6 +401,10 @@ struct v4l2_ioctl_ops { > > struct v4l2_format *f); > > int (*vidioc_s_fmt_meta_out)(struct file *file, void *fh, > > struct v4l2_format *f); > > + int (*vidioc_s_fmt_audio_cap)(struct file *file, void *fh, > > + struct v4l2_format *f); > > + int (*vidioc_s_fmt_audio_out)(struct file *file, void *fh, > > + struct v4l2_format *f); > > > > /* VIDIOC_TRY_FMT handlers */ > > int (*vidioc_try_fmt_vid_cap)(struct file *file, void *fh, > > @@ -405,6 +435,10 @@ struct v4l2_ioctl_ops { > > struct v4l2_format *f); > > int (*vidioc_try_fmt_meta_out)(struct file *file, void *fh, > > struct v4l2_format *f); > > + int (*vidioc_try_fmt_audio_cap)(struct file *file, void *fh, > > + struct v4l2_format *f); > > + int (*vidioc_try_fmt_audio_out)(struct file *file, void *fh, > > + struct v4l2_format *f); > > > > /* Buffer handlers */ > > int (*vidioc_reqbufs)(struct file *file, void *fh, > > diff --git a/include/uapi/linux/videodev2.h b/include/uapi/linux/videodev2.h > > index aee75eb9e686..a7af28f4c8c3 100644 > > --- a/include/uapi/linux/videodev2.h > > +++ b/include/uapi/linux/videodev2.h > > @@ -153,6 +153,8 @@ enum v4l2_buf_type { > > V4L2_BUF_TYPE_SDR_OUTPUT = 12, > > V4L2_BUF_TYPE_META_CAPTURE = 13, > > V4L2_BUF_TYPE_META_OUTPUT = 14, > > + V4L2_BUF_TYPE_AUDIO_CAPTURE = 15, > > + V4L2_BUF_TYPE_AUDIO_OUTPUT = 16, > > /* Deprecated, do not use */ > > V4L2_BUF_TYPE_PRIVATE = 0x80, > > }; > > @@ -169,6 +171,7 @@ enum v4l2_buf_type { > > || (type) == V4L2_BUF_TYPE_VBI_OUTPUT \ > > || (type) == V4L2_BUF_TYPE_SLICED_VBI_OUTPUT \ > > || (type) == V4L2_BUF_TYPE_SDR_OUTPUT \ > > + || (type) == V4L2_BUF_TYPE_AUDIO_OUTPUT \ > > || (type) == V4L2_BUF_TYPE_META_OUTPUT) > > > > #define V4L2_TYPE_IS_CAPTURE(type) (!V4L2_TYPE_IS_OUTPUT(type)) > > @@ -2404,6 +2407,20 @@ struct v4l2_meta_format { > > __u32 buffersize; > > } __attribute__ ((packed)); > > > > +/** > > + * struct v4l2_audio_format - audio data format definition > > + * @rate: sample rate > > + * @format: sample format > > + * @channels: channel numbers > > + * @buffersize: maximum size in bytes required for data > > + */ > > +struct v4l2_audio_format { > > + __u32 rate; > > + __u32 format; > > + __u32 channels; > > + __u32 buffersize; > > +} __attribute__ ((packed)); > > + > > /** > > * struct v4l2_format - stream data format > > * @type: enum v4l2_buf_type; type of the data stream > > @@ -2412,6 +2429,7 @@ struct v4l2_meta_format { > > * @win: definition of an overlaid image > > * @vbi: raw VBI capture or output parameters > > * @sliced: sliced VBI capture or output parameters > > + * @audio: definition of an audio format > > * @raw_data: placeholder for future extensions and custom formats > > * @fmt: union of @pix, @pix_mp, @win, @vbi, @sliced, @sdr, @meta > > * and @raw_data > > @@ -2426,6 +2444,7 @@ struct v4l2_format { > > struct v4l2_sliced_vbi_format sliced; /* V4L2_BUF_TYPE_SLICED_VBI_CAPTURE */ > > struct v4l2_sdr_format sdr; /* V4L2_BUF_TYPE_SDR_CAPTURE */ > > struct v4l2_meta_format meta; /* V4L2_BUF_TYPE_META_CAPTURE */ > > + struct v4l2_audio_format audio; /* V4L2_BUF_TYPE_AUDIO_CAPTURE */ > > __u8 raw_data[200]; /* user-defined */ > > } fmt; > > }; > > -- > > 2.34.1 > > > > -- > Sakari Ailus >
On Mon, 03 Jul 2023 11:54:22 +0200, Shengjiu Wang wrote: > > > Hi Sakari > > On Fri, Jun 30, 2023 at 6:05 PM Sakari Ailus <sakari.ailus@iki.fi> wrote: > > Hi Shengjiu, > > On Thu, Jun 29, 2023 at 09:37:48AM +0800, Shengjiu Wang wrote: > > Audio signal processing has the requirement for memory to > > memory similar as Video. > > > > This patch is to add this support in v4l2 framework, defined > > new buffer type V4L2_BUF_TYPE_AUDIO_CAPTURE and > > V4L2_BUF_TYPE_AUDIO_OUTPUT, defined new format v4l2_audio_format > > for audio case usage. > > Why are you proposing to add this to V4L2 framework instead of doing this > within ALSA? > > Also cc Hans and Jacopo. > > There is no such memory to memory interface defined in ALSA. Seems > ALSA is not designed for M2M cases. There is no restriction to implement memory-to-memory capture in ALSA framework. It'd be a matter of the setup of PCM capture source, and you can create a corresponding kcontrol element to switch the mode or assign a dedicated PCM substream, for example. It's just that there was little demand for that. I'm not much against adding the audio capture feature to V4L2, though, if it really makes sense. But creating a crafted /dev/audio* doesn't look like a great idea to me, at least. thanks, Takashi
On Mon, Jul 03, 2023 at 02:07:10PM +0200, Takashi Iwai wrote: > Shengjiu Wang wrote: > > There is no such memory to memory interface defined in ALSA. Seems > > ALSA is not designed for M2M cases. > There is no restriction to implement memory-to-memory capture in ALSA > framework. It'd be a matter of the setup of PCM capture source, and > you can create a corresponding kcontrol element to switch the mode or > assign a dedicated PCM substream, for example. It's just that there > was little demand for that. Yeah, it's not a terrible idea. We might use it more if we ever get better support for DSP audio, routing between the DSP and external devices if driven from the CPU would be a memory to memory thing. > I'm not much against adding the audio capture feature to V4L2, > though, if it really makes sense. But creating a crafted /dev/audio* > doesn't look like a great idea to me, at least. I've still not looked at the code at all.
On 03/07/2023 14:53, Mark Brown wrote: > On Mon, Jul 03, 2023 at 02:07:10PM +0200, Takashi Iwai wrote: >> Shengjiu Wang wrote: > >>> There is no such memory to memory interface defined in ALSA. Seems >>> ALSA is not designed for M2M cases. > >> There is no restriction to implement memory-to-memory capture in ALSA >> framework. It'd be a matter of the setup of PCM capture source, and >> you can create a corresponding kcontrol element to switch the mode or >> assign a dedicated PCM substream, for example. It's just that there >> was little demand for that. > > Yeah, it's not a terrible idea. We might use it more if we ever get > better support for DSP audio, routing between the DSP and external > devices if driven from the CPU would be a memory to memory thing. > >> I'm not much against adding the audio capture feature to V4L2, >> though, if it really makes sense. But creating a crafted /dev/audio* >> doesn't look like a great idea to me, at least. > > I've still not looked at the code at all. My main concern is that these cross-subsystem drivers are a pain to maintain. So there have to be good reasons to do this. Also it is kind of weird to have to use the V4L2 API in userspace to deal with a specific audio conversion. Quite unexpected. But in the end, that's a decision I can't make. So I wait for that feedback. Note that if the decision is made that this can use V4L2, then there is quite a lot more that needs to be done: documentation, new compliance tests, etc. It's adding a new API, and that comes with additional work... Regards, Hans
On Mon, 03 Jul 2023 15:12:55 +0200, Hans Verkuil wrote: > > On 03/07/2023 14:53, Mark Brown wrote: > > On Mon, Jul 03, 2023 at 02:07:10PM +0200, Takashi Iwai wrote: > >> Shengjiu Wang wrote: > > > >>> There is no such memory to memory interface defined in ALSA. Seems > >>> ALSA is not designed for M2M cases. > > > >> There is no restriction to implement memory-to-memory capture in ALSA > >> framework. It'd be a matter of the setup of PCM capture source, and > >> you can create a corresponding kcontrol element to switch the mode or > >> assign a dedicated PCM substream, for example. It's just that there > >> was little demand for that. > > > > Yeah, it's not a terrible idea. We might use it more if we ever get > > better support for DSP audio, routing between the DSP and external > > devices if driven from the CPU would be a memory to memory thing. > > > >> I'm not much against adding the audio capture feature to V4L2, > >> though, if it really makes sense. But creating a crafted /dev/audio* > >> doesn't look like a great idea to me, at least. > > > > I've still not looked at the code at all. > > My main concern is that these cross-subsystem drivers are a pain to > maintain. So there have to be good reasons to do this. > > Also it is kind of weird to have to use the V4L2 API in userspace to > deal with a specific audio conversion. Quite unexpected. > > But in the end, that's a decision I can't make. > > So I wait for that feedback. Note that if the decision is made that this > can use V4L2, then there is quite a lot more that needs to be done: > documentation, new compliance tests, etc. It's adding a new API, and that > comes with additional work... All agreed. Especially in this case, it doesn't have to be in V4L2 API, as it seems. (Though, the support of audio on V4L2 might be useful if it's closely tied with the a stream. But that's another story.) thanks, Takashi
On Mon, Jul 03, 2023 at 03:12:55PM +0200, Hans Verkuil wrote: > My main concern is that these cross-subsystem drivers are a pain to > maintain. So there have to be good reasons to do this. > Also it is kind of weird to have to use the V4L2 API in userspace to > deal with a specific audio conversion. Quite unexpected. > But in the end, that's a decision I can't make. > So I wait for that feedback. Note that if the decision is made that this > can use V4L2, then there is quite a lot more that needs to be done: > documentation, new compliance tests, etc. It's adding a new API, and that > comes with additional work... Absolutely, I agree with all of this - my impression was that the target here would be bypass of audio streams to/from a v4l2 device, without bouncing through an application layer. If it's purely for audio usage with no other tie to v4l2 then involving v4l2 does just seem like complication.
On Mon, Jul 3, 2023 at 9:25 PM Takashi Iwai <tiwai@suse.de> wrote: > On Mon, 03 Jul 2023 15:12:55 +0200, > Hans Verkuil wrote: > > > > On 03/07/2023 14:53, Mark Brown wrote: > > > On Mon, Jul 03, 2023 at 02:07:10PM +0200, Takashi Iwai wrote: > > >> Shengjiu Wang wrote: > > > > > >>> There is no such memory to memory interface defined in ALSA. Seems > > >>> ALSA is not designed for M2M cases. > > > > > >> There is no restriction to implement memory-to-memory capture in ALSA > > >> framework. It'd be a matter of the setup of PCM capture source, and > > >> you can create a corresponding kcontrol element to switch the mode or > > >> assign a dedicated PCM substream, for example. It's just that there > > >> was little demand for that. > > > > > > Yeah, it's not a terrible idea. We might use it more if we ever get > > > better support for DSP audio, routing between the DSP and external > > > devices if driven from the CPU would be a memory to memory thing. > > > > > >> I'm not much against adding the audio capture feature to V4L2, > > >> though, if it really makes sense. But creating a crafted /dev/audio* > > >> doesn't look like a great idea to me, at least. > > > > > > I've still not looked at the code at all. > > > > My main concern is that these cross-subsystem drivers are a pain to > > maintain. So there have to be good reasons to do this. > > > > Also it is kind of weird to have to use the V4L2 API in userspace to > > deal with a specific audio conversion. Quite unexpected. > > > > But in the end, that's a decision I can't make. > > > > So I wait for that feedback. Note that if the decision is made that this > > can use V4L2, then there is quite a lot more that needs to be done: > > documentation, new compliance tests, etc. It's adding a new API, and that > > comes with additional work... > > All agreed. Especially in this case, it doesn't have to be in V4L2 > API, as it seems. > > (Though, the support of audio on V4L2 might be useful if it's closely > tied with the a stream. But that's another story.) > audio is a stream, for this m2m audio case, V4L2 is the best choice I found. I know there is API change for V4L2, but V4L2 is a good framework for memory to memory, I think it is worth to do this change. if implement this M2M case in ALSA, we may need to create a sound card and open it twice for playback and capture, it is complicated to do this, and I am not sure if there is any other issue besides these. I can't find an example in the ALSA framework for this case. best regards wang shengjiu > > > thanks, > > Takashi >
On Tue, Jul 4, 2023 at 1:59 AM Mark Brown <broonie@kernel.org> wrote: > On Mon, Jul 03, 2023 at 03:12:55PM +0200, Hans Verkuil wrote: > > > My main concern is that these cross-subsystem drivers are a pain to > > maintain. So there have to be good reasons to do this. > > > Also it is kind of weird to have to use the V4L2 API in userspace to > > deal with a specific audio conversion. Quite unexpected. > > > But in the end, that's a decision I can't make. > > > So I wait for that feedback. Note that if the decision is made that this > > can use V4L2, then there is quite a lot more that needs to be done: > > documentation, new compliance tests, etc. It's adding a new API, and that > > comes with additional work... > > Absolutely, I agree with all of this - my impression was that the target > here would be bypass of audio streams to/from a v4l2 device, without > bouncing through an application layer. If it's purely for audio usage > with no other tie to v4l2 then involving v4l2 does just seem like > complication. > This audio use case is using the v4l2 application layer. in the user space I need to call below v4l2 ioctls to implement the feature: VIDIOC_QUERYCAP VIDIOC_TRY_FMT VIDIOC_S_FMT VIDIOC_REQBUFS VIDIOC_QUERYBUF VIDIOC_STREAMON VIDIOC_QBUF VIDIOC_DQBUF VIDIOC_STREAMOFF why the driver was put in the ALSA, because previously we implemented the ASRC M2P (memory to peripheral) in ALSA, so I think it is better to add M2M driver in ALSA. The hardware IP is the same. The compatible string is the same. Best regards Wang Shengjiu
Hi Mark On Tue, Jul 4, 2023 at 12:03 PM Shengjiu Wang <shengjiu.wang@gmail.com> wrote: > > > On Tue, Jul 4, 2023 at 1:59 AM Mark Brown <broonie@kernel.org> wrote: > >> On Mon, Jul 03, 2023 at 03:12:55PM +0200, Hans Verkuil wrote: >> >> > My main concern is that these cross-subsystem drivers are a pain to >> > maintain. So there have to be good reasons to do this. >> >> > Also it is kind of weird to have to use the V4L2 API in userspace to >> > deal with a specific audio conversion. Quite unexpected. >> >> > But in the end, that's a decision I can't make. >> >> > So I wait for that feedback. Note that if the decision is made that this >> > can use V4L2, then there is quite a lot more that needs to be done: >> > documentation, new compliance tests, etc. It's adding a new API, and >> that >> > comes with additional work... >> >> Absolutely, I agree with all of this - my impression was that the target >> here would be bypass of audio streams to/from a v4l2 device, without >> bouncing through an application layer. If it's purely for audio usage >> with no other tie to v4l2 then involving v4l2 does just seem like >> complication. >> > > This audio use case is using the v4l2 application layer. in the user space > I need to call below v4l2 ioctls to implement the feature: > VIDIOC_QUERYCAP > VIDIOC_TRY_FMT > VIDIOC_S_FMT > VIDIOC_REQBUFS > VIDIOC_QUERYBUF > VIDIOC_STREAMON > VIDIOC_QBUF > VIDIOC_DQBUF > VIDIOC_STREAMOFF > > why the driver was put in the ALSA, because previously we implemented > the ASRC M2P (memory to peripheral) in ALSA, so I think it is better to > add M2M driver in ALSA. The hardware IP is the same. The compatible > string is the same. > > > Could you please share more of your ideas about this patch? and could you please check further about this implementation. I tried to find a good interface in ALSA for this m2m request, but didn't find one, then I try the V4L2, find it is good this audio case. but it needs to extend the V4L2 API. I have no idea how to go on, could you please recommend? best regards wang shengjiu > >
Hi Mark On Fri, Jul 7, 2023 at 11:13 AM Shengjiu Wang <shengjiu.wang@gmail.com> wrote: > Hi Mark > > On Tue, Jul 4, 2023 at 12:03 PM Shengjiu Wang <shengjiu.wang@gmail.com> > wrote: > >> >> >> On Tue, Jul 4, 2023 at 1:59 AM Mark Brown <broonie@kernel.org> wrote: >> >>> On Mon, Jul 03, 2023 at 03:12:55PM +0200, Hans Verkuil wrote: >>> >>> > My main concern is that these cross-subsystem drivers are a pain to >>> > maintain. So there have to be good reasons to do this. >>> >>> > Also it is kind of weird to have to use the V4L2 API in userspace to >>> > deal with a specific audio conversion. Quite unexpected. >>> >>> > But in the end, that's a decision I can't make. >>> >>> > So I wait for that feedback. Note that if the decision is made that >>> this >>> > can use V4L2, then there is quite a lot more that needs to be done: >>> > documentation, new compliance tests, etc. It's adding a new API, and >>> that >>> > comes with additional work... >>> >>> Absolutely, I agree with all of this - my impression was that the target >>> here would be bypass of audio streams to/from a v4l2 device, without >>> bouncing through an application layer. If it's purely for audio usage >>> with no other tie to v4l2 then involving v4l2 does just seem like >>> complication. >>> >> >> This audio use case is using the v4l2 application layer. in the user space >> I need to call below v4l2 ioctls to implement the feature: >> VIDIOC_QUERYCAP >> VIDIOC_TRY_FMT >> VIDIOC_S_FMT >> VIDIOC_REQBUFS >> VIDIOC_QUERYBUF >> VIDIOC_STREAMON >> VIDIOC_QBUF >> VIDIOC_DQBUF >> VIDIOC_STREAMOFF >> >> why the driver was put in the ALSA, because previously we implemented >> the ASRC M2P (memory to peripheral) in ALSA, so I think it is better to >> add M2M driver in ALSA. The hardware IP is the same. The compatible >> string is the same. >> >> >> Could you please share more of your ideas about this patch? and could > you please check further about this implementation. > > I tried to find a good interface in ALSA for this m2m request, but didn't > find one, then I try the V4L2, find it is good this audio case. > > but it needs to extend the V4L2 API. > > I have no idea how to go on, could you please recommend? > > Should I implement the asrc m2m driver as a separate v4l2 driver? And move it to the /driver/media folder ? In ALSA part, just need register the platform device. The bridge between ALSA and V4L2 framework can be the header file in /include/sound/ Does it sound better? Best regards Wang Shengjiu
diff --git a/drivers/media/common/videobuf2/videobuf2-v4l2.c b/drivers/media/common/videobuf2/videobuf2-v4l2.c index c7a54d82a55e..12f2be2773a2 100644 --- a/drivers/media/common/videobuf2/videobuf2-v4l2.c +++ b/drivers/media/common/videobuf2/videobuf2-v4l2.c @@ -785,6 +785,10 @@ int vb2_create_bufs(struct vb2_queue *q, struct v4l2_create_buffers *create) case V4L2_BUF_TYPE_META_OUTPUT: requested_sizes[0] = f->fmt.meta.buffersize; break; + case V4L2_BUF_TYPE_AUDIO_CAPTURE: + case V4L2_BUF_TYPE_AUDIO_OUTPUT: + requested_sizes[0] = f->fmt.audio.buffersize; + break; default: return -EINVAL; } diff --git a/drivers/media/v4l2-core/v4l2-dev.c b/drivers/media/v4l2-core/v4l2-dev.c index f81279492682..67484f4c6eaf 100644 --- a/drivers/media/v4l2-core/v4l2-dev.c +++ b/drivers/media/v4l2-core/v4l2-dev.c @@ -553,6 +553,7 @@ static void determine_valid_ioctls(struct video_device *vdev) bool is_tch = vdev->vfl_type == VFL_TYPE_TOUCH; bool is_meta = vdev->vfl_type == VFL_TYPE_VIDEO && (vdev->device_caps & meta_caps); + bool is_audio = vdev->vfl_type == VFL_TYPE_AUDIO; bool is_rx = vdev->vfl_dir != VFL_DIR_TX; bool is_tx = vdev->vfl_dir != VFL_DIR_RX; bool is_io_mc = vdev->device_caps & V4L2_CAP_IO_MC; @@ -664,6 +665,19 @@ static void determine_valid_ioctls(struct video_device *vdev) SET_VALID_IOCTL(ops, VIDIOC_S_FMT, vidioc_s_fmt_meta_out); SET_VALID_IOCTL(ops, VIDIOC_TRY_FMT, vidioc_try_fmt_meta_out); } + if (is_audio && is_rx) { + /* audio capture specific ioctls */ + SET_VALID_IOCTL(ops, VIDIOC_ENUM_FMT, vidioc_enum_fmt_audio_cap); + SET_VALID_IOCTL(ops, VIDIOC_G_FMT, vidioc_g_fmt_audio_cap); + SET_VALID_IOCTL(ops, VIDIOC_S_FMT, vidioc_s_fmt_audio_cap); + SET_VALID_IOCTL(ops, VIDIOC_TRY_FMT, vidioc_try_fmt_audio_cap); + } else if (is_audio && is_tx) { + /* audio output specific ioctls */ + SET_VALID_IOCTL(ops, VIDIOC_ENUM_FMT, vidioc_enum_fmt_audio_out); + SET_VALID_IOCTL(ops, VIDIOC_G_FMT, vidioc_g_fmt_audio_out); + SET_VALID_IOCTL(ops, VIDIOC_S_FMT, vidioc_s_fmt_audio_out); + SET_VALID_IOCTL(ops, VIDIOC_TRY_FMT, vidioc_try_fmt_audio_out); + } if (is_vbi) { /* vbi specific ioctls */ if ((is_rx && (ops->vidioc_g_fmt_vbi_cap || @@ -927,6 +941,9 @@ int __video_register_device(struct video_device *vdev, case VFL_TYPE_TOUCH: name_base = "v4l-touch"; break; + case VFL_TYPE_AUDIO: + name_base = "audio"; + break; default: pr_err("%s called with unknown type: %d\n", __func__, type); diff --git a/drivers/media/v4l2-core/v4l2-ioctl.c b/drivers/media/v4l2-core/v4l2-ioctl.c index a858acea6547..26bc4b0d8ef0 100644 --- a/drivers/media/v4l2-core/v4l2-ioctl.c +++ b/drivers/media/v4l2-core/v4l2-ioctl.c @@ -188,6 +188,8 @@ const char *v4l2_type_names[] = { [V4L2_BUF_TYPE_SDR_OUTPUT] = "sdr-out", [V4L2_BUF_TYPE_META_CAPTURE] = "meta-cap", [V4L2_BUF_TYPE_META_OUTPUT] = "meta-out", + [V4L2_BUF_TYPE_AUDIO_CAPTURE] = "audio-cap", + [V4L2_BUF_TYPE_AUDIO_OUTPUT] = "audio-out", }; EXPORT_SYMBOL(v4l2_type_names); @@ -276,6 +278,7 @@ static void v4l_print_format(const void *arg, bool write_only) const struct v4l2_sliced_vbi_format *sliced; const struct v4l2_window *win; const struct v4l2_meta_format *meta; + const struct v4l2_audio_format *audio; u32 pixelformat; u32 planes; unsigned i; @@ -346,6 +349,12 @@ static void v4l_print_format(const void *arg, bool write_only) pr_cont(", dataformat=%p4cc, buffersize=%u\n", &pixelformat, meta->buffersize); break; + case V4L2_BUF_TYPE_AUDIO_CAPTURE: + case V4L2_BUF_TYPE_AUDIO_OUTPUT: + audio = &p->fmt.audio; + pr_cont(", rate=%u, format=%u, channels=%u, buffersize=%u\n", + audio->rate, audio->format, audio->channels, audio->buffersize); + break; } } @@ -927,6 +936,7 @@ static int check_fmt(struct file *file, enum v4l2_buf_type type) bool is_tch = vfd->vfl_type == VFL_TYPE_TOUCH; bool is_meta = vfd->vfl_type == VFL_TYPE_VIDEO && (vfd->device_caps & meta_caps); + bool is_audio = vfd->vfl_type == VFL_TYPE_AUDIO; bool is_rx = vfd->vfl_dir != VFL_DIR_TX; bool is_tx = vfd->vfl_dir != VFL_DIR_RX; @@ -992,6 +1002,14 @@ static int check_fmt(struct file *file, enum v4l2_buf_type type) if (is_meta && is_tx && ops->vidioc_g_fmt_meta_out) return 0; break; + case V4L2_BUF_TYPE_AUDIO_CAPTURE: + if (is_audio && is_rx && ops->vidioc_g_fmt_audio_cap) + return 0; + break; + case V4L2_BUF_TYPE_AUDIO_OUTPUT: + if (is_audio && is_tx && ops->vidioc_g_fmt_audio_out) + return 0; + break; default: break; } @@ -1592,6 +1610,16 @@ static int v4l_enum_fmt(const struct v4l2_ioctl_ops *ops, break; ret = ops->vidioc_enum_fmt_meta_out(file, fh, arg); break; + case V4L2_BUF_TYPE_AUDIO_CAPTURE: + if (unlikely(!ops->vidioc_enum_fmt_audio_cap)) + break; + ret = ops->vidioc_enum_fmt_audio_cap(file, fh, arg); + break; + case V4L2_BUF_TYPE_AUDIO_OUTPUT: + if (unlikely(!ops->vidioc_enum_fmt_audio_out)) + break; + ret = ops->vidioc_enum_fmt_audio_out(file, fh, arg); + break; } if (ret == 0) v4l_fill_fmtdesc(p); @@ -1668,6 +1696,10 @@ static int v4l_g_fmt(const struct v4l2_ioctl_ops *ops, return ops->vidioc_g_fmt_meta_cap(file, fh, arg); case V4L2_BUF_TYPE_META_OUTPUT: return ops->vidioc_g_fmt_meta_out(file, fh, arg); + case V4L2_BUF_TYPE_AUDIO_CAPTURE: + return ops->vidioc_g_fmt_audio_cap(file, fh, arg); + case V4L2_BUF_TYPE_AUDIO_OUTPUT: + return ops->vidioc_g_fmt_audio_out(file, fh, arg); } return -EINVAL; } @@ -1779,6 +1811,16 @@ static int v4l_s_fmt(const struct v4l2_ioctl_ops *ops, break; memset_after(p, 0, fmt.meta); return ops->vidioc_s_fmt_meta_out(file, fh, arg); + case V4L2_BUF_TYPE_AUDIO_CAPTURE: + if (unlikely(!ops->vidioc_s_fmt_audio_cap)) + break; + memset_after(p, 0, fmt.audio); + return ops->vidioc_s_fmt_audio_cap(file, fh, arg); + case V4L2_BUF_TYPE_AUDIO_OUTPUT: + if (unlikely(!ops->vidioc_s_fmt_audio_out)) + break; + memset_after(p, 0, fmt.audio); + return ops->vidioc_s_fmt_audio_out(file, fh, arg); } return -EINVAL; } @@ -1887,6 +1929,16 @@ static int v4l_try_fmt(const struct v4l2_ioctl_ops *ops, break; memset_after(p, 0, fmt.meta); return ops->vidioc_try_fmt_meta_out(file, fh, arg); + case V4L2_BUF_TYPE_AUDIO_CAPTURE: + if (unlikely(!ops->vidioc_try_fmt_audio_cap)) + break; + memset_after(p, 0, fmt.audio); + return ops->vidioc_try_fmt_audio_cap(file, fh, arg); + case V4L2_BUF_TYPE_AUDIO_OUTPUT: + if (unlikely(!ops->vidioc_try_fmt_audio_out)) + break; + memset_after(p, 0, fmt.audio); + return ops->vidioc_try_fmt_audio_out(file, fh, arg); } return -EINVAL; } diff --git a/include/media/v4l2-dev.h b/include/media/v4l2-dev.h index e0a13505f88d..0924e6d1dab1 100644 --- a/include/media/v4l2-dev.h +++ b/include/media/v4l2-dev.h @@ -30,6 +30,7 @@ * @VFL_TYPE_SUBDEV: for V4L2 subdevices * @VFL_TYPE_SDR: for Software Defined Radio tuners * @VFL_TYPE_TOUCH: for touch sensors + * @VFL_TYPE_AUDIO: for audio input/output devices * @VFL_TYPE_MAX: number of VFL types, must always be last in the enum */ enum vfl_devnode_type { @@ -39,6 +40,7 @@ enum vfl_devnode_type { VFL_TYPE_SUBDEV, VFL_TYPE_SDR, VFL_TYPE_TOUCH, + VFL_TYPE_AUDIO, VFL_TYPE_MAX /* Shall be the last one */ }; diff --git a/include/media/v4l2-ioctl.h b/include/media/v4l2-ioctl.h index edb733f21604..f840cf740ce1 100644 --- a/include/media/v4l2-ioctl.h +++ b/include/media/v4l2-ioctl.h @@ -45,6 +45,12 @@ struct v4l2_fh; * @vidioc_enum_fmt_meta_out: pointer to the function that implements * :ref:`VIDIOC_ENUM_FMT <vidioc_enum_fmt>` ioctl logic * for metadata output + * @vidioc_enum_fmt_audio_cap: pointer to the function that implements + * :ref:`VIDIOC_ENUM_FMT <vidioc_enum_fmt>` ioctl logic + * for audio capture + * @vidioc_enum_fmt_audio_out: pointer to the function that implements + * :ref:`VIDIOC_ENUM_FMT <vidioc_enum_fmt>` ioctl logic + * for audio output * @vidioc_g_fmt_vid_cap: pointer to the function that implements * :ref:`VIDIOC_G_FMT <vidioc_g_fmt>` ioctl logic for video capture * in single plane mode @@ -79,6 +85,10 @@ struct v4l2_fh; * :ref:`VIDIOC_G_FMT <vidioc_g_fmt>` ioctl logic for metadata capture * @vidioc_g_fmt_meta_out: pointer to the function that implements * :ref:`VIDIOC_G_FMT <vidioc_g_fmt>` ioctl logic for metadata output + * @vidioc_g_fmt_audio_cap: pointer to the function that implements + * :ref:`VIDIOC_G_FMT <vidioc_g_fmt>` ioctl logic for audio capture + * @vidioc_g_fmt_audio_out: pointer to the function that implements + * :ref:`VIDIOC_G_FMT <vidioc_g_fmt>` ioctl logic for audio output * @vidioc_s_fmt_vid_cap: pointer to the function that implements * :ref:`VIDIOC_S_FMT <vidioc_g_fmt>` ioctl logic for video capture * in single plane mode @@ -113,6 +123,10 @@ struct v4l2_fh; * :ref:`VIDIOC_S_FMT <vidioc_g_fmt>` ioctl logic for metadata capture * @vidioc_s_fmt_meta_out: pointer to the function that implements * :ref:`VIDIOC_S_FMT <vidioc_g_fmt>` ioctl logic for metadata output + * @vidioc_s_fmt_audio_cap: pointer to the function that implements + * :ref:`VIDIOC_S_FMT <vidioc_g_fmt>` ioctl logic for audio capture + * @vidioc_s_fmt_audio_out: pointer to the function that implements + * :ref:`VIDIOC_S_FMT <vidioc_g_fmt>` ioctl logic for audio output * @vidioc_try_fmt_vid_cap: pointer to the function that implements * :ref:`VIDIOC_TRY_FMT <vidioc_g_fmt>` ioctl logic for video capture * in single plane mode @@ -149,6 +163,10 @@ struct v4l2_fh; * :ref:`VIDIOC_TRY_FMT <vidioc_g_fmt>` ioctl logic for metadata capture * @vidioc_try_fmt_meta_out: pointer to the function that implements * :ref:`VIDIOC_TRY_FMT <vidioc_g_fmt>` ioctl logic for metadata output + * @vidioc_try_fmt_audio_cap: pointer to the function that implements + * :ref:`VIDIOC_TRY_FMT <vidioc_g_fmt>` ioctl logic for audio capture + * @vidioc_try_fmt_audio_out: pointer to the function that implements + * :ref:`VIDIOC_TRY_FMT <vidioc_g_fmt>` ioctl logic for audio output * @vidioc_reqbufs: pointer to the function that implements * :ref:`VIDIOC_REQBUFS <vidioc_reqbufs>` ioctl * @vidioc_querybuf: pointer to the function that implements @@ -315,6 +333,10 @@ struct v4l2_ioctl_ops { struct v4l2_fmtdesc *f); int (*vidioc_enum_fmt_meta_out)(struct file *file, void *fh, struct v4l2_fmtdesc *f); + int (*vidioc_enum_fmt_audio_cap)(struct file *file, void *fh, + struct v4l2_fmtdesc *f); + int (*vidioc_enum_fmt_audio_out)(struct file *file, void *fh, + struct v4l2_fmtdesc *f); /* VIDIOC_G_FMT handlers */ int (*vidioc_g_fmt_vid_cap)(struct file *file, void *fh, @@ -345,6 +367,10 @@ struct v4l2_ioctl_ops { struct v4l2_format *f); int (*vidioc_g_fmt_meta_out)(struct file *file, void *fh, struct v4l2_format *f); + int (*vidioc_g_fmt_audio_cap)(struct file *file, void *fh, + struct v4l2_format *f); + int (*vidioc_g_fmt_audio_out)(struct file *file, void *fh, + struct v4l2_format *f); /* VIDIOC_S_FMT handlers */ int (*vidioc_s_fmt_vid_cap)(struct file *file, void *fh, @@ -375,6 +401,10 @@ struct v4l2_ioctl_ops { struct v4l2_format *f); int (*vidioc_s_fmt_meta_out)(struct file *file, void *fh, struct v4l2_format *f); + int (*vidioc_s_fmt_audio_cap)(struct file *file, void *fh, + struct v4l2_format *f); + int (*vidioc_s_fmt_audio_out)(struct file *file, void *fh, + struct v4l2_format *f); /* VIDIOC_TRY_FMT handlers */ int (*vidioc_try_fmt_vid_cap)(struct file *file, void *fh, @@ -405,6 +435,10 @@ struct v4l2_ioctl_ops { struct v4l2_format *f); int (*vidioc_try_fmt_meta_out)(struct file *file, void *fh, struct v4l2_format *f); + int (*vidioc_try_fmt_audio_cap)(struct file *file, void *fh, + struct v4l2_format *f); + int (*vidioc_try_fmt_audio_out)(struct file *file, void *fh, + struct v4l2_format *f); /* Buffer handlers */ int (*vidioc_reqbufs)(struct file *file, void *fh, diff --git a/include/uapi/linux/videodev2.h b/include/uapi/linux/videodev2.h index aee75eb9e686..a7af28f4c8c3 100644 --- a/include/uapi/linux/videodev2.h +++ b/include/uapi/linux/videodev2.h @@ -153,6 +153,8 @@ enum v4l2_buf_type { V4L2_BUF_TYPE_SDR_OUTPUT = 12, V4L2_BUF_TYPE_META_CAPTURE = 13, V4L2_BUF_TYPE_META_OUTPUT = 14, + V4L2_BUF_TYPE_AUDIO_CAPTURE = 15, + V4L2_BUF_TYPE_AUDIO_OUTPUT = 16, /* Deprecated, do not use */ V4L2_BUF_TYPE_PRIVATE = 0x80, }; @@ -169,6 +171,7 @@ enum v4l2_buf_type { || (type) == V4L2_BUF_TYPE_VBI_OUTPUT \ || (type) == V4L2_BUF_TYPE_SLICED_VBI_OUTPUT \ || (type) == V4L2_BUF_TYPE_SDR_OUTPUT \ + || (type) == V4L2_BUF_TYPE_AUDIO_OUTPUT \ || (type) == V4L2_BUF_TYPE_META_OUTPUT) #define V4L2_TYPE_IS_CAPTURE(type) (!V4L2_TYPE_IS_OUTPUT(type)) @@ -2404,6 +2407,20 @@ struct v4l2_meta_format { __u32 buffersize; } __attribute__ ((packed)); +/** + * struct v4l2_audio_format - audio data format definition + * @rate: sample rate + * @format: sample format + * @channels: channel numbers + * @buffersize: maximum size in bytes required for data + */ +struct v4l2_audio_format { + __u32 rate; + __u32 format; + __u32 channels; + __u32 buffersize; +} __attribute__ ((packed)); + /** * struct v4l2_format - stream data format * @type: enum v4l2_buf_type; type of the data stream @@ -2412,6 +2429,7 @@ struct v4l2_meta_format { * @win: definition of an overlaid image * @vbi: raw VBI capture or output parameters * @sliced: sliced VBI capture or output parameters + * @audio: definition of an audio format * @raw_data: placeholder for future extensions and custom formats * @fmt: union of @pix, @pix_mp, @win, @vbi, @sliced, @sdr, @meta * and @raw_data @@ -2426,6 +2444,7 @@ struct v4l2_format { struct v4l2_sliced_vbi_format sliced; /* V4L2_BUF_TYPE_SLICED_VBI_CAPTURE */ struct v4l2_sdr_format sdr; /* V4L2_BUF_TYPE_SDR_CAPTURE */ struct v4l2_meta_format meta; /* V4L2_BUF_TYPE_META_CAPTURE */ + struct v4l2_audio_format audio; /* V4L2_BUF_TYPE_AUDIO_CAPTURE */ __u8 raw_data[200]; /* user-defined */ } fmt; };
Audio signal processing has the requirement for memory to memory similar as Video. This patch is to add this support in v4l2 framework, defined new buffer type V4L2_BUF_TYPE_AUDIO_CAPTURE and V4L2_BUF_TYPE_AUDIO_OUTPUT, defined new format v4l2_audio_format for audio case usage. The created audio device is named "/dev/audioX". Signed-off-by: Shengjiu Wang <shengjiu.wang@nxp.com> --- .../media/common/videobuf2/videobuf2-v4l2.c | 4 ++ drivers/media/v4l2-core/v4l2-dev.c | 17 ++++++ drivers/media/v4l2-core/v4l2-ioctl.c | 52 +++++++++++++++++++ include/media/v4l2-dev.h | 2 + include/media/v4l2-ioctl.h | 34 ++++++++++++ include/uapi/linux/videodev2.h | 19 +++++++ 6 files changed, 128 insertions(+)