Linux v4l2子系统(2)：v4l2框架分析

v4l为Video子系统提供统一框架，驱动通过将v4l2_dev/v4l2_subdev注册到Video子系统，在用户空间创建设备节点。

使用如下命令在kernel(5.10.110)生成driver-api.pdf帮助文件：

make SPHINXDIRS="driver-api" pdfdocs

在《CHAPTER FIFTYTWO MEDIA SUBSYSTEM KERNEL INTERNAL API》介绍了v4l2 subsystem。

或者参考《1. Video4Linux devices — The Linux Kernel documentation》。

1 v4l子系统初始化

static int __init videodev_init(void)
{
    dev_t dev = MKDEV(VIDEO_MAJOR, 0);
    int ret;

    pr_info("Linux video capture interface: v2.00\n");
    ret = register_chrdev_region(dev, VIDEO_NUM_DEVICES, VIDEO_NAME);--注册主设备号为81的video设备区间，子设备数量为256，名称为video4linux。
    if (ret < 0) {
        pr_warn("videodev: unable to get major %d\n",
                VIDEO_MAJOR);
        return ret;
    }

    ret = class_register(&video_class);--注册名称为video4linux名称的video类。
    if (ret < 0) {
        unregister_chrdev_region(dev, VIDEO_NUM_DEVICES);
        pr_warn("video_dev: class_register failed\n");
        return -EIO;
    }

    return 0;
}

所有video类型设备位于/sys/class/video4linux，并且没有设备都会有如下属性：

name：对应struce video_device的name。

dev_debug：使能不通模块的调试信息。

Definition	Mask	Description
V4L2_DEV_DEBUG_IOCTL	0x01	Log the ioctl name and error code. VIDIOC_(D)QBUF ioctls are only logged if bit 0x08 is also set.
V4L2_DEV_DEBUG_IOCTL_ARG	0x02	Log the ioctl name arguments and error code. VIDIOC_(D)QBUF ioctls are only logged if bit 0x08 is also set.
V4L2_DEV_DEBUG_FOP	0x04	Log the fle operations open, release, read, write, mmap and get_unmapped_area. The read and write operations are only logged if bit 0x08 is also set.
V4L2_DEV_DEBUG_STREAMING	0x08	Log the read and write fle operations and the VIDIOC_QBUF and VIDIOC_DQBUF ioctls.
V4L2_DEV_DEBUG_POLL	0x10	Log the poll fle operation.
V4L2_DEV_DEBUG_CTRL	0x20	Log error and messages in the control operations

index：设备序号。

static struct class video_class = {
    .name = VIDEO_NAME,
    .dev_groups = video_device_groups,
};

static struct attribute *video_device_attrs[] = {
    &dev_attr_name.attr,
    &dev_attr_dev_debug.attr,
    &dev_attr_index.attr,
    NULL,
};
ATTRIBUTE_GROUPS(video_device);

2 video_device

struct video_device用于创建和管理v4l2设备节点：

struct video_device
{
#if defined(CONFIG_MEDIA_CONTROLLER)
    struct media_entity entity;--该设备在Media子系统中对应的Entity。
    struct media_intf_devnode *intf_devnode;--该video_device的设备在Media子系统中对应的Interface。
    struct media_pipeline pipe;
#endif
    const struct v4l2_file_operations *fops;--v4l子系统中对v4l2设备的操作函数集。
    u32 device_caps;

    /* sysfs */
    struct device dev;
    struct cdev *cdev;
    struct v4l2_device *v4l2_dev;
    struct device *dev_parent;
    struct v4l2_ctrl_handler *ctrl_handler;
    struct vb2_queue *queue;--此设备所使用的vb2_queue。
    struct v4l2_prio_state *prio;

    /* device info */
    char name[32];
    enum vfl_devnode_type vfl_type;
    enum vfl_devnode_direction vfl_dir;
    int minor;
    u16 num;
    unsigned long flags;
    int index;

    /* V4L2 file handles */
    spinlock_t        fh_lock;
    struct list_head    fh_list;
    int dev_debug;--调试配置，可以通过设备的dev_debug节点配置。
    v4l2_std_id tvnorms;

    /* callbacks */
    void (*release)(struct video_device *vdev);
    const struct v4l2_ioctl_ops *ioctl_ops;--ioctl扩展。
...
};

struct v4l2_file_operations是对v4l2设备进行操作的函数集：

struct v4l2_file_operations {
    struct module *owner;
    ssize_t (*read) (struct file *, char __user *, size_t, loff_t *);
    ssize_t (*write) (struct file *, const char __user *, size_t, loff_t *);
    __poll_t (*poll) (struct file *, struct poll_table_struct *);
    long (*unlocked_ioctl) (struct file *, unsigned int, unsigned long);
#ifdef CONFIG_COMPAT
    long (*compat_ioctl32) (struct file *, unsigned int, unsigned long);
#endif
    unsigned long (*get_unmapped_area) (struct file *, unsigned long,
                unsigned long, unsigned long, unsigned long);
    int (*mmap) (struct file *, struct vm_area_struct *);
    int (*open) (struct file *);
    int (*release) (struct file *);

    ANDROID_KABI_RESERVE(1);
};

video_device主要API有：

struct video_device * __must_check video_device_alloc(void);--分配video_device。
static inline int __must_check video_register_device(struct video_device *vdev,
                             enum vfl_devnode_type type,
                             int nr)--注册v4l设备，根据type生成设备节点。
void video_unregister_device(struct video_device *vdev);--注销v4l设备。
void video_device_release(struct video_device *vdev);--释放video_device数据。

__video_register_device是注册v4l设备的主体：

int __video_register_device(struct video_device *vdev,
                enum vfl_devnode_type type,
                int nr, int warn_if_nr_in_use,
                struct module *owner)
{
...
    spin_lock_init(&vdev->fh_lock);
    INIT_LIST_HEAD(&vdev->fh_list);

    /* Part 1: check device type */--根据type生成设备节点，名称根据节点类型不同而异。
    switch (type) {
    case VFL_TYPE_VIDEO:
        name_base = "video";
        break;
...
    default:
...
    }
...
    /* Part 2: find a free minor, device node number and device index. */--根据类型不同minor基准也不同，并找到合适的minor号。
#ifdef CONFIG_VIDEO_FIXED_MINOR_RANGES
    switch (type) {
    case VFL_TYPE_VIDEO:
        minor_offset = 0;
        minor_cnt = 64;
        break;
...
    }
#endif
...
/* Part 3: Initialize the character device */--分配并初始化video字符设备，操作函数集为v4l2_fops。
    vdev->cdev = cdev_alloc();
    if (vdev->cdev == NULL) {
        ret = -ENOMEM;
        goto cleanup;
    }
    vdev->cdev->ops = &v4l2_fops;
    vdev->cdev->owner = owner;
    ret = cdev_add(vdev->cdev, MKDEV(VIDEO_MAJOR, vdev->minor), 1);
...
    /* Part 4: register the device with sysfs */--注册为video4linux类设备，创建设备节点。
    vdev->dev.class = &video_class;
    vdev->dev.devt = MKDEV(VIDEO_MAJOR, vdev->minor);
    vdev->dev.parent = vdev->dev_parent;
    dev_set_name(&vdev->dev, "%s%d", name_base, vdev->num);
    ret = device_register(&vdev->dev);
    if (ret < 0) {
        pr_err("%s: device_register failed\n", __func__);
        goto cleanup;
    }
    vdev->dev.release = v4l2_device_release;--当设备引用计数为0时调用，释放资源。

    if (nr != -1 && nr != vdev->num && warn_if_nr_in_use)
        pr_warn("%s: requested %s%d, got %s\n", __func__,
            name_base, nr, video_device_node_name(vdev));

    /* Increase v4l2_device refcount */
    v4l2_device_get(vdev->v4l2_dev);

    /* Part 5: Register the entity. */
    ret = video_register_media_controller(vdev);--如果定义了vdev->v4l2_dev->mdev，则注册Entity到Media，并且创建Media设备节点。

    /* Part 6: Activate this minor. The char device can now be used. */
    set_bit(V4L2_FL_REGISTERED, &vdev->flags);--video_device设备标志置为V4L2_FL_REGISTERED。

    return 0;
...
}

类似/dev/videoX设备节点系统调用对应的函数集：

static const struct file_operations v4l2_fops = {
    .owner = THIS_MODULE,
    .read = v4l2_read,
    .write = v4l2_write,
    .open = v4l2_open,
    .get_unmapped_area = v4l2_get_unmapped_area,
    .mmap = v4l2_mmap,
    .unlocked_ioctl = v4l2_ioctl,
#ifdef CONFIG_COMPAT
    .compat_ioctl = v4l2_compat_ioctl32,
#endif
    .release = v4l2_release,
    .poll = v4l2_poll,
    .llseek = no_llseek,
};

3 v4l2_device

v4l2_device表示一个v4l2设备实例，分别和设备节点、Media子系统、subdev等关联。

struct v4l2_device {
    struct device *dev;--对应的device设备节点数据结构。
    struct media_device *mdev;--作为Media设备注册到Media子系统中。
    struct list_head subdevs;--下属的v4l2_subdev列表。
    spinlock_t lock;
    char name[V4L2_DEVICE_NAME_SIZE];
    void (*notify)(struct v4l2_subdev *sd,
            unsigned int notification, void *arg);--v4l2_subdev的通知。
    struct v4l2_ctrl_handler *ctrl_handler;
    struct v4l2_prio_state prio;
    struct kref ref;
    void (*release)(struct v4l2_device *v4l2_dev);--应用计数为0时被调用。
};

struct v4l2_device用于表示v4l2的sub-device：

struct v4l2_subdev {
#if defined(CONFIG_MEDIA_CONTROLLER)
    struct media_entity entity;--subdev在Media子系统中作为Entity。
#endif
    struct list_head list;--统一v4l2_device下的subdev列表。
    struct module *owner;
    bool owner_v4l2_dev;
    u32 flags;--标志位，V4L2_SUBDEV_FL_HAS_DEVNODE表示要为此subdev创建一个设备节点。
    struct v4l2_device *v4l2_dev;
    const struct v4l2_subdev_ops *ops;--v4l2_subdev私有的操作函数差异体现在v4l2_subdev_ops中。
    const struct v4l2_subdev_internal_ops *internal_ops;
    struct v4l2_ctrl_handler *ctrl_handler;
    char name[V4L2_SUBDEV_NAME_SIZE];
    u32 grp_id;
    void *dev_priv;
    void *host_priv;
    struct video_device *devnode;--在v4l2子系统中实例。
    struct device *dev;
    struct fwnode_handle *fwnode;
    struct list_head async_list;--连接到全局subdev_list列表中。
    struct v4l2_async_subdev *asd;
    struct v4l2_async_notifier *notifier;
    struct v4l2_async_notifier *subdev_notifier;
    struct v4l2_subdev_platform_data *pdata;
};

其中v4l2_subdev_ops函数集包括：Core操作函数集，以及针对特定类型设备的操作函数集。

struct v4l2_subdev_ops {
    const struct v4l2_subdev_core_ops    *core;
    const struct v4l2_subdev_tuner_ops    *tuner;
    const struct v4l2_subdev_audio_ops    *audio;
    const struct v4l2_subdev_video_ops    *video;
...
};
struct v4l2_subdev_core_ops {
    int (*log_status)(struct v4l2_subdev *sd);
    int (*s_io_pin_config)(struct v4l2_subdev *sd, size_t n,
                      struct v4l2_subdev_io_pin_config *pincfg);
...
};
struct v4l2_subdev_video_ops {
    int (*s_routing)(struct v4l2_subdev *sd, u32 input, u32 output, u32 config);
...
};

v4l2_subdev_init()注册时将v4l2_subdev_ops和v4l2_subdev关联。

v4l2_subdev_call()优先使用v4l2_subdev_call_wrappers的函数，然后使用特定v4l2_subdev的v4l2_subdev_ops。

#define v4l2_subdev_call(sd, o, f, args...)                \
    ({                                \
        struct v4l2_subdev *__sd = (sd);            \
        int __result;                        \
        if (!__sd)                        \
            __result = -ENODEV;                \
        else if (!(__sd->ops->o && __sd->ops->o->f))        \
            __result = -ENOIOCTLCMD;            \
        else if (v4l2_subdev_call_wrappers.o &&            \
             v4l2_subdev_call_wrappers.o->f)        \
            __result = v4l2_subdev_call_wrappers.o->f(    \
                            __sd, ##args);    \
        else                            \
            __result = __sd->ops->o->f(__sd, ##args);    \
        __result;                        \
    })

v4l2_dev和v4l2_subdev相关函数主要包括：

v4l2注册注销：v4l2_device_register、v4l2_device_unregister。
v4l2引用计数：v4l2_device_get、v4l2_device_put。
subdev注册注销，设备创建：v4l2_device_register_subdev、v4l2_device_unregister_subdev、v4l2_device_register_subdev_nodes等。

static inline void v4l2_device_get(struct v4l2_device *v4l2_dev)
int v4l2_device_put(struct v4l2_device *v4l2_dev);

int __must_check v4l2_device_register(struct device *dev,
                      struct v4l2_device *v4l2_dev);--初始化v4l2_dev成员，若dev->driver_data为空则指向v4l2_dev。
int v4l2_device_set_name(struct v4l2_device *v4l2_dev, const char *basename,
             atomic_t *instance);--配置v4l2_device的那么。
void v4l2_device_disconnect(struct v4l2_device *v4l2_dev);--对于支持热插拔的USB设备，设置状态位disconnected。
void v4l2_device_unregister(struct v4l2_device *v4l2_dev);--去注册所有sub-devices，以及其他相关资源。
int __must_check v4l2_device_register_subdev(struct v4l2_device *v4l2_dev,
                         struct v4l2_subdev *sd);--将v4l2_subdev和v4l2_dev关联，创建Media Entity。
void v4l2_device_unregister_subdev(struct v4l2_subdev *sd);--将v4l2_subdev和其所属的v4l2_dev解除关联，并且注销相关Media Entity、释放相关资源等。

int __must_check
__v4l2_device_register_subdev_nodes(struct v4l2_device *v4l2_dev,
                    bool read_only);
static inline int __must_check
v4l2_device_register_subdev_nodes(struct v4l2_device *v4l2_dev)--遍历v4l2_dev的所有v4l2_subdev，为具有V4L2_SUBDEV_FL_HAS_DEVNODE标志位的v4l2_subdev创建设备节点。
{
    return __v4l2_device_register_subdev_nodes(v4l2_dev, false);
}
static inline int __must_check
v4l2_device_register_ro_subdev_nodes(struct v4l2_device *v4l2_dev)
{
    return __v4l2_device_register_subdev_nodes(v4l2_dev, true);
}

v4l2_device_register_subdev_nodes()为v4l2_device下所有v4l2_subdev：

注册到v4l2子系统，创建/dev/v4l-subdevX节点。
建立Media Graph中Entity到设备节点链接。

int __v4l2_device_register_subdev_nodes(struct v4l2_device *v4l2_dev,
                    bool read_only)
{
...
    list_for_each_entry(sd, &v4l2_dev->subdevs, list) {--遍历v4l2_dev->subdevs上所有v4l2_subdev。
        if (!(sd->flags & V4L2_SUBDEV_FL_HAS_DEVNODE))
            continue;
...
        vdev = kzalloc(sizeof(*vdev), GFP_KERNEL);--为v4l2_subdev创建video_device。
...
        video_set_drvdata(vdev, sd);
        strscpy(vdev->name, sd->name, sizeof(vdev->name));
        vdev->dev_parent = sd->dev;
        vdev->v4l2_dev = v4l2_dev;
        vdev->fops = &v4l2_subdev_fops;--v4l2_subdev类型的设备操作函数集v4l2_subdev_fops。
        vdev->release = v4l2_device_release_subdev_node;
        vdev->ctrl_handler = sd->ctrl_handler;
        if (read_only)
            set_bit(V4L2_FL_SUBDEV_RO_DEVNODE, &vdev->flags);
        err = __video_register_device(vdev, VFL_TYPE_SUBDEV, -1, 1,
                          sd->owner);--将v4l2_subdev类型设备注册到v4l2子系统，创建的设备名称为/dev/v4l-subdevX。
...
        sd->devnode = vdev;
#if defined(CONFIG_MEDIA_CONTROLLER)
        sd->entity.info.dev.major = VIDEO_MAJOR;
        sd->entity.info.dev.minor = vdev->minor;
        if (vdev->v4l2_dev->mdev) {
            struct media_link *link;
            link = media_create_intf_link(&sd->entity,
                              &vdev->intf_devnode->intf,
                              MEDIA_LNK_FL_ENABLED |
                              MEDIA_LNK_FL_IMMUTABLE);--如果对应v4l2_subdev设备创建了设备节点，则在Media子系统中建立和此设备节点关联的Entity。
...
        }
#endif
    }
    return 0;
...
}

v4l2_subdev对应的v4l2_file_operations函数集为：

const struct v4l2_file_operations v4l2_subdev_fops = {
    .owner = THIS_MODULE,
    .open = subdev_open,
    .unlocked_ioctl = subdev_ioctl,
#ifdef CONFIG_COMPAT
    .compat_ioctl32 = subdev_compat_ioctl32,
#endif
    .release = subdev_close,
    .poll = subdev_poll,
};

对于v4l2_device和v4l2_subdev对应的操作函数关系如下：

有填充色的表示私有函数集。

4 v4l2_fh

参考《1.6. V4L2 File handlers — The Linux Kernel documentation》。

struct v4l2_fh用于v4l2框架中file handle保存特殊数据。如果video_device->flags置位V4L2_FL_USES_V4L2_FH，则表示file->private_data指向v4l2_fh。

struct v4l2_fh {
    struct list_head    list;
    struct video_device    *vdev;
    struct v4l2_ctrl_handler *ctrl_handler;
    enum v4l2_priority    prio;

    /* Events */
    wait_queue_head_t    wait;
    struct mutex        subscribe_lock;
    struct list_head    subscribed;
    struct list_head    available;
    unsigned int        navailable;
    u32            sequence;

    struct v4l2_m2m_ctx    *m2m_ctx;
};

v4l2_fh对应的API有：

void v4l2_fh_init(struct v4l2_fh *fh, struct video_device *vdev);
void v4l2_fh_add(struct v4l2_fh *fh);
int v4l2_fh_open(struct file *filp);
int v4l2_fh_release(struct file *filp);
void v4l2_fh_del(struct v4l2_fh *fh);
void v4l2_fh_exit(struct v4l2_fh *fh);

5 v4l2_ctrl

参考《1.13. V4L2 Controls — The Linux Kernel documentation》。

struct v4l2_ctrl是对某一个属性可控制属性的抽象，通过v4l2_ctrl对属性进行配置。

可配置的属性定义位于include\uapi\linux\v4l2-controls.h中。

struct v4l2_ctrl {
    /* Administrative fields */
    struct list_head node;
    struct list_head ev_subs;
    struct v4l2_ctrl_handler *handler;--拥有此v4l2_ctrl的handler。
    struct v4l2_ctrl **cluster;
    unsigned int ncontrols;
...
    const struct v4l2_ctrl_ops *ops;
    const struct v4l2_ctrl_type_ops *type_ops;
    u32 id;
    const char *name;
    enum v4l2_ctrl_type type;
    s64 minimum, maximum, default_value;--此v4l2_ctrl的最小、最大和默认值。
...
    unsigned long flags;
    void *priv;
    s32 val;
    struct {
        s32 val;
    } cur;
...
};

struct v4l2_ctrl_handler是一系列v4l2_ctrl的集合

struct v4l2_ctrl_handler {
    struct mutex _lock;
    struct mutex *lock;
    struct list_head ctrls;--所属的v4l2_ctrl列表。
    struct list_head ctrl_refs;
    struct v4l2_ctrl_ref *cached;
    struct v4l2_ctrl_ref **buckets;
    v4l2_ctrl_notify_fnc notify;
    void *notify_priv;
...
};

struct v4l2_ctrl_ops是v4l2_ctrl对应的操作函数：

struct v4l2_ctrl_ops {
    int (*g_volatile_ctrl)(struct v4l2_ctrl *ctrl);
    int (*try_ctrl)(struct v4l2_ctrl *ctrl);--测试control值是否有效。
    int (*s_ctrl)(struct v4l2_ctrl *ctrl);--设置control值。

    ANDROID_KABI_RESERVE(1);
};

v4l2_ctrl相关API有：

#define v4l2_ctrl_handler_init(hdl, nr_of_controls_hint)        \
    v4l2_ctrl_handler_init_class(hdl, nr_of_controls_hint, NULL, NULL)
void v4l2_ctrl_handler_free(struct v4l2_ctrl_handler *hdl);

int v4l2_ctrl_handler_setup(struct v4l2_ctrl_handler *hdl);
struct v4l2_ctrl *v4l2_ctrl_new_std(struct v4l2_ctrl_handler *hdl,
                    const struct v4l2_ctrl_ops *ops,
                    u32 id, s64 min, s64 max, u64 step,
                    s64 def);
struct v4l2_ctrl *v4l2_ctrl_new_std_menu(struct v4l2_ctrl_handler *hdl,
                     const struct v4l2_ctrl_ops *ops,
                     u32 id, u8 max, u64 mask, u8 def);
struct v4l2_ctrl *v4l2_ctrl_new_int_menu(struct v4l2_ctrl_handler *hdl,
                     const struct v4l2_ctrl_ops *ops,
                     u32 id, u8 max, u8 def,
                     const s64 *qmenu_int);
int v4l2_ctrl_add_handler(struct v4l2_ctrl_handler *hdl,
              struct v4l2_ctrl_handler *add,
              v4l2_ctrl_filter filter,
              bool from_other_dev);