V4l2 driver layer analysis

One 、Camera V4L2 Driver layer analysis

Linux The video input equipment in the system mainly includes the following four parts ：

1. Character device driver ：V4L2 It's a character device , It has all the characteristics of character device , Expose interface to user space ;
2.V4L2 Drive the core ： It is mainly to build a standard video device driver framework in the kernel , Provide unified interface function for video operation ;
3. platform V4L2 Device drivers ： stay V4L2 Within the framework of , According to the characteristics of the platform, the platform related V4L2 Drive part , Including registration video_device and v4l2_dev;
4. Concrete sensor drive ： Main power on 、 Provide working clock 、 Video image clipping 、 flow IO Turn on, etc , Implement various device control methods for the upper layer to call and register v4l2_subdev.

V4L2 The core source code is located in drivers/media/v4l2-core, According to the function, it can be divided into four categories ：

1. Character device module ： from v4l2-dev.c Realization , Main function application character main equipment number 、 register class And to provide video device Registration and logout and other related functions .
2.V4L2 Basic framework ： from v4l2-device.c、v4l2-subdev.c、v4l2-fh.c、v4l2-ctrls.c Wait for the file to build V4L2 Basic framework .
3.videobuf management
from videobuf2-core.c、videobuf2-dma-contig.c、videobuf2-dma-sg.c、videobuf2-memops.c、videobuf2-vmalloc.c、v4l2-mem2mem.c Wait for the file to be implemented , complete videobuffer The distribution of 、 Manage and log off .
4.Ioctl frame ： from v4l2-ioctl.c File implementation , structure V4L2ioctl Framework .

• establish v4l2_device Structure , Fill in the information , adopt v4l2_device_register Method to register with the system and create video Device node . //“kernel/msm-4.19/drivers/media/v4l2-core/v4l2-device.c”

• establish media_device Structure , Fill in the information , adopt media_device_register Register with the system , And create media Device node , And assign it to v4l2_device Medium mdev. //“kernel/msm-4.19/drivers/media/media-device.c”

• establish v4l2_subdev Structure , Fill in the information , adopt v4l2_device_register_subdev Register with the system , And mount it to v4l2_device In the device //“kernel/msm-4.19/drivers/media/v4l2-core/v4l2-device.c”

• Create the corresponding media_entity, And pass media_device_register_entity Method to add it to media controller Unified management in . //“kernel/msm-4.19/drivers/media/media-device.c”

Two 、V4L2 Basic framework

2.1 /media/v4l2-core/v4l2-dev.c

In this file , Mainly responsible for creating /sys/classs/video4linux Catalog , When a device is registered , Create the corresponding /dev/videox 、/dev/vbix、/dev/radiox、/dev/subdevx Equal node .

The main work is as follows ：
1. Put the character device number (81,0) To (81,255) During this time 256 Word order equipment number , All applications are v4l2 Use ,name=video4linux
2. register /sys/classs/video4linux Catalog

@ kernel/msm-4.4/drivers/media/v4l2-core/v4l2-dev.c

static struct class video_class = {
	.name = VIDEO_NAME,		// video4linux
	.dev_groups = video_device_groups,
};

static int __init videodev_init(void)
{
	dev_t dev = MKDEV(VIDEO_MAJOR, 0);	// VIDEO_MAJOR: 81
	printk(KERN_INFO "Linux video capture interface: v2.00\n");
	
	// 1.  Put the character device number (81,0)  To  (81,255)  During this time 256 Word order equipment number , All applications are  v4l2  Use ,name=video4linux
	ret = register_chrdev_region(dev, VIDEO_NUM_DEVICES, VIDEO_NAME); //VIDEO_NUM_DEVICES: 256  VIDEO_NAME:"video4linux"
	======> int register_chrdev_region(dev_t from, unsigned count, const char *name)

	// 2.  register  /sys/classs/video4linux  Catalog 
	ret = class_register(&video_class);

	return 0;
}

2.2 register V4L2 equipment __video_register_device()

The current device needs to be registered as v4l2 subdev when , Would call video_register_device() Function to register ：

"kernel/msm-4.19/drivers/media/platform/msm/camera_v2/camera/camera.c"
int camera_init_v4l2(struct device *dev, unsigned int *session)
{
	struct msm_video_device *pvdev;
	struct v4l2_device *v4l2_dev = NULL;
	int rc = 0;

	pvdev = kzalloc(sizeof(struct msm_video_device),
		GFP_KERNEL);
	if (WARN_ON(!pvdev)) {
		rc = -ENOMEM;
		goto init_end;
	}

	pvdev->vdev = video_device_alloc(); // Distribute video_device Memory 
	if (WARN_ON(!pvdev->vdev)) {
		rc = -ENOMEM;
		goto video_fail;
	}

	v4l2_dev = kzalloc(sizeof(struct v4l2_device), GFP_KERNEL);  // Distribute v4l2_dev  Memory 
	if (WARN_ON(!v4l2_dev)) {
		rc = -ENOMEM;
		goto v4l2_fail;
	}

#if defined(CONFIG_MEDIA_CONTROLLER)
	v4l2_dev->mdev = kzalloc(sizeof(struct media_device),   // Distribute media_device  Memory 
							 GFP_KERNEL);
	if (!v4l2_dev->mdev) {
		rc = -ENOMEM;
		goto mdev_fail;
	}
	media_device_init(v4l2_dev->mdev);
	strlcpy(v4l2_dev->mdev->model, MSM_CAMERA_NAME,
			sizeof(v4l2_dev->mdev->model));     //model  by msm_camera

	v4l2_dev->mdev->dev = dev;

	rc = media_device_register(v4l2_dev->mdev);  //media_device  register 
	if (WARN_ON(rc < 0))
		goto media_fail;

	rc = media_entity_pads_init(&pvdev->vdev->entity, 0, NULL); // establish media_entity And media_pad Links between ：
	if (WARN_ON(rc < 0))
		goto entity_fail;
	pvdev->vdev->entity.function = QCAMERA_VNODE_GROUP_ID;
#endif

	v4l2_dev->notify = NULL;
	pvdev->vdev->v4l2_dev = v4l2_dev;

	rc = v4l2_device_register(dev, pvdev->vdev->v4l2_dev);  //  Set the parent device to dev , The information is based on the incoming parameters .  for example :dv4l2_dev->name =qcom,camera ca0c000.qcom,cci:qcom,c
	if (WARN_ON(rc < 0))
		goto register_fail;

	strlcpy(pvdev->vdev->name, "msm-sensor", sizeof(pvdev->vdev->name));
	pvdev->vdev->release  = video_device_release;
	pvdev->vdev->fops     = &camera_v4l2_fops;         //  To configure  video_device  Character device operation function 
	pvdev->vdev->ioctl_ops = &camera_v4l2_ioctl_ops;	//  To configure  v4l2 IOCTRL
	pvdev->vdev->minor     = -1;
	pvdev->vdev->vfl_type  = VFL_TYPE_GRABBER;
	pvdev->vdev->device_caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_STREAMING;
	rc = video_register_device(pvdev->vdev,
		VFL_TYPE_GRABBER, -1);   call __video_register_device()
		

kernel/msm-4.19/drivers/media/platform/msm/camera_v2/msm.c
static int msm_probe(struct platform_device *pdev)
{
    struct msm_video_device *pvdev = NULL;
    static struct dentry *cam_debugfs_root;
    int rc = 0;
    // 1.  Initialize a  v4l2_device  Type of structure , And allocate the structure memory 
    msm_v4l2_dev = kzalloc(sizeof(*msm_v4l2_dev),
        GFP_KERNEL);
    if (WARN_ON(!msm_v4l2_dev)) {
        rc = -ENOMEM;
        goto probe_end;
    }

    pvdev = kzalloc(sizeof(struct msm_video_device),
        GFP_KERNEL);
    if (WARN_ON(!pvdev)) {
        rc = -ENOMEM;
        goto pvdev_fail;
    }
    // 2.  Distribute  video_device  Structure memory 
    pvdev->vdev = video_device_alloc();
    if (WARN_ON(!pvdev->vdev)) {
        rc = -ENOMEM;
        goto video_fail;
    }

#if defined(CONFIG_MEDIA_CONTROLLER)
    // 3.  Distribute  media_device  Structure memory 
    msm_v4l2_dev->mdev = kzalloc(sizeof(struct media_device),
        GFP_KERNEL);
    if (!msm_v4l2_dev->mdev) {
        rc = -ENOMEM;
        goto mdev_fail;
    }
    // 4. initialization  media_device  Structure 
    media_device_init(msm_v4l2_dev->mdev);
    strlcpy(msm_v4l2_dev->mdev->model, MSM_CONFIGURATION_NAME,sizeof(msm_v4l2_dev->mdev->model));  //MSM_CONFIGURATION_NAME = "msm_config"  In the code open  node , Will compare whether it is smsm_config

    msm_v4l2_dev->mdev->dev = &(pdev->dev);
    // 5.  register  media_device ,  The use of  v4l2 
    rc = media_device_register(msm_v4l2_dev->mdev);
/**
media_device_register()
    media_devnode_register () 
        device_initialize()  // initialization media  establish mediaX
        cdev_init()      // Initialize character device 
        cdev_device_add() // 
    device_create_file(&devnode->dev, &dev_attr_model)  //  Node created  sys/devices/platform/soc/ca00000.qcom,msm-cam/media0/model

*/
    if (WARN_ON(rc < 0))
        goto media_fail;

    if (WARN_ON((rc == media_entity_pads_init(&pvdev->vdev->entity,
            0, NULL)) < 0))
        goto entity_fail;

    pvdev->vdev->entity.function = QCAMERA_VNODE_GROUP_ID;
#endif

    msm_v4l2_dev->notify = msm_sd_notify;

    pvdev->vdev->v4l2_dev = msm_v4l2_dev;
    // 6.  Set the parent device to  pdev->dev （ That is to say  qcom,msm-cam  Device information for ）
    rc = v4l2_device_register(&(pdev->dev), pvdev->vdev->v4l2_dev);
/**
int v4l2_device_register(struct device *dev, struct v4l2_device *v4l2_dev)
{
    if (v4l2_dev == NULL)
        return -EINVAL;

    INIT_LIST_HEAD(&v4l2_dev->subdevs);
    spin_lock_init(&v4l2_dev->lock);
    v4l2_prio_init(&v4l2_dev->prio);
    kref_init(&v4l2_dev->ref);
    get_device(dev);
    v4l2_dev->dev = dev;
    if (dev == NULL) {
        /* If dev == NULL, then name must be filled in by the caller */
        if (WARN_ON(!v4l2_dev->name[0]))
            return -EINVAL;
        return 0;
    }

    /* Set name to driver name + device name if it is empty. */
    if (!v4l2_dev->name[0])
       snprintf(v4l2_dev->name, sizeof(v4l2_dev->name), "%s %s",
            dev->driver->name, dev_name(dev));
    printk("v4l2_dev->name =%s \n",v4l2_dev->name); //log  Information v4l2_dev->name =msm ca00000.qcom,msm-cam . dev->driver->name  The name set in the driver ,dev_name(dev) dtsi  Medium lable
    if (!dev_get_drvdata(dev))
        dev_set_drvdata(dev, v4l2_dev);
    return 0;
}

*/
    if (WARN_ON(rc < 0))
        goto register_fail;
    // 7.  register  video_device equipment  
    strlcpy(pvdev->vdev->name, "msm-config", sizeof(pvdev->vdev->name));
    pvdev->vdev->release  = video_device_release;
    pvdev->vdev->fops     = &msm_fops;
    pvdev->vdev->ioctl_ops = &g_msm_ioctl_ops;
    pvdev->vdev->minor     = -1;
    pvdev->vdev->vfl_type  = VFL_TYPE_GRABBER;
    rc = video_register_device(pvdev->vdev,
        VFL_TYPE_GRABBER, -1);   //  node  /dev/vdieoX

2.3 __video_register_device

With "qcom,msm-cam" For example , When it is registered , Delivered nr = -1, Explain that the allocation starts from the first , That is to say /dev/video0.
But if there is anything else to do first video_register_device . /dev/video0 Can be other values . You can view nodes
/sys/class/video4linux # cat video0/name
sde_rotator
because "platform/msm/sde/rotator/sde_rotator_dev.c" Execute first , Incoming nr = -1, so /dev/video0 by sde_rotator

 kernel/msm-4.4/drivers/media/v4l2-core/v4l2-dev.c
/**
 *	__video_register_device - register video4linux devices
 *	@vdev: video device structure we want to register
 *	@type: type of device to register
 *	@nr:   which device node number (0 == /dev/video0, 1 == /dev/video1, ... -1 == first free)
 *	@warn_if_nr_in_use: warn if the desired device node number was already in use and another number was chosen instead.
 *	@owner: module that owns the video device node
 *
 *	The registration code assigns minor numbers and device node numbersbased on the requested type and registers the new device node with the kernel.
 *
 *	This function assumes that struct video_device was zeroed when it was allocated and does not contain any stale date.
 *
 *	An error is returned if no free minor or device node number could be found, or if the registration of the device node failed.
 *
 *	Zero is returned on success.
 *
 *	Valid types are
 *	%VFL_TYPE_GRABBER - A frame grabber
 *	%VFL_TYPE_VBI - Vertical blank data (undecoded)
 *	%VFL_TYPE_RADIO - A radio card
 *	%VFL_TYPE_SUBDEV - A subdevice
 *	%VFL_TYPE_SDR - Software Defined Radio
 */
int __video_register_device(struct video_device *vdev, int type, int nr, int warn_if_nr_in_use, struct module *owner)
{
	int minor_cnt = VIDEO_NUM_DEVICES;
	const char *name_base;

	/* A minor value of -1 marks this video device as never having been registered */
	vdev->minor = -1;
	
	// 1.  initialization  fh->list
	/* v4l2_fh support */
	INIT_LIST_HEAD(&vdev->fh_list);

	// 2.  Check the type of equipment 
	/* Part 1: check device type */
	switch (type) {
	case VFL_TYPE_GRABBER: 	name_base = "video";  	break;
	case VFL_TYPE_VBI: 		name_base = "vbi"; 		break;
	case VFL_TYPE_RADIO: 	name_base = "radio"; 	break;
	case VFL_TYPE_SUBDEV:	name_base = "v4l-subdev";break;
	case VFL_TYPE_SDR: 		name_base = "swradio";	break; 		/* Use device name 'swradio' because 'sdr' was already taken. */
	}

	vdev->vfl_type = type;	// VFL_TYPE_GRABBER
	vdev->cdev = NULL;

	// 3.  Look for one that is not in use   Secondary device number ,  The main device number is  81,（0～63  by video）(128,191  by sub-dev)
	/* Part 2: find a free minor, device node number and device index. */
	/* Keep the ranges for the first four types for historical reasons.
	 * Newer devices (not yet in place) should use the range  of 128-191 and just pick the first free minor there (new style). */
	switch (type) {
	case VFL_TYPE_GRABBER: 	minor_offset = 0;	minor_cnt = 64; break;
	case VFL_TYPE_RADIO:	minor_offset = 64; 	minor_cnt = 64; break;
	case VFL_TYPE_VBI:		minor_offset = 224;	minor_cnt = 32;	break;
	default:				minor_offset = 128;	minor_cnt = 64;	break;
	}
	/* Pick a device node number */
	nr = devnode_find(vdev, nr == -1 ? 0 : nr, minor_cnt);
	if (nr == minor_cnt)
		nr = devnode_find(vdev, 0, minor_cnt);

	/* The device node number and minor numbers are independent, so
	   we just find the first free minor number. */
	for (i = 0; i < VIDEO_NUM_DEVICES; i++)
		if (video_device[i] == NULL)
			break;

	vdev->minor = i + minor_offset;
	vdev->num = nr;
	devnode_set(vdev);

	// 4.  obtain  index, Register the currently required  video_device  The equipment is kept in  video_device[] In the global array 
	vdev->index = get_index(vdev);
	video_device[vdev->minor] = vdev;

	// 5.  Assign the corresponding character device  /dev/video0, Character device number , It's the one in front  （81,minor）
	/* Part 3: Initialize the character device */
	vdev->cdev = cdev_alloc();

	vdev->cdev->ops = &v4l2_fops;
	vdev->cdev->owner = owner;
	ret = cdev_add(vdev->cdev, MKDEV(VIDEO_MAJOR, vdev->minor), 1);

	// 6.  Assign the corresponding sys node  /sys/class/video4linux/video0
	/* Part 4: register the device with sysfs */
	vdev->dev.class = &video_class;
	vdev->dev.devt = MKDEV(VIDEO_MAJOR, vdev->minor);
	vdev->dev.parent = vdev->dev_parent;
	dev_set_name(&vdev->dev, "%s%d", name_base, vdev->num);
	ret = device_register(&vdev->dev);

	// 7.  register release  Function called on 
	/* Register the release callback that will be called when the last reference to the device goes away. */
	vdev->dev.release = v4l2_device_release;

	/* Increase v4l2_device refcount */
	v4l2_device_get(vdev->v4l2_dev);

	// 8.  Will be  v4l2 subdevice  Think of it as a  entity  Sign up to  media device
	/* Part 5: Register the entity. */
	if (vdev->v4l2_dev->mdev && vdev->vfl_type != VFL_TYPE_SUBDEV) {
		vdev->entity.type = MEDIA_ENT_T_DEVNODE_V4L;
		vdev->entity.name = vdev->name;
		vdev->entity.info.dev.major = VIDEO_MAJOR;
		vdev->entity.info.dev.minor = vdev->minor;
		ret = media_device_register_entity(vdev->v4l2_dev->mdev,&vdev->entity);
	}

	/* Part 6: Activate this minor. The char device can now be used. */
	set_bit(V4L2_FL_REGISTERED, &vdev->flags);

	return 0;
}
EXPORT_SYMBOL(__video_register_device);

2.3.1 Character device operation function v4l2_fops

static const struct file_operations v4l2_fops = {
	.owner = THIS_MODULE,
	.read = v4l2_read,
	.write = v4l2_write,
	.open = v4l2_open,
	.get_unmapped_area = v4l2_get_unmapped_area,
	.mmap = v4l2_mmap,
	.unlocked_ioctl = v4l2_ioctl,
#ifdef CONFIG_COMPAT
	.compat_ioctl = v4l2_compat_ioctl32,
#endif
	.release = v4l2_release,
	.poll = v4l2_poll,
	.llseek = no_llseek,
};

Create success /dev/video0 After node , When you want to open the corresponding node , Would call fops Corresponding operation function , The corresponding code is assigned at the time of registration .

"kernel/msm-4.19/drivers/media/platform/msm/camera_v2/msm.c"
strlcpy(pvdev->vdev->name, "msm-config", sizeof(pvdev->vdev->name));
pvdev->vdev->release  = video_device_release;
pvdev->vdev->fops     = &msm_fops;
pvdev->vdev->ioctl_ops = &g_msm_ioctl_ops;

static struct v4l2_file_operations msm_fops = {
    .owner  = THIS_MODULE,
    .open   = msm_open,
    .poll   = msm_poll,
    .release = msm_close,
    .unlocked_ioctl   = video_ioctl2,
#ifdef CONFIG_COMPAT
    .compat_ioctl32 = video_ioctl2,
#endif
};

"kernel/msm-4.19/drivers/media/platform/msm/camera_v2/camera/camera.c"
strlcpy(pvdev->vdev->name, "msm-sensor", sizeof(pvdev->vdev->name));
pvdev->vdev->release  = video_device_release;
pvdev->vdev->fops     = &camera_v4l2_fops;
pvdev->vdev->ioctl_ops = &camera_v4l2_ioctl_ops;


static struct v4l2_file_operations camera_v4l2_fops = {
    .owner   = THIS_MODULE,
    .open   = camera_v4l2_open,
    .poll   = camera_v4l2_poll,
    .release = camera_v4l2_close,
    .unlocked_ioctl   = video_ioctl2,
#ifdef CONFIG_COMPAT
    .compat_ioctl32 = camera_v4l2_compat_ioctl,
#endif
};

log Information

02-11 06:44:53.201     0     0 W         : v4l2_open
02-11 06:44:53.201     0     0 W         :  msm_open

02-11 06:44:54.756     0     0 W         : v4l2_open
02-11 06:44:54.756     0     0 W         : camera_v4l2_open

 according to open  Different nodes call different v4l2_file_operations 

2.3.2 v4l2_ioctrl

kernel/msm-4.4/drivers/media/v4l2-core/v4l2-compat-ioctl32.c //iotctrl  Empathy open , There is a default v4l2  Of  ioctrl ,  With corresponding drive ioctrl
long v4l2_compat_ioctl32(struct file *file, unsigned int cmd, unsigned long arg)
{
	if (_IOC_TYPE(cmd) == 'V' && _IOC_NR(cmd) < BASE_VIDIOC_PRIVATE)
		ret = do_video_ioctl(file, cmd, arg);
	else if (vdev->fops->compat_ioctl32)
		ret = vdev->fops->compat_ioctl32(file, cmd, arg);
	return ret;
}

2.4 Register sub devices /media/v4l2-core/v4l2-subdev.c

When there is sub-dev Need to register to v4l2 when , call v4l2_device_register_subdev() function .
The final call __video_register_device(), Pass parameters VFL_TYPE_SUBDEV, The description is to register sub_dev equipment .

Reference resources :https://blog.csdn.net/Ciellee/article/details/105483079