Linux内核之ISP驱动流程分析

ISP驱动分析

Linux版本: 4.19

芯片平台: RK3399/RK3288

源码路径:

drivers/media/platform/rk-isp10/cif_isp10_v4l2.c

drivers/media/platform/rk-isp10/cif_isp10.c

(1)装载和卸载函数

//DTS匹配

static const struct of_device_id cif_isp10_v4l2_of_match[] = {

{.compatible = "rockchip,rk3288-cif-isp",

.data = (void *)&rk3288_cfg},

{.compatible = "rockchip,rk3399-cif-isp",

.data = (void *)&rk3399_cfg},

{},

};

static struct platform_driver cif_isp10_v4l2_plat_drv = {

.driver = {

.name = DRIVER_NAME,

.of_match_table = of_match_ptr(cif_isp10_v4l2_of_match),

.pm = &cif_isp10_dev_pm_ops,

},

.probe = cif_isp10_v4l2_drv_probe,

.remove = cif_isp10_v4l2_drv_remove,

.suspend = cif_isp10_v4l2_drv_suspend,

.resume = cif_isp10_v4l2_drv_resume,

};

static int cif_isp10_v4l2_init(void)

{

int ret;

g_cif_isp10_v4l2_dev_cnt = 0;

ret = platform_driver_register(&cif_isp10_v4l2_plat_drv); //注册platform_driver

if (ret) {

cif_isp10_pltfrm_pr_err(NULL,

"cannot register platform driver, failed with %d\n",

ret);

return -ENODEV;

}

return ret;

}

static void __exit cif_isp10_v4l2_exit(void)

{

platform_driver_unregister(&cif_isp10_v4l2_plat_drv);

}

上面就是简单地注册了一个platform设备。

(2)probe()

static int cif_isp10_v4l2_drv_probe(struct platform_device *pdev)

{

const struct of_device_id *match;

struct device_node *node = pdev->dev.of_node;

struct cif_isp10_device *dev = NULL;

struct cif_isp10_v4l2_device *cif_isp10_v4l2_dev;

int ret;

//........

//分配cif_isp10_v4l2_device

cif_isp10_v4l2_dev = devm_kzalloc(

&pdev->dev,

sizeof(struct cif_isp10_v4l2_device),

GFP_KERNEL);

//.....

match = of_match_node(cif_isp10_v4l2_of_match, node); //获取匹配的是RK3288还是RK3399

dev = cif_isp10_create(&pdev->dev, //创建cif_isp10_device

cif_isp10_v4l2_event,

cif_isp10_v4l2_requeue_bufs,

(struct pltfrm_soc_cfg *)match->data);

//......

dev->dev_id = g_cif_isp10_v4l2_dev_cnt;

dev->isp_dev.dev_id = &dev->dev_id;

dev->nodes = (void *)cif_isp10_v4l2_dev;

dev->isp_state = CIF_ISP10_STATE_IDLE;

spin_lock_init(&dev->vbq_lock);

spin_lock_init(&dev->vbreq_lock);

spin_lock_init(&dev->iowrite32_verify_lock);

spin_lock_init(&dev->isp_state_lock);

init_waitqueue_head(&dev->isp_stop_wait);

mutex_init(&dev->api_mutex);

ret = v4l2_device_register(dev->dev, &dev->v4l2_dev); //注册v4l2_device

//......

ret = cif_isp10_v4l2_register_video_device( //注册video_device, 即生成/dev/videox节点，该节点具有VIDEO_OVERLAY功能

dev,

&cif_isp10_v4l2_dev->node[SP_DEV].vdev, //SP:selfpath

SP_VDEV_NAME,

V4L2_CAP_VIDEO_OVERLAY,

CIF_ISP10_V4L2_SP_DEV_MAJOR,

&cif_isp10_v4l2_fops,

&cif_isp10_v4l2_sp_ioctlops);

if (ret)

goto err;

ret = register_cifisp_device(&dev->isp_dev, //注册ISP video_device

&cif_isp10_v4l2_dev->node[ISP_DEV].vdev,

&dev->v4l2_dev,

dev->config.base_addr);

if (ret)

goto err;

ret = cif_isp10_v4l2_register_video_device( //注册video_device, 即生成/dev/videox节点，该节点具有VIDEO_CAPTURE功能

dev,

&cif_isp10_v4l2_dev->node[MP_DEV].vdev, //MP:mainpath

MP_VDEV_NAME,

V4L2_CAP_VIDEO_CAPTURE,

CIF_ISP10_V4L2_MP_DEV_MAJOR,

&cif_isp10_v4l2_fops,

&cif_isp10_v4l2_mp_ioctlops);

if (ret)

goto err;

ret = cif_isp10_v4l2_register_video_device( //注册video_device, 即生成/dev/videox节点，该节点具有VIDEO_OUTPUT功能

dev,

&cif_isp10_v4l2_dev->node[DMA_DEV].vdev,

DMA_VDEV_NAME,

V4L2_CAP_VIDEO_OUTPUT,

CIF_ISP10_V4L2_DMA_DEV_MAJOR,

&cif_isp10_v4l2_fops,

&cif_isp10_v4l2_dma_ioctlops);

if (ret)

goto err;

cif_isp10_v4l2_register_imgsrc_subdev( //注册v4l2_subdev，关联v4l2_device和v4l2_subdev

dev);

pm_runtime_enable(&pdev->dev);

g_cif_isp10_v4l2_dev[g_cif_isp10_v4l2_dev_cnt] =

cif_isp10_v4l2_dev;

g_cif_isp10_v4l2_dev_cnt++;

return 0;

err:

cif_isp10_destroy(dev);

return ret;

}

上面主要做了:

(1)创建和初始化cif_isp10_device，该结构体中保存着从DTS中解析出来的信息。

(2)注册v4l2_device

(3)注册了4个video_device：

rkisp1_ispdev：ISP设备

rkisp1_selfpath: 图像捕获设备

rkisp1_mainpath: 图像捕获设备，用于高分辨率

rkisp1_dmapath: DMA设备

(4)注册v4l2_subdev, 将v4l2_device和v4l2_subdev关联到一起。

*注意: 应用层就是通过访问video_device生成的节点来进行操作Camera，所以video_device注册时指定了很多ioctl函数。

(3)创建cif_isp10_device

struct cif_isp10_device *cif_isp10_create(

CIF_ISP10_PLTFRM_DEVICE pdev,

void (*sof_event)(struct cif_isp10_device *dev, __u32 frame_sequence),

void (*requeue_bufs)(struct cif_isp10_device *dev,

enum cif_isp10_stream_id stream_id),

struct pltfrm_soc_cfg *soc_cfg)

{

int ret;

struct cif_isp10_device *dev;

cif_isp10_pltfrm_pr_dbg(NULL, "\n");

//分配结构体

dev = kzalloc(sizeof(*dev), GFP_KERNEL);

//......

dev->sof_event = sof_event;

dev->requeue_bufs = requeue_bufs;

ret = cif_isp10_pltfrm_dev_init(dev,

&pdev, &dev->config.base_addr); //平台初始化(重映射寄存器地址，申请中断等)

cif_isp10_pltfrm_soc_init(dev, soc_cfg); //soc相关初始化(ISP,DPHY等时钟初始化)

ret = cif_isp10_img_srcs_init(dev); //初始化图像源，即ISP连接的Camera

ret = cif_isp10_register_isrs(dev); //注册中断处理函数 (ISR, Interrupt Service Routine),

//......

dev->pm_state = CIF_ISP10_PM_STATE_OFF;

dev->sp_stream.state = CIF_ISP10_STATE_DISABLED;

dev->sp_stream.id = CIF_ISP10_STREAM_SP;

dev->mp_stream.state = CIF_ISP10_STATE_DISABLED;

dev->mp_stream.id = CIF_ISP10_STREAM_MP;

dev->dma_stream.state = CIF_ISP10_STATE_DISABLED;

dev->dma_stream.id = CIF_ISP10_STREAM_DMA;

dev->config.mi_config.async_updt = 0;

(void)cif_isp10_init(dev, CIF_ISP10_ALL_STREAMS); //初始化所有的流(SP,MP,DMA)

cif_isp10_pltfrm_event_init(dev->dev, &dev->dma_stream.done);

cif_isp10_pltfrm_event_init(dev->dev, &dev->sp_stream.done);

cif_isp10_pltfrm_event_init(dev->dev, &dev->mp_stream.done);

dev->img_src_exps.exp_valid_frms[VALID_FR_EXP_T_INDEX] = 4;

dev->img_src_exps.exp_valid_frms[VALID_FR_EXP_G_INDEX] = 4;

dev->img_src_exps.inited = false;

mutex_init(&dev->img_src_exps.mutex);

memset(&dev->img_src_exps.data, 0x00, sizeof(dev->img_src_exps.data));

spin_lock_init(&dev->img_src_exps.lock);

INIT_LIST_HEAD(&dev->img_src_exps.list);

dev->vs_wq = alloc_workqueue("cif isp10 vs workqueue",

WQ_UNBOUND | WQ_MEM_RECLAIM, 1);

/* TBD: clean this up */

init_output_formats();

return dev;

//省略异常处理.....

}

上面是主要的初始化，包括dts解析，时钟设置，关联Camera等配置。

主要的几个初始化函数就是上面注释的位置，我们分析一下 cif_isp10_img_srcs_init，它会获取ISP上关联的Camera。

static int cif_isp10_img_srcs_init(

struct cif_isp10_device *dev)

{

int ret = 0;

memset(dev->img_src_array, 0x00, sizeof(dev->img_src_array));

dev->img_src_cnt = cif_isp10_pltfrm_get_img_src_device(dev->dev, //获取ISP上的Camera，最多10个

dev->img_src_array, CIF_ISP10_NUM_INPUTS);

if (dev->img_src_cnt > 0)

return 0;

dev->img_src_cnt = 0;

ret = -EFAULT;

cif_isp10_pltfrm_pr_err(dev->dev,

"failed with error %d\n", ret);

return ret;

}

int cif_isp10_pltfrm_get_img_src_device(

struct device *dev,

struct cif_isp10_img_src **img_src_array,

unsigned int array_len)

{

struct device_node *node = NULL;

struct device_node *camera_list_node = NULL;

struct i2c_client *client = NULL;

int ret = 0;

int index, size = 0;

const __be32 *phandle;

int num_cameras = 0;

struct cif_isp10_device *cif_isp10_dev = dev_get_drvdata(dev);

node = of_node_get(dev->of_node);

//.......

//获取ISP上关联的Camera

phandle = of_get_property(node,

"rockchip,camera-modules-attached", &size);

//.......

for (index = 0; index < size / sizeof(*phandle); index++) {

camera_list_node = of_parse_phandle(node,

"rockchip,camera-modules-attached", index);

of_node_put(node);

//......

//判断是不是I2C subdev，是的话就加入到数组中

if (!strcmp(camera_list_node->type,

"v4l2-i2c-subdev")) {

client = of_find_i2c_device_by_node(

camera_list_node);

//......

} else {

//......

continue;

}

//加到数组中

img_src_array[num_cameras] =

cif_isp10_img_src_to_img_src(

&client->dev,

&(cif_isp10_dev->soc_cfg));

if (!IS_ERR_OR_NULL(img_src_array[num_cameras])) {

cif_isp10_pltfrm_pr_info(dev,

"%s attach to cif isp10 img_src_array[%d]\n",

cif_isp10_img_src_g_name(

img_src_array[num_cameras]),

num_cameras);

num_cameras++;

if (num_cameras >= array_len) {

cif_isp10_pltfrm_pr_err(dev,

"cif isp10 isn't support > %d 'camera modules attached'\n",

array_len);

break;

}

}

}

return num_cameras;

//省略异常处理.....

}

上面就是获取DTS中的如下定义:

&cif_isp0 {

rockchip,camera-modules-attached = <&camera0>;

status = "okay";

};

获取完后保存在数组中，cif_isp10_v4l2_register_imgsrc_subdev函数中会将这些v4l2_subdev和v4l2_device关联。

(4)ioctl

应用层调用ioctl会先调用到v4l2_file_operations中的unlocked_ioctl或compat_ioctl32，然后最终会调用到v4l2_ioctl_ops中的各个ioctl。

所以应用层对Camera的控制主要就是通过ioctl，我们随便找两个看看:

查询V4L2功能

static int v4l2_querycap(struct file *file,

void *priv, struct v4l2_capability *cap)

{

struct vb2_queue *queue = to_vb2_queue(file);

struct video_device *vdev = video_devdata(file);

struct cif_isp10_device *dev = to_cif_isp10_device(queue);

u32 stream_ids = to_stream_id(file); //获取id

strcpy(cap->driver, DRIVER_NAME);

strlcpy(cap->card, vdev->name, sizeof(cap->card));

snprintf(cap->bus_info, sizeof(cap->bus_info),

"platform:" DRIVER_NAME "-%03i",

dev->dev_id);

//根据ID(SP,MP,DMA)返回对应的功能

if (stream_ids == CIF_ISP10_STREAM_SP) {

cap->capabilities = V4L2_CAP_VIDEO_CAPTURE |

V4L2_CAP_STREAMING;

cap->device_caps |= V4L2_CAP_VIDEO_CAPTURE |

V4L2_CAP_STREAMING;

} else if (stream_ids == CIF_ISP10_STREAM_MP) {

cap->capabilities = V4L2_CAP_VIDEO_CAPTURE |

V4L2_CAP_STREAMING;

cap->device_caps |= V4L2_CAP_VIDEO_CAPTURE |

V4L2_CAP_STREAMING;

}

else if (stream_ids == CIF_ISP10_STREAM_DMA)

cap->capabilities = V4L2_CAP_VIDEO_M2M_MPLANE |

V4L2_CAP_VIDEO_M2M;

cap->capabilities |= V4L2_CAP_DEVICE_CAPS;

cap->device_caps |= V4L2_CAP_DEVICE_CAPS;

return 0;

}

这只是某个ioctl的处理函数，在内部还有非常多，我们就不一一举例了。因为ISP和Camera已经关联在一起了，所以ISP中的ioctl会去调用Camera驱动中的ioctl。这也就串联起来了！

建议大家可以去网上找个Camera拍照的应用demo，就会清楚为什么驱动会分析到ioctl了。

2. 打开数据流

static int cif_isp10_v4l2_streamon(

struct file *file,

void *priv,

enum v4l2_buf_type buf_type)

{

//......

//打开buffer, 准备接收数据流

ret = vb2_streamon(queue, buf_type);

//开启Camera数据流。让数据流从Camera流到ISP

ret = cif_isp10_streamon(dev, stream_ids);

if (IS_ERR_VALUE(ret)) {

goto err;

}

return 0;

//......

}

开启队列中的buffer，然后调用Camera中的接口开启数据流，让数据流从Camera流到ISP。

int cif_isp10_streamon(

struct cif_isp10_device *dev,

u32 stream_ids)

{

int ret = 0;

bool streamon_sp = stream_ids & CIF_ISP10_STREAM_SP;

bool streamon_mp = stream_ids & CIF_ISP10_STREAM_MP;

bool streamon_dma = stream_ids & CIF_ISP10_STREAM_DMA;

//......

stream_ids = 0;

if (streamon_mp && dev->mp_stream.updt_cfg)

stream_ids |= CIF_ISP10_STREAM_MP;

if (streamon_sp && dev->sp_stream.updt_cfg)

stream_ids |= CIF_ISP10_STREAM_SP;

ret = cif_isp10_config_cif(dev, stream_ids);

if (IS_ERR_VALUE(ret))

goto err;

//开启数据传输

ret = cif_isp10_start(dev, streamon_sp, streamon_mp);

if (IS_ERR_VALUE(ret))

goto err;

//......

}

static int cif_isp10_start(

struct cif_isp10_device *dev,

bool start_sp,

bool start_mp)

{

if (!CIF_ISP10_INP_IS_DMA(dev->config.input_sel)) {

//调用Camera中的ioctl开启数据流

mutex_lock(&dev->img_src_exps.mutex);

cif_isp10_img_src_ioctl(dev->img_src,

RK_VIDIOC_SENSOR_MODE_DATA,

&dev->img_src_exps.data[0].data);

cif_isp10_img_src_ioctl(dev->img_src,

RK_VIDIOC_SENSOR_MODE_DATA,

&dev->img_src_exps.data[1].data);

dev->img_src_exps.data[0].v_frame_id = 0;

dev->img_src_exps.data[1].v_frame_id = 0;

mutex_unlock(&dev->img_src_exps.mutex);

//.......

}

//.......

return ret;

}

long cif_isp10_img_src_ioctl(

struct cif_isp10_img_src *img_src,

unsigned int cmd,

void *arg)

{

if (!img_src) {

cif_isp10_pltfrm_pr_err(NULL, "img_src is NULL\n");

return -EINVAL;

}

//调用Camera的ioctl

return img_src->ops->ioctl(img_src->img_src, cmd, arg);

}

通过上面的一些列调用关系可以看出，最终调用了Camera的ioctl。这里img_src指的就是sensor。数据就开始从Camera一直流向ISP。

总结

我们分析了ISP驱动的一个大致流程，Camera的很多核心算法不是放在驱动上的，大部分都是放在应用层上面的。所以我们在驱动上看到的更多是一些控制，参数配置等接口。



审核编辑：刘清