linux 核间通讯rpmsg架构分析
以imx8为例
在最底层硬件上，A核和M核通讯是靠硬件来进行的，称为MU，如图
Linux RPMsg 是在virtio framework上实现的一个消息传递机制
VirtIO 是一个用来实现“虚拟IO”的通用框架，典型虚拟的pci，网卡，磁盘等虚拟设备，kvm等都使用了这个技术
与virtio对应的还有一个virtio-ring，其实现了 virtio 的具体通信机制和数据流程
virtio 层属于控制层，负责前后端之间的通知机制（kick，notify）和控制流程，而 virtio-vring 则负责具体数据流转发
从整体架构上看，关系如下：
最底层有platform_bus，负责从dts获取配置来初始化相关对象，如virtio_device，初始化其config操作函数列表以及devID等，同时注册到virtio_bus
dts相关配置：
&rpmsg{
/*
* 64K for one rpmsg instance:
*/
vdev-nums = <2>;
reg = <0x0 0x90000000 0x0 0x20000>;
status = "okay";
};
主要初始化过程在imx_rpmsg_probe中，关键操作有：
注册MU相关的硬件中断
ret = request_irq(irq, imx_mu_rpmsg_isr, IRQF_EARLY_RESUME | IRQF_SHARED,"imx-mu-rpmsg", rpdev);
初始化MU硬件
ret = imx_rpmsg_mu_init(rpdev);
创建工作队列用于处理MU中断数据
INIT_DELAYED_WORK(&(rpdev->rpmsg_work), rpmsg_work_handler);
创建通知链用于对接virtio queue
BLOCKING_INIT_NOTIFIER_HEAD(&(rpdev->notifier));
初始化virtio_device并注册
for (j = 0; j < rpdev->vdev_nums; j++) {
pr_debug("%s rpdev%d vdev%d: vring0 0x%x, vring1 0x%x\n",
__func__, rpdev->core_id, rpdev->vdev_nums,
rpdev->ivdev[j].vring[0],
rpdev->ivdev[j].vring[1]);
rpdev->ivdev[j].vdev.id.device = VIRTIO_ID_RPMSG;
rpdev->ivdev[j].vdev.config = &imx_rpmsg_config_ops;
rpdev->ivdev[j].vdev.dev.parent = &pdev->dev;
rpdev->ivdev[j].vdev.dev.release = imx_rpmsg_vproc_release;
rpdev->ivdev[j].base_vq_id = j * 2;
ret = register_virtio_device(&rpdev->ivdev[j].vdev);
if (ret) {
pr_err("%s failed to register rpdev: %d\n",
__func__, ret);
return ret;
}
}
值得注意的是virtio_device的config结构 rpdev->ivdev[j].vdev.config = &imx_rpmsg_config_ops;
static struct virtio_config_ops imx_rpmsg_config_ops = {
.get_features = imx_rpmsg_get_features,
.finalize_features = imx_rpmsg_finalize_features,
.find_vqs = imx_rpmsg_find_vqs,
.del_vqs = imx_rpmsg_del_vqs,
.reset = imx_rpmsg_reset,
.set_status = imx_rpmsg_set_status,
.get_status = imx_rpmsg_get_status,
};
imx_rpmsg_find_vqs
--> rp_find_vq
-->ioremap_nocache
-->vring_new_virtqueue(...imx_rpmsg_notify, callback...)
需要注意的是callback的注册过程，在rpmsg_bus中
rpmsg_probe
-->vq_callback_t *vq_cbs[] = { rpmsg_recv_done, rpmsg_xmit_done };
-->virtio_find_vqs(vdev, 2, vqs, vq_cbs, names, NULL);
在此处注册的imx_rpmsg_notify 和 callback 将被virtio_bus框架所调用
中间virtio_bus承上启下，并负责提供统一标准的virtio queue操作接口，如virtqueue_add，virtqueue_kick等
针对struct virtqueue，对外只有一个callback函数，用于表示queue的数据变化
struct virtqueue {
struct list_head list;
void (*callback)(struct virtqueue *vq);
const char *name;
struct virtio_device *vdev;
unsigned int index;
unsigned int num_free;
void *priv;
};
其实virtqueue只是提供一层标准queue的操作接口，其具体实现依靠vring_virtqueue
struct vring_virtqueue {
struct virtqueue vq;
/* Actual memory layout for this queue */
struct vring vring
{
//queue的具体实现
unsigned int num;
struct vring_desc *desc;
struct vring_avail *avail;
struct vring_used *used;
};
/* How to notify other side. FIXME: commonalize hcalls! */
bool (*notify)(struct virtqueue *vq);
...
/* Per-descriptor state. */
struct vring_desc_state desc_state[];
};
其触发过程在vring_interrupt
irqreturn_t vring_interrupt(int irq, void *_vq)
{
##对外只有virtqueue，找到其包装vring_virtqueue
struct vring_virtqueue *vq = to_vvq(_vq);
if (!more_used(vq)) {
pr_debug("virtqueue interrupt with no work for %p\n", vq);
return IRQ_NONE;
}
if (unlikely(vq->broken))
return IRQ_HANDLED;
pr_debug("virtqueue callback for %p (%p)\n", vq, vq->vq.callback);
##调用virtqueue的callback
if (vq->vq.callback)
vq->vq.callback(&vq->vq);
return IRQ_HANDLED;
}
结合中断，整体流程如下：
imx_mu_rpmsg_isr
-->rpmsg_work_handler
-->vring_interrupt
-->virtqueue.callback
关于vring_virtqueue，包含一个notify，用于通知queue有变化
virtqueue_add 和 virtqueue_kick 以及 virtqueue_notify 都能够触发notify
最终notify的实现在imx_rpmsg_notify，其内容为设置MU寄存器，发送数据
/* kick the remote processor, and let it know which virtqueue to poke at */
static bool imx_rpmsg_notify(struct virtqueue *vq)
{
unsigned int mu_rpmsg = 0;
struct imx_rpmsg_vq_info *rpvq = vq->priv;
mu_rpmsg = rpvq->vq_id << 16;
mutex_lock(&rpvq->rpdev->lock);
/*
* Send the index of the triggered virtqueue as the mu payload.
* Use the timeout MU send message here.
* Since that M4 core may not be loaded, and the first MSG may
* not be handled by M4 when multi-vdev is enabled.
* To make sure that the message wound't be discarded when M4
* is running normally or in the suspend mode. Only use
* the timeout mechanism by the first notify when the vdev is
* registered.
*/
if (unlikely(rpvq->rpdev->first_notify > 0)) {
rpvq->rpdev->first_notify--;
MU_SendMessageTimeout(rpvq->rpdev->mu_base, 1, mu_rpmsg, 200);
} else {
MU_SendMessage(rpvq->rpdev->mu_base, 1, mu_rpmsg);
}
mutex_unlock(&rpvq->rpdev->lock);
return true;
}
最上面可看成基于rpmsg的应用，挂载到rpmsg_bus总线，针对rpmsg也有对应的标准操作接口，如rpmsg_send，rpmsg_sendto，rpmsg_poll等等
在rpmsg_bus这一层，还有一个rpmsg_endpoint概念，其对应有一个rpmsg_endpoint_ops，包含send，send_to等接口，目前还未对其深入研究
static const struct rpmsg_endpoint_ops virtio_endpoint_ops = {
.destroy_ept = virtio_rpmsg_destroy_ept,
.send = virtio_rpmsg_send,
.sendto = virtio_rpmsg_sendto,
.send_offchannel = virtio_rpmsg_send_offchannel,
.trysend = virtio_rpmsg_trysend,
.trysendto = virtio_rpmsg_trysendto,
.trysend_offchannel = virtio_rpmsg_trysend_offchannel,
};
发送流程为
rpmsg_send
-->rpmsg_endpoint.ops->send
-->virtio_rpmsg_send
-->virtqueue_add_outbuf 往queue填充数据
-->virtqueue_kick 通知对端
-->virtqueue_notify
-->imx_rpmsg_notify
-->MU_REG_WRITE
rpmsg_bus总线默认提供两个回调rpmsg_recv_done和rpmsg_xmit_done以便通知给上层rpmsg应用,分别表示收到数据及发送完成
接收处理流程：
imx_mu_rpmsg_isr
-->rpmsg_work_handler
-->vring_interrupt
-->virtqueue.callback
-->rpmsg_recv_done or rpmsg_xmit_done
/* called when an rx buffer is used, and it's time to digest a message */
static void rpmsg_recv_done(struct virtqueue *rvq)
{
struct virtproc_info *vrp = rvq->vdev->priv;
struct device *dev = &rvq->vdev->dev;
struct rpmsg_hdr *msg;
unsigned int len, msgs_received = 0;
int err;
msg = virtqueue_get_buf(rvq, &len);
if (!msg) {
dev_err(dev, "uhm, incoming signal, but no used buffer ?\n");
return;
}
while (msg) {
err = rpmsg_recv_single(vrp, dev, msg, len);
if (err)
break;
msgs_received++;
msg = virtqueue_get_buf(rvq, &len);
}
dev_dbg(dev, "Received %u messages\n", msgs_received);
/* tell the remote processor we added another available rx buffer */
if (msgs_received)
## 通知接收queue
virtqueue_kick(vrp->rvq);
}
static void rpmsg_xmit_done(struct virtqueue *svq)
{
struct virtproc_info *vrp = svq->vdev->priv;
dev_dbg(&svq->vdev->dev, "%s\n", __func__);
/* wake up potential senders that are waiting for a tx buffer */
wake_up_interruptible(&vrp->sendq);
}
整体过程如下：
