IP_PKTINFO选项

 UDP 在多网卡的情况下，可能会发生服务器端回复报文源地址不对的情况。目前这种情况在portal  对接ac的时候经常出现

所以问题就是：需要保存本次udp 请求的local ip remote ip

IP_PKTINFO 这个选项就是让内核在 socket 中保存 IP 报文的信息，当然也包括了报文的源地址和目的地址。IP_PKTINFO 和 msghdr 的关系可以在这个 stackoverflow 中找到：

https://stackoverflow.com/questions/3062205/setting-the-source-ip-for-a-udp-socket。

 The code in the link uses IP_PKTINFO (or IP_RECVDSTADDR depending on the platform) to get the destination address of a UDP message from the ancillary cmsg(3) data. Paraphrased here:

struct msghdr msg;
struct cmsghdr *cmsg;
struct in_addr addr;
// after recvmsg(sd, &msg, flags);
for(cmsg = CMSG_FIRSTHDR(&msg);
    cmsg != NULL;
    cmsg = CMSG_NXTHDR(&msg, cmsg)) {
  if (cmsg->cmsg_level == IPPROTO_IP && cmsg->cmsg_type == IP_PKTINFO) {
    addr = ((struct in_pktinfo*)CMSG_DATA(cmsg))->ipi_addr;
    printf("message received on address %s\n", inet_ntoa(addr));
  }
}

Gene, your question asked how to set the source address on outgoing packets. With IP_PKTINFO it is possible to set the ipi_spec_dst field of the struct in_pktinfo in the ancillary data passed to sendmsg(2). See the post referenced above, cmsg(3), and sendmsg(2) for guidelines on how to create and manipulate the ancillary data in a struct msghdr. An example (no guarantee here) might be: 

struct msghdr msg;
struct cmsghdr *cmsg;
struct in_pktinfo *pktinfo;
// after initializing msghdr & control data to CMSG_SPACE(sizeof(struct in_pktinfo))
cmsg = CMSG_FIRSTHDR(&msg);
cmsg->cmsg_level = IPPROTO_IP;
cmsg->cmsg_type = IP_PKTINFO;
cmsg->cmsg_len = CMSG_LEN(sizeof(struct in_pktinfo));
pktinfo = (struct in_pktinfo*) CMSG_DATA(cmsg);
pktinfo->ipi_ifindex = src_interface_index;
pktinfo->ipi_spec_dst = src_addr;
// bytes_sent = sendmsg(sd, &msg, flags);

  结构体in_pktinfo如下，其中ipi_ifindex表示接收报文的接口索引；成员ipi_spec_dst表示报文的本地地址；而ipi_addr表示报文头部的目的地址。在使用sendmsg结构发送报文时，如果ipi_spec_dst字段不为零，在内核中查找路由时，将使用此指定地址为源地址进行查询。再者，如果ipi_ifindex不为零，在查找路由时，此索引所对应的接口的索引作为查找路由时的出接口索引。

struct in_pktinfo {
        unsigned int   ipi_ifindex;  /* Interface index */
        struct in_addr ipi_spec_dst; /* Local address  是UDP包中路由目的地址（the destination in_addr）*/  
        struct in_addr ipi_addr;     /* Header Destination
                                        address是UDP包中的头标识目的地址（the receiving interface in_addr） */
    };

 

IP_PKTINFO信息

如下的UDP协议接收函数udp_recvmsg，如果inet_sock套接口cmsg_flags标志不为空，使用函数ip_cmsg_recv_offset添加控制信息。

ip_cmsg_recv_offset--->
    if (flags & IP_CMSG_PKTINFO) {
		ip_cmsg_recv_pktinfo(msg, skb);

static void ip_cmsg_recv_pktinfo(struct msghdr *msg, struct sk_buff *skb)
{
	struct in_pktinfo info = *PKTINFO_SKB_CB(skb);

	info.ipi_addr.s_addr = ip_hdr(skb)->daddr;

	put_cmsg(msg, SOL_IP, IP_PKTINFO, sizeof(info), &info);
}

  ipi_addr中保存了报文IP头部的目的地址。结构体in_pktinfo中的其它两个字段保存在skb的回调cb字段，参见宏PKTINFO_SKB_CB。

其余的几个字段在ipv4_pktinfo_prepare 中填充

/**
 * ipv4_pktinfo_prepare - transfer some info from rtable to skb
 * @sk: socket
 * @skb: buffer
 *
 * To support IP_CMSG_PKTINFO option, we store rt_iif and specific
 * destination in skb->cb[] before dst drop.
 * This way, receiver doesn't make cache line misses to read rtable.
 */
void ipv4_pktinfo_prepare(const struct sock *sk, struct sk_buff *skb)
{
	struct in_pktinfo *pktinfo = PKTINFO_SKB_CB(skb);
	bool prepare = (inet_sk(sk)->cmsg_flags & IP_CMSG_PKTINFO) ||
		       ipv6_sk_rxinfo(sk);

	if (prepare && skb_rtable(skb)) {
		/* skb->cb is overloaded: prior to this point it is IP{6}CB
		 * which has interface index (iif) as the first member of the
		 * underlying inet{6}_skb_parm struct. This code then overlays
		 * PKTINFO_SKB_CB and in_pktinfo also has iif as the first
		 * element so the iif is picked up from the prior IPCB. If iif
		 * is the loopback interface, then return the sending interface
		 * (e.g., process binds socket to eth0 for Tx which is
		 * redirected to loopback in the rtable/dst).
		 */
		struct rtable *rt = skb_rtable(skb);
		bool l3slave = ipv4_l3mdev_skb(IPCB(skb)->flags);//indicating a Layer 3 slave device

		if (pktinfo->ipi_ifindex == LOOPBACK_IFINDEX)
			pktinfo->ipi_ifindex = inet_iif(skb);
        //ipi_ifindex索引等于环回接口LOOPBACK_IFINDEX的情况，如果路由缓存中的rt_iif有值，inet_iif函数返回此值作为原始发送接口索引，
        //否者，使用skb接口中的skb_iif值，此值在接收路径的函数__netif_receive_skb_core中赋值。
		else if (l3slave && rt && rt->rt_iif)
			pktinfo->ipi_ifindex = rt->rt_iif;

		pktinfo->ipi_spec_dst.s_addr = fib_compute_spec_dst(skb);
	} else {
		pktinfo->ipi_ifindex = 0;
		pktinfo->ipi_spec_dst.s_addr = 0;
	}
	skb_dst_drop(skb);
}

/*
ipi_ifindex索引等于环回接口LOOPBACK_IFINDEX的情况，如果路由缓存中的rt_iif有值，inet_iif函数返回此值作为原始发送接口索引，
否者，使用skb接口中的skb_iif值，此值在接收路径的函数__netif_receive_skb_core中赋值。
*/
static inline int inet_iif(const struct sk_buff *skb)
{
	struct rtable *rt = skb_rtable(skb);
	if (rt && rt->rt_iif)
		return rt->rt_iif;
	return skb->skb_iif;
}
__be32 fib_compute_spec_dst(struct sk_buff *skb)
{
	struct net_device *dev = skb->dev;
	struct in_device *in_dev;
	struct fib_result res;
	struct rtable *rt;
	struct net *net;
	int scope;
/*对于路由缓存中仅有RTCF_LOCAL本地标志，而没有广播和多播标志的情况，返回报文IP头部中的目的地址，
可见此时in_pktinfo结构中的ipi_spec_dst就和ipi_addr成员的值相同
	*/
	rt = skb_rtable(skb);
	if ((rt->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST | RTCF_LOCAL)) ==
	    RTCF_LOCAL)
		return ip_hdr(skb)->daddr;

/*
否则，如果路由缓存显示为广播或者多播属性，并且如果报文的源IP地址不是零网地址，使用此地址作为目的地址，执行路由查找，
返回查找到的路由中的源地址（FIB_RES_PREFSRC）。其它情况下，使用函数inet_select_addr，也是以报文IP头中的源地址作为目的地址，查找相符的源地址。
*/
	in_dev = __in_dev_get_rcu(dev);

	net = dev_net(dev);

	scope = RT_SCOPE_UNIVERSE;
	if (!ipv4_is_zeronet(ip_hdr(skb)->saddr)) {
		bool vmark = in_dev && IN_DEV_SRC_VMARK(in_dev);
		struct flowi4 fl4 = {
			.flowi4_iif = LOOPBACK_IFINDEX,
			.flowi4_oif = l3mdev_master_ifindex_rcu(dev),
			.daddr = ip_hdr(skb)->saddr,
			.flowi4_tos = RT_TOS(ip_hdr(skb)->tos),
			.flowi4_scope = scope,
			.flowi4_mark = vmark ? skb->mark : 0,
		};
		if (!fib_lookup(net, &fl4, &res, 0))
			return fib_result_prefsrc(net, &res);
	} else {
		scope = RT_SCOPE_LINK;
	}

	return inet_select_addr(dev, ip_hdr(skb)->saddr, scope);
}

 

UDP_SEND IP_PKTINFO控制信息

如下以UDP协议为例，在函数udp_sendmsg中，如果控制消息的长度不为零，首先处理UDP层的控制消息，如果其返回值大于零，表明需要进行IP层控制消息的处理，调用处理函数ip_cmsg_send。

在函数ip_cmsg_send中，将in_pktinfo结构中的成员ipi_ifindex和ipi_spec_dst赋值给ipcm_cookie结构的成员ofi和addr变量。

case IP_PKTINFO:
		{
			struct in_pktinfo *info;
			if (cmsg->cmsg_len != CMSG_LEN(sizeof(struct in_pktinfo)))
				return -EINVAL;
			info = (struct in_pktinfo *)CMSG_DATA(cmsg);
			if (info->ipi_ifindex)
				ipc->oif = info->ipi_ifindex;
			ipc->addr = info->ipi_spec_dst.s_addr;
			break;
		}

继续看函数udp_sendmsg的后续处理，对于flowi4结构类型变量fl4，其成员flowi4_oif和saddr，赋予了ipc中的oif和addr值，作为查询路由时使用的出接口和源IP地址。

saddr = ipc.addr;
if (!rt) {
		struct net *net = sock_net(sk);
		__u8 flow_flags = inet_sk_flowi_flags(sk);

		fl4 = &fl4_stack;

		flowi4_init_output(fl4, ipc.oif, ipc.sockc.mark, tos,
				   RT_SCOPE_UNIVERSE, sk->sk_protocol,
				   flow_flags,
				   faddr, saddr, dport, inet->inet_sport,
				   sk->sk_uid);

		security_sk_classify_flow(sk, flowi4_to_flowi(fl4));
		rt = ip_route_output_flow(net, fl4, sk);
		if (IS_ERR(rt)) {
			err = PTR_ERR(rt);
			rt = NULL;
			if (err == -ENETUNREACH)
				IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
			goto out;
		}

		err = -EACCES;
		if ((rt->rt_flags & RTCF_BROADCAST) &&
		    !sock_flag(sk, SOCK_BROADCAST))
			goto out;
		if (connected)
			sk_dst_set(sk, dst_clone(&rt->dst));
	}