IPVS的ICMP报文处理-由内到外
这里主要明与NAT/Masq转发模式相关的ICMP报文处理，但也会提及由于出错引发的IPVS系统主动发送的ICMP报文。
1.ICMP由外到内处理流程入口
入口函数ip_vs_in实质上挂载在netfilter的2个hook点上，分别为：NF_INET_LOCAL_IN和NF_INET_LOCAL_OUT。第一个hook点作用于目的地址为本机的报文；后者作用于由本机发送的报文。此函数用于处理IPVS由外到内的请求报文，当然也包括ICMP报文。如果协议号为IPPROTO_ICMP/IPPROTO_ICMPV6，分别使用函数ip_vs_in_icmp、ip_vs_in_icmp_v6进行处理。
static unsigned int ip_vs_in(struct netns_ipvs *ipvs, unsigned int hooknum, struct sk_buff *skb, int af) { struct ip_vs_iphdr iph; struct ip_vs_protocol *pp; struct ip_vs_proto_data *pd; struct ip_vs_conn *cp; #ifdef CONFIG_IP_VS_IPV6 if (af == AF_INET6) { if (unlikely(iph.protocol == IPPROTO_ICMPV6)) { int verdict = ip_vs_in_icmp_v6(ipvs, skb, &related, hooknum, &iph); if (related) return verdict; } } else #endif if (unlikely(iph.protocol == IPPROTO_ICMP)) { int verdict = ip_vs_in_icmp(ipvs, skb, &related, hooknum); if (related) return verdict; } /* Protocol supported? */ pd = ip_vs_proto_data_get(ipvs, iph.protocol); if (unlikely(!pd)) return NF_ACCEPT;
如果上述的ip_vs_in_icmp函数未能进行ICMP处理，在随后的协议查找中也会失败，因为IPVS不支持ICMP协议。
2.IPVS由外到内的ICMP处理
函数ip_vs_in_icmp目前仅处理三种类型的ICMP报文：ICMP_DEST_UNREACH、ICMP_SOURCE_QUENCH和ICMP_TIME_EXCEEDED。如果不是这三种类型，设置为不相关联的ICMP，结束处理。
static int ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, unsigned int hooknum) { struct icmphdr _icmph, *ic; struct iphdr _ciph, *cih; /* The ip header contained within the ICMP */ struct ip_vs_iphdr ciph; struct ip_vs_conn *cp; struct ip_vs_protocol *pp; struct ip_vs_proto_data *pd; *related = 1; iph = ip_hdr(skb); offset = ihl = iph->ihl * 4; ic = skb_header_pointer(skb, offset, sizeof(_icmph), &_icmph); /* * Work through seeing if this is for us. * These checks are supposed to be in an order that means easy things are checked first to speed up processing.... however * this means that some packets will manage to get a long way down this stack and then be rejected, but that's life. */ if ((ic->type != ICMP_DEST_UNREACH) && (ic->type != ICMP_SOURCE_QUENCH) && (ic->type != ICMP_TIME_EXCEEDED)) { *related = 0; return NF_ACCEPT; }
接下来，找到ICMP报文中内层的IP报文。在这里，先检查以下内层的是不是IPIP协议报文，如果是IPIP协议，进行合法性检查，最后，偏移到最内层的IP报头处。
/* Now find the contained IP header */ offset += sizeof(_icmph); cih = skb_header_pointer(skb, offset, sizeof(_ciph), &_ciph); /* Special case for errors for IPIP packets */ ipip = false; if (cih->protocol == IPPROTO_IPIP) { if (unlikely(cih->frag_off & htons(IP_OFFSET))) return NF_ACCEPT; /* Error for our IPIP must arrive at LOCAL_IN */ if (!(skb_rtable(skb)->rt_flags & RTCF_LOCAL)) return NF_ACCEPT; offset += cih->ihl * 4; cih = skb_header_pointer(skb, offset, sizeof(_ciph), &_ciph); if (cih == NULL) return NF_ACCEPT; /* The packet looks wrong, ignore */ ipip = true; }
之后根据找到的最内层IP报头中的协议字段，来查找相应的IPVS协议数据结构,进而找到协议结构。为了完整加解密的需要，AH/ESP协议要求报文不能分片（dont_defag）。
根据其中的IP头部信息，查找IPVS连接。如果找到的话，表明此ICMP报文是由之前客户端的请求报文所触发的，由真实服务器回复的ICMP报文。就有函数handle_response_icmp处理。
pd = ip_vs_proto_data_get(ipvs, cih->protocol); if (!pd) return NF_ACCEPT; pp = pd->pp; /* Is the embedded protocol header present? */ if (unlikely(cih->frag_off & htons(IP_OFFSET) && pp->dont_defrag)) return NF_ACCEPT;
对于找不到关联IPVS连接的ICMP报文，默认是不进行处理的，这可通过PROC文件/proc/sys/net/ipv4/vs/schedule_icmp进行更改。如果其为真，IPVS系统将尝试将此ICMP报文调度的选择的目的服务器。
offset2 = offset; ip_vs_fill_iph_skb_icmp(AF_INET, skb, offset, !ipip, &ciph); offset = ciph.len; /* The embedded headers contain source and dest in reverse order. For IPIP this is error for request, not for reply. */ cp = pp->conn_in_get(ipvs, AF_INET, skb, &ciph); if (!cp) { if (!sysctl_schedule_icmp(ipvs)) return NF_ACCEPT; if (!ip_vs_try_to_schedule(ipvs, AF_INET, skb, pd, &v, &cp, &ciph)) return v; new_cp = true; } verdict = NF_DROP; /* Ensure the checksum is correct */ if (!skb_csum_unnecessary(skb) && ip_vs_checksum_complete(skb, ihl)) { /* Failed checksum! */ IP_VS_DBG(1, "Incoming ICMP: failed checksum from %pI4!\n", &iph->saddr); goto out; }
对于原报文是IPIP协议报文的特殊情况，即IPVS在隧道转发模式下，接收到的ICMP错误报文，如果ICMP的类型为ICMP_DEST_UNREACH，并且代码为ICMP_FRAG_NEEDED（需要分片），从ICMP报文中取出要求的MTU值，作为路径MTU更新到对应的路由表项中。
if (ipip) { __be32 info = ic->un.gateway; __u8 type = ic->type; __u8 code = ic->code; /* Update the MTU */ if (ic->type == ICMP_DEST_UNREACH && ic->code == ICMP_FRAG_NEEDED) { struct ip_vs_dest *dest = cp->dest; u32 mtu = ntohs(ic->un.frag.mtu); __be16 frag_off = cih->frag_off; /* Strip outer IP and ICMP, go to IPIP header */ if (pskb_pull(skb, ihl + sizeof(_icmph)) == NULL) goto ignore_ipip; offset2 -= ihl + sizeof(_icmph); skb_reset_network_header(skb); IP_VS_DBG(12, "ICMP for IPIP %pI4->%pI4: mtu=%u\n", &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr, mtu); ipv4_update_pmtu(skb, ipvs->net, mtu, 0, 0, 0, 0); /* Client uses PMTUD? */ if (!(frag_off & htons(IP_DF))) goto ignore_ipip; /* Prefer the resulting PMTU */ if (dest) { struct ip_vs_dest_dst *dest_dst; dest_dst = rcu_dereference(dest->dest_dst); if (dest_dst) mtu = dst_mtu(dest_dst->dst_cache); } if (mtu > 68 + sizeof(struct iphdr)) mtu -= sizeof(struct iphdr); info = htonl(mtu); }
此处，去掉此ICMP报文的最外层IP头，ICMP头部以及IPIP头部，仅保留原始的客户端IP请求报文，使用icmp_send函数发送ICMP报文到最初的客户端。除去以上的ICMP分片进行了处理，其它类型的ICMP报文，未做处理。
/* Strip outer IP, ICMP and IPIP, go to IP header of original request. */ if (pskb_pull(skb, offset2) == NULL) goto ignore_ipip; skb_reset_network_header(skb); IP_VS_DBG(12, "Sending ICMP for %pI4->%pI4: t=%u, c=%u, i=%u\n", &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr, type, code, ntohl(info)); icmp_send(skb, type, code, info); /* ICMP can be shorter but anyways, account it */ ip_vs_out_stats(cp, skb); ignore_ipip: consume_skb(skb); verdict = NF_STOLEN; goto out; }
函数的最后，对于内层IP头部协议字段为：IPPROTO_TCP、IPPROTO_UDP和IPPROTO_SCTP的报文，offset偏移到四层头部的源端口和目的端口处，调用ip_vs_icmp_xmit函数转发ICMP报文。
/* do the statistics and put it back */ ip_vs_in_stats(cp, skb); if (IPPROTO_TCP == cih->protocol || IPPROTO_UDP == cih->protocol || IPPROTO_SCTP == cih->protocol) offset += 2 * sizeof(__u16); verdict = ip_vs_icmp_xmit(skb, cp, pp, offset, hooknum, &ciph); out:
3.ICMP报文发送
对于除NAT/Masq转发模式之外的其它模式，由于不需要进行地址或者端口的转换，直接调用IPVS连接的发送函数packet_xmit处理。
int ip_vs_icmp_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, struct ip_vs_protocol *pp, int offset, unsigned int hooknum, struct ip_vs_iphdr *iph) { /* The ICMP packet for VS/TUN, VS/DR and LOCALNODE will be forwarded directly here, because there is no need to translate address/port back */ if (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ) { if (cp->packet_xmit) rc = cp->packet_xmit(skb, cp, pp, iph); else rc = NF_ACCEPT; /* do not touch skb anymore */ atomic_inc(&cp->in_pkts); goto out; }
对于转发NF_INET_FORWARD的hook点，在查找路由时使用IP_VS_RT_MODE_NON_LOCAL标志，表示不允许结果是到本机的路由。
/* mangle and send the packet here (only for VS/NAT) */ was_input = rt_is_input_route(skb_rtable(skb)); /* LOCALNODE from FORWARD hook is not supported */ rt_mode = (hooknum != NF_INET_FORWARD) ? IP_VS_RT_MODE_LOCAL | IP_VS_RT_MODE_NON_LOCAL | IP_VS_RT_MODE_RDR : IP_VS_RT_MODE_NON_LOCAL; local = __ip_vs_get_out_rt(cp->ipvs, cp->af, skb, cp->dest, cp->daddr.ip, rt_mode, NULL, iph); if (local < 0) goto tx_error; rt = skb_rtable(skb);
如果此连接是由同步进程接收到的，并且前面路由查询的结果目的是发往本机，而且netfilter系统已经创建了连接跟踪结构，结束处理返回。
/* Avoid duplicate tuple in reply direction for NAT traffic to local address when connection is sync-ed */ #if IS_ENABLED(CONFIG_NF_CONNTRACK) if (cp->flags & IP_VS_CONN_F_SYNC && local) { enum ip_conntrack_info ctinfo; struct nf_conn *ct = nf_ct_get(skb, &ctinfo); if (ct) { IP_VS_DBG(10, "%s(): stopping DNAT to local address %pI4\n", __func__, &cp->daddr.ip); goto tx_error; } } #endif
以下判断，对于原始报文路由到本机，目的IP为回环地址，并且以上查询到的出口路由也是发送本机的报文，停止DNAT处理。
/* From world but DNAT to loopback address? */
if (local && ipv4_is_loopback(cp->daddr.ip) && was_input) {
IP_VS_DBG(1, "%s(): stopping DNAT to loopback %pI4\n", __func__, &cp->daddr.ip);
goto tx_error;
}
函数ip_vs_nat_icmp执行ICMP报文的DNAT转换，最终由函数ip_vs_nat_send_or_cont执行发送操作。
/* copy-on-write the packet before mangling it */
if (!skb_make_writable(skb, offset))
goto tx_error;
if (skb_cow(skb, rt->dst.dev->hard_header_len))
goto tx_error;
ip_vs_nat_icmp(skb, pp, cp, 0);
/* Another hack: avoid icmp_send in ip_fragment */
skb->ignore_df = 1;
rc = ip_vs_nat_send_or_cont(NFPROTO_IPV4, skb, cp, local);
goto out;
4.ICMP的DNAT转换
函数ip_vs_nat_icmp负责对ICMP报文进行DNAT处理。由于当前的处理报文是由外部到内部，inout参数为0。修改报文的IP头部的目的地址，和ICMP内层IP报文的源IP地址（因为内层IP表示原方向报文），同时更新IP头部校验和。
void ip_vs_nat_icmp(struct sk_buff *skb, struct ip_vs_protocol *pp, struct ip_vs_conn *cp, int inout) { struct iphdr *iph = ip_hdr(skb); unsigned int icmp_offset = iph->ihl*4; struct icmphdr *icmph = (struct icmphdr *)(skb_network_header(skb) + icmp_offset); struct iphdr *ciph = (struct iphdr *)(icmph + 1); if (inout) { iph->saddr = cp->vaddr.ip; ip_send_check(iph); ciph->daddr = cp->vaddr.ip; ip_send_check(ciph); } else { iph->daddr = cp->daddr.ip; ip_send_check(iph); ciph->saddr = cp->daddr.ip; ip_send_check(ciph); }
随后，对于4层协议IPPROTO_TCP、IPPROTO_UDP和IPPROTO_SCTP，如果报文为由外到内，修改ICMP内部4层头中源端口号（还原为发送时真实服务器的端口号）。
/* the TCP/UDP/SCTP port */
if (IPPROTO_TCP == ciph->protocol || IPPROTO_UDP == ciph->protocol || IPPROTO_SCTP == ciph->protocol) {
__be16 *ports = (void *)ciph + ciph->ihl*4;
if (inout)
ports[1] = cp->vport;
else
ports[0] = cp->dport;
}
/* And finally the ICMP checksum */
icmph->checksum = 0;
icmph->checksum = ip_vs_checksum_complete(skb, icmp_offset);
skb->ip_summed = CHECKSUM_UNNECESSARY;
5.NAT发送
函数ip_vs_nat_send_or_cont执行最后的发送操作。在此阶段，如果连接没有设置连接跟踪标志IP_VS_CONN_F_NFCT，释放建立的连接跟踪结构；否则，更新连接跟踪信息。默认情况下IPVS不会为新连接添加标志IP_VS_CONN_F_NFCT，即不会保留连接跟踪信息，但是可通过PROC文件：/proc/sys/net/ipv4/vs/conntrack 修改此默认行为。
/* return NF_STOLEN (sent) or NF_ACCEPT if local=1 (not sent) */ static inline int ip_vs_nat_send_or_cont(int pf, struct sk_buff *skb, struct ip_vs_conn *cp, int local) { int ret = NF_STOLEN; skb->ipvs_property = 1; if (likely(!(cp->flags & IP_VS_CONN_F_NFCT))) ip_vs_notrack(skb); else ip_vs_update_conntrack(skb, cp, 1);
如果目的地址非本地，或者目的端口变化，或者目的地址有变化，任何一种情况发送都将导致缓存的sock结构失效。最后，对于非本地目的地址的报文，在调用NF_INET_LOCAL_OUT点的hook函数之后，由dst_output发出。
/* Remove the early_demux association unless it's bound for the exact same port and address on this host after translation. */ if (!local || cp->vport != cp->dport || !ip_vs_addr_equal(cp->af, &cp->vaddr, &cp->daddr)) ip_vs_drop_early_demux_sk(skb); if (!local) { skb_forward_csum(skb); NF_HOOK(pf, NF_INET_LOCAL_OUT, cp->ipvs->net, NULL, skb, NULL, skb_dst(skb)->dev, dst_output); } else ret = NF_ACCEPT;
6.ICMP在NF_INET_FORWARD上的处理
另外，看一下IPVS在注册netfilter的hook点的定义结构ip_vs_ops，除了以上的hook的ip_vs_in函数，在hook点NF_INET_FORWARD上，注册了ip_vs_forward_icmp函数，用于处理目的地址为0.0.0.0/0的ICMP报文。
static const struct nf_hook_ops ip_vs_ops[] = { /* After packet filtering (but before ip_vs_out_icmp), catch icmp destined for 0.0.0.0/0, which is for incoming IPVS connections */ { .hook = ip_vs_forward_icmp, .pf = NFPROTO_IPV4, .hooknum = NF_INET_FORWARD, .priority = 99, }, }
由于使用fwmark配置的IPVS虚拟服务，iptables的MARK功能不能进行标记。所以在NF_INET_FORWARD进行处理。
# iptables -A PREROUTING -t mangle -d 207.175.44.110/31 -j MARK --set-mark 1
内核版本 4.15
转载: https://blog.csdn.net/sinat_20184565/article/details/102410231
