Daniel Borkmann | da5bab0 | 2013-06-08 12:56:03 +0200 | [diff] [blame] | 1 | /* |
| 2 | * IPV4 GSO/GRO offload support |
| 3 | * Linux INET implementation |
| 4 | * |
| 5 | * This program is free software; you can redistribute it and/or |
| 6 | * modify it under the terms of the GNU General Public License |
| 7 | * as published by the Free Software Foundation; either version |
| 8 | * 2 of the License, or (at your option) any later version. |
| 9 | * |
| 10 | * UDPv4 GSO support |
| 11 | */ |
| 12 | |
| 13 | #include <linux/skbuff.h> |
| 14 | #include <net/udp.h> |
| 15 | #include <net/protocol.h> |
| 16 | |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 17 | static DEFINE_SPINLOCK(udp_offload_lock); |
Shlomo Pongratz | a1d0cd8 | 2014-01-22 15:23:29 +0200 | [diff] [blame] | 18 | static struct udp_offload_priv __rcu *udp_offload_base __read_mostly; |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 19 | |
Shlomo Pongratz | a664a4f | 2014-02-02 15:42:10 +0200 | [diff] [blame] | 20 | #define udp_deref_protected(X) rcu_dereference_protected(X, lockdep_is_held(&udp_offload_lock)) |
| 21 | |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 22 | struct udp_offload_priv { |
| 23 | struct udp_offload *offload; |
| 24 | struct rcu_head rcu; |
| 25 | struct udp_offload_priv __rcu *next; |
| 26 | }; |
| 27 | |
Tom Herbert | 8bce6d7 | 2014-09-29 20:22:29 -0700 | [diff] [blame] | 28 | static struct sk_buff *__skb_udp_tunnel_segment(struct sk_buff *skb, |
| 29 | netdev_features_t features, |
| 30 | struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb, |
| 31 | netdev_features_t features), |
Tom Herbert | 4bcb877 | 2014-11-04 09:06:52 -0800 | [diff] [blame^] | 32 | __be16 new_protocol, bool is_ipv6) |
Tom Herbert | 155e010 | 2014-07-13 19:49:56 -0700 | [diff] [blame] | 33 | { |
| 34 | struct sk_buff *segs = ERR_PTR(-EINVAL); |
| 35 | u16 mac_offset = skb->mac_header; |
| 36 | int mac_len = skb->mac_len; |
| 37 | int tnl_hlen = skb_inner_mac_header(skb) - skb_transport_header(skb); |
| 38 | __be16 protocol = skb->protocol; |
| 39 | netdev_features_t enc_features; |
| 40 | int udp_offset, outer_hlen; |
| 41 | unsigned int oldlen; |
Tom Herbert | 4bcb877 | 2014-11-04 09:06:52 -0800 | [diff] [blame^] | 42 | bool need_csum = !!(skb_shinfo(skb)->gso_type & |
| 43 | SKB_GSO_UDP_TUNNEL_CSUM); |
| 44 | bool offload_csum = false, dont_encap = need_csum; |
Tom Herbert | 155e010 | 2014-07-13 19:49:56 -0700 | [diff] [blame] | 45 | |
| 46 | oldlen = (u16)~skb->len; |
| 47 | |
| 48 | if (unlikely(!pskb_may_pull(skb, tnl_hlen))) |
| 49 | goto out; |
| 50 | |
| 51 | skb->encapsulation = 0; |
| 52 | __skb_pull(skb, tnl_hlen); |
| 53 | skb_reset_mac_header(skb); |
| 54 | skb_set_network_header(skb, skb_inner_network_offset(skb)); |
| 55 | skb->mac_len = skb_inner_network_offset(skb); |
Tom Herbert | 8bce6d7 | 2014-09-29 20:22:29 -0700 | [diff] [blame] | 56 | skb->protocol = new_protocol; |
Tom Herbert | 4bcb877 | 2014-11-04 09:06:52 -0800 | [diff] [blame^] | 57 | skb->encap_hdr_csum = need_csum; |
Tom Herbert | 155e010 | 2014-07-13 19:49:56 -0700 | [diff] [blame] | 58 | |
Tom Herbert | 4bcb877 | 2014-11-04 09:06:52 -0800 | [diff] [blame^] | 59 | /* Try to offload checksum if possible */ |
| 60 | offload_csum = !!(need_csum && |
| 61 | (skb->dev->features & |
| 62 | (is_ipv6 ? NETIF_F_V6_CSUM : NETIF_F_V4_CSUM))); |
Tom Herbert | 155e010 | 2014-07-13 19:49:56 -0700 | [diff] [blame] | 63 | |
| 64 | /* segment inner packet. */ |
Florian Westphal | 1e16aa3 | 2014-10-20 13:49:16 +0200 | [diff] [blame] | 65 | enc_features = skb->dev->hw_enc_features & features; |
Tom Herbert | 8bce6d7 | 2014-09-29 20:22:29 -0700 | [diff] [blame] | 66 | segs = gso_inner_segment(skb, enc_features); |
Himangi Saraogi | 2744644 | 2014-07-27 12:38:38 +0530 | [diff] [blame] | 67 | if (IS_ERR_OR_NULL(segs)) { |
Tom Herbert | 155e010 | 2014-07-13 19:49:56 -0700 | [diff] [blame] | 68 | skb_gso_error_unwind(skb, protocol, tnl_hlen, mac_offset, |
| 69 | mac_len); |
| 70 | goto out; |
| 71 | } |
| 72 | |
| 73 | outer_hlen = skb_tnl_header_len(skb); |
| 74 | udp_offset = outer_hlen - tnl_hlen; |
| 75 | skb = segs; |
| 76 | do { |
| 77 | struct udphdr *uh; |
| 78 | int len; |
Tom Herbert | 4bcb877 | 2014-11-04 09:06:52 -0800 | [diff] [blame^] | 79 | __be32 delta; |
Tom Herbert | 155e010 | 2014-07-13 19:49:56 -0700 | [diff] [blame] | 80 | |
Tom Herbert | 4bcb877 | 2014-11-04 09:06:52 -0800 | [diff] [blame^] | 81 | if (dont_encap) { |
| 82 | skb->encapsulation = 0; |
| 83 | skb->ip_summed = CHECKSUM_NONE; |
| 84 | } else { |
| 85 | /* Only set up inner headers if we might be offloading |
| 86 | * inner checksum. |
| 87 | */ |
| 88 | skb_reset_inner_headers(skb); |
| 89 | skb->encapsulation = 1; |
| 90 | } |
Tom Herbert | 155e010 | 2014-07-13 19:49:56 -0700 | [diff] [blame] | 91 | |
| 92 | skb->mac_len = mac_len; |
Tom Herbert | 4bcb877 | 2014-11-04 09:06:52 -0800 | [diff] [blame^] | 93 | skb->protocol = protocol; |
Tom Herbert | 155e010 | 2014-07-13 19:49:56 -0700 | [diff] [blame] | 94 | |
| 95 | skb_push(skb, outer_hlen); |
| 96 | skb_reset_mac_header(skb); |
| 97 | skb_set_network_header(skb, mac_len); |
| 98 | skb_set_transport_header(skb, udp_offset); |
| 99 | len = skb->len - udp_offset; |
| 100 | uh = udp_hdr(skb); |
| 101 | uh->len = htons(len); |
| 102 | |
Tom Herbert | 4bcb877 | 2014-11-04 09:06:52 -0800 | [diff] [blame^] | 103 | if (!need_csum) |
| 104 | continue; |
Tom Herbert | 155e010 | 2014-07-13 19:49:56 -0700 | [diff] [blame] | 105 | |
Tom Herbert | 4bcb877 | 2014-11-04 09:06:52 -0800 | [diff] [blame^] | 106 | delta = htonl(oldlen + len); |
| 107 | |
| 108 | uh->check = ~csum_fold((__force __wsum) |
| 109 | ((__force u32)uh->check + |
| 110 | (__force u32)delta)); |
| 111 | |
| 112 | if (offload_csum) { |
| 113 | skb->ip_summed = CHECKSUM_PARTIAL; |
| 114 | skb->csum_start = skb_transport_header(skb) - skb->head; |
| 115 | skb->csum_offset = offsetof(struct udphdr, check); |
| 116 | } else { |
Tom Herbert | 155e010 | 2014-07-13 19:49:56 -0700 | [diff] [blame] | 117 | uh->check = gso_make_checksum(skb, ~uh->check); |
| 118 | |
| 119 | if (uh->check == 0) |
| 120 | uh->check = CSUM_MANGLED_0; |
| 121 | } |
Tom Herbert | 155e010 | 2014-07-13 19:49:56 -0700 | [diff] [blame] | 122 | } while ((skb = skb->next)); |
| 123 | out: |
| 124 | return segs; |
| 125 | } |
| 126 | |
Tom Herbert | 8bce6d7 | 2014-09-29 20:22:29 -0700 | [diff] [blame] | 127 | struct sk_buff *skb_udp_tunnel_segment(struct sk_buff *skb, |
| 128 | netdev_features_t features, |
| 129 | bool is_ipv6) |
| 130 | { |
| 131 | __be16 protocol = skb->protocol; |
| 132 | const struct net_offload **offloads; |
| 133 | const struct net_offload *ops; |
| 134 | struct sk_buff *segs = ERR_PTR(-EINVAL); |
| 135 | struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb, |
| 136 | netdev_features_t features); |
| 137 | |
| 138 | rcu_read_lock(); |
| 139 | |
| 140 | switch (skb->inner_protocol_type) { |
| 141 | case ENCAP_TYPE_ETHER: |
| 142 | protocol = skb->inner_protocol; |
| 143 | gso_inner_segment = skb_mac_gso_segment; |
| 144 | break; |
| 145 | case ENCAP_TYPE_IPPROTO: |
| 146 | offloads = is_ipv6 ? inet6_offloads : inet_offloads; |
| 147 | ops = rcu_dereference(offloads[skb->inner_ipproto]); |
| 148 | if (!ops || !ops->callbacks.gso_segment) |
| 149 | goto out_unlock; |
| 150 | gso_inner_segment = ops->callbacks.gso_segment; |
| 151 | break; |
| 152 | default: |
| 153 | goto out_unlock; |
| 154 | } |
| 155 | |
| 156 | segs = __skb_udp_tunnel_segment(skb, features, gso_inner_segment, |
Tom Herbert | 4bcb877 | 2014-11-04 09:06:52 -0800 | [diff] [blame^] | 157 | protocol, is_ipv6); |
Tom Herbert | 8bce6d7 | 2014-09-29 20:22:29 -0700 | [diff] [blame] | 158 | |
| 159 | out_unlock: |
| 160 | rcu_read_unlock(); |
| 161 | |
| 162 | return segs; |
| 163 | } |
| 164 | |
Daniel Borkmann | da5bab0 | 2013-06-08 12:56:03 +0200 | [diff] [blame] | 165 | static struct sk_buff *udp4_ufo_fragment(struct sk_buff *skb, |
| 166 | netdev_features_t features) |
| 167 | { |
| 168 | struct sk_buff *segs = ERR_PTR(-EINVAL); |
| 169 | unsigned int mss; |
Wei-Chun Chao | 7a7ffba | 2013-12-26 13:10:22 -0800 | [diff] [blame] | 170 | __wsum csum; |
Tom Herbert | f71470b | 2014-09-20 14:52:29 -0700 | [diff] [blame] | 171 | struct udphdr *uh; |
| 172 | struct iphdr *iph; |
Wei-Chun Chao | 7a7ffba | 2013-12-26 13:10:22 -0800 | [diff] [blame] | 173 | |
| 174 | if (skb->encapsulation && |
Tom Herbert | 0f4f4ff | 2014-06-04 17:20:16 -0700 | [diff] [blame] | 175 | (skb_shinfo(skb)->gso_type & |
| 176 | (SKB_GSO_UDP_TUNNEL|SKB_GSO_UDP_TUNNEL_CSUM))) { |
Tom Herbert | 8bce6d7 | 2014-09-29 20:22:29 -0700 | [diff] [blame] | 177 | segs = skb_udp_tunnel_segment(skb, features, false); |
Wei-Chun Chao | 7a7ffba | 2013-12-26 13:10:22 -0800 | [diff] [blame] | 178 | goto out; |
| 179 | } |
Daniel Borkmann | da5bab0 | 2013-06-08 12:56:03 +0200 | [diff] [blame] | 180 | |
Tom Herbert | f71470b | 2014-09-20 14:52:29 -0700 | [diff] [blame] | 181 | if (!pskb_may_pull(skb, sizeof(struct udphdr))) |
| 182 | goto out; |
| 183 | |
Daniel Borkmann | da5bab0 | 2013-06-08 12:56:03 +0200 | [diff] [blame] | 184 | mss = skb_shinfo(skb)->gso_size; |
| 185 | if (unlikely(skb->len <= mss)) |
| 186 | goto out; |
| 187 | |
| 188 | if (skb_gso_ok(skb, features | NETIF_F_GSO_ROBUST)) { |
| 189 | /* Packet is from an untrusted source, reset gso_segs. */ |
| 190 | int type = skb_shinfo(skb)->gso_type; |
| 191 | |
| 192 | if (unlikely(type & ~(SKB_GSO_UDP | SKB_GSO_DODGY | |
| 193 | SKB_GSO_UDP_TUNNEL | |
Tom Herbert | 0f4f4ff | 2014-06-04 17:20:16 -0700 | [diff] [blame] | 194 | SKB_GSO_UDP_TUNNEL_CSUM | |
Eric Dumazet | cb32f51 | 2013-10-19 11:42:57 -0700 | [diff] [blame] | 195 | SKB_GSO_IPIP | |
Tom Herbert | 4749c09 | 2014-06-04 17:20:23 -0700 | [diff] [blame] | 196 | SKB_GSO_GRE | SKB_GSO_GRE_CSUM | |
| 197 | SKB_GSO_MPLS) || |
Daniel Borkmann | da5bab0 | 2013-06-08 12:56:03 +0200 | [diff] [blame] | 198 | !(type & (SKB_GSO_UDP)))) |
| 199 | goto out; |
| 200 | |
| 201 | skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(skb->len, mss); |
| 202 | |
| 203 | segs = NULL; |
| 204 | goto out; |
| 205 | } |
| 206 | |
Wei-Chun Chao | 7a7ffba | 2013-12-26 13:10:22 -0800 | [diff] [blame] | 207 | /* Do software UFO. Complete and fill in the UDP checksum as |
| 208 | * HW cannot do checksum of UDP packets sent as multiple |
| 209 | * IP fragments. |
| 210 | */ |
Tom Herbert | f71470b | 2014-09-20 14:52:29 -0700 | [diff] [blame] | 211 | |
| 212 | uh = udp_hdr(skb); |
| 213 | iph = ip_hdr(skb); |
| 214 | |
| 215 | uh->check = 0; |
| 216 | csum = skb_checksum(skb, 0, skb->len, 0); |
| 217 | uh->check = udp_v4_check(skb->len, iph->saddr, iph->daddr, csum); |
| 218 | if (uh->check == 0) |
| 219 | uh->check = CSUM_MANGLED_0; |
| 220 | |
Wei-Chun Chao | 7a7ffba | 2013-12-26 13:10:22 -0800 | [diff] [blame] | 221 | skb->ip_summed = CHECKSUM_NONE; |
| 222 | |
Daniel Borkmann | da5bab0 | 2013-06-08 12:56:03 +0200 | [diff] [blame] | 223 | /* Fragment the skb. IP headers of the fragments are updated in |
| 224 | * inet_gso_segment() |
| 225 | */ |
Wei-Chun Chao | 7a7ffba | 2013-12-26 13:10:22 -0800 | [diff] [blame] | 226 | segs = skb_segment(skb, features); |
Daniel Borkmann | da5bab0 | 2013-06-08 12:56:03 +0200 | [diff] [blame] | 227 | out: |
| 228 | return segs; |
| 229 | } |
| 230 | |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 231 | int udp_add_offload(struct udp_offload *uo) |
| 232 | { |
Or Gerlitz | b5aaab1 | 2014-01-29 18:08:59 +0200 | [diff] [blame] | 233 | struct udp_offload_priv *new_offload = kzalloc(sizeof(*new_offload), GFP_ATOMIC); |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 234 | |
| 235 | if (!new_offload) |
| 236 | return -ENOMEM; |
| 237 | |
| 238 | new_offload->offload = uo; |
| 239 | |
| 240 | spin_lock(&udp_offload_lock); |
Shlomo Pongratz | a664a4f | 2014-02-02 15:42:10 +0200 | [diff] [blame] | 241 | new_offload->next = udp_offload_base; |
| 242 | rcu_assign_pointer(udp_offload_base, new_offload); |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 243 | spin_unlock(&udp_offload_lock); |
| 244 | |
| 245 | return 0; |
| 246 | } |
| 247 | EXPORT_SYMBOL(udp_add_offload); |
| 248 | |
| 249 | static void udp_offload_free_routine(struct rcu_head *head) |
| 250 | { |
| 251 | struct udp_offload_priv *ou_priv = container_of(head, struct udp_offload_priv, rcu); |
| 252 | kfree(ou_priv); |
| 253 | } |
| 254 | |
| 255 | void udp_del_offload(struct udp_offload *uo) |
| 256 | { |
| 257 | struct udp_offload_priv __rcu **head = &udp_offload_base; |
| 258 | struct udp_offload_priv *uo_priv; |
| 259 | |
| 260 | spin_lock(&udp_offload_lock); |
| 261 | |
Shlomo Pongratz | a664a4f | 2014-02-02 15:42:10 +0200 | [diff] [blame] | 262 | uo_priv = udp_deref_protected(*head); |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 263 | for (; uo_priv != NULL; |
Shlomo Pongratz | a664a4f | 2014-02-02 15:42:10 +0200 | [diff] [blame] | 264 | uo_priv = udp_deref_protected(*head)) { |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 265 | if (uo_priv->offload == uo) { |
Shlomo Pongratz | a664a4f | 2014-02-02 15:42:10 +0200 | [diff] [blame] | 266 | rcu_assign_pointer(*head, |
| 267 | udp_deref_protected(uo_priv->next)); |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 268 | goto unlock; |
| 269 | } |
| 270 | head = &uo_priv->next; |
| 271 | } |
Shlomo Pongratz | a1d0cd8 | 2014-01-22 15:23:29 +0200 | [diff] [blame] | 272 | pr_warn("udp_del_offload: didn't find offload for port %d\n", ntohs(uo->port)); |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 273 | unlock: |
| 274 | spin_unlock(&udp_offload_lock); |
| 275 | if (uo_priv != NULL) |
| 276 | call_rcu(&uo_priv->rcu, udp_offload_free_routine); |
| 277 | } |
| 278 | EXPORT_SYMBOL(udp_del_offload); |
| 279 | |
Tom Herbert | 57c67ff | 2014-08-22 13:34:44 -0700 | [diff] [blame] | 280 | struct sk_buff **udp_gro_receive(struct sk_buff **head, struct sk_buff *skb, |
| 281 | struct udphdr *uh) |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 282 | { |
| 283 | struct udp_offload_priv *uo_priv; |
| 284 | struct sk_buff *p, **pp = NULL; |
Tom Herbert | 57c67ff | 2014-08-22 13:34:44 -0700 | [diff] [blame] | 285 | struct udphdr *uh2; |
| 286 | unsigned int off = skb_gro_offset(skb); |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 287 | int flush = 1; |
| 288 | |
| 289 | if (NAPI_GRO_CB(skb)->udp_mark || |
Tom Herbert | 662880f | 2014-08-27 21:26:56 -0700 | [diff] [blame] | 290 | (skb->ip_summed != CHECKSUM_PARTIAL && |
| 291 | NAPI_GRO_CB(skb)->csum_cnt == 0 && |
| 292 | !NAPI_GRO_CB(skb)->csum_valid)) |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 293 | goto out; |
| 294 | |
| 295 | /* mark that this skb passed once through the udp gro layer */ |
| 296 | NAPI_GRO_CB(skb)->udp_mark = 1; |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 297 | |
| 298 | rcu_read_lock(); |
| 299 | uo_priv = rcu_dereference(udp_offload_base); |
| 300 | for (; uo_priv != NULL; uo_priv = rcu_dereference(uo_priv->next)) { |
| 301 | if (uo_priv->offload->port == uh->dest && |
| 302 | uo_priv->offload->callbacks.gro_receive) |
| 303 | goto unflush; |
| 304 | } |
| 305 | goto out_unlock; |
| 306 | |
| 307 | unflush: |
| 308 | flush = 0; |
| 309 | |
| 310 | for (p = *head; p; p = p->next) { |
| 311 | if (!NAPI_GRO_CB(p)->same_flow) |
| 312 | continue; |
| 313 | |
| 314 | uh2 = (struct udphdr *)(p->data + off); |
Tom Herbert | 57c67ff | 2014-08-22 13:34:44 -0700 | [diff] [blame] | 315 | |
| 316 | /* Match ports and either checksums are either both zero |
| 317 | * or nonzero. |
| 318 | */ |
| 319 | if ((*(u32 *)&uh->source != *(u32 *)&uh2->source) || |
| 320 | (!uh->check ^ !uh2->check)) { |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 321 | NAPI_GRO_CB(p)->same_flow = 0; |
| 322 | continue; |
| 323 | } |
| 324 | } |
| 325 | |
| 326 | skb_gro_pull(skb, sizeof(struct udphdr)); /* pull encapsulating udp header */ |
Tom Herbert | 6bae1d4 | 2014-06-10 18:54:26 -0700 | [diff] [blame] | 327 | skb_gro_postpull_rcsum(skb, uh, sizeof(struct udphdr)); |
Tom Herbert | afe9332 | 2014-09-17 12:25:57 -0700 | [diff] [blame] | 328 | NAPI_GRO_CB(skb)->proto = uo_priv->offload->ipproto; |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 329 | pp = uo_priv->offload->callbacks.gro_receive(head, skb); |
| 330 | |
| 331 | out_unlock: |
| 332 | rcu_read_unlock(); |
| 333 | out: |
| 334 | NAPI_GRO_CB(skb)->flush |= flush; |
| 335 | return pp; |
| 336 | } |
| 337 | |
Tom Herbert | 57c67ff | 2014-08-22 13:34:44 -0700 | [diff] [blame] | 338 | static struct sk_buff **udp4_gro_receive(struct sk_buff **head, |
| 339 | struct sk_buff *skb) |
| 340 | { |
| 341 | struct udphdr *uh = udp_gro_udphdr(skb); |
| 342 | |
Tom Herbert | 2abb7cd | 2014-08-31 15:12:43 -0700 | [diff] [blame] | 343 | if (unlikely(!uh)) |
| 344 | goto flush; |
Tom Herbert | 57c67ff | 2014-08-22 13:34:44 -0700 | [diff] [blame] | 345 | |
Tom Herbert | 2abb7cd | 2014-08-31 15:12:43 -0700 | [diff] [blame] | 346 | /* Don't bother verifying checksum if we're going to flush anyway. */ |
Scott Wood | 2d8f7e2 | 2014-09-10 21:23:18 -0500 | [diff] [blame] | 347 | if (NAPI_GRO_CB(skb)->flush) |
Tom Herbert | 2abb7cd | 2014-08-31 15:12:43 -0700 | [diff] [blame] | 348 | goto skip; |
| 349 | |
| 350 | if (skb_gro_checksum_validate_zero_check(skb, IPPROTO_UDP, uh->check, |
| 351 | inet_gro_compute_pseudo)) |
| 352 | goto flush; |
| 353 | else if (uh->check) |
| 354 | skb_gro_checksum_try_convert(skb, IPPROTO_UDP, uh->check, |
| 355 | inet_gro_compute_pseudo); |
| 356 | skip: |
Tom Herbert | efc98d0 | 2014-10-03 15:48:08 -0700 | [diff] [blame] | 357 | NAPI_GRO_CB(skb)->is_ipv6 = 0; |
Tom Herbert | 57c67ff | 2014-08-22 13:34:44 -0700 | [diff] [blame] | 358 | return udp_gro_receive(head, skb, uh); |
Tom Herbert | 2abb7cd | 2014-08-31 15:12:43 -0700 | [diff] [blame] | 359 | |
| 360 | flush: |
| 361 | NAPI_GRO_CB(skb)->flush = 1; |
| 362 | return NULL; |
Tom Herbert | 57c67ff | 2014-08-22 13:34:44 -0700 | [diff] [blame] | 363 | } |
| 364 | |
| 365 | int udp_gro_complete(struct sk_buff *skb, int nhoff) |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 366 | { |
| 367 | struct udp_offload_priv *uo_priv; |
| 368 | __be16 newlen = htons(skb->len - nhoff); |
| 369 | struct udphdr *uh = (struct udphdr *)(skb->data + nhoff); |
| 370 | int err = -ENOSYS; |
| 371 | |
| 372 | uh->len = newlen; |
| 373 | |
| 374 | rcu_read_lock(); |
| 375 | |
| 376 | uo_priv = rcu_dereference(udp_offload_base); |
| 377 | for (; uo_priv != NULL; uo_priv = rcu_dereference(uo_priv->next)) { |
| 378 | if (uo_priv->offload->port == uh->dest && |
| 379 | uo_priv->offload->callbacks.gro_complete) |
| 380 | break; |
| 381 | } |
| 382 | |
Tom Herbert | afe9332 | 2014-09-17 12:25:57 -0700 | [diff] [blame] | 383 | if (uo_priv != NULL) { |
| 384 | NAPI_GRO_CB(skb)->proto = uo_priv->offload->ipproto; |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 385 | err = uo_priv->offload->callbacks.gro_complete(skb, nhoff + sizeof(struct udphdr)); |
Tom Herbert | afe9332 | 2014-09-17 12:25:57 -0700 | [diff] [blame] | 386 | } |
Or Gerlitz | b582ef0 | 2014-01-20 13:59:19 +0200 | [diff] [blame] | 387 | |
| 388 | rcu_read_unlock(); |
| 389 | return err; |
| 390 | } |
| 391 | |
Eric Dumazet | 72bb17b | 2014-09-09 08:29:12 -0700 | [diff] [blame] | 392 | static int udp4_gro_complete(struct sk_buff *skb, int nhoff) |
Tom Herbert | 57c67ff | 2014-08-22 13:34:44 -0700 | [diff] [blame] | 393 | { |
| 394 | const struct iphdr *iph = ip_hdr(skb); |
| 395 | struct udphdr *uh = (struct udphdr *)(skb->data + nhoff); |
| 396 | |
| 397 | if (uh->check) |
| 398 | uh->check = ~udp_v4_check(skb->len - nhoff, iph->saddr, |
| 399 | iph->daddr, 0); |
| 400 | |
| 401 | return udp_gro_complete(skb, nhoff); |
| 402 | } |
| 403 | |
Daniel Borkmann | da5bab0 | 2013-06-08 12:56:03 +0200 | [diff] [blame] | 404 | static const struct net_offload udpv4_offload = { |
| 405 | .callbacks = { |
Daniel Borkmann | da5bab0 | 2013-06-08 12:56:03 +0200 | [diff] [blame] | 406 | .gso_segment = udp4_ufo_fragment, |
Tom Herbert | 57c67ff | 2014-08-22 13:34:44 -0700 | [diff] [blame] | 407 | .gro_receive = udp4_gro_receive, |
| 408 | .gro_complete = udp4_gro_complete, |
Daniel Borkmann | da5bab0 | 2013-06-08 12:56:03 +0200 | [diff] [blame] | 409 | }, |
| 410 | }; |
| 411 | |
| 412 | int __init udpv4_offload_init(void) |
| 413 | { |
| 414 | return inet_add_offload(&udpv4_offload, IPPROTO_UDP); |
| 415 | } |