1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * net/core/dst_cache.c - dst entry cache 4 * 5 * Copyright (c) 2016 Paolo Abeni <pabeni@redhat.com> 6 */ 7 8 #include <linux/kernel.h> 9 #include <linux/percpu.h> 10 #include <net/dst_cache.h> 11 #include <net/route.h> 12 #if IS_ENABLED(CONFIG_IPV6) 13 #include <net/ip6_fib.h> 14 #endif 15 #include <uapi/linux/in.h> 16 17 struct dst_cache_pcpu { 18 unsigned long refresh_ts; 19 struct dst_entry *dst; 20 u32 cookie; 21 union { 22 struct in_addr in_saddr; 23 struct in6_addr in6_saddr; 24 }; 25 }; 26 dst_cache_per_cpu_dst_set(struct dst_cache_pcpu * dst_cache,struct dst_entry * dst,u32 cookie)27 static void dst_cache_per_cpu_dst_set(struct dst_cache_pcpu *dst_cache, 28 struct dst_entry *dst, u32 cookie) 29 { 30 DEBUG_NET_WARN_ON_ONCE(!in_softirq()); 31 dst_release(dst_cache->dst); 32 if (dst) 33 dst_hold(dst); 34 35 dst_cache->cookie = cookie; 36 dst_cache->dst = dst; 37 } 38 dst_cache_per_cpu_get(struct dst_cache * dst_cache,struct dst_cache_pcpu * idst)39 static struct dst_entry *dst_cache_per_cpu_get(struct dst_cache *dst_cache, 40 struct dst_cache_pcpu *idst) 41 { 42 struct dst_entry *dst; 43 44 DEBUG_NET_WARN_ON_ONCE(!in_softirq()); 45 dst = idst->dst; 46 if (!dst) 47 goto fail; 48 49 /* the cache already hold a dst reference; it can't go away */ 50 dst_hold(dst); 51 52 if (unlikely(!time_after(idst->refresh_ts, 53 READ_ONCE(dst_cache->reset_ts)) || 54 (dst->obsolete && !dst->ops->check(dst, idst->cookie)))) { 55 dst_cache_per_cpu_dst_set(idst, NULL, 0); 56 dst_release(dst); 57 goto fail; 58 } 59 return dst; 60 61 fail: 62 idst->refresh_ts = jiffies; 63 return NULL; 64 } 65 dst_cache_get(struct dst_cache * dst_cache)66 struct dst_entry *dst_cache_get(struct dst_cache *dst_cache) 67 { 68 if (!dst_cache->cache) 69 return NULL; 70 71 return dst_cache_per_cpu_get(dst_cache, this_cpu_ptr(dst_cache->cache)); 72 } 73 EXPORT_SYMBOL_GPL(dst_cache_get); 74 dst_cache_get_ip4(struct dst_cache * dst_cache,__be32 * saddr)75 struct rtable *dst_cache_get_ip4(struct dst_cache *dst_cache, __be32 *saddr) 76 { 77 struct dst_cache_pcpu *idst; 78 struct dst_entry *dst; 79 80 if (!dst_cache->cache) 81 return NULL; 82 83 idst = this_cpu_ptr(dst_cache->cache); 84 dst = dst_cache_per_cpu_get(dst_cache, idst); 85 if (!dst) 86 return NULL; 87 88 *saddr = idst->in_saddr.s_addr; 89 return dst_rtable(dst); 90 } 91 EXPORT_SYMBOL_GPL(dst_cache_get_ip4); 92 dst_cache_set_ip4(struct dst_cache * dst_cache,struct dst_entry * dst,__be32 saddr)93 void dst_cache_set_ip4(struct dst_cache *dst_cache, struct dst_entry *dst, 94 __be32 saddr) 95 { 96 struct dst_cache_pcpu *idst; 97 98 if (!dst_cache->cache) 99 return; 100 101 idst = this_cpu_ptr(dst_cache->cache); 102 dst_cache_per_cpu_dst_set(idst, dst, 0); 103 idst->in_saddr.s_addr = saddr; 104 } 105 EXPORT_SYMBOL_GPL(dst_cache_set_ip4); 106 107 #if IS_ENABLED(CONFIG_IPV6) dst_cache_set_ip6(struct dst_cache * dst_cache,struct dst_entry * dst,const struct in6_addr * saddr)108 void dst_cache_set_ip6(struct dst_cache *dst_cache, struct dst_entry *dst, 109 const struct in6_addr *saddr) 110 { 111 struct dst_cache_pcpu *idst; 112 113 if (!dst_cache->cache) 114 return; 115 116 idst = this_cpu_ptr(dst_cache->cache); 117 dst_cache_per_cpu_dst_set(idst, dst, 118 rt6_get_cookie(dst_rt6_info(dst))); 119 idst->in6_saddr = *saddr; 120 } 121 EXPORT_SYMBOL_GPL(dst_cache_set_ip6); 122 dst_cache_get_ip6(struct dst_cache * dst_cache,struct in6_addr * saddr)123 struct dst_entry *dst_cache_get_ip6(struct dst_cache *dst_cache, 124 struct in6_addr *saddr) 125 { 126 struct dst_cache_pcpu *idst; 127 struct dst_entry *dst; 128 129 if (!dst_cache->cache) 130 return NULL; 131 132 idst = this_cpu_ptr(dst_cache->cache); 133 dst = dst_cache_per_cpu_get(dst_cache, idst); 134 if (!dst) 135 return NULL; 136 137 *saddr = idst->in6_saddr; 138 return dst; 139 } 140 EXPORT_SYMBOL_GPL(dst_cache_get_ip6); 141 #endif 142 dst_cache_init(struct dst_cache * dst_cache,gfp_t gfp)143 int dst_cache_init(struct dst_cache *dst_cache, gfp_t gfp) 144 { 145 dst_cache->cache = alloc_percpu_gfp(struct dst_cache_pcpu, 146 gfp | __GFP_ZERO); 147 if (!dst_cache->cache) 148 return -ENOMEM; 149 150 dst_cache_reset(dst_cache); 151 return 0; 152 } 153 EXPORT_SYMBOL_GPL(dst_cache_init); 154 dst_cache_destroy(struct dst_cache * dst_cache)155 void dst_cache_destroy(struct dst_cache *dst_cache) 156 { 157 int i; 158 159 if (!dst_cache->cache) 160 return; 161 162 for_each_possible_cpu(i) 163 dst_release(per_cpu_ptr(dst_cache->cache, i)->dst); 164 165 free_percpu(dst_cache->cache); 166 } 167 EXPORT_SYMBOL_GPL(dst_cache_destroy); 168 dst_cache_reset_now(struct dst_cache * dst_cache)169 void dst_cache_reset_now(struct dst_cache *dst_cache) 170 { 171 int i; 172 173 if (!dst_cache->cache) 174 return; 175 176 dst_cache_reset(dst_cache); 177 for_each_possible_cpu(i) { 178 struct dst_cache_pcpu *idst = per_cpu_ptr(dst_cache->cache, i); 179 struct dst_entry *dst = idst->dst; 180 181 idst->cookie = 0; 182 idst->dst = NULL; 183 dst_release(dst); 184 } 185 } 186 EXPORT_SYMBOL_GPL(dst_cache_reset_now); 187