1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * net/core/dst_cache.c - dst entry cache
4 *
5 * Copyright (c) 2016 Paolo Abeni <pabeni@redhat.com>
6 */
7
8 #include <linux/kernel.h>
9 #include <linux/percpu.h>
10 #include <net/dst_cache.h>
11 #include <net/route.h>
12 #if IS_ENABLED(CONFIG_IPV6)
13 #include <net/ip6_fib.h>
14 #endif
15 #include <uapi/linux/in.h>
16
17 struct dst_cache_pcpu {
18 unsigned long refresh_ts;
19 struct dst_entry *dst;
20 u32 cookie;
21 union {
22 struct in_addr in_saddr;
23 struct in6_addr in6_saddr;
24 };
25 };
26
dst_cache_per_cpu_dst_set(struct dst_cache_pcpu * dst_cache,struct dst_entry * dst,u32 cookie)27 static void dst_cache_per_cpu_dst_set(struct dst_cache_pcpu *dst_cache,
28 struct dst_entry *dst, u32 cookie)
29 {
30 DEBUG_NET_WARN_ON_ONCE(!in_softirq());
31 dst_release(dst_cache->dst);
32 if (dst)
33 dst_hold(dst);
34
35 dst_cache->cookie = cookie;
36 dst_cache->dst = dst;
37 }
38
dst_cache_per_cpu_get(struct dst_cache * dst_cache,struct dst_cache_pcpu * idst)39 static struct dst_entry *dst_cache_per_cpu_get(struct dst_cache *dst_cache,
40 struct dst_cache_pcpu *idst)
41 {
42 struct dst_entry *dst;
43
44 DEBUG_NET_WARN_ON_ONCE(!in_softirq());
45 dst = idst->dst;
46 if (!dst)
47 goto fail;
48
49 /* the cache already hold a dst reference; it can't go away */
50 dst_hold(dst);
51
52 if (unlikely(!time_after(idst->refresh_ts,
53 READ_ONCE(dst_cache->reset_ts)) ||
54 (dst->obsolete && !dst->ops->check(dst, idst->cookie)))) {
55 dst_cache_per_cpu_dst_set(idst, NULL, 0);
56 dst_release(dst);
57 goto fail;
58 }
59 return dst;
60
61 fail:
62 idst->refresh_ts = jiffies;
63 return NULL;
64 }
65
dst_cache_get(struct dst_cache * dst_cache)66 struct dst_entry *dst_cache_get(struct dst_cache *dst_cache)
67 {
68 if (!dst_cache->cache)
69 return NULL;
70
71 return dst_cache_per_cpu_get(dst_cache, this_cpu_ptr(dst_cache->cache));
72 }
73 EXPORT_SYMBOL_GPL(dst_cache_get);
74
dst_cache_get_ip4(struct dst_cache * dst_cache,__be32 * saddr)75 struct rtable *dst_cache_get_ip4(struct dst_cache *dst_cache, __be32 *saddr)
76 {
77 struct dst_cache_pcpu *idst;
78 struct dst_entry *dst;
79
80 if (!dst_cache->cache)
81 return NULL;
82
83 idst = this_cpu_ptr(dst_cache->cache);
84 dst = dst_cache_per_cpu_get(dst_cache, idst);
85 if (!dst)
86 return NULL;
87
88 *saddr = idst->in_saddr.s_addr;
89 return dst_rtable(dst);
90 }
91 EXPORT_SYMBOL_GPL(dst_cache_get_ip4);
92
dst_cache_set_ip4(struct dst_cache * dst_cache,struct dst_entry * dst,__be32 saddr)93 void dst_cache_set_ip4(struct dst_cache *dst_cache, struct dst_entry *dst,
94 __be32 saddr)
95 {
96 struct dst_cache_pcpu *idst;
97
98 if (!dst_cache->cache)
99 return;
100
101 idst = this_cpu_ptr(dst_cache->cache);
102 dst_cache_per_cpu_dst_set(idst, dst, 0);
103 idst->in_saddr.s_addr = saddr;
104 }
105 EXPORT_SYMBOL_GPL(dst_cache_set_ip4);
106
107 #if IS_ENABLED(CONFIG_IPV6)
dst_cache_set_ip6(struct dst_cache * dst_cache,struct dst_entry * dst,const struct in6_addr * saddr)108 void dst_cache_set_ip6(struct dst_cache *dst_cache, struct dst_entry *dst,
109 const struct in6_addr *saddr)
110 {
111 struct dst_cache_pcpu *idst;
112
113 if (!dst_cache->cache)
114 return;
115
116 idst = this_cpu_ptr(dst_cache->cache);
117 dst_cache_per_cpu_dst_set(idst, dst,
118 rt6_get_cookie(dst_rt6_info(dst)));
119 idst->in6_saddr = *saddr;
120 }
121 EXPORT_SYMBOL_GPL(dst_cache_set_ip6);
122
dst_cache_get_ip6(struct dst_cache * dst_cache,struct in6_addr * saddr)123 struct dst_entry *dst_cache_get_ip6(struct dst_cache *dst_cache,
124 struct in6_addr *saddr)
125 {
126 struct dst_cache_pcpu *idst;
127 struct dst_entry *dst;
128
129 if (!dst_cache->cache)
130 return NULL;
131
132 idst = this_cpu_ptr(dst_cache->cache);
133 dst = dst_cache_per_cpu_get(dst_cache, idst);
134 if (!dst)
135 return NULL;
136
137 *saddr = idst->in6_saddr;
138 return dst;
139 }
140 EXPORT_SYMBOL_GPL(dst_cache_get_ip6);
141 #endif
142
dst_cache_init(struct dst_cache * dst_cache,gfp_t gfp)143 int dst_cache_init(struct dst_cache *dst_cache, gfp_t gfp)
144 {
145 dst_cache->cache = alloc_percpu_gfp(struct dst_cache_pcpu,
146 gfp | __GFP_ZERO);
147 if (!dst_cache->cache)
148 return -ENOMEM;
149
150 dst_cache_reset(dst_cache);
151 return 0;
152 }
153 EXPORT_SYMBOL_GPL(dst_cache_init);
154
dst_cache_destroy(struct dst_cache * dst_cache)155 void dst_cache_destroy(struct dst_cache *dst_cache)
156 {
157 int i;
158
159 if (!dst_cache->cache)
160 return;
161
162 for_each_possible_cpu(i)
163 dst_release(per_cpu_ptr(dst_cache->cache, i)->dst);
164
165 free_percpu(dst_cache->cache);
166 }
167 EXPORT_SYMBOL_GPL(dst_cache_destroy);
168
dst_cache_reset_now(struct dst_cache * dst_cache)169 void dst_cache_reset_now(struct dst_cache *dst_cache)
170 {
171 int i;
172
173 if (!dst_cache->cache)
174 return;
175
176 dst_cache_reset(dst_cache);
177 for_each_possible_cpu(i) {
178 struct dst_cache_pcpu *idst = per_cpu_ptr(dst_cache->cache, i);
179 struct dst_entry *dst = idst->dst;
180
181 idst->cookie = 0;
182 idst->dst = NULL;
183 dst_release(dst);
184 }
185 }
186 EXPORT_SYMBOL_GPL(dst_cache_reset_now);
187