GeezerVison is showing codebrowser.dev — spinlock_rt.c source code [linux/kernel/locking/spinlock_rt.c] - Codebrowser

Contact

1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* PREEMPT_RT substitution for spin/rw_locks
4	*
5	* spinlocks and rwlocks on RT are based on rtmutexes, with a few twists to
6	* resemble the non RT semantics:
7	*
8	* - Contrary to plain rtmutexes, spinlocks and rwlocks are state
9	* preserving. The task state is saved before blocking on the underlying
10	* rtmutex, and restored when the lock has been acquired. Regular wakeups
11	* during that time are redirected to the saved state so no wake up is
12	* missed.
13	*
14	* - Non RT spin/rwlocks disable preemption and eventually interrupts.
15	* Disabling preemption has the side effect of disabling migration and
16	* preventing RCU grace periods.
17	*
18	* The RT substitutions explicitly disable migration and take
19	* rcu_read_lock() across the lock held section.
20	*/
21	#include <linux/spinlock.h>
22	#include <linux/export.h>
23
24	#define RT_MUTEX_BUILD_SPINLOCKS
25	#include "rtmutex.c"
26
27	/*
28	* __might_resched() skips the state check as rtlocks are state
29	* preserving. Take RCU nesting into account as spin/read/write_lock() can
30	* legitimately nest into an RCU read side critical section.
31	*/
32	#define RTLOCK_RESCHED_OFFSETS \
33	(rcu_preempt_depth() << MIGHT_RESCHED_RCU_SHIFT)
34
35	#define rtlock_might_resched() \
36	__might_resched(__FILE__, __LINE__, RTLOCK_RESCHED_OFFSETS)
37
38	static __always_inline void rtlock_lock(struct rt_mutex_base *rtm)
39	{
40	lockdep_assert(!current->pi_blocked_on);
41
42	if (unlikely(!rt_mutex_cmpxchg_acquire(rtm, NULL, current)))
43	rtlock_slowlock(lock: rtm);
44	}
45
46	static __always_inline void __rt_spin_lock(spinlock_t *lock)
47	{
48	rtlock_might_resched();
49	rtlock_lock(rtm: &lock->lock);
50	rcu_read_lock();
51	migrate_disable();
52	}
53
54	void __sched rt_spin_lock(spinlock_t *lock) __acquires(RCU)
55	{
56	spin_acquire(&lock->dep_map, `0`, `0`, _RET_IP_);
57	__rt_spin_lock(lock);
58	}
59	EXPORT_SYMBOL(rt_spin_lock);
60
61	#ifdef CONFIG_DEBUG_LOCK_ALLOC
62	void __sched rt_spin_lock_nested(spinlock_t lock, int* subclass)
63	{
64	spin_acquire(&lock->dep_map, subclass, `0`, _RET_IP_);
65	__rt_spin_lock(lock);
66	}
67	EXPORT_SYMBOL(rt_spin_lock_nested);
68
69	void __sched rt_spin_lock_nest_lock(spinlock_t *lock,
70	struct lockdep_map *nest_lock)
71	{
72	spin_acquire_nest(&lock->dep_map, `0`, `0`, nest_lock, _RET_IP_);
73	__rt_spin_lock(lock);
74	}
75	EXPORT_SYMBOL(rt_spin_lock_nest_lock);
76	#endif
77
78	void __sched rt_spin_unlock(spinlock_t *lock) __releases(RCU)
79	{
80	spin_release(&lock->dep_map, _RET_IP_);
81	migrate_enable();
82	rcu_read_unlock();
83
84	if (unlikely(!rt_mutex_cmpxchg_release(&lock->lock, current, NULL)))
85	rt_mutex_slowunlock(lock: &lock->lock);
86	}
87	EXPORT_SYMBOL(rt_spin_unlock);
88
89	/*
90	* Wait for the lock to get unlocked: instead of polling for an unlock
91	* (like raw spinlocks do), lock and unlock, to force the kernel to
92	* schedule if there's contention:
93	*/
94	void __sched rt_spin_lock_unlock(spinlock_t *lock)
95	{
96	spin_lock(lock);
97	spin_unlock(lock);
98	}
99	EXPORT_SYMBOL(rt_spin_lock_unlock);
100
101	static __always_inline int __rt_spin_trylock(spinlock_t *lock)
102	{
103	int ret = `1`;
104
105	if (unlikely(!rt_mutex_cmpxchg_acquire(&lock->lock, NULL, current)))
106	ret = rt_mutex_slowtrylock(lock: &lock->lock);
107
108	if (ret) {
109	spin_acquire(&lock->dep_map, `0`, `1`, _RET_IP_);
110	rcu_read_lock();
111	migrate_disable();
112	}
113	return ret;
114	}
115
116	int __sched rt_spin_trylock(spinlock_t *lock)
117	{
118	return __rt_spin_trylock(lock);
119	}
120	EXPORT_SYMBOL(rt_spin_trylock);
121
122	int __sched rt_spin_trylock_bh(spinlock_t *lock)
123	{
124	int ret;
125
126	local_bh_disable();
127	ret = __rt_spin_trylock(lock);
128	if (!ret)
129	local_bh_enable();
130	return ret;
131	}
132	EXPORT_SYMBOL(rt_spin_trylock_bh);
133
134	#ifdef CONFIG_DEBUG_LOCK_ALLOC
135	void __rt_spin_lock_init(spinlock_t lock, const* char *name,
136	struct lock_class_key *key, bool percpu)
137	{
138	u8 type = percpu ? LD_LOCK_PERCPU : LD_LOCK_NORMAL;
139
140	debug_check_no_locks_freed(from: (void )lock, len: sizeof(lock));
141	lockdep_init_map_type(lock: &lock->dep_map, name, key, subclass: `0`, inner: LD_WAIT_CONFIG,
142	outer: LD_WAIT_INV, lock_type: type);
143	}
144	EXPORT_SYMBOL(__rt_spin_lock_init);
145	#endif
146
147	/*
148	* RT-specific reader/writer locks
149	*/
150	#define rwbase_set_and_save_current_state(state) \
151	current_save_and_set_rtlock_wait_state()
152
153	#define rwbase_restore_current_state() \
154	current_restore_rtlock_saved_state()
155
156	static __always_inline int
157	rwbase_rtmutex_lock_state(struct rt_mutex_base rtm, unsigned* int state)
158	{
159	if (unlikely(!rt_mutex_cmpxchg_acquire(rtm, NULL, current)))
160	rtlock_slowlock(lock: rtm);
161	return `0`;
162	}
163
164	static __always_inline int
165	rwbase_rtmutex_slowlock_locked(struct rt_mutex_base rtm, unsigned* int state,
166	struct wake_q_head *wake_q)
167	{
168	rtlock_slowlock_locked(lock: rtm, wake_q);
169	return `0`;
170	}
171
172	static __always_inline void rwbase_rtmutex_unlock(struct rt_mutex_base *rtm)
173	{
174	if (likely(rt_mutex_cmpxchg_acquire(rtm, current, NULL)))
175	return;
176
177	rt_mutex_slowunlock(lock: rtm);
178	}
179
180	static __always_inline int rwbase_rtmutex_trylock(struct rt_mutex_base *rtm)
181	{
182	if (likely(rt_mutex_cmpxchg_acquire(rtm, NULL, current)))
183	return `1`;
184
185	return rt_mutex_slowtrylock(lock: rtm);
186	}
187
188	#define rwbase_signal_pending_state(state, current) (0)
189
190	#define rwbase_pre_schedule()
191
192	#define rwbase_schedule() \
193	schedule_rtlock()
194
195	#define rwbase_post_schedule()
196
197	#include "rwbase_rt.c"
198	/*
199	* The common functions which get wrapped into the rwlock API.
200	*/
201	int __sched rt_read_trylock(rwlock_t *rwlock)
202	{
203	int ret;
204
205	ret = rwbase_read_trylock(rwb: &rwlock->rwbase);
206	if (ret) {
207	rwlock_acquire_read(&rwlock->dep_map, `0`, `1`, _RET_IP_);
208	rcu_read_lock();
209	migrate_disable();
210	}
211	return ret;
212	}
213	EXPORT_SYMBOL(rt_read_trylock);
214
215	int __sched rt_write_trylock(rwlock_t *rwlock)
216	{
217	int ret;
218
219	ret = rwbase_write_trylock(rwb: &rwlock->rwbase);
220	if (ret) {
221	rwlock_acquire(&rwlock->dep_map, `0`, `1`, _RET_IP_);
222	rcu_read_lock();
223	migrate_disable();
224	}
225	return ret;
226	}
227	EXPORT_SYMBOL(rt_write_trylock);
228
229	void __sched rt_read_lock(rwlock_t *rwlock) __acquires(RCU)
230	{
231	rtlock_might_resched();
232	rwlock_acquire_read(&rwlock->dep_map, `0`, `0`, _RET_IP_);
233	rwbase_read_lock(rwb: &rwlock->rwbase, TASK_RTLOCK_WAIT);
234	rcu_read_lock();
235	migrate_disable();
236	}
237	EXPORT_SYMBOL(rt_read_lock);
238
239	void __sched rt_write_lock(rwlock_t *rwlock) __acquires(RCU)
240	{
241	rtlock_might_resched();
242	rwlock_acquire(&rwlock->dep_map, `0`, `0`, _RET_IP_);
243	rwbase_write_lock(rwb: &rwlock->rwbase, TASK_RTLOCK_WAIT);
244	rcu_read_lock();
245	migrate_disable();
246	}
247	EXPORT_SYMBOL(rt_write_lock);
248
249	#ifdef CONFIG_DEBUG_LOCK_ALLOC
250	void __sched rt_write_lock_nested(rwlock_t rwlock, int* subclass) __acquires(RCU)
251	{
252	rtlock_might_resched();
253	rwlock_acquire(&rwlock->dep_map, subclass, `0`, _RET_IP_);
254	rwbase_write_lock(rwb: &rwlock->rwbase, TASK_RTLOCK_WAIT);
255	rcu_read_lock();
256	migrate_disable();
257	}
258	EXPORT_SYMBOL(rt_write_lock_nested);
259	#endif
260
261	void __sched rt_read_unlock(rwlock_t *rwlock) __releases(RCU)
262	{
263	rwlock_release(&rwlock->dep_map, _RET_IP_);
264	migrate_enable();
265	rcu_read_unlock();
266	rwbase_read_unlock(rwb: &rwlock->rwbase, TASK_RTLOCK_WAIT);
267	}
268	EXPORT_SYMBOL(rt_read_unlock);
269
270	void __sched rt_write_unlock(rwlock_t *rwlock) __releases(RCU)
271	{
272	rwlock_release(&rwlock->dep_map, _RET_IP_);
273	rcu_read_unlock();
274	migrate_enable();
275	rwbase_write_unlock(rwb: &rwlock->rwbase);
276	}
277	EXPORT_SYMBOL(rt_write_unlock);
278
279	#ifdef CONFIG_DEBUG_LOCK_ALLOC
280	void __rt_rwlock_init(rwlock_t rwlock, const* char *name,
281	struct lock_class_key *key)
282	{
283	debug_check_no_locks_freed(from: (void )rwlock, len: sizeof(rwlock));
284	lockdep_init_map_wait(lock: &rwlock->dep_map, name, key, subclass: `0`, inner: LD_WAIT_CONFIG);
285	}
286	EXPORT_SYMBOL(__rt_rwlock_init);
287	#endif
288

spinlock_rt.c