blob: d2233de9a86e564f297e01305ddbc41392b3d2ee [file] [log] [blame]
Linus Torvalds2f4f12e2013-09-02 11:58:20 -07001#include <linux/export.h>
2#include <linux/lockref.h>
3
Peter Zijlstra57f42572013-11-14 14:31:54 -08004#if USE_CMPXCHG_LOCKREF
Linus Torvaldsbc08b442013-09-02 12:12:15 -07005
6/*
Will Deacond2212b42013-09-26 17:27:00 +01007 * Allow weakly-ordered memory architectures to provide barrier-less
8 * cmpxchg semantics for lockref updates.
9 */
10#ifndef cmpxchg64_relaxed
11# define cmpxchg64_relaxed cmpxchg64
12#endif
13
14/*
Linus Torvaldsbc08b442013-09-02 12:12:15 -070015 * Note that the "cmpxchg()" reloads the "old" value for the
16 * failure case.
17 */
18#define CMPXCHG_LOOP(CODE, SUCCESS) do { \
19 struct lockref old; \
20 BUILD_BUG_ON(sizeof(old) != 8); \
21 old.lock_count = ACCESS_ONCE(lockref->lock_count); \
22 while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) { \
23 struct lockref new = old, prev = old; \
24 CODE \
Will Deacond2212b42013-09-26 17:27:00 +010025 old.lock_count = cmpxchg64_relaxed(&lockref->lock_count, \
26 old.lock_count, \
27 new.lock_count); \
Linus Torvaldsbc08b442013-09-02 12:12:15 -070028 if (likely(old.lock_count == prev.lock_count)) { \
29 SUCCESS; \
30 } \
Davidlohr Bueso3a6bfbc2014-06-29 15:09:33 -070031 cpu_relax_lowlatency(); \
Linus Torvaldsbc08b442013-09-02 12:12:15 -070032 } \
33} while (0)
34
35#else
36
37#define CMPXCHG_LOOP(CODE, SUCCESS) do { } while (0)
38
39#endif
40
Linus Torvalds2f4f12e2013-09-02 11:58:20 -070041/**
42 * lockref_get - Increments reference count unconditionally
Linus Torvalds44a0cf92013-09-07 15:30:29 -070043 * @lockref: pointer to lockref structure
Linus Torvalds2f4f12e2013-09-02 11:58:20 -070044 *
45 * This operation is only valid if you already hold a reference
46 * to the object, so you know the count cannot be zero.
47 */
48void lockref_get(struct lockref *lockref)
49{
Linus Torvaldsbc08b442013-09-02 12:12:15 -070050 CMPXCHG_LOOP(
51 new.count++;
52 ,
53 return;
54 );
55
Linus Torvalds2f4f12e2013-09-02 11:58:20 -070056 spin_lock(&lockref->lock);
57 lockref->count++;
58 spin_unlock(&lockref->lock);
59}
60EXPORT_SYMBOL(lockref_get);
61
62/**
63 * lockref_get_not_zero - Increments count unless the count is 0
Linus Torvalds44a0cf92013-09-07 15:30:29 -070064 * @lockref: pointer to lockref structure
Linus Torvalds2f4f12e2013-09-02 11:58:20 -070065 * Return: 1 if count updated successfully or 0 if count was zero
66 */
67int lockref_get_not_zero(struct lockref *lockref)
68{
Linus Torvaldsbc08b442013-09-02 12:12:15 -070069 int retval;
70
71 CMPXCHG_LOOP(
72 new.count++;
73 if (!old.count)
74 return 0;
75 ,
76 return 1;
77 );
Linus Torvalds2f4f12e2013-09-02 11:58:20 -070078
79 spin_lock(&lockref->lock);
Linus Torvaldsbc08b442013-09-02 12:12:15 -070080 retval = 0;
Linus Torvalds2f4f12e2013-09-02 11:58:20 -070081 if (lockref->count) {
82 lockref->count++;
83 retval = 1;
84 }
85 spin_unlock(&lockref->lock);
86 return retval;
87}
88EXPORT_SYMBOL(lockref_get_not_zero);
89
90/**
91 * lockref_get_or_lock - Increments count unless the count is 0
Linus Torvalds44a0cf92013-09-07 15:30:29 -070092 * @lockref: pointer to lockref structure
Linus Torvalds2f4f12e2013-09-02 11:58:20 -070093 * Return: 1 if count updated successfully or 0 if count was zero
94 * and we got the lock instead.
95 */
96int lockref_get_or_lock(struct lockref *lockref)
97{
Linus Torvaldsbc08b442013-09-02 12:12:15 -070098 CMPXCHG_LOOP(
99 new.count++;
100 if (!old.count)
101 break;
102 ,
103 return 1;
104 );
105
Linus Torvalds2f4f12e2013-09-02 11:58:20 -0700106 spin_lock(&lockref->lock);
107 if (!lockref->count)
108 return 0;
109 lockref->count++;
110 spin_unlock(&lockref->lock);
111 return 1;
112}
113EXPORT_SYMBOL(lockref_get_or_lock);
114
115/**
116 * lockref_put_or_lock - decrements count unless count <= 1 before decrement
Linus Torvalds44a0cf92013-09-07 15:30:29 -0700117 * @lockref: pointer to lockref structure
Linus Torvalds2f4f12e2013-09-02 11:58:20 -0700118 * Return: 1 if count updated successfully or 0 if count <= 1 and lock taken
119 */
120int lockref_put_or_lock(struct lockref *lockref)
121{
Linus Torvaldsbc08b442013-09-02 12:12:15 -0700122 CMPXCHG_LOOP(
123 new.count--;
124 if (old.count <= 1)
125 break;
126 ,
127 return 1;
128 );
129
Linus Torvalds2f4f12e2013-09-02 11:58:20 -0700130 spin_lock(&lockref->lock);
131 if (lockref->count <= 1)
132 return 0;
133 lockref->count--;
134 spin_unlock(&lockref->lock);
135 return 1;
136}
137EXPORT_SYMBOL(lockref_put_or_lock);
Linus Torvaldse7d33bb2013-09-07 15:49:18 -0700138
139/**
140 * lockref_mark_dead - mark lockref dead
141 * @lockref: pointer to lockref structure
142 */
143void lockref_mark_dead(struct lockref *lockref)
144{
145 assert_spin_locked(&lockref->lock);
146 lockref->count = -128;
147}
Steven Whitehousee66cf162013-10-15 15:18:08 +0100148EXPORT_SYMBOL(lockref_mark_dead);
Linus Torvaldse7d33bb2013-09-07 15:49:18 -0700149
150/**
151 * lockref_get_not_dead - Increments count unless the ref is dead
152 * @lockref: pointer to lockref structure
153 * Return: 1 if count updated successfully or 0 if lockref was dead
154 */
155int lockref_get_not_dead(struct lockref *lockref)
156{
157 int retval;
158
159 CMPXCHG_LOOP(
160 new.count++;
161 if ((int)old.count < 0)
162 return 0;
163 ,
164 return 1;
165 );
166
167 spin_lock(&lockref->lock);
168 retval = 0;
169 if ((int) lockref->count >= 0) {
170 lockref->count++;
171 retval = 1;
172 }
173 spin_unlock(&lockref->lock);
174 return retval;
175}
176EXPORT_SYMBOL(lockref_get_not_dead);