| 1 | /*␊ |
| 2 | * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.␊ |
| 3 | *␊ |
| 4 | * @APPLE_LICENSE_HEADER_START@␊ |
| 5 | * ␊ |
| 6 | * This file contains Original Code and/or Modifications of Original Code␊ |
| 7 | * as defined in and that are subject to the Apple Public Source License␊ |
| 8 | * Version 2.0 (the 'License'). You may not use this file except in␊ |
| 9 | * compliance with the License. Please obtain a copy of the License at␊ |
| 10 | * http://www.opensource.apple.com/apsl/ and read it before using this␊ |
| 11 | * file.␊ |
| 12 | * ␊ |
| 13 | * The Original Code and all software distributed under the License are␊ |
| 14 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER␊ |
| 15 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,␊ |
| 16 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,␊ |
| 17 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.␊ |
| 18 | * Please see the License for the specific language governing rights and␊ |
| 19 | * limitations under the License.␊ |
| 20 | * ␊ |
| 21 | * @APPLE_LICENSE_HEADER_END@␊ |
| 22 | */␊ |
| 23 | ␊ |
| 24 | #ifndef _OSATOMIC_H_␊ |
| 25 | #define _OSATOMIC_H_␊ |
| 26 | ␊ |
| 27 | #include <stddef.h>␊ |
| 28 | #include <sys/cdefs.h>␊ |
| 29 | #include <stdint.h>␊ |
| 30 | #include <stdbool.h>␊ |
| 31 | ␊ |
| 32 | /* These are the preferred versions of the atomic and synchronization operations.␊ |
| 33 | * Their implementation is customized at boot time for the platform, including␊ |
| 34 | * late-breaking errata fixes as necessary. They are thread safe.␊ |
| 35 | *␊ |
| 36 | * WARNING: all addresses passed to these functions must be "naturally aligned", ie␊ |
| 37 | * int32_t's must be 32-bit aligned (low 2 bits of address zero), and int64_t's␊ |
| 38 | * must be 64-bit aligned (low 3 bits of address zero.)␊ |
| 39 | *␊ |
| 40 | * Note that some versions of the atomic functions incorporate memory barriers,␊ |
| 41 | * and some do not. Barriers strictly order memory access on a weakly-ordered␊ |
| 42 | * architecture such as PPC. All loads and stores executed in sequential program␊ |
| 43 | * order before the barrier will complete before any load or store executed after␊ |
| 44 | * the barrier. On a uniprocessor, the barrier operation is typically a nop.␊ |
| 45 | * On a multiprocessor, the barrier can be quite expensive on some platforms,␊ |
| 46 | * eg PPC.␊ |
| 47 | *␊ |
| 48 | * Most code will want to use the barrier functions to insure that memory shared␊ |
| 49 | * between threads is properly synchronized. For example, if you want to initialize␊ |
| 50 | * a shared data structure and then atomically increment a variable to indicate␊ |
| 51 | * that the initialization is complete, then you must use OSAtomicIncrement32Barrier()␊ |
| 52 | * to ensure that the stores to your data structure complete before the atomic add.␊ |
| 53 | * Likewise, the consumer of that data structure must use OSAtomicDecrement32Barrier(),␊ |
| 54 | * in order to ensure that their loads of the structure are not executed before␊ |
| 55 | * the atomic decrement. On the other hand, if you are simply incrementing a global␊ |
| 56 | * counter, then it is safe and potentially faster to use OSAtomicIncrement32().␊ |
| 57 | *␊ |
| 58 | * If you are unsure which version to use, prefer the barrier variants as they are␊ |
| 59 | * safer.␊ |
| 60 | *␊ |
| 61 | * The spinlock and queue operations always incorporate a barrier.␊ |
| 62 | */ ␊ |
| 63 | __BEGIN_DECLS␊ |
| 64 | ␊ |
| 65 | ␊ |
| 66 | /* Arithmetic functions. They return the new value.␊ |
| 67 | */␊ |
| 68 | int32_t␉OSAtomicAdd32( int32_t __theAmount, volatile int32_t *__theValue );␊ |
| 69 | int32_t␉OSAtomicAdd32Barrier( int32_t __theAmount, volatile int32_t *__theValue );␊ |
| 70 | ␊ |
| 71 | __inline static␊ |
| 72 | int32_t␉OSAtomicIncrement32( volatile int32_t *__theValue )␊ |
| 73 | { return OSAtomicAdd32( 1, __theValue); }␊ |
| 74 | __inline static␊ |
| 75 | int32_t␉OSAtomicIncrement32Barrier( volatile int32_t *__theValue )␊ |
| 76 | { return OSAtomicAdd32Barrier( 1, __theValue); }␊ |
| 77 | ␊ |
| 78 | __inline static␊ |
| 79 | int32_t␉OSAtomicDecrement32( volatile int32_t *__theValue )␊ |
| 80 | { return OSAtomicAdd32( -1, __theValue); }␊ |
| 81 | __inline static␊ |
| 82 | int32_t␉OSAtomicDecrement32Barrier( volatile int32_t *__theValue )␊ |
| 83 | { return OSAtomicAdd32Barrier( -1, __theValue); }␊ |
| 84 | ␊ |
| 85 | #if defined(__ppc64__) || defined(__i386__) || defined(__x86_64__) || defined(__arm__)␊ |
| 86 | ␊ |
| 87 | int64_t␉OSAtomicAdd64( int64_t __theAmount, volatile int64_t *__theValue );␊ |
| 88 | int64_t␉OSAtomicAdd64Barrier( int64_t __theAmount, volatile int64_t *__theValue );␊ |
| 89 | ␊ |
| 90 | __inline static␊ |
| 91 | int64_t␉OSAtomicIncrement64( volatile int64_t *__theValue )␊ |
| 92 | { return OSAtomicAdd64( 1, __theValue); }␊ |
| 93 | __inline static␊ |
| 94 | int64_t␉OSAtomicIncrement64Barrier( volatile int64_t *__theValue )␊ |
| 95 | { return OSAtomicAdd64Barrier( 1, __theValue); }␊ |
| 96 | ␊ |
| 97 | __inline static␊ |
| 98 | int64_t␉OSAtomicDecrement64( volatile int64_t *__theValue )␊ |
| 99 | { return OSAtomicAdd64( -1, __theValue); }␊ |
| 100 | __inline static␊ |
| 101 | int64_t␉OSAtomicDecrement64Barrier( volatile int64_t *__theValue )␊ |
| 102 | { return OSAtomicAdd64Barrier( -1, __theValue); }␊ |
| 103 | ␊ |
| 104 | #endif /* defined(__ppc64__) || defined(__i386__) || defined(__x86_64__) || defined(__arm__) */␊ |
| 105 | ␊ |
| 106 | ␊ |
| 107 | /* Boolean functions (and, or, xor.) These come in four versions for each operation:␊ |
| 108 | * with and without barriers, and returning the old or new value of the operation.␊ |
| 109 | * The "Orig" versions return the original value, ie before the operation, the non-Orig␊ |
| 110 | * versions return the value after the operation. All are layered on top of␊ |
| 111 | * compare-and-swap.␊ |
| 112 | */␊ |
| 113 | int32_t␉OSAtomicOr32( uint32_t __theMask, volatile uint32_t *__theValue );␊ |
| 114 | int32_t␉OSAtomicOr32Barrier( uint32_t __theMask, volatile uint32_t *__theValue );␊ |
| 115 | int32_t␉OSAtomicOr32Orig( uint32_t __theMask, volatile uint32_t *__theValue );␊ |
| 116 | int32_t␉OSAtomicOr32OrigBarrier( uint32_t __theMask, volatile uint32_t *__theValue );␊ |
| 117 | ␊ |
| 118 | int32_t␉OSAtomicAnd32( uint32_t __theMask, volatile uint32_t *__theValue ); ␊ |
| 119 | int32_t␉OSAtomicAnd32Barrier( uint32_t __theMask, volatile uint32_t *__theValue ); ␊ |
| 120 | int32_t␉OSAtomicAnd32Orig( uint32_t __theMask, volatile uint32_t *__theValue ); ␊ |
| 121 | int32_t␉OSAtomicAnd32OrigBarrier( uint32_t __theMask, volatile uint32_t *__theValue ); ␊ |
| 122 | ␊ |
| 123 | int32_t␉OSAtomicXor32( uint32_t __theMask, volatile uint32_t *__theValue );␊ |
| 124 | int32_t␉OSAtomicXor32Barrier( uint32_t __theMask, volatile uint32_t *__theValue );␊ |
| 125 | int32_t␉OSAtomicXor32Orig( uint32_t __theMask, volatile uint32_t *__theValue );␊ |
| 126 | int32_t␉OSAtomicXor32OrigBarrier( uint32_t __theMask, volatile uint32_t *__theValue );␊ |
| 127 | ␊ |
| 128 | ␊ |
| 129 | /* Compare and swap. They return true if the swap occured. There are several versions,␊ |
| 130 | * depending on data type and whether or not a barrier is used.␊ |
| 131 | */␊ |
| 132 | bool OSAtomicCompareAndSwap32( int32_t __oldValue, int32_t __newValue, volatile int32_t *__theValue );␊ |
| 133 | bool OSAtomicCompareAndSwap32Barrier( int32_t __oldValue, int32_t __newValue, volatile int32_t *__theValue );␊ |
| 134 | bool␉OSAtomicCompareAndSwapPtr( void *__oldValue, void *__newValue, void * volatile *__theValue );␊ |
| 135 | bool␉OSAtomicCompareAndSwapPtrBarrier( void *__oldValue, void *__newValue, void * volatile *__theValue );␊ |
| 136 | bool␉OSAtomicCompareAndSwapInt( int __oldValue, int __newValue, volatile int *__theValue );␊ |
| 137 | bool␉OSAtomicCompareAndSwapIntBarrier( int __oldValue, int __newValue, volatile int *__theValue );␊ |
| 138 | bool␉OSAtomicCompareAndSwapLong( long __oldValue, long __newValue, volatile long *__theValue );␊ |
| 139 | bool␉OSAtomicCompareAndSwapLongBarrier( long __oldValue, long __newValue, volatile long *__theValue );␊ |
| 140 | ␊ |
| 141 | #if defined(__ppc64__) || defined(__i386__) || defined(__x86_64__) || defined(__arm__)␊ |
| 142 | ␊ |
| 143 | bool OSAtomicCompareAndSwap64( int64_t __oldValue, int64_t __newValue, volatile int64_t *__theValue );␊ |
| 144 | bool OSAtomicCompareAndSwap64Barrier( int64_t __oldValue, int64_t __newValue, volatile int64_t *__theValue );␊ |
| 145 | ␊ |
| 146 | #endif /* defined(__ppc64__) || defined(__i386__) || defined(__x86_64__) || defined(__arm__) */␊ |
| 147 | ␊ |
| 148 | ␊ |
| 149 | /* Test and set. They return the original value of the bit, and operate on bit (0x80>>(n&7))␊ |
| 150 | * in byte ((char*)theAddress + (n>>3)).␊ |
| 151 | */␊ |
| 152 | bool OSAtomicTestAndSet( uint32_t __n, volatile void *__theAddress );␊ |
| 153 | bool OSAtomicTestAndSetBarrier( uint32_t __n, volatile void *__theAddress );␊ |
| 154 | bool OSAtomicTestAndClear( uint32_t __n, volatile void *__theAddress );␊ |
| 155 | bool OSAtomicTestAndClearBarrier( uint32_t __n, volatile void *__theAddress );␊ |
| 156 | ␊ |
| 157 | ␊ |
| 158 | /* Spinlocks. These use memory barriers as required to synchronize access to shared␊ |
| 159 | * memory protected by the lock. The lock operation spins, but employs various strategies␊ |
| 160 | * to back off if the lock is held, making it immune to most priority-inversion livelocks.␊ |
| 161 | * The try operation immediately returns false if the lock was held, true if it took the␊ |
| 162 | * lock. The convention is that unlocked is zero, locked is nonzero.␊ |
| 163 | */␊ |
| 164 | #define␉OS_SPINLOCK_INIT 0␊ |
| 165 | ␊ |
| 166 | typedef int32_t OSSpinLock;␊ |
| 167 | ␊ |
| 168 | bool OSSpinLockTry( volatile OSSpinLock *__lock );␊ |
| 169 | void OSSpinLockLock( volatile OSSpinLock *__lock );␊ |
| 170 | void OSSpinLockUnlock( volatile OSSpinLock *__lock );␊ |
| 171 | ␊ |
| 172 | ␊ |
| 173 | /* Lockless atomic enqueue and dequeue. These routines manipulate singly␊ |
| 174 | * linked LIFO lists. Ie, a dequeue will return the most recently enqueued␊ |
| 175 | * element, or NULL if the list is empty. The "offset" parameter is the offset␊ |
| 176 | * in bytes of the link field within the data structure being queued. The␊ |
| 177 | * link field should be a pointer type. Memory barriers are incorporated as ␊ |
| 178 | * needed to permit thread-safe access to the queue element.␊ |
| 179 | */␊ |
| 180 | #if defined(__x86_64__)␊ |
| 181 | ␊ |
| 182 | typedef volatile struct {␊ |
| 183 | ␉void␉*opaque1;␊ |
| 184 | ␉long␉ opaque2;␊ |
| 185 | } OSQueueHead __attribute__ ((aligned (16)));␊ |
| 186 | ␊ |
| 187 | #else␊ |
| 188 | ␊ |
| 189 | typedef volatile struct {␊ |
| 190 | ␉void␉*opaque1;␊ |
| 191 | ␉long␉ opaque2;␊ |
| 192 | } OSQueueHead;␊ |
| 193 | ␊ |
| 194 | #endif␊ |
| 195 | ␊ |
| 196 | #define␉OS_ATOMIC_QUEUE_INIT␉{ NULL, 0 }␊ |
| 197 | ␊ |
| 198 | void OSAtomicEnqueue( OSQueueHead *__list, void *__new, size_t __offset);␊ |
| 199 | void* OSAtomicDequeue( OSQueueHead *__list, size_t __offset);␊ |
| 200 | ␊ |
| 201 | ␊ |
| 202 | /* Memory barrier. It is both a read and write barrier.␊ |
| 203 | */␊ |
| 204 | void OSMemoryBarrier( void );␊ |
| 205 | ␊ |
| 206 | ␊ |
| 207 | __END_DECLS␊ |
| 208 | ␊ |
| 209 | #endif /* _OSATOMIC_H_ */␊ |
| 210 | |
