Support 1 and 2 byte cmpset on x86

[markj]

Summary

This is intended for use in the LinuxKPI - the DRM code needs it.

[kib]

sys/i386/include/atomic.h

222 ↗

(On Diff #26470)

There is %zN construct, which prints opcode suffix for operand N. I think it is supported by clang. If you use it, you would avoid passing the instruction mnemonic to macros. Also, I suspect that it would work without suffix at all.

[markj]

sys/i386/include/atomic.h

222 ↗

(On Diff #26470)

With clang 4.0 I get "error: invalid operand in inline asm," though it works with gcc 4.9. But indeed, it looks like the suffixes aren't needed anyway.

[markj]

Don't specify the operand type, it can be inferred.

head/share/man/man9/atomic.9

 .\" NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 .\" DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 .\" THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 .\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 .\" THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .\"
 .\" $FreeBSD$
 .\"
-.Dd Jan 3, 2017
+.Dd March 22, 2017
 .Dt ATOMIC 9
 .Os
 .Sh NAME
 .Nm atomic_add ,
 .Nm atomic_clear ,
 .Nm atomic_cmpset ,
 .Nm atomic_fcmpset ,
 .Nm atomic_fetchadd ,

 if (*dst == old) {
         *dst = new;
         return (1);
 } else
         return (0);
 .Ed
 .El
 .Pp
-The
+Some architectures do not implement the
 .Fn atomic_cmpset
-functions are not implemented for the types
+functions for the types
 .Dq Li char ,
 .Dq Li short ,
 .Dq Li 8 ,
 and
 .Dq Li 16 .
 .Bl -hang
 .It Fn atomic_fcmpset dst *old new
 .El

 cache line by other CPU.
 In this case
 .Fn atomic_fcmpset
 function also returns
 .Dv false ,
 despite
 .Dl *old == *dst .
 .Pp
-The
+Some architectures do not implement the
 .Fn atomic_fcmpset
-functions are not implemented for the types
+functions for the types
 .Dq Li char ,
 .Dq Li short ,
 .Dq Li 8 ,
 and
 .Dq Li 16 .
 .Bl -hang
 .It Fn atomic_fetchadd p v
 .Bd -literal -compact

head/sys/amd64/include/atomic.h

  * Kernel modules call real functions which are built into the kernel.
  * This allows kernel modules to be portable between UP and SMP systems.
  */
 #if defined(KLD_MODULE) || !defined(__GNUCLIKE_ASM)
 #define ATOMIC_ASM(NAME, TYPE, OP, CONS, V)                     \
 void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v);  \
 void atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v)
 
+int        atomic_cmpset_char(volatile u_char *dst, u_char expect, u_char src);
+int        atomic_cmpset_short(volatile u_short *dst, u_short expect, u_short src);
 int        atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src);
 int        atomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src);
+int        atomic_fcmpset_char(volatile u_char *dst, u_char *expect, u_char src);
+int        atomic_fcmpset_short(volatile u_short *dst, u_short *expect,
+            u_short src);
 int        atomic_fcmpset_int(volatile u_int *dst, u_int *expect, u_int src);
 int        atomic_fcmpset_long(volatile u_long *dst, u_long *expect, u_long src);
 u_int        atomic_fetchadd_int(volatile u_int *p, u_int v);
 u_long        atomic_fetchadd_long(volatile u_long *p, u_long v);
 int        atomic_testandset_int(volatile u_int *p, u_int v);
 int        atomic_testandset_long(volatile u_long *p, u_int v);
 int        atomic_testandclear_int(volatile u_int *p, u_int v);
 int        atomic_testandclear_long(volatile u_long *p, u_int v);
atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\
         __asm __volatile(MPLOCKED OP                    \
         : "+m" (*p)                                   \
         : CONS (V)                                      \
         : "memory", "cc");                          \
 }                                                       \
 struct __hack
 
 /*
- * Atomic compare and set, used by the mutex functions
+ * Atomic compare and set, used by the mutex functions.
  *
- * if (*dst == expect) *dst = src (all 32 bit words)
+ * cmpset:
+ *      if (*dst == expect)
+ *              *dst = src
  *
- * Returns 0 on failure, non-zero on success
+ * fcmpset:
+ *      if (*dst == *expect)
+ *              *dst = src
+ *      else
+ *              *expect = *dst
+ *
+ * Returns 0 on failure, non-zero on success.
  */
-
-static __inline int
-atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src)
-{
-        u_char res;
-
-        __asm __volatile(
-        "  " MPLOCKED "          "
-        "  cmpxchgl %3,%1 ;        "
-        "       sete       %0 ;            "
-        "# atomic_cmpset_int"
-        : "=q" (res),                        /* 0 */
-          "+m" (*dst),                        /* 1 */
-          "+a" (expect)                       /* 2 */
-        : "r" (src)                        /* 3 */
-        : "memory", "cc");
-        return (res);
+#define ATOMIC_CMPSET(TYPE)                             \
+static __inline int                                     \
+atomic_cmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE expect, u_##TYPE src) \
+{                                                       \
+        u_char res;                                     \
+                                                        \
+        __asm __volatile(                               \
+        "  " MPLOCKED "          "          \
+        "  cmpxchg %3,%1 ; "                  \
+        "  sete    %0 ;            "          \
+        "# atomic_cmpset_" #TYPE "       "          \
+        : "=q" (res),                 /* 0 */         \
+          "+m" (*dst),                        /* 1 */         \
+          "+a" (expect)                       /* 2 */         \
+        : "r" (src)                   /* 3 */         \
+        : "memory", "cc");                          \
+        return (res);                                   \
+}                                                       \
+                                                        \
+static __inline int                                     \
+atomic_fcmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE *expect, u_##TYPE src) \
+{                                                       \
+        u_char res;                                     \
+                                                        \
+        __asm __volatile(                               \
+        "  " MPLOCKED "          "          \
+        "  cmpxchg %3,%1 ;         "          \
+        "  sete    %0 ;            "          \
+        "# atomic_fcmpset_" #TYPE "      "          \
+        : "=q" (res),                 /* 0 */         \
+          "+m" (*dst),                        /* 1 */         \
+          "+a" (*expect)              /* 2 */         \
+        : "r" (src)                   /* 3 */         \
+        : "memory", "cc");                          \
+        return (res);                                   \
 }
 
-static __inline int
-atomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src)
-{
-        u_char res;
+ATOMIC_CMPSET(char);
+ATOMIC_CMPSET(short);
+ATOMIC_CMPSET(int);
+ATOMIC_CMPSET(long);
 
-        __asm __volatile(
-        "  " MPLOCKED "          "
-        "  cmpxchgq %3,%1 ;        "
-        "       sete       %0 ;            "
-        "# atomic_cmpset_long"
-        : "=q" (res),                        /* 0 */
-          "+m" (*dst),                        /* 1 */
-          "+a" (expect)                       /* 2 */
-        : "r" (src)                        /* 3 */
-        : "memory", "cc");
-        return (res);
-}
-
-static __inline int
-atomic_fcmpset_int(volatile u_int *dst, u_int *expect, u_int src)
-{
-        u_char res;
-
-        __asm __volatile(
-        "  " MPLOCKED "          "
-        "  cmpxchgl %3,%1 ;        "
-        "       sete       %0 ;            "
-        "# atomic_fcmpset_int"
-        : "=r" (res),                        /* 0 */
-          "+m" (*dst),                        /* 1 */
-          "+a" (*expect)              /* 2 */
-        : "r" (src)                        /* 3 */
-        : "memory", "cc");
-        return (res);
-}
-
-static __inline int
-atomic_fcmpset_long(volatile u_long *dst, u_long *expect, u_long src)
-{
-        u_char res;
-
-        __asm __volatile(
-        "  " MPLOCKED "          "
-        "  cmpxchgq %3,%1 ;        "
-        "       sete       %0 ;            "
-        "# atomic_fcmpset_long"
-        : "=r" (res),                        /* 0 */
-          "+m" (*dst),                        /* 1 */
-          "+a" (*expect)              /* 2 */
-        : "r" (src)                        /* 3 */
-        : "memory", "cc");
-        return (res);
-}
-
 /*
  * Atomically add the value of v to the integer pointed to by p and return
  * the previous value of *p.
  */
 static __inline u_int
 atomic_fetchadd_int(volatile u_int *p, u_int v)
 {
 

 #define atomic_set_acq_char             atomic_set_barr_char
 #define atomic_set_rel_char             atomic_set_barr_char
 #define atomic_clear_acq_char           atomic_clear_barr_char
 #define atomic_clear_rel_char           atomic_clear_barr_char
 #define atomic_add_acq_char             atomic_add_barr_char
 #define atomic_add_rel_char             atomic_add_barr_char
 #define atomic_subtract_acq_char        atomic_subtract_barr_char
 #define atomic_subtract_rel_char        atomic_subtract_barr_char
+#define atomic_cmpset_acq_char          atomic_cmpset_char
+#define atomic_cmpset_rel_char          atomic_cmpset_char
+#define atomic_fcmpset_acq_char         atomic_fcmpset_char
+#define atomic_fcmpset_rel_char         atomic_fcmpset_char
 
 #define atomic_set_acq_short            atomic_set_barr_short
 #define atomic_set_rel_short            atomic_set_barr_short
 #define atomic_clear_acq_short          atomic_clear_barr_short
 #define atomic_clear_rel_short          atomic_clear_barr_short
 #define atomic_add_acq_short            atomic_add_barr_short
 #define atomic_add_rel_short            atomic_add_barr_short
 #define atomic_subtract_acq_short       atomic_subtract_barr_short
 #define atomic_subtract_rel_short       atomic_subtract_barr_short
+#define atomic_cmpset_acq_short         atomic_cmpset_short
+#define atomic_cmpset_rel_short         atomic_cmpset_short
+#define atomic_fcmpset_acq_short        atomic_fcmpset_short
+#define atomic_fcmpset_rel_short        atomic_fcmpset_short
 
 #define atomic_set_acq_int              atomic_set_barr_int
 #define atomic_set_rel_int              atomic_set_barr_int
 #define atomic_clear_acq_int            atomic_clear_barr_int
 #define atomic_clear_rel_int            atomic_clear_barr_int
 #define atomic_add_acq_int              atomic_add_barr_int
 #define atomic_add_rel_int              atomic_add_barr_int
 #define atomic_subtract_acq_int         atomic_subtract_barr_int
 #define atomic_subtract_rel_int         atomic_subtract_barr_int
 #define atomic_cmpset_acq_int           atomic_cmpset_int
 #define atomic_cmpset_rel_int           atomic_cmpset_int
-#define atomic_fcmpset_acq_int  atomic_fcmpset_int
-#define atomic_fcmpset_rel_int  atomic_fcmpset_int
+#define atomic_fcmpset_acq_int          atomic_fcmpset_int
+#define atomic_fcmpset_rel_int          atomic_fcmpset_int
 
 #define atomic_set_acq_long             atomic_set_barr_long
 #define atomic_set_rel_long             atomic_set_barr_long
 #define atomic_clear_acq_long           atomic_clear_barr_long
 #define atomic_clear_rel_long           atomic_clear_barr_long
 #define atomic_add_acq_long             atomic_add_barr_long
 #define atomic_add_rel_long             atomic_add_barr_long
 #define atomic_subtract_acq_long        atomic_subtract_barr_long
 #define atomic_subtract_rel_long        atomic_subtract_barr_long
 #define atomic_cmpset_acq_long          atomic_cmpset_long
 #define atomic_cmpset_rel_long          atomic_cmpset_long
-#define atomic_fcmpset_acq_long atomic_fcmpset_long
-#define atomic_fcmpset_rel_long atomic_fcmpset_long
+#define atomic_fcmpset_acq_long         atomic_fcmpset_long
+#define atomic_fcmpset_rel_long         atomic_fcmpset_long
 
 #define atomic_readandclear_int(p)      atomic_swap_int(p, 0)
 #define atomic_readandclear_long(p)     atomic_swap_long(p, 0)
 
 /* Operations on 8-bit bytes. */
 #define atomic_set_8            atomic_set_char
 #define atomic_set_acq_8        atomic_set_acq_char
 #define atomic_set_rel_8        atomic_set_rel_char
 #define atomic_clear_8          atomic_clear_char
 #define atomic_clear_acq_8      atomic_clear_acq_char
 #define atomic_clear_rel_8      atomic_clear_rel_char
 #define atomic_add_8            atomic_add_char
 #define atomic_add_acq_8        atomic_add_acq_char
 #define atomic_add_rel_8        atomic_add_rel_char
 #define atomic_subtract_8       atomic_subtract_char
 #define atomic_subtract_acq_8   atomic_subtract_acq_char
 #define atomic_subtract_rel_8   atomic_subtract_rel_char
 #define atomic_load_acq_8       atomic_load_acq_char
 #define atomic_store_rel_8      atomic_store_rel_char
+#define atomic_cmpset_8         atomic_cmpset_char
+#define atomic_cmpset_acq_8     atomic_cmpset_acq_char
+#define atomic_cmpset_rel_8     atomic_cmpset_rel_char
+#define atomic_fcmpset_8        atomic_fcmpset_char
+#define atomic_fcmpset_acq_8    atomic_fcmpset_acq_char
+#define atomic_fcmpset_rel_8    atomic_fcmpset_rel_char
 
 /* Operations on 16-bit words. */
 #define atomic_set_16           atomic_set_short
 #define atomic_set_acq_16       atomic_set_acq_short
 #define atomic_set_rel_16       atomic_set_rel_short
 #define atomic_clear_16         atomic_clear_short
 #define atomic_clear_acq_16     atomic_clear_acq_short
 #define atomic_clear_rel_16     atomic_clear_rel_short
 #define atomic_add_16           atomic_add_short
 #define atomic_add_acq_16       atomic_add_acq_short
 #define atomic_add_rel_16       atomic_add_rel_short
 #define atomic_subtract_16      atomic_subtract_short
 #define atomic_subtract_acq_16  atomic_subtract_acq_short
 #define atomic_subtract_rel_16  atomic_subtract_rel_short
 #define atomic_load_acq_16      atomic_load_acq_short
 #define atomic_store_rel_16     atomic_store_rel_short
+#define atomic_cmpset_16        atomic_cmpset_short
+#define atomic_cmpset_acq_16    atomic_cmpset_acq_short
+#define atomic_cmpset_rel_16    atomic_cmpset_rel_short
+#define atomic_fcmpset_16       atomic_fcmpset_short
+#define atomic_fcmpset_acq_16   atomic_fcmpset_acq_short
+#define atomic_fcmpset_rel_16   atomic_fcmpset_rel_short
 
 /* Operations on 32-bit double words. */
 #define atomic_set_32           atomic_set_int
 #define atomic_set_acq_32       atomic_set_acq_int
 #define atomic_set_rel_32       atomic_set_rel_int
 #define atomic_clear_32         atomic_clear_int
 #define atomic_clear_acq_32     atomic_clear_acq_int
 #define atomic_clear_rel_32     atomic_clear_rel_int