diff --git a/crypto/openssl/crypto/aes/asm/aesv8-armx.pl b/crypto/openssl/crypto/aes/asm/aesv8-armx.pl --- a/crypto/openssl/crypto/aes/asm/aesv8-armx.pl +++ b/crypto/openssl/crypto/aes/asm/aesv8-armx.pl @@ -120,6 +120,8 @@ .Lenc_key: ___ $code.=<<___ if ($flavour =~ /64/); + AARCH64_VALID_CALL_TARGET + // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! add x29,sp,#0 ___ @@ -295,7 +297,7 @@ ${prefix}_set_decrypt_key: ___ $code.=<<___ if ($flavour =~ /64/); - .inst 0xd503233f // paciasp + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! add x29,sp,#0 ___ @@ -339,7 +341,7 @@ ___ $code.=<<___ if ($flavour =~ /64/); ldp x29,x30,[sp],#16 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret ___ $code.=<<___; @@ -359,6 +361,11 @@ .type ${prefix}_${dir}crypt,%function .align 5 ${prefix}_${dir}crypt: +___ +$code.=<<___ if ($flavour =~ /64/); + AARCH64_VALID_CALL_TARGET +___ +$code.=<<___; ldr $rounds,[$key,#240] vld1.32 {$rndkey0},[$key],#16 vld1.8 {$inout},[$inp] @@ -442,6 +449,7 @@ ${prefix}_ecb_encrypt: ___ $code.=<<___ if ($flavour =~ /64/); + AARCH64_VALID_CALL_TARGET subs $len,$len,#16 // Original input data size bigger than 16, jump to big size processing. b.ne .Lecb_big_size @@ -1236,6 +1244,8 @@ ${prefix}_cbc_encrypt: ___ $code.=<<___ if ($flavour =~ /64/); + AARCH64_VALID_CALL_TARGET + // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! add x29,sp,#0 ___ @@ -1764,6 +1774,8 @@ ${prefix}_ctr32_encrypt_blocks: ___ $code.=<<___ if ($flavour =~ /64/); + AARCH64_VALID_CALL_TARGET + // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! add x29,sp,#0 ___ @@ -2256,6 +2268,7 @@ ${prefix}_xts_encrypt: ___ $code.=<<___ if ($flavour =~ /64/); + AARCH64_VALID_CALL_TARGET cmp $len,#16 // Original input data size bigger than 16, jump to big size processing. b.ne .Lxts_enc_big_size @@ -2930,6 +2943,7 @@ .type ${prefix}_xts_decrypt,%function .align 5 ${prefix}_xts_decrypt: + AARCH64_VALID_CALL_TARGET ___ $code.=<<___ if ($flavour =~ /64/); cmp $len,#16 diff --git a/crypto/openssl/crypto/aes/asm/vpaes-armv8.pl b/crypto/openssl/crypto/aes/asm/vpaes-armv8.pl --- a/crypto/openssl/crypto/aes/asm/vpaes-armv8.pl +++ b/crypto/openssl/crypto/aes/asm/vpaes-armv8.pl @@ -53,6 +53,8 @@ *STDOUT=*OUT; $code.=<<___; +#include "arm_arch.h" + .text .type _vpaes_consts,%object @@ -259,7 +261,7 @@ .type vpaes_encrypt,%function .align 4 vpaes_encrypt: - .inst 0xd503233f // paciasp + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -269,7 +271,7 @@ st1 {v0.16b}, [$out] ldp x29,x30,[sp],#16 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size vpaes_encrypt,.-vpaes_encrypt @@ -492,7 +494,7 @@ .type vpaes_decrypt,%function .align 4 vpaes_decrypt: - .inst 0xd503233f // paciasp + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -502,7 +504,7 @@ st1 {v0.16b}, [$out] ldp x29,x30,[sp],#16 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size vpaes_decrypt,.-vpaes_decrypt @@ -673,7 +675,7 @@ .type _vpaes_schedule_core,%function .align 4 _vpaes_schedule_core: - .inst 0xd503233f // paciasp + AARCH64_SIGN_LINK_REGISTER stp x29, x30, [sp,#-16]! add x29,sp,#0 @@ -838,7 +840,7 @@ eor v6.16b, v6.16b, v6.16b // vpxor %xmm6, %xmm6, %xmm6 eor v7.16b, v7.16b, v7.16b // vpxor %xmm7, %xmm7, %xmm7 ldp x29, x30, [sp],#16 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size _vpaes_schedule_core,.-_vpaes_schedule_core @@ -1051,7 +1053,7 @@ .type vpaes_set_encrypt_key,%function .align 4 vpaes_set_encrypt_key: - .inst 0xd503233f // paciasp + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! add x29,sp,#0 stp d8,d9,[sp,#-16]! // ABI spec says so @@ -1067,7 +1069,7 @@ ldp d8,d9,[sp],#16 ldp x29,x30,[sp],#16 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size vpaes_set_encrypt_key,.-vpaes_set_encrypt_key @@ -1075,7 +1077,7 @@ .type vpaes_set_decrypt_key,%function .align 4 vpaes_set_decrypt_key: - .inst 0xd503233f // paciasp + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! add x29,sp,#0 stp d8,d9,[sp,#-16]! // ABI spec says so @@ -1095,7 +1097,7 @@ ldp d8,d9,[sp],#16 ldp x29,x30,[sp],#16 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size vpaes_set_decrypt_key,.-vpaes_set_decrypt_key ___ @@ -1108,11 +1110,11 @@ .type vpaes_cbc_encrypt,%function .align 4 vpaes_cbc_encrypt: + AARCH64_SIGN_LINK_REGISTER cbz $len, .Lcbc_abort cmp w5, #0 // check direction b.eq vpaes_cbc_decrypt - .inst 0xd503233f // paciasp stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -1135,15 +1137,16 @@ st1 {v0.16b}, [$ivec] // write ivec ldp x29,x30,[sp],#16 - .inst 0xd50323bf // autiasp .Lcbc_abort: + AARCH64_VALIDATE_LINK_REGISTER ret .size vpaes_cbc_encrypt,.-vpaes_cbc_encrypt .type vpaes_cbc_decrypt,%function .align 4 vpaes_cbc_decrypt: - .inst 0xd503233f // paciasp + // Not adding AARCH64_SIGN_LINK_REGISTER here because vpaes_cbc_decrypt is jumped to + // only from vpaes_cbc_encrypt which has already signed the return address. stp x29,x30,[sp,#-16]! add x29,sp,#0 stp d8,d9,[sp,#-16]! // ABI spec says so @@ -1185,7 +1188,7 @@ ldp d10,d11,[sp],#16 ldp d8,d9,[sp],#16 ldp x29,x30,[sp],#16 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size vpaes_cbc_decrypt,.-vpaes_cbc_decrypt ___ @@ -1195,7 +1198,7 @@ .type vpaes_ecb_encrypt,%function .align 4 vpaes_ecb_encrypt: - .inst 0xd503233f // paciasp + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! add x29,sp,#0 stp d8,d9,[sp,#-16]! // ABI spec says so @@ -1229,7 +1232,7 @@ ldp d10,d11,[sp],#16 ldp d8,d9,[sp],#16 ldp x29,x30,[sp],#16 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size vpaes_ecb_encrypt,.-vpaes_ecb_encrypt @@ -1237,7 +1240,7 @@ .type vpaes_ecb_decrypt,%function .align 4 vpaes_ecb_decrypt: - .inst 0xd503233f // paciasp + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! add x29,sp,#0 stp d8,d9,[sp,#-16]! // ABI spec says so @@ -1271,7 +1274,7 @@ ldp d10,d11,[sp],#16 ldp d8,d9,[sp],#16 ldp x29,x30,[sp],#16 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size vpaes_ecb_decrypt,.-vpaes_ecb_decrypt ___ diff --git a/crypto/openssl/crypto/aes/build.info b/crypto/openssl/crypto/aes/build.info --- a/crypto/openssl/crypto/aes/build.info +++ b/crypto/openssl/crypto/aes/build.info @@ -116,6 +116,7 @@ GENERATE[aesv8-armx.S]=asm/aesv8-armx.pl INCLUDE[aesv8-armx.o]=.. GENERATE[vpaes-armv8.S]=asm/vpaes-armv8.pl +INCLUDE[vpaes-armv8.o]=.. GENERATE[aes-armv4.S]=asm/aes-armv4.pl INCLUDE[aes-armv4.o]=.. diff --git a/crypto/openssl/crypto/arm64cpuid.pl b/crypto/openssl/crypto/arm64cpuid.pl --- a/crypto/openssl/crypto/arm64cpuid.pl +++ b/crypto/openssl/crypto/arm64cpuid.pl @@ -31,6 +31,7 @@ .globl _armv7_neon_probe .type _armv7_neon_probe,%function _armv7_neon_probe: + AARCH64_VALID_CALL_TARGET orr v15.16b, v15.16b, v15.16b ret .size _armv7_neon_probe,.-_armv7_neon_probe @@ -38,6 +39,7 @@ .globl _armv7_tick .type _armv7_tick,%function _armv7_tick: + AARCH64_VALID_CALL_TARGET #ifdef __APPLE__ mrs x0, CNTPCT_EL0 #else @@ -49,6 +51,7 @@ .globl _armv8_aes_probe .type _armv8_aes_probe,%function _armv8_aes_probe: + AARCH64_VALID_CALL_TARGET aese v0.16b, v0.16b ret .size _armv8_aes_probe,.-_armv8_aes_probe @@ -56,6 +59,7 @@ .globl _armv8_sha1_probe .type _armv8_sha1_probe,%function _armv8_sha1_probe: + AARCH64_VALID_CALL_TARGET sha1h s0, s0 ret .size _armv8_sha1_probe,.-_armv8_sha1_probe @@ -63,6 +67,7 @@ .globl _armv8_sha256_probe .type _armv8_sha256_probe,%function _armv8_sha256_probe: + AARCH64_VALID_CALL_TARGET sha256su0 v0.4s, v0.4s ret .size _armv8_sha256_probe,.-_armv8_sha256_probe @@ -70,6 +75,7 @@ .globl _armv8_pmull_probe .type _armv8_pmull_probe,%function _armv8_pmull_probe: + AARCH64_VALID_CALL_TARGET pmull v0.1q, v0.1d, v0.1d ret .size _armv8_pmull_probe,.-_armv8_pmull_probe @@ -77,6 +83,7 @@ .globl _armv8_sha512_probe .type _armv8_sha512_probe,%function _armv8_sha512_probe: + AARCH64_VALID_CALL_TARGET .long 0xcec08000 // sha512su0 v0.2d,v0.2d ret .size _armv8_sha512_probe,.-_armv8_sha512_probe @@ -84,6 +91,7 @@ .globl _armv8_cpuid_probe .type _armv8_cpuid_probe,%function _armv8_cpuid_probe: + AARCH64_VALID_CALL_TARGET mrs x0, midr_el1 ret .size _armv8_cpuid_probe,.-_armv8_cpuid_probe @@ -92,6 +100,7 @@ .type OPENSSL_cleanse,%function .align 5 OPENSSL_cleanse: + AARCH64_VALID_CALL_TARGET cbz x1,.Lret // len==0? cmp x1,#15 b.hi .Lot // len>15 @@ -123,6 +132,7 @@ .type CRYPTO_memcmp,%function .align 4 CRYPTO_memcmp: + AARCH64_VALID_CALL_TARGET eor w3,w3,w3 cbz x2,.Lno_data // len==0? cmp x2,#16 diff --git a/crypto/openssl/crypto/arm_arch.h b/crypto/openssl/crypto/arm_arch.h --- a/crypto/openssl/crypto/arm_arch.h +++ b/crypto/openssl/crypto/arm_arch.h @@ -126,4 +126,62 @@ # define MIDR_IS_CPU_MODEL(midr, imp, partnum) \ (((midr) & MIDR_CPU_MODEL_MASK) == MIDR_CPU_MODEL(imp, partnum)) + +#if defined(__ASSEMBLER__) + + /* + * Support macros for + * - Armv8.3-A Pointer Authentication and + * - Armv8.5-A Branch Target Identification + * features which require emitting a .note.gnu.property section with the + * appropriate architecture-dependent feature bits set. + * Read more: "ELF for the ArmĀ® 64-bit Architecture" + */ + +# if defined(__ARM_FEATURE_BTI_DEFAULT) && __ARM_FEATURE_BTI_DEFAULT == 1 +# define GNU_PROPERTY_AARCH64_BTI (1 << 0) /* Has Branch Target Identification */ +# define AARCH64_VALID_CALL_TARGET hint #34 /* BTI 'c' */ +# else +# define GNU_PROPERTY_AARCH64_BTI 0 /* No Branch Target Identification */ +# define AARCH64_VALID_CALL_TARGET +# endif + +# if defined(__ARM_FEATURE_PAC_DEFAULT) && \ + (__ARM_FEATURE_PAC_DEFAULT & 1) == 1 /* Signed with A-key */ +# define GNU_PROPERTY_AARCH64_POINTER_AUTH \ + (1 << 1) /* Has Pointer Authentication */ +# define AARCH64_SIGN_LINK_REGISTER hint #25 /* PACIASP */ +# define AARCH64_VALIDATE_LINK_REGISTER hint #29 /* AUTIASP */ +# elif defined(__ARM_FEATURE_PAC_DEFAULT) && \ + (__ARM_FEATURE_PAC_DEFAULT & 2) == 2 /* Signed with B-key */ +# define GNU_PROPERTY_AARCH64_POINTER_AUTH \ + (1 << 1) /* Has Pointer Authentication */ +# define AARCH64_SIGN_LINK_REGISTER hint #27 /* PACIBSP */ +# define AARCH64_VALIDATE_LINK_REGISTER hint #31 /* AUTIBSP */ +# else +# define GNU_PROPERTY_AARCH64_POINTER_AUTH 0 /* No Pointer Authentication */ +# if GNU_PROPERTY_AARCH64_BTI != 0 +# define AARCH64_SIGN_LINK_REGISTER AARCH64_VALID_CALL_TARGET +# else +# define AARCH64_SIGN_LINK_REGISTER +# endif +# define AARCH64_VALIDATE_LINK_REGISTER +# endif + +# if GNU_PROPERTY_AARCH64_POINTER_AUTH != 0 || GNU_PROPERTY_AARCH64_BTI != 0 + .pushsection .note.gnu.property, "a"; + .balign 8; + .long 4; + .long 0x10; + .long 0x5; + .asciz "GNU"; + .long 0xc0000000; /* GNU_PROPERTY_AARCH64_FEATURE_1_AND */ + .long 4; + .long (GNU_PROPERTY_AARCH64_POINTER_AUTH | GNU_PROPERTY_AARCH64_BTI); + .long 0; + .popsection; +# endif + +# endif /* defined __ASSEMBLER__ */ + #endif diff --git a/crypto/openssl/crypto/bn/asm/armv8-mont.pl b/crypto/openssl/crypto/bn/asm/armv8-mont.pl --- a/crypto/openssl/crypto/bn/asm/armv8-mont.pl +++ b/crypto/openssl/crypto/bn/asm/armv8-mont.pl @@ -67,8 +67,8 @@ $num="x5"; # int num); $code.=<<___; +#include "arm_arch.h" #ifndef __KERNEL__ -# include "arm_arch.h" .extern OPENSSL_armv8_rsa_neonized .hidden OPENSSL_armv8_rsa_neonized #endif @@ -78,6 +78,7 @@ .type bn_mul_mont,%function .align 5 bn_mul_mont: + AARCH64_SIGN_LINK_REGISTER .Lbn_mul_mont: tst $num,#3 b.ne .Lmul_mont @@ -288,6 +289,7 @@ mov x0,#1 ldp x23,x24,[x29,#48] ldr x29,[sp],#64 + AARCH64_VALIDATE_LINK_REGISTER ret .size bn_mul_mont,.-bn_mul_mont ___ @@ -309,6 +311,8 @@ .type bn_mul8x_mont_neon,%function .align 5 bn_mul8x_mont_neon: + // Not adding AARCH64_SIGN_LINK_REGISTER here because bn_mul8x_mont_neon is jumped to + // only from bn_mul_mont which has already signed the return address. stp x29,x30,[sp,#-80]! mov x16,sp stp d8,d9,[sp,#16] @@ -649,6 +653,7 @@ ldp d10,d11,[sp,#32] ldp d8,d9,[sp,#16] ldr x29,[sp],#80 + AARCH64_VALIDATE_LINK_REGISTER ret // bx lr .size bn_mul8x_mont_neon,.-bn_mul8x_mont_neon @@ -671,7 +676,8 @@ cmp $ap,$bp b.ne __bn_mul4x_mont .Lsqr8x_mont: - .inst 0xd503233f // paciasp + // Not adding AARCH64_SIGN_LINK_REGISTER here because __bn_sqr8x_mont is jumped to + // only from bn_mul_mont which has already signed the return address. stp x29,x30,[sp,#-128]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -1425,7 +1431,8 @@ ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldr x29,[sp],#128 - .inst 0xd50323bf // autiasp + // x30 is loaded earlier + AARCH64_VALIDATE_LINK_REGISTER ret .size __bn_sqr8x_mont,.-__bn_sqr8x_mont ___ @@ -1449,7 +1456,8 @@ .type __bn_mul4x_mont,%function .align 5 __bn_mul4x_mont: - .inst 0xd503233f // paciasp + // Not adding AARCH64_SIGN_LINK_REGISTER here because __bn_mul4x_mont is jumped to + // only from bn_mul_mont (or __bn_sqr8x_mont from bn_mul_mont) which has already signed the return address. stp x29,x30,[sp,#-128]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -1883,7 +1891,8 @@ ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldr x29,[sp],#128 - .inst 0xd50323bf // autiasp + // x30 loaded earlier + AARCH64_VALIDATE_LINK_REGISTER ret .size __bn_mul4x_mont,.-__bn_mul4x_mont ___ diff --git a/crypto/openssl/crypto/chacha/asm/chacha-armv8.pl b/crypto/openssl/crypto/chacha/asm/chacha-armv8.pl --- a/crypto/openssl/crypto/chacha/asm/chacha-armv8.pl +++ b/crypto/openssl/crypto/chacha/asm/chacha-armv8.pl @@ -132,8 +132,8 @@ } $code.=<<___; +#include "arm_arch.h" #ifndef __KERNEL__ -# include "arm_arch.h" .extern OPENSSL_armcap_P .hidden OPENSSL_armcap_P #endif @@ -153,6 +153,7 @@ .type ChaCha20_ctr32,%function .align 5 ChaCha20_ctr32: + AARCH64_SIGN_LINK_REGISTER cbz $len,.Labort cmp $len,#192 b.lo .Lshort @@ -165,7 +166,6 @@ #endif .Lshort: - .inst 0xd503233f // paciasp stp x29,x30,[sp,#-96]! add x29,sp,#0 @@ -285,8 +285,8 @@ ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#96 - .inst 0xd50323bf // autiasp .Labort: + AARCH64_VALIDATE_LINK_REGISTER ret .align 4 @@ -342,7 +342,7 @@ ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#96 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size ChaCha20_ctr32,.-ChaCha20_ctr32 ___ @@ -432,8 +432,8 @@ .type ChaCha20_neon,%function .align 5 ChaCha20_neon: + AARCH64_SIGN_LINK_REGISTER .LChaCha20_neon: - .inst 0xd503233f // paciasp stp x29,x30,[sp,#-96]! add x29,sp,#0 @@ -667,7 +667,7 @@ ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#96 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .align 4 @@ -799,7 +799,7 @@ ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#96 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size ChaCha20_neon,.-ChaCha20_neon ___ @@ -844,7 +844,7 @@ .type ChaCha20_512_neon,%function .align 5 ChaCha20_512_neon: - .inst 0xd503233f // paciasp + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-96]! add x29,sp,#0 @@ -1268,7 +1268,7 @@ ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#96 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size ChaCha20_512_neon,.-ChaCha20_512_neon ___ diff --git a/crypto/openssl/crypto/ec/asm/ecp_nistz256-armv8.pl b/crypto/openssl/crypto/ec/asm/ecp_nistz256-armv8.pl --- a/crypto/openssl/crypto/ec/asm/ecp_nistz256-armv8.pl +++ b/crypto/openssl/crypto/ec/asm/ecp_nistz256-armv8.pl @@ -122,7 +122,7 @@ .type ecp_nistz256_to_mont,%function .align 6 ecp_nistz256_to_mont: - .inst 0xd503233f // paciasp + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-32]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -138,7 +138,7 @@ ldp x19,x20,[sp,#16] ldp x29,x30,[sp],#32 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size ecp_nistz256_to_mont,.-ecp_nistz256_to_mont @@ -147,7 +147,7 @@ .type ecp_nistz256_from_mont,%function .align 4 ecp_nistz256_from_mont: - .inst 0xd503233f // paciasp + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-32]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -163,7 +163,7 @@ ldp x19,x20,[sp,#16] ldp x29,x30,[sp],#32 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size ecp_nistz256_from_mont,.-ecp_nistz256_from_mont @@ -173,7 +173,7 @@ .type ecp_nistz256_mul_mont,%function .align 4 ecp_nistz256_mul_mont: - .inst 0xd503233f // paciasp + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-32]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -188,7 +188,7 @@ ldp x19,x20,[sp,#16] ldp x29,x30,[sp],#32 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size ecp_nistz256_mul_mont,.-ecp_nistz256_mul_mont @@ -197,7 +197,7 @@ .type ecp_nistz256_sqr_mont,%function .align 4 ecp_nistz256_sqr_mont: - .inst 0xd503233f // paciasp + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-32]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -211,7 +211,7 @@ ldp x19,x20,[sp,#16] ldp x29,x30,[sp],#32 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size ecp_nistz256_sqr_mont,.-ecp_nistz256_sqr_mont @@ -221,7 +221,7 @@ .type ecp_nistz256_add,%function .align 4 ecp_nistz256_add: - .inst 0xd503233f // paciasp + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -235,7 +235,7 @@ bl __ecp_nistz256_add ldp x29,x30,[sp],#16 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size ecp_nistz256_add,.-ecp_nistz256_add @@ -244,7 +244,7 @@ .type ecp_nistz256_div_by_2,%function .align 4 ecp_nistz256_div_by_2: - .inst 0xd503233f // paciasp + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -256,7 +256,7 @@ bl __ecp_nistz256_div_by_2 ldp x29,x30,[sp],#16 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size ecp_nistz256_div_by_2,.-ecp_nistz256_div_by_2 @@ -265,7 +265,7 @@ .type ecp_nistz256_mul_by_2,%function .align 4 ecp_nistz256_mul_by_2: - .inst 0xd503233f // paciasp + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -281,7 +281,7 @@ bl __ecp_nistz256_add // ret = a+a // 2*a ldp x29,x30,[sp],#16 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size ecp_nistz256_mul_by_2,.-ecp_nistz256_mul_by_2 @@ -290,7 +290,7 @@ .type ecp_nistz256_mul_by_3,%function .align 4 ecp_nistz256_mul_by_3: - .inst 0xd503233f // paciasp + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -317,7 +317,7 @@ bl __ecp_nistz256_add // ret += a // 2*a+a=3*a ldp x29,x30,[sp],#16 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size ecp_nistz256_mul_by_3,.-ecp_nistz256_mul_by_3 @@ -327,7 +327,7 @@ .type ecp_nistz256_sub,%function .align 4 ecp_nistz256_sub: - .inst 0xd503233f // paciasp + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -339,7 +339,7 @@ bl __ecp_nistz256_sub_from ldp x29,x30,[sp],#16 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size ecp_nistz256_sub,.-ecp_nistz256_sub @@ -348,7 +348,7 @@ .type ecp_nistz256_neg,%function .align 4 ecp_nistz256_neg: - .inst 0xd503233f // paciasp + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -363,7 +363,7 @@ bl __ecp_nistz256_sub_from ldp x29,x30,[sp],#16 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size ecp_nistz256_neg,.-ecp_nistz256_neg @@ -724,7 +724,7 @@ .type ecp_nistz256_point_double,%function .align 5 ecp_nistz256_point_double: - .inst 0xd503233f // paciasp + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-96]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -859,7 +859,7 @@ ldp x19,x20,[x29,#16] ldp x21,x22,[x29,#32] ldp x29,x30,[sp],#96 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size ecp_nistz256_point_double,.-ecp_nistz256_point_double ___ @@ -882,7 +882,7 @@ .type ecp_nistz256_point_add,%function .align 5 ecp_nistz256_point_add: - .inst 0xd503233f // paciasp + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-96]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -1117,7 +1117,7 @@ ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#96 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size ecp_nistz256_point_add,.-ecp_nistz256_point_add ___ @@ -1139,7 +1139,7 @@ .type ecp_nistz256_point_add_affine,%function .align 5 ecp_nistz256_point_add_affine: - .inst 0xd503233f // paciasp + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-80]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -1328,7 +1328,7 @@ ldp x23,x24,[x29,#48] ldp x25,x26,[x29,#64] ldp x29,x30,[sp],#80 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size ecp_nistz256_point_add_affine,.-ecp_nistz256_point_add_affine ___ @@ -1346,6 +1346,8 @@ .type ecp_nistz256_ord_mul_mont,%function .align 4 ecp_nistz256_ord_mul_mont: + AARCH64_VALID_CALL_TARGET + // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-64]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -1487,6 +1489,8 @@ .type ecp_nistz256_ord_sqr_mont,%function .align 4 ecp_nistz256_ord_sqr_mont: + AARCH64_VALID_CALL_TARGET + // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-64]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -1641,6 +1645,8 @@ .type ecp_nistz256_scatter_w5,%function .align 4 ecp_nistz256_scatter_w5: + AARCH64_VALID_CALL_TARGET + // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -1703,6 +1709,8 @@ .type ecp_nistz256_gather_w5,%function .align 4 ecp_nistz256_gather_w5: + AARCH64_VALID_CALL_TARGET + // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -1780,6 +1788,8 @@ .type ecp_nistz256_scatter_w7,%function .align 4 ecp_nistz256_scatter_w7: + AARCH64_VALID_CALL_TARGET + // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -1824,6 +1834,8 @@ .type ecp_nistz256_gather_w7,%function .align 4 ecp_nistz256_gather_w7: + AARCH64_VALID_CALL_TARGET + // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! add x29,sp,#0 diff --git a/crypto/openssl/crypto/modes/asm/aes-gcm-armv8_64.pl b/crypto/openssl/crypto/modes/asm/aes-gcm-armv8_64.pl --- a/crypto/openssl/crypto/modes/asm/aes-gcm-armv8_64.pl +++ b/crypto/openssl/crypto/modes/asm/aes-gcm-armv8_64.pl @@ -257,6 +257,7 @@ .type aes_gcm_enc_128_kernel,%function .align 4 aes_gcm_enc_128_kernel: + AARCH64_VALID_CALL_TARGET cbz x1, .L128_enc_ret stp x19, x20, [sp, #-112]! mov x16, x4 @@ -1142,6 +1143,7 @@ .type aes_gcm_dec_128_kernel,%function .align 4 aes_gcm_dec_128_kernel: + AARCH64_VALID_CALL_TARGET cbz x1, .L128_dec_ret stp x19, x20, [sp, #-112]! mov x16, x4 @@ -2099,6 +2101,7 @@ .type aes_gcm_enc_192_kernel,%function .align 4 aes_gcm_enc_192_kernel: + AARCH64_VALID_CALL_TARGET cbz x1, .L192_enc_ret stp x19, x20, [sp, #-112]! mov x16, x4 @@ -3035,6 +3038,7 @@ .type aes_gcm_dec_192_kernel,%function .align 4 aes_gcm_dec_192_kernel: + AARCH64_VALID_CALL_TARGET cbz x1, .L192_dec_ret stp x19, x20, [sp, #-112]! mov x16, x4 @@ -4042,6 +4046,7 @@ .type aes_gcm_enc_256_kernel,%function .align 4 aes_gcm_enc_256_kernel: + AARCH64_VALID_CALL_TARGET cbz x1, .L256_enc_ret stp x19, x20, [sp, #-112]! mov x16, x4 @@ -5026,6 +5031,7 @@ .type aes_gcm_dec_256_kernel,%function .align 4 aes_gcm_dec_256_kernel: + AARCH64_VALID_CALL_TARGET cbz x1, .L256_dec_ret stp x19, x20, [sp, #-112]! mov x16, x4 diff --git a/crypto/openssl/crypto/modes/asm/ghashv8-armx.pl b/crypto/openssl/crypto/modes/asm/ghashv8-armx.pl --- a/crypto/openssl/crypto/modes/asm/ghashv8-armx.pl +++ b/crypto/openssl/crypto/modes/asm/ghashv8-armx.pl @@ -107,6 +107,11 @@ .type gcm_init_v8,%function .align 4 gcm_init_v8: +___ +$code.=<<___ if ($flavour =~ /64/); + AARCH64_VALID_CALL_TARGET +___ +$code.=<<___; vld1.64 {$t1},[x1] @ load input H vmov.i8 $xC2,#0xe1 vshl.i64 $xC2,$xC2,#57 @ 0xc2.0 @@ -214,6 +219,11 @@ .type gcm_gmult_v8,%function .align 4 gcm_gmult_v8: +___ +$code.=<<___ if ($flavour =~ /64/); + AARCH64_VALID_CALL_TARGET +___ +$code.=<<___; vld1.64 {$t1},[$Xi] @ load Xi vmov.i8 $xC2,#0xe1 vld1.64 {$H-$Hhl},[$Htbl] @ load twisted H, ... @@ -268,6 +278,7 @@ gcm_ghash_v8: ___ $code.=<<___ if ($flavour =~ /64/); + AARCH64_VALID_CALL_TARGET cmp $len,#64 b.hs .Lgcm_ghash_v8_4x ___ diff --git a/crypto/openssl/crypto/poly1305/asm/poly1305-armv8.pl b/crypto/openssl/crypto/poly1305/asm/poly1305-armv8.pl --- a/crypto/openssl/crypto/poly1305/asm/poly1305-armv8.pl +++ b/crypto/openssl/crypto/poly1305/asm/poly1305-armv8.pl @@ -72,6 +72,7 @@ .type poly1305_init,%function .align 5 poly1305_init: + AARCH64_VALID_CALL_TARGET cmp $inp,xzr stp xzr,xzr,[$ctx] // zero hash value stp xzr,xzr,[$ctx,#16] // [along with is_base2_26] @@ -119,6 +120,9 @@ .align 5 poly1305_blocks: .Lpoly1305_blocks: + // The symbol .Lpoly1305_blocks is not a .globl symbol + // but a pointer to it is returned by poly1305_init + AARCH64_VALID_CALL_TARGET ands $len,$len,#-16 b.eq .Lno_data @@ -184,6 +188,9 @@ .align 5 poly1305_emit: .Lpoly1305_emit: + // The symbol .poly1305_emit is not a .globl symbol + // but a pointer to it is returned by poly1305_init + AARCH64_VALID_CALL_TARGET ldp $h0,$h1,[$ctx] // load hash base 2^64 ldr $h2,[$ctx,#16] ldp $t0,$t1,[$nonce] // load nonce @@ -291,13 +298,16 @@ .align 5 poly1305_blocks_neon: .Lpoly1305_blocks_neon: + // The symbol .Lpoly1305_blocks_neon is not a .globl symbol + // but a pointer to it is returned by poly1305_init + AARCH64_VALID_CALL_TARGET ldr $is_base2_26,[$ctx,#24] cmp $len,#128 b.hs .Lblocks_neon cbz $is_base2_26,.Lpoly1305_blocks .Lblocks_neon: - .inst 0xd503233f // paciasp + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-80]! add x29,sp,#0 @@ -867,7 +877,7 @@ .Lno_data_neon: ldr x29,[sp],#80 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size poly1305_blocks_neon,.-poly1305_blocks_neon @@ -875,6 +885,9 @@ .align 5 poly1305_emit_neon: .Lpoly1305_emit_neon: + // The symbol .Lpoly1305_emit_neon is not a .globl symbol + // but a pointer to it is returned by poly1305_init + AARCH64_VALID_CALL_TARGET ldr $is_base2_26,[$ctx,#24] cbz $is_base2_26,poly1305_emit diff --git a/crypto/openssl/crypto/sha/asm/keccak1600-armv8.pl b/crypto/openssl/crypto/sha/asm/keccak1600-armv8.pl --- a/crypto/openssl/crypto/sha/asm/keccak1600-armv8.pl +++ b/crypto/openssl/crypto/sha/asm/keccak1600-armv8.pl @@ -80,6 +80,8 @@ [ 18, 2, 61, 56, 14 ]); $code.=<<___; +#include "arm_arch.h" + .text .align 8 // strategic alignment and padding that allows to use @@ -125,7 +127,7 @@ .align 5 KeccakF1600_int: adr $C[2],iotas - .inst 0xd503233f // paciasp + AARCH64_SIGN_LINK_REGISTER stp $C[2],x30,[sp,#16] // 32 bytes on top are mine b .Loop .align 4 @@ -297,14 +299,14 @@ bne .Loop ldr x30,[sp,#24] - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size KeccakF1600_int,.-KeccakF1600_int .type KeccakF1600,%function .align 5 KeccakF1600: - .inst 0xd503233f // paciasp + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-128]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -354,7 +356,7 @@ ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#128 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size KeccakF1600,.-KeccakF1600 @@ -362,7 +364,7 @@ .type SHA3_absorb,%function .align 5 SHA3_absorb: - .inst 0xd503233f // paciasp + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-128]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -460,7 +462,7 @@ ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#128 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size SHA3_absorb,.-SHA3_absorb ___ @@ -471,7 +473,7 @@ .type SHA3_squeeze,%function .align 5 SHA3_squeeze: - .inst 0xd503233f // paciasp + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-48]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -534,7 +536,7 @@ ldp x19,x20,[sp,#16] ldp x21,x22,[sp,#32] ldp x29,x30,[sp],#48 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size SHA3_squeeze,.-SHA3_squeeze ___ @@ -653,7 +655,7 @@ .type KeccakF1600_cext,%function .align 5 KeccakF1600_cext: - .inst 0xd503233f // paciasp + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-80]! add x29,sp,#0 stp d8,d9,[sp,#16] // per ABI requirement @@ -686,7 +688,7 @@ ldp d12,d13,[sp,#48] ldp d14,d15,[sp,#64] ldr x29,[sp],#80 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size KeccakF1600_cext,.-KeccakF1600_cext ___ @@ -699,7 +701,7 @@ .type SHA3_absorb_cext,%function .align 5 SHA3_absorb_cext: - .inst 0xd503233f // paciasp + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-80]! add x29,sp,#0 stp d8,d9,[sp,#16] // per ABI requirement @@ -771,7 +773,7 @@ ldp d12,d13,[sp,#48] ldp d14,d15,[sp,#64] ldp x29,x30,[sp],#80 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size SHA3_absorb_cext,.-SHA3_absorb_cext ___ @@ -783,7 +785,7 @@ .type SHA3_squeeze_cext,%function .align 5 SHA3_squeeze_cext: - .inst 0xd503233f // paciasp + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! add x29,sp,#0 mov x9,$ctx @@ -839,7 +841,7 @@ .Lsqueeze_done_ce: ldr x29,[sp],#16 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size SHA3_squeeze_cext,.-SHA3_squeeze_cext ___ diff --git a/crypto/openssl/crypto/sha/asm/sha1-armv8.pl b/crypto/openssl/crypto/sha/asm/sha1-armv8.pl --- a/crypto/openssl/crypto/sha/asm/sha1-armv8.pl +++ b/crypto/openssl/crypto/sha/asm/sha1-armv8.pl @@ -175,8 +175,8 @@ } $code.=<<___; +#include "arm_arch.h" #ifndef __KERNEL__ -# include "arm_arch.h" .extern OPENSSL_armcap_P .hidden OPENSSL_armcap_P #endif @@ -187,11 +187,13 @@ .type sha1_block_data_order,%function .align 6 sha1_block_data_order: + AARCH64_VALID_CALL_TARGET adrp x16,OPENSSL_armcap_P ldr w16,[x16,#:lo12:OPENSSL_armcap_P] tst w16,#ARMV8_SHA1 b.ne .Lv8_entry + // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-96]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -253,6 +255,7 @@ .align 6 sha1_block_armv8: .Lv8_entry: + // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! add x29,sp,#0 diff --git a/crypto/openssl/crypto/sha/asm/sha512-armv8.pl b/crypto/openssl/crypto/sha/asm/sha512-armv8.pl --- a/crypto/openssl/crypto/sha/asm/sha512-armv8.pl +++ b/crypto/openssl/crypto/sha/asm/sha512-armv8.pl @@ -190,8 +190,8 @@ } $code.=<<___; +#include "arm_arch.h" #ifndef __KERNEL__ -# include "arm_arch.h" .extern OPENSSL_armcap_P .hidden OPENSSL_armcap_P #endif @@ -202,6 +202,7 @@ .type $func,%function .align 6 $func: + AARCH64_VALID_CALL_TARGET #ifndef __KERNEL__ adrp x16,OPENSSL_armcap_P ldr w16,[x16,#:lo12:OPENSSL_armcap_P] @@ -218,7 +219,7 @@ ___ $code.=<<___; #endif - .inst 0xd503233f // paciasp + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-128]! add x29,sp,#0 @@ -280,7 +281,7 @@ ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#128 - .inst 0xd50323bf // autiasp + AARCH64_VALIDATE_LINK_REGISTER ret .size $func,.-$func @@ -370,6 +371,7 @@ .align 6 sha256_block_armv8: .Lv8_entry: + // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -632,7 +634,9 @@ .type sha256_block_neon,%function .align 4 sha256_block_neon: + AARCH64_VALID_CALL_TARGET .Lneon_entry: + // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later stp x29, x30, [sp, #-16]! mov x29, sp sub sp,sp,#16*4 @@ -743,6 +747,7 @@ .align 6 sha512_block_armv8: .Lv8_entry: + // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later stp x29,x30,[sp,#-16]! add x29,sp,#0 diff --git a/crypto/openssl/crypto/sha/build.info b/crypto/openssl/crypto/sha/build.info --- a/crypto/openssl/crypto/sha/build.info +++ b/crypto/openssl/crypto/sha/build.info @@ -153,6 +153,7 @@ GENERATE[sha512-armv8.S]=asm/sha512-armv8.pl INCLUDE[sha512-armv8.o]=.. GENERATE[keccak1600-armv8.S]=asm/keccak1600-armv8.pl +INCLUDE[keccak1600-armv8.o]=.. GENERATE[sha1-s390x.S]=asm/sha1-s390x.pl INCLUDE[sha1-s390x.o]=..