commit 8b0b142b31397c0d179b1d20642e9f7f49c229b3 Merge: 3119cac49725 e7be843b4a16 Author: Pierre Pronchery Date: Thu Aug 7 15:50:32 2025 +0200 Merge commit '1095efe41feed8ea5a6fe5ca123c347ae0914801' Approved by: philip (mentor) Sponsored by: Alpha-Omega Beach Cleaning Project Sponsored by: The FreeBSD Foundation diff --git a/crypto/openssl/crypto/aes/asm/aes-x86_64.pl b/crypto/openssl/crypto/aes/asm/aes-x86_64.pl index 25f7ded947ed..8c7b205a0512 100755 --- a/crypto/openssl/crypto/aes/asm/aes-x86_64.pl +++ b/crypto/openssl/crypto/aes/asm/aes-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2005-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2005-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -2221,6 +2221,7 @@ ___ } $code.=<<___; +.section .rodata align=64 .align 64 .LAES_Te: ___ @@ -2643,6 +2644,7 @@ $code.=<<___; .long 0x1b1b1b1b, 0x1b1b1b1b, 0, 0 .asciz "AES for x86_64, CRYPTOGAMS by " .align 64 +.previous ___ # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, diff --git a/crypto/openssl/crypto/aes/asm/aesni-sha1-x86_64.pl b/crypto/openssl/crypto/aes/asm/aesni-sha1-x86_64.pl index dbe33a3f1a02..4e8fa1d753d7 100755 --- a/crypto/openssl/crypto/aes/asm/aesni-sha1-x86_64.pl +++ b/crypto/openssl/crypto/aes/asm/aesni-sha1-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2011-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2011-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -1738,6 +1738,7 @@ ___ }}} } $code.=<<___; +.section .rodata align=64 .align 64 K_XX_XX: .long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 # K_00_19 @@ -1749,6 +1750,7 @@ K_XX_XX: .asciz "AESNI-CBC+SHA1 stitch for x86_64, CRYPTOGAMS by " .align 64 +.previous ___ if ($shaext) {{{ ($in0,$out,$len,$key,$ivp,$ctx,$inp)=("%rdi","%rsi","%rdx","%rcx","%r8","%r9","%r10"); diff --git a/crypto/openssl/crypto/aes/asm/aesni-sha256-x86_64.pl b/crypto/openssl/crypto/aes/asm/aesni-sha256-x86_64.pl index 5521766a6a7d..39d29ddbb022 100755 --- a/crypto/openssl/crypto/aes/asm/aesni-sha256-x86_64.pl +++ b/crypto/openssl/crypto/aes/asm/aesni-sha256-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2013-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2013-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -168,6 +168,7 @@ $code.=<<___; .cfi_endproc .size $func,.-$func +.section .rodata align=64 .align 64 .type $TABLE,\@object $TABLE: @@ -210,6 +211,7 @@ $TABLE: .long 0,0,0,0, 0,0,0,0 .asciz "AESNI-CBC+SHA256 stitch for x86_64, CRYPTOGAMS by " .align 64 +.previous ___ ###################################################################### diff --git a/crypto/openssl/crypto/aes/asm/aesni-x86.pl b/crypto/openssl/crypto/aes/asm/aesni-x86.pl index d8cc378b5f9b..7cf838db170b 100755 --- a/crypto/openssl/crypto/aes/asm/aesni-x86.pl +++ b/crypto/openssl/crypto/aes/asm/aesni-x86.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2009-2022 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2009-2020 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy diff --git a/crypto/openssl/crypto/aes/asm/aesni-x86_64.pl b/crypto/openssl/crypto/aes/asm/aesni-x86_64.pl index 09c8f78890ed..bff33f2b8dcd 100755 --- a/crypto/openssl/crypto/aes/asm/aesni-x86_64.pl +++ b/crypto/openssl/crypto/aes/asm/aesni-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2009-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2009-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -4743,6 +4743,7 @@ ___ } $code.=<<___; +.section .rodata align=64 .align 64 .Lbswap_mask: .byte 15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0 @@ -4765,6 +4766,7 @@ $code.=<<___; .asciz "AES for Intel AES-NI, CRYPTOGAMS by " .align 64 +.previous ___ # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, diff --git a/crypto/openssl/crypto/aes/asm/aesp8-ppc.pl b/crypto/openssl/crypto/aes/asm/aesp8-ppc.pl index f7f78d04b0e1..38b9405a283b 100755 --- a/crypto/openssl/crypto/aes/asm/aesp8-ppc.pl +++ b/crypto/openssl/crypto/aes/asm/aesp8-ppc.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2014-2024 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2014-2020 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy diff --git a/crypto/openssl/crypto/aes/asm/aesv8-armx.pl b/crypto/openssl/crypto/aes/asm/aesv8-armx.pl index 0aa14da30ac0..0fe126a5ef2f 100755 --- a/crypto/openssl/crypto/aes/asm/aesv8-armx.pl +++ b/crypto/openssl/crypto/aes/asm/aesv8-armx.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2014-2023 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2014-2025 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -110,6 +110,11 @@ my ($zero,$rcon,$mask,$in0,$in1,$tmp,$key)= $flavour=~/64/? map("q$_",(0..6)) : map("q$_",(0..3,8..10)); +# +# This file generates .s file for 64-bit and 32-bit CPUs. +# We don't implement .rodata on 32-bit CPUs yet. +# +$code.=".rodata\n" if ($flavour =~ /64/); $code.=<<___; .align 5 .type .Lrcon,%object @@ -118,7 +123,10 @@ $code.=<<___; .long 0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d // rotate-n-splat .long 0x1b,0x1b,0x1b,0x1b .size .Lrcon, . - .Lrcon +___ +$code.=".previous\n" if ($flavour =~ /64/); +$code.=<<___; .globl ${prefix}_set_encrypt_key .type ${prefix}_set_encrypt_key,%function .align 5 @@ -145,7 +153,15 @@ $code.=<<___; tst $bits,#0x3f b.ne .Lenc_key_abort +___ +$code.=<<___ if ($flavour =~ /64/); + adrp $ptr,.Lrcon + add $ptr,$ptr,:lo12:.Lrcon +___ +$code.=<<___ if ($flavour !~ /64/); adr $ptr,.Lrcon +___ +$code.=<<___; cmp $bits,#192 veor $zero,$zero,$zero @@ -423,7 +439,7 @@ ___ # If lsize < 3*16 bytes, treat them as the tail, interleave the # two blocks AES instructions. # There is one special case, if the original input data size dsize -# = 16 bytes, we will treat it seperately to improve the +# = 16 bytes, we will treat it separately to improve the # performance: one independent code block without LR, FP load and # store, just looks like what the original ECB implementation does. @@ -1758,6 +1774,755 @@ $code.=<<___; .size ${prefix}_cbc_encrypt,.-${prefix}_cbc_encrypt ___ }}} + +{{{ +my ($inp,$out,$len,$key,$ivp)=map("x$_",(0..4)); +my ($rounds,$roundsx,$cnt,$key_)=("w5","x5","w6","x7"); +my ($ctr,$tctr0,$tctr1,$tctr2)=map("w$_",(8..10,12)); +my ($tctr3,$tctr4,$tctr5,$tctr6)=map("w$_",(11,13..15)); +my ($tctr7,$tctr8,$tctr9,$tctr10,$tctr11)=map("w$_",(19..23)); + +# q0-q7 => v0-v7; q8-q23 => v16-v31; q24-q31 => v8-v15 +my ($ivec,$rndlast,$rndping,$rndpang)=map("q$_",(0..3)); +my ($in0,$in1,$in2,$in3,$in4,$in5)=map("q$_",(4..9)); +my ($in6,$in7,$in8,$in9,$in10,$in11)=map("q$_",(10..15)); +my ($dat0,$dat1,$dat2,$dat3,$dat4,$dat5)=map("q$_",(16..21)); +my ($dat6,$dat7,$dat8,$dat9,$dat10,$dat11)=map("q$_",(22..27)); +my ($tmp0,$tmp1,$tmp2)=map("q$_",(25..27)); + +#q_X => qX, for ldp & stp +my ($in0q,$in1q,$in2q,$in3q)=map("q_$_",(4..7)); +my ($in4q,$in5q,$in6q,$in7q,$in8q,$in9q,$in10q,$in11q)=map("q_$_",(16..23)); + +my ($dat8d,$dat9d,$dat10d,$dat11d)=map("d$_",(8..11)); + +$code.=<<___ if ($flavour =~ /64/); +.globl ${prefix}_ctr32_encrypt_blocks_unroll12_eor3 +.type ${prefix}_ctr32_encrypt_blocks_unroll12_eor3,%function +.align 5 +${prefix}_ctr32_encrypt_blocks_unroll12_eor3: + AARCH64_VALID_CALL_TARGET + // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. + stp x29,x30,[sp,#-80]! + stp d8,d9,[sp, #16] + stp d10,d11,[sp, #32] + stp d12,d13,[sp, #48] + stp d14,d15,[sp, #64] + add x29,sp,#0 + + ldr $rounds,[$key,#240] + + ldr $ctr, [$ivp, #12] +#ifdef __AARCH64EB__ + vld1.8 {$dat0},[$ivp] +#else + vld1.32 {$dat0},[$ivp] +#endif + vld1.32 {$rndping-$rndpang},[$key] // load key schedule... + sub $rounds,$rounds,#4 + cmp $len,#2 + add $key_,$key,$roundsx,lsl#4 // pointer to last round key + sub $rounds,$rounds,#2 + add $key_, $key_, #64 + vld1.32 {$rndlast},[$key_] + add $key_,$key,#32 + mov $cnt,$rounds +#ifndef __AARCH64EB__ + rev $ctr, $ctr +#endif + + vorr $dat1,$dat0,$dat0 + add $tctr1, $ctr, #1 + vorr $dat2,$dat0,$dat0 + add $ctr, $ctr, #2 + vorr $ivec,$dat0,$dat0 + rev $tctr1, $tctr1 + vmov.32 ${dat1}[3],$tctr1 + b.ls .Lctr32_tail_unroll + cmp $len,#6 + rev $tctr2, $ctr + sub $len,$len,#3 // bias + vmov.32 ${dat2}[3],$tctr2 + b.lo .Loop3x_ctr32_unroll + cmp $len,#9 + vorr $dat3,$dat0,$dat0 + add $tctr3, $ctr, #1 + vorr $dat4,$dat0,$dat0 + add $tctr4, $ctr, #2 + rev $tctr3, $tctr3 + vorr $dat5,$dat0,$dat0 + add $ctr, $ctr, #3 + rev $tctr4, $tctr4 + vmov.32 ${dat3}[3],$tctr3 + rev $tctr5, $ctr + vmov.32 ${dat4}[3],$tctr4 + vmov.32 ${dat5}[3],$tctr5 + sub $len,$len,#3 + b.lo .Loop6x_ctr32_unroll + + // push regs to stack when 12 data chunks are interleaved + stp x19,x20,[sp,#-16]! + stp x21,x22,[sp,#-16]! + stp x23,x24,[sp,#-16]! + stp $dat8d,$dat9d,[sp,#-32]! + stp $dat10d,$dat11d,[sp,#-32]! + + add $tctr6,$ctr,#1 + add $tctr7,$ctr,#2 + add $tctr8,$ctr,#3 + add $tctr9,$ctr,#4 + add $tctr10,$ctr,#5 + add $ctr,$ctr,#6 + vorr $dat6,$dat0,$dat0 + rev $tctr6,$tctr6 + vorr $dat7,$dat0,$dat0 + rev $tctr7,$tctr7 + vorr $dat8,$dat0,$dat0 + rev $tctr8,$tctr8 + vorr $dat9,$dat0,$dat0 + rev $tctr9,$tctr9 + vorr $dat10,$dat0,$dat0 + rev $tctr10,$tctr10 + vorr $dat11,$dat0,$dat0 + rev $tctr11,$ctr + + sub $len,$len,#6 // bias + vmov.32 ${dat6}[3],$tctr6 + vmov.32 ${dat7}[3],$tctr7 + vmov.32 ${dat8}[3],$tctr8 + vmov.32 ${dat9}[3],$tctr9 + vmov.32 ${dat10}[3],$tctr10 + vmov.32 ${dat11}[3],$tctr11 + b .Loop12x_ctr32_unroll + +.align 4 +.Loop12x_ctr32_unroll: + aese $dat0,$rndping + aesmc $dat0,$dat0 + aese $dat1,$rndping + aesmc $dat1,$dat1 + aese $dat2,$rndping + aesmc $dat2,$dat2 + aese $dat3,$rndping + aesmc $dat3,$dat3 + aese $dat4,$rndping + aesmc $dat4,$dat4 + aese $dat5,$rndping + aesmc $dat5,$dat5 + aese $dat6,$rndping + aesmc $dat6,$dat6 + aese $dat7,$rndping + aesmc $dat7,$dat7 + aese $dat8,$rndping + aesmc $dat8,$dat8 + aese $dat9,$rndping + aesmc $dat9,$dat9 + aese $dat10,$rndping + aesmc $dat10,$dat10 + aese $dat11,$rndping + aesmc $dat11,$dat11 + vld1.32 {$rndping},[$key_],#16 + subs $cnt,$cnt,#2 + aese $dat0,$rndpang + aesmc $dat0,$dat0 + aese $dat1,$rndpang + aesmc $dat1,$dat1 + aese $dat2,$rndpang + aesmc $dat2,$dat2 + aese $dat3,$rndpang + aesmc $dat3,$dat3 + aese $dat4,$rndpang + aesmc $dat4,$dat4 + aese $dat5,$rndpang + aesmc $dat5,$dat5 + aese $dat6,$rndpang + aesmc $dat6,$dat6 + aese $dat7,$rndpang + aesmc $dat7,$dat7 + aese $dat8,$rndpang + aesmc $dat8,$dat8 + aese $dat9,$rndpang + aesmc $dat9,$dat9 + aese $dat10,$rndpang + aesmc $dat10,$dat10 + aese $dat11,$rndpang + aesmc $dat11,$dat11 + vld1.32 {$rndpang},[$key_],#16 + b.gt .Loop12x_ctr32_unroll + + aese $dat0,$rndping + aesmc $dat0,$dat0 + aese $dat1,$rndping + aesmc $dat1,$dat1 + aese $dat2,$rndping + aesmc $dat2,$dat2 + aese $dat3,$rndping + aesmc $dat3,$dat3 + aese $dat4,$rndping + aesmc $dat4,$dat4 + aese $dat5,$rndping + aesmc $dat5,$dat5 + aese $dat6,$rndping + aesmc $dat6,$dat6 + aese $dat7,$rndping + aesmc $dat7,$dat7 + aese $dat8,$rndping + aesmc $dat8,$dat8 + aese $dat9,$rndping + aesmc $dat9,$dat9 + aese $dat10,$rndping + aesmc $dat10,$dat10 + aese $dat11,$rndping + aesmc $dat11,$dat11 + vld1.32 {$rndping},[$key_],#16 + + aese $dat0,$rndpang + aesmc $dat0,$dat0 + aese $dat1,$rndpang + aesmc $dat1,$dat1 + aese $dat2,$rndpang + aesmc $dat2,$dat2 + aese $dat3,$rndpang + aesmc $dat3,$dat3 + aese $dat4,$rndpang + aesmc $dat4,$dat4 + aese $dat5,$rndpang + aesmc $dat5,$dat5 + aese $dat6,$rndpang + aesmc $dat6,$dat6 + aese $dat7,$rndpang + aesmc $dat7,$dat7 + aese $dat8,$rndpang + aesmc $dat8,$dat8 + aese $dat9,$rndpang + aesmc $dat9,$dat9 + aese $dat10,$rndpang + aesmc $dat10,$dat10 + aese $dat11,$rndpang + aesmc $dat11,$dat11 + vld1.32 {$rndpang},[$key_],#16 + + aese $dat0,$rndping + aesmc $dat0,$dat0 + add $tctr0,$ctr,#1 + add $tctr1,$ctr,#2 + aese $dat1,$rndping + aesmc $dat1,$dat1 + add $tctr2,$ctr,#3 + add $tctr3,$ctr,#4 + aese $dat2,$rndping + aesmc $dat2,$dat2 + add $tctr4,$ctr,#5 + add $tctr5,$ctr,#6 + rev $tctr0,$tctr0 + aese $dat3,$rndping + aesmc $dat3,$dat3 + add $tctr6,$ctr,#7 + add $tctr7,$ctr,#8 + rev $tctr1,$tctr1 + rev $tctr2,$tctr2 + aese $dat4,$rndping + aesmc $dat4,$dat4 + add $tctr8,$ctr,#9 + add $tctr9,$ctr,#10 + rev $tctr3,$tctr3 + rev $tctr4,$tctr4 + aese $dat5,$rndping + aesmc $dat5,$dat5 + add $tctr10,$ctr,#11 + add $tctr11,$ctr,#12 + rev $tctr5,$tctr5 + rev $tctr6,$tctr6 + aese $dat6,$rndping + aesmc $dat6,$dat6 + rev $tctr7,$tctr7 + rev $tctr8,$tctr8 + aese $dat7,$rndping + aesmc $dat7,$dat7 + rev $tctr9,$tctr9 + rev $tctr10,$tctr10 + aese $dat8,$rndping + aesmc $dat8,$dat8 + rev $tctr11,$tctr11 + aese $dat9,$rndping + aesmc $dat9,$dat9 + aese $dat10,$rndping + aesmc $dat10,$dat10 + aese $dat11,$rndping + aesmc $dat11,$dat11 + vld1.32 {$rndping},[$key_],#16 + + aese $dat0,$rndpang + aesmc $dat0,$dat0 + aese $dat1,$rndpang + aesmc $dat1,$dat1 + aese $dat2,$rndpang + aesmc $dat2,$dat2 + aese $dat3,$rndpang + aesmc $dat3,$dat3 + vld1.8 {$in0,$in1,$in2,$in3},[$inp],#64 + aese $dat4,$rndpang + aesmc $dat4,$dat4 + aese $dat5,$rndpang + aesmc $dat5,$dat5 + aese $dat6,$rndpang + aesmc $dat6,$dat6 + aese $dat7,$rndpang + aesmc $dat7,$dat7 + vld1.8 {$in4,$in5,$in6,$in7},[$inp],#64 + aese $dat8,$rndpang + aesmc $dat8,$dat8 + aese $dat9,$rndpang + aesmc $dat9,$dat9 + aese $dat10,$rndpang + aesmc $dat10,$dat10 + aese $dat11,$rndpang + aesmc $dat11,$dat11 + vld1.8 {$in8,$in9,$in10,$in11},[$inp],#64 + vld1.32 {$rndpang},[$key_],#16 + + mov $key_, $key + aese $dat0,$rndping + aesmc $dat0,$dat0 + aese $dat1,$rndping + aesmc $dat1,$dat1 + aese $dat2,$rndping + aesmc $dat2,$dat2 + aese $dat3,$rndping + aesmc $dat3,$dat3 + aese $dat4,$rndping + aesmc $dat4,$dat4 + aese $dat5,$rndping + aesmc $dat5,$dat5 + aese $dat6,$rndping + aesmc $dat6,$dat6 + aese $dat7,$rndping + aesmc $dat7,$dat7 + aese $dat8,$rndping + aesmc $dat8,$dat8 + aese $dat9,$rndping + aesmc $dat9,$dat9 + aese $dat10,$rndping + aesmc $dat10,$dat10 + aese $dat11,$rndping + aesmc $dat11,$dat11 + vld1.32 {$rndping},[$key_],#16 // re-pre-load rndkey[0] + + aese $dat0,$rndpang + eor3 $in0,$in0,$rndlast,$dat0 + vorr $dat0,$ivec,$ivec + aese $dat1,$rndpang + eor3 $in1,$in1,$rndlast,$dat1 + vorr $dat1,$ivec,$ivec + aese $dat2,$rndpang + eor3 $in2,$in2,$rndlast,$dat2 + vorr $dat2,$ivec,$ivec + aese $dat3,$rndpang + eor3 $in3,$in3,$rndlast,$dat3 + vorr $dat3,$ivec,$ivec + aese $dat4,$rndpang + eor3 $in4,$in4,$rndlast,$dat4 + vorr $dat4,$ivec,$ivec + aese $dat5,$rndpang + eor3 $in5,$in5,$rndlast,$dat5 + vorr $dat5,$ivec,$ivec + aese $dat6,$rndpang + eor3 $in6,$in6,$rndlast,$dat6 + vorr $dat6,$ivec,$ivec + aese $dat7,$rndpang + eor3 $in7,$in7,$rndlast,$dat7 + vorr $dat7,$ivec,$ivec + aese $dat8,$rndpang + eor3 $in8,$in8,$rndlast,$dat8 + vorr $dat8,$ivec,$ivec + aese $dat9,$rndpang + eor3 $in9,$in9,$rndlast,$dat9 + vorr $dat9,$ivec,$ivec + aese $dat10,$rndpang + eor3 $in10,$in10,$rndlast,$dat10 + vorr $dat10,$ivec,$ivec + aese $dat11,$rndpang + eor3 $in11,$in11,$rndlast,$dat11 + vorr $dat11,$ivec,$ivec + vld1.32 {$rndpang},[$key_],#16 // re-pre-load rndkey[1] + + vmov.32 ${dat0}[3],$tctr0 + vmov.32 ${dat1}[3],$tctr1 + vmov.32 ${dat2}[3],$tctr2 + vmov.32 ${dat3}[3],$tctr3 + vst1.8 {$in0,$in1,$in2,$in3},[$out],#64 + vmov.32 ${dat4}[3],$tctr4 + vmov.32 ${dat5}[3],$tctr5 + vmov.32 ${dat6}[3],$tctr6 + vmov.32 ${dat7}[3],$tctr7 + vst1.8 {$in4,$in5,$in6,$in7},[$out],#64 + vmov.32 ${dat8}[3],$tctr8 + vmov.32 ${dat9}[3],$tctr9 + vmov.32 ${dat10}[3],$tctr10 + vmov.32 ${dat11}[3],$tctr11 + vst1.8 {$in8,$in9,$in10,$in11},[$out],#64 + + mov $cnt,$rounds + + add $ctr,$ctr,#12 + subs $len,$len,#12 + b.hs .Loop12x_ctr32_unroll + + // pop regs from stack when 12 data chunks are interleaved + ldp $dat10d,$dat11d,[sp],#32 + ldp $dat8d,$dat9d,[sp],#32 + ldp x23,x24,[sp],#16 + ldp x21,x22,[sp],#16 + ldp x19,x20,[sp],#16 + + add $len,$len,#12 + cbz $len,.Lctr32_done_unroll + sub $ctr,$ctr,#12 + + cmp $len,#2 + b.ls .Lctr32_tail_unroll + + cmp $len,#6 + sub $len,$len,#3 // bias + add $ctr,$ctr,#3 + b.lo .Loop3x_ctr32_unroll + + sub $len,$len,#3 + add $ctr,$ctr,#3 + b.lo .Loop6x_ctr32_unroll + +.align 4 +.Loop6x_ctr32_unroll: + aese $dat0,$rndping + aesmc $dat0,$dat0 + aese $dat1,$rndping + aesmc $dat1,$dat1 + aese $dat2,$rndping + aesmc $dat2,$dat2 + aese $dat3,$rndping + aesmc $dat3,$dat3 + aese $dat4,$rndping + aesmc $dat4,$dat4 + aese $dat5,$rndping + aesmc $dat5,$dat5 + vld1.32 {$rndping},[$key_],#16 + subs $cnt,$cnt,#2 + aese $dat0,$rndpang + aesmc $dat0,$dat0 + aese $dat1,$rndpang + aesmc $dat1,$dat1 + aese $dat2,$rndpang + aesmc $dat2,$dat2 + aese $dat3,$rndpang + aesmc $dat3,$dat3 + aese $dat4,$rndpang + aesmc $dat4,$dat4 + aese $dat5,$rndpang + aesmc $dat5,$dat5 + vld1.32 {$rndpang},[$key_],#16 + b.gt .Loop6x_ctr32_unroll + + aese $dat0,$rndping + aesmc $dat0,$dat0 + aese $dat1,$rndping + aesmc $dat1,$dat1 + aese $dat2,$rndping + aesmc $dat2,$dat2 + aese $dat3,$rndping + aesmc $dat3,$dat3 + aese $dat4,$rndping + aesmc $dat4,$dat4 + aese $dat5,$rndping + aesmc $dat5,$dat5 + vld1.32 {$rndping},[$key_],#16 + + aese $dat0,$rndpang + aesmc $dat0,$dat0 + aese $dat1,$rndpang + aesmc $dat1,$dat1 + aese $dat2,$rndpang + aesmc $dat2,$dat2 + aese $dat3,$rndpang + aesmc $dat3,$dat3 + aese $dat4,$rndpang + aesmc $dat4,$dat4 + aese $dat5,$rndpang + aesmc $dat5,$dat5 + vld1.32 {$rndpang},[$key_],#16 + + aese $dat0,$rndping + aesmc $dat0,$dat0 + add $tctr0,$ctr,#1 + add $tctr1,$ctr,#2 + aese $dat1,$rndping + aesmc $dat1,$dat1 + add $tctr2,$ctr,#3 + add $tctr3,$ctr,#4 + aese $dat2,$rndping + aesmc $dat2,$dat2 + add $tctr4,$ctr,#5 + add $tctr5,$ctr,#6 + rev $tctr0,$tctr0 + aese $dat3,$rndping + aesmc $dat3,$dat3 + rev $tctr1,$tctr1 + rev $tctr2,$tctr2 + aese $dat4,$rndping + aesmc $dat4,$dat4 + rev $tctr3,$tctr3 + rev $tctr4,$tctr4 + aese $dat5,$rndping + aesmc $dat5,$dat5 + rev $tctr5,$tctr5 + vld1.32 {$rndping},[$key_],#16 + + aese $dat0,$rndpang + aesmc $dat0,$dat0 + aese $dat1,$rndpang + aesmc $dat1,$dat1 + vld1.8 {$in0,$in1,$in2,$in3},[$inp],#64 + aese $dat2,$rndpang + aesmc $dat2,$dat2 + aese $dat3,$rndpang + aesmc $dat3,$dat3 + vld1.8 {$in4,$in5},[$inp],#32 + aese $dat4,$rndpang + aesmc $dat4,$dat4 + aese $dat5,$rndpang + aesmc $dat5,$dat5 + vld1.32 {$rndpang},[$key_],#16 + + mov $key_, $key + aese $dat0,$rndping + aesmc $dat0,$dat0 + aese $dat1,$rndping + aesmc $dat1,$dat1 + aese $dat2,$rndping + aesmc $dat2,$dat2 + aese $dat3,$rndping + aesmc $dat3,$dat3 + aese $dat4,$rndping + aesmc $dat4,$dat4 + aese $dat5,$rndping + aesmc $dat5,$dat5 + vld1.32 {$rndping},[$key_],#16 // re-pre-load rndkey[0] + + aese $dat0,$rndpang + eor3 $in0,$in0,$rndlast,$dat0 + aese $dat1,$rndpang + eor3 $in1,$in1,$rndlast,$dat1 + aese $dat2,$rndpang + eor3 $in2,$in2,$rndlast,$dat2 + aese $dat3,$rndpang + eor3 $in3,$in3,$rndlast,$dat3 + aese $dat4,$rndpang + eor3 $in4,$in4,$rndlast,$dat4 + aese $dat5,$rndpang + eor3 $in5,$in5,$rndlast,$dat5 + vld1.32 {$rndpang},[$key_],#16 // re-pre-load rndkey[1] + + vorr $dat0,$ivec,$ivec + vorr $dat1,$ivec,$ivec + vorr $dat2,$ivec,$ivec + vorr $dat3,$ivec,$ivec + vorr $dat4,$ivec,$ivec + vorr $dat5,$ivec,$ivec + + vmov.32 ${dat0}[3],$tctr0 + vmov.32 ${dat1}[3],$tctr1 + vst1.8 {$in0,$in1,$in2,$in3},[$out],#64 + vmov.32 ${dat2}[3],$tctr2 + vmov.32 ${dat3}[3],$tctr3 + vst1.8 {$in4,$in5},[$out],#32 + vmov.32 ${dat4}[3],$tctr4 + vmov.32 ${dat5}[3],$tctr5 + + cbz $len,.Lctr32_done_unroll + mov $cnt,$rounds + + cmp $len,#2 + b.ls .Lctr32_tail_unroll + + sub $len,$len,#3 // bias + add $ctr,$ctr,#3 + b .Loop3x_ctr32_unroll + +.align 4 +.Loop3x_ctr32_unroll: + aese $dat0,$rndping + aesmc $dat0,$dat0 + aese $dat1,$rndping + aesmc $dat1,$dat1 + aese $dat2,$rndping + aesmc $dat2,$dat2 + vld1.32 {$rndping},[$key_],#16 + subs $cnt,$cnt,#2 + aese $dat0,$rndpang + aesmc $dat0,$dat0 + aese $dat1,$rndpang + aesmc $dat1,$dat1 + aese $dat2,$rndpang + aesmc $dat2,$dat2 + vld1.32 {$rndpang},[$key_],#16 + b.gt .Loop3x_ctr32_unroll + + aese $dat0,$rndping + aesmc $tmp0,$dat0 + aese $dat1,$rndping + aesmc $tmp1,$dat1 + vld1.8 {$in0,$in1,$in2},[$inp],#48 + vorr $dat0,$ivec,$ivec + aese $dat2,$rndping + aesmc $dat2,$dat2 + vld1.32 {$rndping},[$key_],#16 + vorr $dat1,$ivec,$ivec + aese $tmp0,$rndpang + aesmc $tmp0,$tmp0 + aese $tmp1,$rndpang + aesmc $tmp1,$tmp1 + aese $dat2,$rndpang + aesmc $tmp2,$dat2 + vld1.32 {$rndpang},[$key_],#16 + vorr $dat2,$ivec,$ivec + add $tctr0,$ctr,#1 + aese $tmp0,$rndping + aesmc $tmp0,$tmp0 + aese $tmp1,$rndping + aesmc $tmp1,$tmp1 + add $tctr1,$ctr,#2 + aese $tmp2,$rndping + aesmc $tmp2,$tmp2 + vld1.32 {$rndping},[$key_],#16 + add $ctr,$ctr,#3 + aese $tmp0,$rndpang + aesmc $tmp0,$tmp0 + aese $tmp1,$rndpang + aesmc $tmp1,$tmp1 + + rev $tctr0,$tctr0 + aese $tmp2,$rndpang + aesmc $tmp2,$tmp2 + vld1.32 {$rndpang},[$key_],#16 + vmov.32 ${dat0}[3], $tctr0 + mov $key_,$key + rev $tctr1,$tctr1 + aese $tmp0,$rndping + aesmc $tmp0,$tmp0 + + aese $tmp1,$rndping + aesmc $tmp1,$tmp1 + vmov.32 ${dat1}[3], $tctr1 + rev $tctr2,$ctr + aese $tmp2,$rndping + aesmc $tmp2,$tmp2 + vmov.32 ${dat2}[3], $tctr2 + + aese $tmp0,$rndpang + aese $tmp1,$rndpang + aese $tmp2,$rndpang + + eor3 $in0,$in0,$rndlast,$tmp0 + vld1.32 {$rndping},[$key_],#16 // re-pre-load rndkey[0] + eor3 $in1,$in1,$rndlast,$tmp1 + mov $cnt,$rounds + eor3 $in2,$in2,$rndlast,$tmp2 + vld1.32 {$rndpang},[$key_],#16 // re-pre-load rndkey[1] + vst1.8 {$in0,$in1,$in2},[$out],#48 + + cbz $len,.Lctr32_done_unroll + +.Lctr32_tail_unroll: + cmp $len,#1 + b.eq .Lctr32_tail_1_unroll + +.Lctr32_tail_2_unroll: + aese $dat0,$rndping + aesmc $dat0,$dat0 + aese $dat1,$rndping + aesmc $dat1,$dat1 + vld1.32 {$rndping},[$key_],#16 + subs $cnt,$cnt,#2 + aese $dat0,$rndpang + aesmc $dat0,$dat0 + aese $dat1,$rndpang + aesmc $dat1,$dat1 + vld1.32 {$rndpang},[$key_],#16 + b.gt .Lctr32_tail_2_unroll + + aese $dat0,$rndping + aesmc $dat0,$dat0 + aese $dat1,$rndping + aesmc $dat1,$dat1 + vld1.32 {$rndping},[$key_],#16 + aese $dat0,$rndpang + aesmc $dat0,$dat0 + aese $dat1,$rndpang + aesmc $dat1,$dat1 + vld1.32 {$rndpang},[$key_],#16 + vld1.8 {$in0,$in1},[$inp],#32 + aese $dat0,$rndping + aesmc $dat0,$dat0 + aese $dat1,$rndping + aesmc $dat1,$dat1 + vld1.32 {$rndping},[$key_],#16 + aese $dat0,$rndpang + aesmc $dat0,$dat0 + aese $dat1,$rndpang + aesmc $dat1,$dat1 + vld1.32 {$rndpang},[$key_],#16 + aese $dat0,$rndping + aesmc $dat0,$dat0 + aese $dat1,$rndping + aesmc $dat1,$dat1 + aese $dat0,$rndpang + aese $dat1,$rndpang + + eor3 $in0,$in0,$rndlast,$dat0 + eor3 $in1,$in1,$rndlast,$dat1 + vst1.8 {$in0,$in1},[$out],#32 + b .Lctr32_done_unroll + +.Lctr32_tail_1_unroll: + aese $dat0,$rndping + aesmc $dat0,$dat0 + vld1.32 {$rndping},[$key_],#16 + subs $cnt,$cnt,#2 + aese $dat0,$rndpang + aesmc $dat0,$dat0 + vld1.32 {$rndpang},[$key_],#16 + b.gt .Lctr32_tail_1_unroll + + aese $dat0,$rndping + aesmc $dat0,$dat0 + vld1.32 {$rndping},[$key_],#16 + aese $dat0,$rndpang + aesmc $dat0,$dat0 + vld1.32 {$rndpang},[$key_],#16 + vld1.8 {$in0},[$inp] + aese $dat0,$rndping + aesmc $dat0,$dat0 + vld1.32 {$rndping},[$key_],#16 + aese $dat0,$rndpang + aesmc $dat0,$dat0 + vld1.32 {$rndpang},[$key_],#16 + aese $dat0,$rndping + aesmc $dat0,$dat0 + aese $dat0,$rndpang + + eor3 $in0,$in0,$rndlast,$dat0 + vst1.8 {$in0},[$out],#16 + +.Lctr32_done_unroll: + ldp d8,d9,[sp, #16] + ldp d10,d11,[sp, #32] + ldp d12,d13,[sp, #48] + ldp d14,d15,[sp, #64] + ldr x29,[sp],#80 + ret +.size ${prefix}_ctr32_encrypt_blocks_unroll12_eor3,.-${prefix}_ctr32_encrypt_blocks_unroll12_eor3 +___ +}}} + {{{ my ($inp,$out,$len,$key,$ivp)=("PTR(0)","PTR(1)","x2","PTR(3)","PTR(4)"); my ($rounds,$cnt,$key_)=("w5","w6","PTR(7)"); @@ -2229,7 +2994,7 @@ ___ # will be processed specially, which be integrated into the 5*16 bytes # loop to improve the efficiency. # There is one special case, if the original input data size dsize -# = 16 bytes, we will treat it seperately to improve the +# = 16 bytes, we will treat it separately to improve the # performance: one independent code block without LR, FP load and # store. # Encryption will process the (length -tailcnt) bytes as mentioned @@ -3568,7 +4333,7 @@ $code.=<<___ if ($flavour =~ /64/); cbnz x2,.Lxts_dec_1st_done vld1.8 {$dat0},[$inp],#16 - // Decrypt the last secod block to get the last plain text block + // Decrypt the last second block to get the last plain text block .Lxts_dec_1st_done: eor $tmpin,$dat0,$iv1 ldr $rounds,[$key1,#240] @@ -3651,7 +4416,8 @@ ___ if ($flavour =~ /64/) { ######## 64-bit code my %opcode = ( "aesd" => 0x4e285800, "aese" => 0x4e284800, - "aesimc"=> 0x4e287800, "aesmc" => 0x4e286800 ); + "aesimc"=> 0x4e287800, "aesmc" => 0x4e286800, + "eor3" => 0xce000000, ); local *unaes = sub { my ($mnemonic,$arg)=@_; @@ -3662,10 +4428,21 @@ if ($flavour =~ /64/) { ######## 64-bit code $mnemonic,$arg; }; + sub unsha3 { + my ($mnemonic,$arg)=@_; + + $arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv#]([0-9\-]+))?)?/ + && + sprintf ".inst\t0x%08x\t//%s %s", + $opcode{$mnemonic}|$1|($2<<5)|($3<<16)|(eval($4)<<10), + $mnemonic,$arg; + } + foreach(split("\n",$code)) { s/\`([^\`]*)\`/eval($1)/geo; - s/\bq([0-9]+)\b/"v".($1<8?$1:$1+8).".16b"/geo; # old->new registers + s/\bq([0-9]+)\b/"v".($1<8?$1:($1<24?$1+8:$1-16)).".16b"/geo; # old->new registers + s/\bq_([0-9]+)\b/"q".$1/geo; # old->new registers s/@\s/\/\//o; # old->new style commentary #s/[v]?(aes\w+)\s+([qv].*)/unaes($1,$2)/geo or @@ -3678,6 +4455,7 @@ if ($flavour =~ /64/) { ######## 64-bit code s/vshr/ushr/o or s/^(\s+)v/$1/o or # strip off v prefix s/\bbx\s+lr\b/ret/o; + s/\b(eor3)\s+(v.*)/unsha3($1,$2)/ge; # fix up remaining legacy suffixes s/\.[ui]?8//o; diff --git a/crypto/openssl/crypto/aes/asm/bsaes-x86_64.pl b/crypto/openssl/crypto/aes/asm/bsaes-x86_64.pl index 6498cfe908c8..7230f760856a 100644 --- a/crypto/openssl/crypto/aes/asm/bsaes-x86_64.pl +++ b/crypto/openssl/crypto/aes/asm/bsaes-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2011-2021 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2011-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -2182,6 +2182,7 @@ $code.=<<___; .align 16 ossl_bsaes_xts_encrypt: .cfi_startproc + endbranch mov %rsp, %rax .Lxts_enc_prologue: push %rbp @@ -2581,6 +2582,7 @@ $code.=<<___; .align 16 ossl_bsaes_xts_decrypt: .cfi_startproc + endbranch mov %rsp, %rax .Lxts_dec_prologue: push %rbp @@ -3003,6 +3005,7 @@ ___ } $code.=<<___; .type _bsaes_const,\@object +.section .rodata align=64 .align 64 _bsaes_const: .LM0ISR: # InvShiftRows constants @@ -3054,9 +3057,9 @@ _bsaes_const: .quad 0x02060a0e03070b0f, 0x0004080c0105090d .L63: .quad 0x6363636363636363, 0x6363636363636363 -.asciz "Bit-sliced AES for x86_64/SSSE3, Emilia Käsper, Peter Schwabe, Andy Polyakov" .align 64 .size _bsaes_const,.-_bsaes_const +.asciz "Bit-sliced AES for x86_64/SSSE3, Emilia Käsper, Peter Schwabe, Andy Polyakov" ___ # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, diff --git a/crypto/openssl/crypto/aes/asm/vpaes-armv8.pl b/crypto/openssl/crypto/aes/asm/vpaes-armv8.pl index e00f824c465f..189bc01ca464 100755 --- a/crypto/openssl/crypto/aes/asm/vpaes-armv8.pl +++ b/crypto/openssl/crypto/aes/asm/vpaes-armv8.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2015-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2015-2025 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -55,7 +55,7 @@ open OUT,"| \"$^X\" $xlate $flavour \"$output\"" $code.=<<___; #include "arm_arch.h" -.text +.rodata .type _vpaes_consts,%object .align 7 // totally strategic alignment @@ -146,6 +146,9 @@ _vpaes_consts: .asciz "Vector Permutation AES for ARMv8, Mike Hamburg (Stanford University)" .size _vpaes_consts,.-_vpaes_consts .align 6 + +.text + ___ { @@ -165,7 +168,8 @@ $code.=<<___; .type _vpaes_encrypt_preheat,%function .align 4 _vpaes_encrypt_preheat: - adr PTR(10), .Lk_inv + adrp PTR(10), .Lk_inv + add PTR(10), PTR(10), :lo12:.Lk_inv movi v17.16b, #0x0f ld1 {v18.2d-v19.2d}, [PTR(10)],#32 // .Lk_inv ld1 {v20.2d-v23.2d}, [PTR(10)],#64 // .Lk_ipt, .Lk_sbo @@ -193,7 +197,8 @@ _vpaes_encrypt_preheat: _vpaes_encrypt_core: mov PTR(9), $key ldr w8, [$key,#240] // pull rounds - adr PTR(11), .Lk_mc_forward+16 + adrp PTR(11), .Lk_mc_forward+16 + add PTR(11), PTR(11), :lo12:.Lk_mc_forward+16 // vmovdqa .Lk_ipt(%rip), %xmm2 # iptlo ld1 {v16.2d}, [PTR(9)], #16 // vmovdqu (%r9), %xmm5 # round0 key and v1.16b, v7.16b, v17.16b // vpand %xmm9, %xmm0, %xmm1 @@ -285,7 +290,8 @@ vpaes_encrypt: _vpaes_encrypt_2x: mov PTR(9), $key ldr w8, [$key,#240] // pull rounds - adr PTR(11), .Lk_mc_forward+16 + adrp PTR(11), .Lk_mc_forward+16 + add PTR(11), PTR(11), :lo12:.Lk_mc_forward+16 // vmovdqa .Lk_ipt(%rip), %xmm2 # iptlo ld1 {v16.2d}, [PTR(9)], #16 // vmovdqu (%r9), %xmm5 # round0 key and v1.16b, v14.16b, v17.16b // vpand %xmm9, %xmm0, %xmm1 @@ -393,9 +399,11 @@ _vpaes_encrypt_2x: .type _vpaes_decrypt_preheat,%function .align 4 _vpaes_decrypt_preheat: - adr PTR(10), .Lk_inv + adrp PTR(10), .Lk_inv + add PTR(10), PTR(10), :lo12:.Lk_inv movi v17.16b, #0x0f - adr PTR(11), .Lk_dipt + adrp PTR(11), .Lk_dipt + add PTR(11), PTR(11), :lo12:.Lk_dipt ld1 {v18.2d-v19.2d}, [PTR(10)],#32 // .Lk_inv ld1 {v20.2d-v23.2d}, [PTR(11)],#64 // .Lk_dipt, .Lk_dsbo ld1 {v24.2d-v27.2d}, [PTR(11)],#64 // .Lk_dsb9, .Lk_dsbd @@ -417,10 +425,12 @@ _vpaes_decrypt_core: // vmovdqa .Lk_dipt(%rip), %xmm2 # iptlo lsl x11, x8, #4 // mov %rax, %r11; shl \$4, %r11 eor x11, x11, #0x30 // xor \$0x30, %r11 - adr PTR(10), .Lk_sr + adrp PTR(10), .Lk_sr + add PTR(10), PTR(10), :lo12:.Lk_sr and x11, x11, #0x30 // and \$0x30, %r11 add PTR(11), PTR(10), x11 - adr PTR(10), .Lk_mc_forward+48 + adrp PTR(10), .Lk_mc_forward+48 + add PTR(10), PTR(10), :lo12:.Lk_mc_forward+48 ld1 {v16.2d}, [PTR(9)],#16 // vmovdqu (%r9), %xmm4 # round0 key and v1.16b, v7.16b, v17.16b // vpand %xmm9, %xmm0, %xmm1 @@ -528,10 +538,12 @@ _vpaes_decrypt_2x: // vmovdqa .Lk_dipt(%rip), %xmm2 # iptlo lsl x11, x8, #4 // mov %rax, %r11; shl \$4, %r11 eor x11, x11, #0x30 // xor \$0x30, %r11 - adr PTR(10), .Lk_sr + adrp PTR(10), .Lk_sr + add PTR(10), PTR(10), :lo12:.Lk_sr and x11, x11, #0x30 // and \$0x30, %r11 add PTR(11), PTR(10), x11 - adr PTR(10), .Lk_mc_forward+48 + adrp PTR(10), .Lk_mc_forward+48 + add PTR(10), PTR(10), :lo12:.Lk_mc_forward+48 ld1 {v16.2d}, [PTR(9)],#16 // vmovdqu (%r9), %xmm4 # round0 key and v1.16b, v14.16b, v17.16b // vpand %xmm9, %xmm0, %xmm1 @@ -668,14 +680,18 @@ $code.=<<___; .type _vpaes_key_preheat,%function .align 4 _vpaes_key_preheat: - adr PTR(10), .Lk_inv + adrp PTR(10), .Lk_inv + add PTR(10), PTR(10), :lo12:.Lk_inv movi v16.16b, #0x5b // .Lk_s63 - adr PTR(11), .Lk_sb1 + adrp PTR(11), .Lk_sb1 + add PTR(11), PTR(11), :lo12:.Lk_sb1 movi v17.16b, #0x0f // .Lk_s0F ld1 {v18.2d-v21.2d}, [PTR(10)] // .Lk_inv, .Lk_ipt - adr PTR(10), .Lk_dksd + adrp PTR(10), .Lk_dksd + add PTR(10), PTR(10), :lo12:.Lk_dksd ld1 {v22.2d-v23.2d}, [PTR(11)] // .Lk_sb1 - adr PTR(11), .Lk_mc_forward + adrp PTR(11), .Lk_mc_forward + add PTR(11), PTR(11), :lo12:.Lk_mc_forward ld1 {v24.2d-v27.2d}, [PTR(10)],#64 // .Lk_dksd, .Lk_dksb ld1 {v28.2d-v31.2d}, [PTR(10)],#64 // .Lk_dkse, .Lk_dks9 ld1 {v8.2d}, [PTR(10)] // .Lk_rcon @@ -699,7 +715,8 @@ _vpaes_schedule_core: bl _vpaes_schedule_transform mov v7.16b, v0.16b // vmovdqa %xmm0, %xmm7 - adr PTR(10), .Lk_sr // lea .Lk_sr(%rip),%r10 + adrp PTR(10), .Lk_sr // lea .Lk_sr(%rip),%r10 + add PTR(10), PTR(10), :lo12:.Lk_sr add PTR(8), PTR(10), x8 cbnz $dir, .Lschedule_am_decrypting @@ -830,12 +847,14 @@ _vpaes_schedule_core: .align 4 .Lschedule_mangle_last: // schedule last round key from xmm0 - adr PTR(11), .Lk_deskew // lea .Lk_deskew(%rip),%r11 # prepare to deskew + adrp PTR(11), .Lk_deskew // lea .Lk_deskew(%rip),%r11 # prepare to deskew + add PTR(11), PTR(11), :lo12:.Lk_deskew cbnz $dir, .Lschedule_mangle_last_dec // encrypting ld1 {v1.2d}, [PTR(8)] // vmovdqa (%r8,%r10),%xmm1 - adr PTR(11), .Lk_opt // lea .Lk_opt(%rip), %r11 # prepare to output transform + adrp PTR(11), .Lk_opt // lea .Lk_opt(%rip), %r11 # prepare to output transform + add PTR(11), PTR(11), :lo12:.Lk_opt add $out, $out, #32 // add \$32, %rdx tbl v0.16b, {v0.16b}, v1.16b // vpshufb %xmm1, %xmm0, %xmm0 # output permute diff --git a/crypto/openssl/crypto/aes/asm/vpaes-x86_64.pl b/crypto/openssl/crypto/aes/asm/vpaes-x86_64.pl index 845528f41ac2..f62818841526 100755 --- a/crypto/openssl/crypto/aes/asm/vpaes-x86_64.pl +++ b/crypto/openssl/crypto/aes/asm/vpaes-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2011-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2011-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -1006,6 +1006,7 @@ _vpaes_preheat: ## ## ######################################################## .type _vpaes_consts,\@object +.section .rodata align=64 .align 64 _vpaes_consts: .Lk_inv: # inv, inva @@ -1101,9 +1102,9 @@ _vpaes_consts: .Lk_dsbo: # decryption sbox final output .quad 0x1387EA537EF94000, 0xC7AA6DB9D4943E2D .quad 0x12D7560F93441D00, 0xCA4B8159D8C58E9C -.asciz "Vector Permutation AES for x86_64/SSSE3, Mike Hamburg (Stanford University)" .align 64 .size _vpaes_consts,.-_vpaes_consts +.asciz "Vector Permutation AES for x86_64/SSSE3, Mike Hamburg (Stanford University)" ___ if ($win64) { diff --git a/crypto/openssl/crypto/bn/asm/armv4-gf2m.pl b/crypto/openssl/crypto/bn/asm/armv4-gf2m.pl index d380c89f1f2b..5733a0174660 100755 --- a/crypto/openssl/crypto/bn/asm/armv4-gf2m.pl +++ b/crypto/openssl/crypto/bn/asm/armv4-gf2m.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2011-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2011-2023 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -325,7 +325,8 @@ $code.=<<___; .align 5 #if __ARM_MAX_ARCH__>=7 -.comm OPENSSL_armcap_P,4,4 +.extern OPENSSL_armcap_P +.hidden OPENSSL_armcap_P #endif ___ diff --git a/crypto/openssl/crypto/bn/asm/armv4-mont.pl b/crypto/openssl/crypto/bn/asm/armv4-mont.pl index c26df751a5eb..6e839d845ca8 100755 --- a/crypto/openssl/crypto/bn/asm/armv4-mont.pl +++ b/crypto/openssl/crypto/bn/asm/armv4-mont.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2007-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2007-2023 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -749,7 +749,8 @@ $code.=<<___; .asciz "Montgomery multiplication for ARMv4/NEON, CRYPTOGAMS by " .align 2 #if __ARM_MAX_ARCH__>=7 -.comm OPENSSL_armcap_P,4,4 +.extern OPENSSL_armcap_P +.hidden OPENSSL_armcap_P #endif ___ diff --git a/crypto/openssl/crypto/bn/asm/armv8-mont.pl b/crypto/openssl/crypto/bn/asm/armv8-mont.pl index 9fe1c46805f5..6acba75c9136 100755 --- a/crypto/openssl/crypto/bn/asm/armv8-mont.pl +++ b/crypto/openssl/crypto/bn/asm/armv8-mont.pl @@ -2051,6 +2051,7 @@ __bn_mul4x_mont: ___ } $code.=<<___; +.rodata .asciz "Montgomery Multiplication for ARMv8, CRYPTOGAMS by " .align 4 ___ diff --git a/crypto/openssl/crypto/bn/asm/mips.pl b/crypto/openssl/crypto/bn/asm/mips.pl index bc18826d08f8..44a6f10d4dbd 100755 --- a/crypto/openssl/crypto/bn/asm/mips.pl +++ b/crypto/openssl/crypto/bn/asm/mips.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2010-2021 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2010-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -802,7 +802,7 @@ $code.=<<___; #if 0 /* - * The bn_div_3_words entry point is re-used for constant-time interface. + * The bn_div_3_words entry point is reused for constant-time interface. * Implementation is retained as historical reference. */ .align 5 diff --git a/crypto/openssl/crypto/bn/asm/ppc64-mont-fixed.pl b/crypto/openssl/crypto/bn/asm/ppc64-mont-fixed.pl index e69de29bb2d1..e27d0ad93d85 100755 --- a/crypto/openssl/crypto/bn/asm/ppc64-mont-fixed.pl +++ b/crypto/openssl/crypto/bn/asm/ppc64-mont-fixed.pl @@ -0,0 +1,583 @@ +#! /usr/bin/env perl +# Copyright 2021-2022 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +# ==================================================================== +# Written by Amitay Isaacs , Martin Schwenke +# & Alastair D'Silva for +# the OpenSSL project. +# ==================================================================== + +# +# Fixed length (n=6), unrolled PPC Montgomery Multiplication +# + +# 2021 +# +# Although this is a generic implementation for unrolling Montgomery +# Multiplication for arbitrary values of n, this is currently only +# used for n = 6 to improve the performance of ECC p384. +# +# Unrolling allows intermediate results to be stored in registers, +# rather than on the stack, improving performance by ~7% compared to +# the existing PPC assembly code. +# +# The ISA 3.0 implementation uses combination multiply/add +# instructions (maddld, maddhdu) to improve performance by an +# additional ~10% on Power 9. +# +# Finally, saving non-volatile registers into volatile vector +# registers instead of onto the stack saves a little more. +# +# On a Power 9 machine we see an overall improvement of ~18%. +# + +use strict; +use warnings; + +my ($flavour, $output, $dir, $xlate); + +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or +die "can't locate ppc-xlate.pl"; + +open STDOUT,"| $^X $xlate $flavour \"$output\"" + or die "can't call $xlate: $!"; + +if ($flavour !~ /64/) { + die "bad flavour ($flavour) - only ppc64 permitted"; +} + +my $SIZE_T= 8; + +# Registers are global so the code is remotely readable + +# Parameters for Montgomery multiplication +my $ze = "r0"; +my $sp = "r1"; +my $toc = "r2"; +my $rp = "r3"; +my $ap = "r4"; +my $bp = "r5"; +my $np = "r6"; +my $n0 = "r7"; +my $num = "r8"; + +my $i = "r9"; +my $c0 = "r10"; +my $bp0 = "r11"; +my $bpi = "r11"; +my $bpj = "r11"; +my $tj = "r12"; +my $apj = "r12"; +my $npj = "r12"; +my $lo = "r14"; +my $c1 = "r14"; + +# Non-volatile registers used for tp[i] +# +# 12 registers are available but the limit on unrolling is 10, +# since registers from $tp[0] to $tp[$n+1] are used. +my @tp = ("r20" .. "r31"); + +# volatile VSRs for saving non-volatile GPRs - faster than stack +my @vsrs = ("v32" .. "v46"); + +package Mont; + +sub new($$) +{ + my ($class, $n) = @_; + + if ($n > 10) { + die "Can't unroll for BN length ${n} (maximum 10)" + } + + my $self = { + code => "", + n => $n, + }; + bless $self, $class; + + return $self; +} + +sub add_code($$) +{ + my ($self, $c) = @_; + + $self->{code} .= $c; +} + +sub get_code($) +{ + my ($self) = @_; + + return $self->{code}; +} + +sub get_function_name($) +{ + my ($self) = @_; + + return "bn_mul_mont_fixed_n" . $self->{n}; +} + +sub get_label($$) +{ + my ($self, $l) = @_; + + return "L" . $l . "_" . $self->{n}; +} + +sub get_labels($@) +{ + my ($self, @labels) = @_; + + my %out = (); + + foreach my $l (@labels) { + $out{"$l"} = $self->get_label("$l"); + } + + return \%out; +} + +sub nl($) +{ + my ($self) = @_; + + $self->add_code("\n"); +} + +sub copy_result($) +{ + my ($self) = @_; + + my ($n) = $self->{n}; + + for (my $j = 0; $j < $n; $j++) { + $self->add_code(<<___); + std $tp[$j],`$j*$SIZE_T`($rp) +___ + } + +} + +sub mul_mont_fixed($) +{ + my ($self) = @_; + + my ($n) = $self->{n}; + my $fname = $self->get_function_name(); + my $label = $self->get_labels("outer", "enter", "sub", "copy", "end"); + + $self->add_code(<<___); + +.globl .${fname} +.align 5 +.${fname}: + +___ + + $self->save_registers(); + + $self->add_code(<<___); + li $ze,0 + ld $n0,0($n0) + + ld $bp0,0($bp) + + ld $apj,0($ap) +___ + + $self->mul_c_0($tp[0], $apj, $bp0, $c0); + + for (my $j = 1; $j < $n - 1; $j++) { + $self->add_code(<<___); + ld $apj,`$j*$SIZE_T`($ap) +___ + $self->mul($tp[$j], $apj, $bp0, $c0); + } + + $self->add_code(<<___); + ld $apj,`($n-1)*$SIZE_T`($ap) +___ + + $self->mul_last($tp[$n-1], $tp[$n], $apj, $bp0, $c0); + + $self->add_code(<<___); + li $tp[$n+1],0 + +___ + + $self->add_code(<<___); + li $i,0 + mtctr $num + b $label->{"enter"} + +.align 4 +$label->{"outer"}: + ldx $bpi,$bp,$i + + ld $apj,0($ap) +___ + + $self->mul_add_c_0($tp[0], $tp[0], $apj, $bpi, $c0); + + for (my $j = 1; $j < $n; $j++) { + $self->add_code(<<___); + ld $apj,`$j*$SIZE_T`($ap) +___ + $self->mul_add($tp[$j], $tp[$j], $apj, $bpi, $c0); + } + + $self->add_code(<<___); + addc $tp[$n],$tp[$n],$c0 + addze $tp[$n+1],$ze +___ + + $self->add_code(<<___); +.align 4 +$label->{"enter"}: + mulld $bpi,$tp[0],$n0 + + ld $npj,0($np) +___ + + $self->mul_add_c_0($lo, $tp[0], $bpi, $npj, $c0); + + for (my $j = 1; $j < $n; $j++) { + $self->add_code(<<___); + ld $npj,`$j*$SIZE_T`($np) +___ + $self->mul_add($tp[$j-1], $tp[$j], $npj, $bpi, $c0); + } + + $self->add_code(<<___); + addc $tp[$n-1],$tp[$n],$c0 + addze $tp[$n],$tp[$n+1] + + addi $i,$i,$SIZE_T + bdnz $label->{"outer"} + + and. $tp[$n],$tp[$n],$tp[$n] + bne $label->{"sub"} + + cmpld $tp[$n-1],$npj + blt $label->{"copy"} + +$label->{"sub"}: +___ + + # + # Reduction + # + + $self->add_code(<<___); + ld $bpj,`0*$SIZE_T`($np) + subfc $c1,$bpj,$tp[0] + std $c1,`0*$SIZE_T`($rp) + +___ + for (my $j = 1; $j < $n - 1; $j++) { + $self->add_code(<<___); + ld $bpj,`$j*$SIZE_T`($np) + subfe $c1,$bpj,$tp[$j] + std $c1,`$j*$SIZE_T`($rp) + +___ + } + + $self->add_code(<<___); + subfe $c1,$npj,$tp[$n-1] + std $c1,`($n-1)*$SIZE_T`($rp) + +___ + + $self->add_code(<<___); + addme. $tp[$n],$tp[$n] + beq $label->{"end"} + +$label->{"copy"}: +___ + + $self->copy_result(); + + $self->add_code(<<___); + +$label->{"end"}: +___ + + $self->restore_registers(); + + $self->add_code(<<___); + li r3,1 + blr +.size .${fname},.-.${fname} +___ + +} + +package Mont::GPR; + +our @ISA = ('Mont'); + +sub new($$) +{ + my ($class, $n) = @_; + + return $class->SUPER::new($n); +} + +sub save_registers($) +{ + my ($self) = @_; + + my $n = $self->{n}; + + $self->add_code(<<___); + std $lo,-8($sp) +___ + + for (my $j = 0; $j <= $n+1; $j++) { + $self->{code}.=<<___; + std $tp[$j],-`($j+2)*8`($sp) +___ + } + + $self->add_code(<<___); + +___ +} + +sub restore_registers($) +{ + my ($self) = @_; + + my $n = $self->{n}; + + $self->add_code(<<___); + ld $lo,-8($sp) +___ + + for (my $j = 0; $j <= $n+1; $j++) { + $self->{code}.=<<___; + ld $tp[$j],-`($j+2)*8`($sp) +___ + } + + $self->{code} .=<<___; + +___ +} + +# Direct translation of C mul() +sub mul($$$$$) +{ + my ($self, $r, $a, $w, $c) = @_; + + $self->add_code(<<___); + mulld $lo,$a,$w + addc $r,$lo,$c + mulhdu $c,$a,$w + addze $c,$c + +___ +} + +# Like mul() but $c is ignored as an input - an optimisation to save a +# preliminary instruction that would set input $c to 0 +sub mul_c_0($$$$$) +{ + my ($self, $r, $a, $w, $c) = @_; + + $self->add_code(<<___); + mulld $r,$a,$w + mulhdu $c,$a,$w + +___ +} + +# Like mul() but does not to the final addition of CA into $c - an +# optimisation to save an instruction +sub mul_last($$$$$$) +{ + my ($self, $r1, $r2, $a, $w, $c) = @_; + + $self->add_code(<<___); + mulld $lo,$a,$w + addc $r1,$lo,$c + mulhdu $c,$a,$w + + addze $r2,$c +___ +} + +# Like C mul_add() but allow $r_out and $r_in to be different +sub mul_add($$$$$$) +{ + my ($self, $r_out, $r_in, $a, $w, $c) = @_; + + $self->add_code(<<___); + mulld $lo,$a,$w + addc $lo,$lo,$c + mulhdu $c,$a,$w + addze $c,$c + addc $r_out,$r_in,$lo + addze $c,$c + +___ +} + +# Like mul_add() but $c is ignored as an input - an optimisation to save a +# preliminary instruction that would set input $c to 0 +sub mul_add_c_0($$$$$$) +{ + my ($self, $r_out, $r_in, $a, $w, $c) = @_; + + $self->add_code(<<___); + mulld $lo,$a,$w + addc $r_out,$r_in,$lo + mulhdu $c,$a,$w + addze $c,$c + +___ +} + +package Mont::GPR_300; + +our @ISA = ('Mont::GPR'); + +sub new($$) +{ + my ($class, $n) = @_; + + my $mont = $class->SUPER::new($n); + + return $mont; +} + +sub get_function_name($) +{ + my ($self) = @_; + + return "bn_mul_mont_300_fixed_n" . $self->{n}; +} + +sub get_label($$) +{ + my ($self, $l) = @_; + + return "L" . $l . "_300_" . $self->{n}; +} + +# Direct translation of C mul() +sub mul($$$$$) +{ + my ($self, $r, $a, $w, $c, $last) = @_; + + $self->add_code(<<___); + maddld $r,$a,$w,$c + maddhdu $c,$a,$w,$c + +___ +} + +# Save the last carry as the final entry +sub mul_last($$$$$) +{ + my ($self, $r1, $r2, $a, $w, $c) = @_; + + $self->add_code(<<___); + maddld $r1,$a,$w,$c + maddhdu $r2,$a,$w,$c + +___ +} + +# Like mul() but $c is ignored as an input - an optimisation to save a +# preliminary instruction that would set input $c to 0 +sub mul_c_0($$$$$) +{ + my ($self, $r, $a, $w, $c) = @_; + + $self->add_code(<<___); + mulld $r,$a,$w + mulhdu $c,$a,$w + +___ +} + +# Like C mul_add() but allow $r_out and $r_in to be different +sub mul_add($$$$$$) +{ + my ($self, $r_out, $r_in, $a, $w, $c) = @_; + + $self->add_code(<<___); + maddld $lo,$a,$w,$c + maddhdu $c,$a,$w,$c + addc $r_out,$r_in,$lo + addze $c,$c + +___ +} + +# Like mul_add() but $c is ignored as an input - an optimisation to save a +# preliminary instruction that would set input $c to 0 +sub mul_add_c_0($$$$$$) +{ + my ($self, $r_out, $r_in, $a, $w, $c) = @_; + + $self->add_code(<<___); + maddld $lo,$a,$w,$r_in + maddhdu $c,$a,$w,$r_in +___ + + if ($r_out ne $lo) { + $self->add_code(<<___); + mr $r_out,$lo +___ + } + + $self->nl(); +} + + +package main; + +my $code; + +$code.=<<___; +.machine "any" +.text +___ + +my $mont; + +$mont = new Mont::GPR(6); +$mont->mul_mont_fixed(); +$code .= $mont->get_code(); + +$mont = new Mont::GPR_300(6); +$mont->mul_mont_fixed(); +$code .= $mont->get_code(); + +$code =~ s/\`([^\`]*)\`/eval $1/gem; + +$code.=<<___; +.asciz "Montgomery Multiplication for PPC by , " +___ + +print $code; +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/bn/asm/rsaz-avx2.pl b/crypto/openssl/crypto/bn/asm/rsaz-avx2.pl index 3d0e342a6b8c..59b9c89b5be2 100755 --- a/crypto/openssl/crypto/bn/asm/rsaz-avx2.pl +++ b/crypto/openssl/crypto/bn/asm/rsaz-avx2.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2013-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2013-2024 The OpenSSL Project Authors. All Rights Reserved. # Copyright (c) 2012, Intel Corporation. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use @@ -1779,6 +1779,7 @@ $code.=<<___; ret .size rsaz_avx2_eligible,.-rsaz_avx2_eligible +.section .rodata align=64 .align 64 .Land_mask: .quad 0x1fffffff,0x1fffffff,0x1fffffff,0x1fffffff @@ -1790,6 +1791,7 @@ $code.=<<___; .long 0,0,0,0, 1,1,1,1 .long 2,2,2,2, 3,3,3,3 .long 4,4,4,4, 4,4,4,4 +.previous .align 64 ___ diff --git a/crypto/openssl/crypto/bn/asm/rsaz-x86_64.pl b/crypto/openssl/crypto/bn/asm/rsaz-x86_64.pl index 5c7d526fa37a..36bf229477aa 100755 --- a/crypto/openssl/crypto/bn/asm/rsaz-x86_64.pl +++ b/crypto/openssl/crypto/bn/asm/rsaz-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2013-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2013-2024 The OpenSSL Project Authors. All Rights Reserved. # Copyright (c) 2012, Intel Corporation. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use @@ -2248,10 +2248,12 @@ $code.=<<___; .cfi_endproc .size rsaz_512_gather4,.-rsaz_512_gather4 +.section .rodata align=64 .align 64 .Linc: .long 0,0, 1,1 .long 2,2, 2,2 +.previous ___ } diff --git a/crypto/openssl/crypto/bn/asm/sparcv8.S b/crypto/openssl/crypto/bn/asm/sparcv8.S index 94487008443e..e5402cc4e26f 100644 --- a/crypto/openssl/crypto/bn/asm/sparcv8.S +++ b/crypto/openssl/crypto/bn/asm/sparcv8.S @@ -381,7 +381,7 @@ bn_add_words: inc 16,%o1 addxcc %g3,%g4,%g3 st %g3,[%o0+4] - + ld [%o1-4],%g3 ld [%o2+12],%g4 inc 16,%o2 @@ -469,7 +469,7 @@ bn_sub_words: inc 16,%o1 subxcc %g3,%g4,%g4 st %g4,[%o0+4] - + ld [%o1-4],%g3 ld [%o2+12],%g4 inc 16,%o2 @@ -1040,7 +1040,7 @@ bn_mul_comba4: addxcc c_2,t_2,c_2 != st c_1,rp(6) !r[6]=c1; st c_2,rp(7) !r[7]=c2; - + ret restore %g0,%g0,%o0 @@ -1448,7 +1448,7 @@ bn_sqr_comba4: addxcc c_2,t_2,c_2 != st c_1,rp(6) !r[6]=c1; st c_2,rp(7) !r[7]=c2; - + ret restore %g0,%g0,%o0 diff --git a/crypto/openssl/crypto/bn/asm/sparcv8plus.S b/crypto/openssl/crypto/bn/asm/sparcv8plus.S index 696dc7b5fe09..8a936f1c72e4 100644 --- a/crypto/openssl/crypto/bn/asm/sparcv8plus.S +++ b/crypto/openssl/crypto/bn/asm/sparcv8plus.S @@ -496,7 +496,7 @@ bn_add_words: inc 16,%o1 addccc %g1,%g2,%g1 stuw %g1,[%o0+4] - + inc 16,%o2 addccc %g3,%g4,%g3 stuw %g3,[%o0+8] @@ -1126,7 +1126,7 @@ bn_mul_comba4: srlx t_1,32,c_12 != stuw t_1,rp(6) !r[6]=c1; stuw c_12,rp(7) !r[7]=c2; - + ret restore %g0,%g0,%o0 @@ -1548,7 +1548,7 @@ bn_sqr_comba4: srlx t_1,32,c_12 stuw t_1,rp(6) !r[6]=c1; stuw c_12,rp(7) !r[7]=c2; - + ret restore %g0,%g0,%o0 diff --git a/crypto/openssl/crypto/bn/asm/x86_64-mont5.pl b/crypto/openssl/crypto/bn/asm/x86_64-mont5.pl index 185d9e76ce23..07dd40d9228c 100755 --- a/crypto/openssl/crypto/bn/asm/x86_64-mont5.pl +++ b/crypto/openssl/crypto/bn/asm/x86_64-mont5.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2011-2022 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2011-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -3577,11 +3577,13 @@ $code.=<<___; ___ } $code.=<<___; +.section .rodata align=64 .align 64 .Linc: .long 0,0, 1,1 .long 2,2, 2,2 .asciz "Montgomery Multiplication with scatter/gather for x86_64, CRYPTOGAMS by " +.previous ___ # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, diff --git a/crypto/openssl/crypto/camellia/asm/cmll-x86_64.pl b/crypto/openssl/crypto/camellia/asm/cmll-x86_64.pl index 7e069614d915..43023a6986b5 100755 --- a/crypto/openssl/crypto/camellia/asm/cmll-x86_64.pl +++ b/crypto/openssl/crypto/camellia/asm/cmll-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2008-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2008-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -657,6 +657,7 @@ sub S0222 { my $i=shift; $i=@SBOX[$i]; $i=($i<<1|$i>>7)&0xff; $i=$i<<16|$i<<8|$i sub S3033 { my $i=shift; $i=@SBOX[$i]; $i=($i>>1|$i<<7)&0xff; $i=$i<<24|$i<<8|$i; sprintf("0x%08x",$i); } $code.=<<___; +.section .rodata align=64 .align 64 .LCamellia_SIGMA: .long 0x3bcc908b, 0xa09e667f, 0x4caa73b2, 0xb67ae858 @@ -682,6 +683,7 @@ $_ivp="40(%rsp)"; $_rsp="48(%rsp)"; $code.=<<___; +.text .globl Camellia_cbc_encrypt .type Camellia_cbc_encrypt,\@function,6 .align 16 diff --git a/crypto/openssl/crypto/chacha/asm/chacha-armv4.pl b/crypto/openssl/crypto/chacha/asm/chacha-armv4.pl index 0aa13519eb83..bb48b6eeb99e 100755 --- a/crypto/openssl/crypto/chacha/asm/chacha-armv4.pl +++ b/crypto/openssl/crypto/chacha/asm/chacha-armv4.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2016-2023 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -1154,7 +1154,8 @@ $code.=<<___; add sp,sp,#4*(16+3) ldmia sp!,{r4-r11,pc} .size ChaCha20_neon,.-ChaCha20_neon -.comm OPENSSL_armcap_P,4,4 +.extern OPENSSL_armcap_P +.hidden OPENSSL_armcap_P #endif ___ }}} diff --git a/crypto/openssl/crypto/chacha/asm/chacha-armv8.pl b/crypto/openssl/crypto/chacha/asm/chacha-armv8.pl index b109dd63f24c..c695d6c35c8b 100755 --- a/crypto/openssl/crypto/chacha/asm/chacha-armv8.pl +++ b/crypto/openssl/crypto/chacha/asm/chacha-armv8.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2016-2025 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -138,9 +138,11 @@ $code.=<<___; #ifndef __KERNEL__ .extern OPENSSL_armcap_P .hidden OPENSSL_armcap_P + +.extern ChaCha20_ctr32_sve #endif -.text +.rodata .align 5 .Lsigma: @@ -151,15 +153,15 @@ $code.=<<___; .long 0x02010003,0x06050407,0x0a09080b,0x0e0d0c0f .asciz "ChaCha20 for ARMv8, CRYPTOGAMS by \@dot-asm" -.globl ChaCha20_ctr32 -.type ChaCha20_ctr32,%function +.text + +.globl ChaCha20_ctr32_dflt +.type ChaCha20_ctr32_dflt,%function .align 5 -ChaCha20_ctr32: +ChaCha20_ctr32_dflt: AARCH64_SIGN_LINK_REGISTER - cbz $len,.Labort cmp $len,#192 b.lo .Lshort - #ifndef __KERNEL__ #ifdef __CHERI_PURE_CAPABILITY__ adrp c17,:got:OPENSSL_armcap_P @@ -169,6 +171,7 @@ ChaCha20_ctr32: adrp x17,OPENSSL_armcap_P ldr w17,[x17,#:lo12:OPENSSL_armcap_P] #endif +.Lcheck_neon: tst w17,#ARMV7_NEON b.ne .LChaCha20_neon #endif @@ -177,7 +180,8 @@ ChaCha20_ctr32: stp PTR(29),PTR(30),[PTRN(sp),#-(12*PTR_WIDTH)]! add PTR(29),PTRN(sp),#0 - adr @p[0],.Lsigma + adrp @p[0],.Lsigma + add @p[0],@p[0],:lo12:.Lsigma stp PTR(19),PTR(20),[PTRN(sp),#(2*PTR_WIDTH)] stp PTR(21),PTR(22),[PTRN(sp),#(4*PTR_WIDTH)] stp PTR(23),PTR(24),[PTRN(sp),#(6*PTR_WIDTH)] @@ -352,6 +356,41 @@ $code.=<<___; ldp PTR(29),PTR(30),[PTRN(sp)],#(12*PTR_WIDTH) AARCH64_VALIDATE_LINK_REGISTER ret +.size ChaCha20_ctr32_dflt,.-ChaCha20_ctr32_dflt + +.globl ChaCha20_ctr32 +.type ChaCha20_ctr32,%function +.align 5 +ChaCha20_ctr32: + AARCH64_SIGN_LINK_REGISTER + cbz $len,.Labort + cmp $len,#192 + b.lo .Lshort +#ifndef __KERNEL__ + adrp x17,OPENSSL_armcap_P + ldr w17,[x17,#:lo12:OPENSSL_armcap_P] + tst w17,#ARMV8_SVE + b.eq .Lcheck_neon + stp x29,x30,[sp,#-16]! + sub sp,sp,#16 + // SVE handling will inevitably increment the counter + // Neon/Scalar code that follows to process tail data needs to + // use new counter, unfortunately the input counter buffer + // pointed to by ctr is meant to be read-only per API contract + // we have to copy the buffer to stack to be writable by SVE + ldp x5,x6,[$ctr] + stp x5,x6,[sp] + mov $ctr,sp + bl ChaCha20_ctr32_sve + cbz $len,1f + bl ChaCha20_ctr32_dflt +1: + add sp,sp,#16 + ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER + ret +#endif + b .Lshort .size ChaCha20_ctr32,.-ChaCha20_ctr32 ___ @@ -445,7 +484,8 @@ ChaCha20_neon: stp PTR(29),PTR(30),[PTRN(sp),#-(12*PTR_WIDTH)]! add PTR(29),PTRN(sp),#0 - adr @p[0],.Lsigma + adrp @p[0],.Lsigma + add @p[0],@p[0],:lo12:.Lsigma stp PTR(19),PTR(20),[PTRN(sp),#(2*PTR_WIDTH)] stp PTR(21),PTR(22),[PTRN(sp),#(4*PTR_WIDTH)] stp PTR(23),PTR(24),[PTRN(sp),#(6*PTR_WIDTH)] @@ -856,7 +896,8 @@ ChaCha20_512_neon: stp PTR(29),PTR(30),[PTRN(sp),#-(12*PTR_WIDTH)]! add PTR(29),PTRN(sp),#0 - adr @p[0],.Lsigma + adrp @p[0],.Lsigma + add @p[0],@p[0],:lo12:.Lsigma stp PTR(19),PTR(20),[PTRN(sp),#(2*PTR_WIDTH)] stp PTR(21),PTR(22),[PTRN(sp),#(4*PTR_WIDTH)] stp PTR(23),PTR(24),[PTRN(sp),#(6*PTR_WIDTH)] diff --git a/crypto/openssl/crypto/chacha/asm/chacha-x86_64.pl b/crypto/openssl/crypto/chacha/asm/chacha-x86_64.pl index cdb900c037fd..5506e33fa2d0 100755 --- a/crypto/openssl/crypto/chacha/asm/chacha-x86_64.pl +++ b/crypto/openssl/crypto/chacha/asm/chacha-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2016-2021 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2016-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -102,6 +102,7 @@ $code.=<<___; .extern OPENSSL_ia32cap_P +.section .rodata align=64 .align 64 .Lzero: .long 0,0,0,0 @@ -133,6 +134,7 @@ $code.=<<___; .Lsigma: .asciz "expand 32-byte k" .asciz "ChaCha20 for x86_64, CRYPTOGAMS by " +.previous ___ sub AUTOLOAD() # thunk [simplified] 32-bit style perlasm diff --git a/crypto/openssl/crypto/ec/asm/ecp_nistz256-armv8.pl b/crypto/openssl/crypto/ec/asm/ecp_nistz256-armv8.pl index 9a9114e37bbb..d81d39443ea6 100755 --- a/crypto/openssl/crypto/ec/asm/ecp_nistz256-armv8.pl +++ b/crypto/openssl/crypto/ec/asm/ecp_nistz256-armv8.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2015-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2015-2025 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -56,7 +56,7 @@ my ($acc6,$acc7)=($apx,$bpx); # used in __ecp_nistz256_sqr_mont $code.=<<___; #include "arm_arch.h" -.text +.rodata ___ ######################################################################## # Convert ecp_nistz256_table.c to layout expected by ecp_nistz_gather_w7 @@ -118,6 +118,8 @@ $code.=<<___; .quad 0xccd1c8aaee00bc4f .asciz "ECP_NISTZ256 for ARMv8, CRYPTOGAMS by " +.text + // void ecp_nistz256_to_mont(BN_ULONG x0[4],const BN_ULONG x1[4]); .globl ecp_nistz256_to_mont .type ecp_nistz256_to_mont,%function @@ -128,12 +130,16 @@ ecp_nistz256_to_mont: add PTR(29),PTRN(sp),#0 stp PTR(19),PTR(20),[PTRN(sp),#(2*PTR_WIDTH)] - ldr $bi,.LRR // bp[0] + adrp $bi,.LRR + ldr $bi,[$bi,:lo12:.LRR] // bp[0] ldp $a0,$a1,[$ap] ldp $a2,$a3,[$ap,#16] - ldr $poly1,.Lpoly+8 - ldr $poly3,.Lpoly+24 - adr $bp,.LRR // &bp[0] + adrp $poly3,.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] + adrp $bp,.LRR // &bp[0] + add $bp,$bp,:lo12:.LRR bl __ecp_nistz256_mul_mont @@ -156,9 +162,12 @@ ecp_nistz256_from_mont: mov $bi,#1 // bp[0] ldp $a0,$a1,[$ap] ldp $a2,$a3,[$ap,#16] - ldr $poly1,.Lpoly+8 - ldr $poly3,.Lpoly+24 - adr $bp,.Lone // &bp[0] + adrp $poly3,.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] + adrp $bp,.Lone // &bp[0] + add $bp,$bp,:lo12:.Lone bl __ecp_nistz256_mul_mont @@ -182,8 +191,10 @@ ecp_nistz256_mul_mont: ldr $bi,[$bp] // bp[0] ldp $a0,$a1,[$ap] ldp $a2,$a3,[$ap,#16] - ldr $poly1,.Lpoly+8 - ldr $poly3,.Lpoly+24 + adrp $poly3,.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] bl __ecp_nistz256_mul_mont @@ -205,8 +216,10 @@ ecp_nistz256_sqr_mont: ldp $a0,$a1,[$ap] ldp $a2,$a3,[$ap,#16] - ldr $poly1,.Lpoly+8 - ldr $poly3,.Lpoly+24 + adrp $poly3,.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] bl __ecp_nistz256_sqr_mont @@ -230,8 +243,10 @@ ecp_nistz256_add: ldp $t0,$t1,[$bp] ldp $acc2,$acc3,[$ap,#16] ldp $t2,$t3,[$bp,#16] - ldr $poly1,.Lpoly+8 - ldr $poly3,.Lpoly+24 + adrp $poly3,.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] bl __ecp_nistz256_add @@ -251,8 +266,10 @@ ecp_nistz256_div_by_2: ldp $acc0,$acc1,[$ap] ldp $acc2,$acc3,[$ap,#16] - ldr $poly1,.Lpoly+8 - ldr $poly3,.Lpoly+24 + adrp $poly3,.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] bl __ecp_nistz256_div_by_2 @@ -272,8 +289,10 @@ ecp_nistz256_mul_by_2: ldp $acc0,$acc1,[$ap] ldp $acc2,$acc3,[$ap,#16] - ldr $poly1,.Lpoly+8 - ldr $poly3,.Lpoly+24 + adrp $poly3,.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] mov $t0,$acc0 mov $t1,$acc1 mov $t2,$acc2 @@ -297,8 +316,10 @@ ecp_nistz256_mul_by_3: ldp $acc0,$acc1,[$ap] ldp $acc2,$acc3,[$ap,#16] - ldr $poly1,.Lpoly+8 - ldr $poly3,.Lpoly+24 + adrp $poly3,.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] mov $t0,$acc0 mov $t1,$acc1 mov $t2,$acc2 @@ -334,8 +355,10 @@ ecp_nistz256_sub: ldp $acc0,$acc1,[$ap] ldp $acc2,$acc3,[$ap,#16] - ldr $poly1,.Lpoly+8 - ldr $poly3,.Lpoly+24 + adrp $poly3,.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] bl __ecp_nistz256_sub_from @@ -358,8 +381,10 @@ ecp_nistz256_neg: mov $acc1,xzr mov $acc2,xzr mov $acc3,xzr - ldr $poly1,.Lpoly+8 - ldr $poly3,.Lpoly+24 + adrp $poly3,.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] bl __ecp_nistz256_sub_from @@ -737,9 +762,11 @@ ecp_nistz256_point_double: mov $rp_real,$rp ldp $acc2,$acc3,[$ap,#48] mov $ap_real,$ap - ldr $poly1,.Lpoly+8 + adrp $poly3,.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] mov $t0,$acc0 - ldr $poly3,.Lpoly+24 + ldr $poly3,[$poly3,#24] mov $t1,$acc1 ldp $a0,$a1,[$ap_real,#64] // forward load for p256_sqr_mont mov $t2,$acc2 @@ -899,8 +926,10 @@ ecp_nistz256_point_add: mov $rp_real,$rp mov $ap_real,$ap mov $bp_real,$bp - ldr $poly1,.Lpoly+8 - ldr $poly3,.Lpoly+24 + adrp $poly3,.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] orr $t0,$a0,$a1 orr $t2,$a2,$a3 orr $in2infty,$t0,$t2 @@ -1154,8 +1183,10 @@ ecp_nistz256_point_add_affine: mov $rp_real,$rp mov $ap_real,$ap mov $bp_real,$bp - ldr $poly1,.Lpoly+8 - ldr $poly3,.Lpoly+24 + adrp $poly3,.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] ldp $a0,$a1,[$ap,#64] // in1_z ldp $a2,$a3,[$ap,#64+16] @@ -1306,7 +1337,8 @@ $code.=<<___; stp $acc2,$acc3,[$rp_real,#$i+16] ___ $code.=<<___ if ($i == 0); - adr $bp_real,.Lone_mont-64 + adrp $bp_real,.Lone_mont-64 + add $bp_real,$bp_real,:lo12:.Lone_mont-64 ___ } $code.=<<___; @@ -1358,7 +1390,8 @@ ecp_nistz256_ord_mul_mont: stp PTR(21),PTR(22),[PTRN(sp),#(4*PTR_WIDTH)] stp PTR(23),PTR(24),[PTRN(sp),#(6*PTR_WIDTH)] - adr $ordp,.Lord + adrp $ordp,.Lord + add $ordp,$ordp,:lo12:.Lord ldr $bi,[$bp] // bp[0] ldp $a0,$a1,[$ap] ldp $a2,$a3,[$ap,#16] @@ -1501,7 +1534,8 @@ ecp_nistz256_ord_sqr_mont: stp PTR(21),PTR(22),[PTRN(sp),#(4*PTR_WIDTH)] stp PTR(23),PTR(24),[PTRN(sp),#(6*PTR_WIDTH)] - adr $ordp,.Lord + adrp $ordp,.Lord + add $ordp,$ordp,:lo12:.Lord ldp $a0,$a1,[$ap] ldp $a2,$a3,[$ap,#16] diff --git a/crypto/openssl/crypto/ec/asm/ecp_nistz256-x86_64.pl b/crypto/openssl/crypto/ec/asm/ecp_nistz256-x86_64.pl index 430b14c86d8d..4da9a149a99a 100755 --- a/crypto/openssl/crypto/ec/asm/ecp_nistz256-x86_64.pl +++ b/crypto/openssl/crypto/ec/asm/ecp_nistz256-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2014-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2014-2024 The OpenSSL Project Authors. All Rights Reserved. # Copyright (c) 2014, Intel Corporation. All Rights Reserved. # Copyright (c) 2015 CloudFlare, Inc. # @@ -85,6 +85,7 @@ $code.=<<___; .extern OPENSSL_ia32cap_P # The polynomial +.section .rodata align=4096 .align 64 .Lpoly: .quad 0xffffffffffffffff, 0x00000000ffffffff, 0x0000000000000000, 0xffffffff00000001 @@ -107,6 +108,7 @@ $code.=<<___; .quad 0xf3b9cac2fc632551, 0xbce6faada7179e84, 0xffffffffffffffff, 0xffffffff00000000 .LordK: .quad 0xccd1c8aaee00bc4f +.previous ___ { @@ -4723,7 +4725,7 @@ close TABLE; die "insane number of elements" if ($#arr != 64*16*37-1); print <<___; -.text +.section .rodata align=4096 .globl ecp_nistz256_precomputed .type ecp_nistz256_precomputed,\@object .align 4096 diff --git a/crypto/openssl/crypto/md5/asm/md5-x86_64.pl b/crypto/openssl/crypto/md5/asm/md5-x86_64.pl index 6625fb7d08aa..4fde03ecabc8 100755 --- a/crypto/openssl/crypto/md5/asm/md5-x86_64.pl +++ b/crypto/openssl/crypto/md5/asm/md5-x86_64.pl @@ -1,6 +1,6 @@ #! /usr/bin/env perl # Author: Marc Bevand -# Copyright 2005-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2005-2025 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -41,7 +41,6 @@ EOF # %r10d = X[k_next] # %r11d = z' (copy of z for the next step) # %r12d = z' (copy of z for the next step) -# Each round2_step() takes about 5.4 clocks (11 instructions, 2.0 IPC) sub round2_step { my ($pos, $dst, $x, $y, $z, $k_next, $T_i, $s) = @_; @@ -53,9 +52,9 @@ sub round2_step lea $T_i($dst,%r10d),$dst /* Const + dst + ... */ and $y, %r11d /* y & (not z) */ mov $k_next*4(%rsi),%r10d /* (NEXT STEP) X[$k_next] */ - or %r11d, %r12d /* (y & (not z)) | (x & z) */ + add %r11d, $dst /* dst += (y & (not z)) */ mov $y, %r11d /* (NEXT STEP) z' = $y */ - add %r12d, $dst /* dst += ... */ + add %r12d, $dst /* dst += (x & z) */ mov $y, %r12d /* (NEXT STEP) z' = $y */ rol \$$s, $dst /* dst <<< s */ add $x, $dst /* dst += x */ diff --git a/crypto/openssl/crypto/modes/asm/aes-gcm-armv8_64.pl b/crypto/openssl/crypto/modes/asm/aes-gcm-armv8_64.pl index bcbd52d18f91..2a97311b1a7c 100755 --- a/crypto/openssl/crypto/modes/asm/aes-gcm-armv8_64.pl +++ b/crypto/openssl/crypto/modes/asm/aes-gcm-armv8_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2019-2025 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -250,12 +250,12 @@ $code.=<<___ if ($flavour !~ /64/); ___ ######################################################################################### -# size_t aes_gcm_enc_128_kernel(const unsigned char *in, -# size_t len, -# unsigned char *out, -# const void *key, +# size_t aes_gcm_enc_128_kernel(const uint8_t * plaintext, +# uint64_t plaintext_length, +# uint8_t * ciphertext, +# uint64_t *Xi, # unsigned char ivec[16], -# u64 *Xi); +# const void *key); # $code.=<<___; .global aes_gcm_enc_128_kernel @@ -1141,12 +1141,12 @@ aes_gcm_enc_128_kernel: ___ ######################################################################################### -# size_t aes_gcm_dec_128_kernel(const unsigned char *in, -# size_t len, -# unsigned char *out, -# const void *key, +# size_t aes_gcm_dec_128_kernel(const uint8_t * ciphertext, +# uint64_t plaintext_length, +# uint8_t * plaintext, +# uint64_t *Xi, # unsigned char ivec[16], -# u64 *Xi); +# const void *key); # $code.=<<___; .global aes_gcm_dec_128_kernel @@ -2105,12 +2105,12 @@ my $rk4v="v22"; my $rk4d="d22"; ######################################################################################### -# size_t aes_gcm_enc_192_kernel(const unsigned char *in, -# size_t len, -# unsigned char *out, -# const void *key, +# size_t aes_gcm_enc_192_kernel(const uint8_t * plaintext, +# uint64_t plaintext_length, +# uint8_t * ciphertext, +# uint64_t *Xi, # unsigned char ivec[16], -# u64 *Xi); +# const void *key); # $code.=<<___; .global aes_gcm_enc_192_kernel @@ -3047,12 +3047,12 @@ aes_gcm_enc_192_kernel: ___ ######################################################################################### -# size_t aes_gcm_dec_192_kernel(const unsigned char *in, -# size_t len, -# unsigned char *out, -# const void *key, +# size_t aes_gcm_dec_192_kernel(const uint8_t * ciphertext, +# uint64_t plaintext_length, +# uint8_t * plaintext, +# uint64_t *Xi, # unsigned char ivec[16], -# u64 *Xi); +# const void *key); # $code.=<<___; .global aes_gcm_dec_192_kernel @@ -4061,12 +4061,12 @@ my $rk4v="v22"; my $rk4d="d22"; ######################################################################################### -# size_t aes_gcm_enc_256_kernel(const unsigned char *in, -# size_t len, -# unsigned char *out, -# const void *key, +# size_t aes_gcm_enc_256_kernel(const uint8_t * plaintext, +# uint64_t plaintext_length, +# uint8_t * ciphertext, +# uint64_t *Xi, # unsigned char ivec[16], -# u64 *Xi); +# const void *key); # $code.=<<___; .global aes_gcm_enc_256_kernel @@ -5051,12 +5051,12 @@ my $t8d="d4"; my $t9="v6"; my $t9d="d6"; ######################################################################################### -# size_t aes_gcm_dec_256_kernel(const unsigned char *in, -# size_t len, -# unsigned char *out, -# const void *key, +# size_t aes_gcm_dec_256_kernel(const uint8_t * ciphertext, +# uint64_t plaintext_length, +# uint8_t * plaintext, +# uint64_t *Xi, # unsigned char ivec[16], -# u64 *Xi); +# const void *key); # $code.=<<___; .global aes_gcm_dec_256_kernel @@ -6072,6 +6072,7 @@ ___ } $code.=<<___; +.rodata .asciz "GHASH for ARMv8, CRYPTOGAMS by " .align 2 #endif diff --git a/crypto/openssl/crypto/modes/asm/aesni-gcm-x86_64.pl b/crypto/openssl/crypto/modes/asm/aesni-gcm-x86_64.pl index eaf4d9c755fb..872e13f8f1a1 100755 --- a/crypto/openssl/crypto/modes/asm/aesni-gcm-x86_64.pl +++ b/crypto/openssl/crypto/modes/asm/aesni-gcm-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2013-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2013-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -943,6 +943,7 @@ $code.=<<___; ___ $code.=<<___; +.section .rodata align=64 .align 64 .Lbswap_mask: .byte 15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0 @@ -955,6 +956,7 @@ $code.=<<___; .Lone_lsb: .byte 1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 .asciz "AES-NI GCM module for x86_64, CRYPTOGAMS by " +.previous .align 64 ___ if ($win64) { diff --git a/crypto/openssl/crypto/modes/asm/ghash-s390x.pl b/crypto/openssl/crypto/modes/asm/ghash-s390x.pl index ba9c5b4a4f4f..0ae6adbf6146 100755 --- a/crypto/openssl/crypto/modes/asm/ghash-s390x.pl +++ b/crypto/openssl/crypto/modes/asm/ghash-s390x.pl @@ -90,25 +90,6 @@ $code.=<<___; .align 32 gcm_gmult_4bit: ___ -$code.=<<___ if(!$softonly && 0); # hardware is slow for single block... - larl %r1,OPENSSL_s390xcap_P - lghi %r0,0 - lg %r1,S390X_KIMD+8(%r1) # load second word of kimd capabilities - # vector - tmhh %r1,0x4000 # check for function 65 - jz .Lsoft_gmult - stg %r0,16($sp) # arrange 16 bytes of zero input - stg %r0,24($sp) - lghi %r0,S390X_GHASH # function 65 - la %r1,0($Xi) # H lies right after Xi in gcm128_context - la $inp,16($sp) - lghi $len,16 - .long 0xb93e0004 # kimd %r0,$inp - brc 1,.-4 # pay attention to "partial completion" - br %r14 -.align 32 -.Lsoft_gmult: -___ $code.=<<___; stm${g} %r6,%r14,6*$SIZE_T($sp) @@ -132,10 +113,21 @@ $code.=<<___ if(!$softonly); # vector tmhh %r0,0x4000 # check for function 65 jz .Lsoft_ghash + # Do not assume this function is called from a gcm128_context. + # This is not true, e.g., for AES-GCM-SIV. + # Parameter Block: + # Chaining Value (XI) 128byte + # Key (Htable[8]) 128byte + lmg %r0,%r1,0($Xi) + stmg %r0,%r1,8($sp) + lmg %r0,%r1,8*16($Htbl) + stmg %r0,%r1,24($sp) + la %r1,8($sp) lghi %r0,S390X_GHASH # function 65 - la %r1,0($Xi) # H lies right after Xi in gcm128_context .long 0xb93e0004 # kimd %r0,$inp brc 1,.-4 # pay attention to "partial completion" + lmg %r0,%r1,8($sp) + stmg %r0,%r1,0($Xi) br %r14 .align 32 .Lsoft_ghash: diff --git a/crypto/openssl/crypto/modes/asm/ghash-x86_64.pl b/crypto/openssl/crypto/modes/asm/ghash-x86_64.pl index 6709f96492ed..6ef8e555d0a2 100755 --- a/crypto/openssl/crypto/modes/asm/ghash-x86_64.pl +++ b/crypto/openssl/crypto/modes/asm/ghash-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2010-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2010-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -534,6 +534,7 @@ $code.=<<___; .align 16 gcm_init_clmul: .cfi_startproc + endbranch .L_init_clmul: ___ $code.=<<___ if ($win64); @@ -1027,6 +1028,7 @@ $code.=<<___; .align 32 gcm_init_avx: .cfi_startproc + endbranch ___ if ($avx) { my ($Htbl,$Xip)=@_4args; @@ -1609,6 +1611,7 @@ ___ } $code.=<<___; +.section .rodata align=64 .align 64 .Lbswap_mask: .byte 15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0 @@ -1662,6 +1665,7 @@ $code.=<<___; .asciz "GHASH for x86_64, CRYPTOGAMS by " .align 64 +.previous ___ # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, diff --git a/crypto/openssl/crypto/modes/asm/ghashv8-armx.pl b/crypto/openssl/crypto/modes/asm/ghashv8-armx.pl index b4862481222e..ac3ec558ef0f 100644 --- a/crypto/openssl/crypto/modes/asm/ghashv8-armx.pl +++ b/crypto/openssl/crypto/modes/asm/ghashv8-armx.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2014-2023 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2014-2025 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -161,6 +161,7 @@ $code.=<<___; ___ if ($flavour =~ /64/) { my ($t3,$Yl,$Ym,$Yh) = map("q$_",(4..7)); +my ($H3,$H34k,$H4,$H5,$H56k,$H6,$H7,$H78k,$H8) = map("q$_",(15..23)); $code.=<<___; @ calculate H^3 and H^4 @@ -195,15 +196,103 @@ $code.=<<___; vpmull.p64 $Yl,$Yl,$xC2 veor $t2,$t2,$Xh veor $t3,$t3,$Yh - veor $H, $Xl,$t2 @ H^3 - veor $H2,$Yl,$t3 @ H^4 - - vext.8 $t0,$H, $H,#8 @ Karatsuba pre-processing - vext.8 $t1,$H2,$H2,#8 - veor $t0,$t0,$H - veor $t1,$t1,$H2 - vext.8 $Hhl,$t0,$t1,#8 @ pack Karatsuba pre-processed - vst1.64 {$H-$H2},[PTR(0)] @ store Htable[3..5] + veor $H3, $Xl,$t2 @ H^3 + veor $H4,$Yl,$t3 @ H^4 + + vext.8 $t0,$H3, $H3,#8 @ Karatsuba pre-processing + vext.8 $t1,$H4,$H4,#8 + vext.8 $t2,$H2,$H2,#8 + veor $t0,$t0,$H3 + veor $t1,$t1,$H4 + veor $t2,$t2,$H2 + vext.8 $H34k,$t0,$t1,#8 @ pack Karatsuba pre-processed + vst1.64 {$H3-$H4},[PTR(0)],#48 @ store Htable[3..5] + + @ calculate H^5 and H^6 + vpmull.p64 $Xl,$H2, $H3 + vpmull.p64 $Yl,$H3,$H3 + vpmull2.p64 $Xh,$H2, $H3 + vpmull2.p64 $Yh,$H3,$H3 + vpmull.p64 $Xm,$t0,$t2 + vpmull.p64 $Ym,$t0,$t0 + + vext.8 $t0,$Xl,$Xh,#8 @ Karatsuba post-processing + vext.8 $t1,$Yl,$Yh,#8 + veor $t2,$Xl,$Xh + veor $Xm,$Xm,$t0 + veor $t3,$Yl,$Yh + veor $Ym,$Ym,$t1 + veor $Xm,$Xm,$t2 + vpmull.p64 $t2,$Xl,$xC2 @ 1st phase + veor $Ym,$Ym,$t3 + vpmull.p64 $t3,$Yl,$xC2 + + vmov $Xh#lo,$Xm#hi @ Xh|Xm - 256-bit result + vmov $Yh#lo,$Ym#hi + vmov $Xm#hi,$Xl#lo @ Xm is rotated Xl + vmov $Ym#hi,$Yl#lo + veor $Xl,$Xm,$t2 + veor $Yl,$Ym,$t3 + + vext.8 $t2,$Xl,$Xl,#8 @ 2nd phase + vext.8 $t3,$Yl,$Yl,#8 + vpmull.p64 $Xl,$Xl,$xC2 + vpmull.p64 $Yl,$Yl,$xC2 + veor $t2,$t2,$Xh + veor $t3,$t3,$Yh + veor $H5,$Xl,$t2 @ H^5 + veor $H6,$Yl,$t3 @ H^6 + + vext.8 $t0,$H5, $H5,#8 @ Karatsuba pre-processing + vext.8 $t1,$H6,$H6,#8 + vext.8 $t2,$H2,$H2,#8 + veor $t0,$t0,$H5 + veor $t1,$t1,$H6 + veor $t2,$t2,$H2 + vext.8 $H56k,$t0,$t1,#8 @ pack Karatsuba pre-processed + vst1.64 {$H5-$H6},[PTR(0)],#48 @ store Htable[6..8] + + @ calculate H^7 and H^8 + vpmull.p64 $Xl,$H2,$H5 + vpmull.p64 $Yl,$H2,$H6 + vpmull2.p64 $Xh,$H2,$H5 + vpmull2.p64 $Yh,$H2,$H6 + vpmull.p64 $Xm,$t0,$t2 + vpmull.p64 $Ym,$t1,$t2 + + vext.8 $t0,$Xl,$Xh,#8 @ Karatsuba post-processing + vext.8 $t1,$Yl,$Yh,#8 + veor $t2,$Xl,$Xh + veor $Xm,$Xm,$t0 + veor $t3,$Yl,$Yh + veor $Ym,$Ym,$t1 + veor $Xm,$Xm,$t2 + vpmull.p64 $t2,$Xl,$xC2 @ 1st phase + veor $Ym,$Ym,$t3 + vpmull.p64 $t3,$Yl,$xC2 + + vmov $Xh#lo,$Xm#hi @ Xh|Xm - 256-bit result + vmov $Yh#lo,$Ym#hi + vmov $Xm#hi,$Xl#lo @ Xm is rotated Xl + vmov $Ym#hi,$Yl#lo + veor $Xl,$Xm,$t2 + veor $Yl,$Ym,$t3 + + vext.8 $t2,$Xl,$Xl,#8 @ 2nd phase + vext.8 $t3,$Yl,$Yl,#8 + vpmull.p64 $Xl,$Xl,$xC2 + vpmull.p64 $Yl,$Yl,$xC2 + veor $t2,$t2,$Xh + veor $t3,$t3,$Yh + veor $H7,$Xl,$t2 @ H^7 + veor $H8,$Yl,$t3 @ H^8 + + vext.8 $t0,$H7,$H7,#8 @ Karatsuba pre-processing + vext.8 $t1,$H8,$H8,#8 + veor $t0,$t0,$H7 + veor $t1,$t1,$H8 + vext.8 $H78k,$t0,$t1,#8 @ pack Karatsuba pre-processed + vst1.64 {$H7-$H8},[PTR(0)] @ store Htable[9..11] ___ } $code.=<<___; @@ -724,6 +813,7 @@ ___ } $code.=<<___; +.rodata .asciz "GHASH for ARMv8, CRYPTOGAMS by " .align 2 #endif diff --git a/crypto/openssl/crypto/poly1305/asm/poly1305-armv4.pl b/crypto/openssl/crypto/poly1305/asm/poly1305-armv4.pl index 041bfd46e699..673ea62ec388 100755 --- a/crypto/openssl/crypto/poly1305/asm/poly1305-armv4.pl +++ b/crypto/openssl/crypto/poly1305/asm/poly1305-armv4.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2016-2023 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -1239,7 +1239,8 @@ $code.=<<___; .asciz "Poly1305 for ARMv4/NEON, CRYPTOGAMS by " .align 2 #if __ARM_MAX_ARCH__>=7 -.comm OPENSSL_armcap_P,4,4 +.extern OPENSSL_armcap_P +.hidden OPENSSL_armcap_P #endif ___ diff --git a/crypto/openssl/crypto/poly1305/asm/poly1305-armv8.pl b/crypto/openssl/crypto/poly1305/asm/poly1305-armv8.pl index 06434a9b60a5..749a5688fb6b 100755 --- a/crypto/openssl/crypto/poly1305/asm/poly1305-armv8.pl +++ b/crypto/openssl/crypto/poly1305/asm/poly1305-armv8.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2016-2023 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2016-2025 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -105,10 +105,14 @@ poly1305_init: tst w17,#ARMV7_NEON - adr $d0p,.Lpoly1305_blocks - adr $r0p,.Lpoly1305_blocks_neon - adr $d1p,.Lpoly1305_emit - adr $r1p,.Lpoly1305_emit_neon + adrp $d0p,poly1305_blocks + add $d0p,$d0p,#:lo12:.Lpoly1305_blocks + adrp $r0p,poly1305_blocks_neon + add $r0p,$r0p,#:lo12:.Lpoly1305_blocks_neon + adrp $d1p,poly1305_emit + add $d1p,$d1p,#:lo12:.Lpoly1305_emit + adrp $r1p,poly1305_emit_neon + add $r1p,$r1p,#:lo12:.Lpoly1305_emit_neon csel $d0p,$d0p,$r0p,eq csel $d1p,$d1p,$r1p,eq @@ -453,7 +457,8 @@ poly1305_blocks_neon: ldr PTR(30),[PTRN(sp),#PTR_WIDTH] add $in2,$inp,#32 - adr $zeros,.Lzeros + adrp $zeros,.Lzeros + add $zeros,$zeros,:lo12:.Lzeros subs $len,$len,#64 csel $in2,$zeros,$in2,lo @@ -465,7 +470,8 @@ poly1305_blocks_neon: .align 4 .Leven_neon: add $in2,$inp,#32 - adr $zeros,.Lzeros + adrp $zeros,.Lzeros + add $zeros,$zeros,:lo12:.Lzeros subs $len,$len,#64 csel $in2,$zeros,$in2,lo @@ -949,6 +955,8 @@ poly1305_emit_neon: ret .size poly1305_emit_neon,.-poly1305_emit_neon +.rodata + .align 5 .Lzeros: .long 0,0,0,0,0,0,0,0 diff --git a/crypto/openssl/crypto/poly1305/asm/poly1305-x86_64.pl b/crypto/openssl/crypto/poly1305/asm/poly1305-x86_64.pl index 4cddca1c514c..305099ca0308 100755 --- a/crypto/openssl/crypto/poly1305/asm/poly1305-x86_64.pl +++ b/crypto/openssl/crypto/poly1305/asm/poly1305-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2016-2023 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2016-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -229,6 +229,7 @@ $code.=<<___; .align 32 poly1305_blocks: .cfi_startproc + endbranch .Lblocks: shr \$4,$len jz .Lno_data # too short @@ -303,6 +304,7 @@ $code.=<<___; .align 32 poly1305_emit: .cfi_startproc + endbranch .Lemit: mov 0($ctx),%r8 # load hash value mov 8($ctx),%r9 @@ -524,6 +526,7 @@ __poly1305_init_avx: .align 32 poly1305_blocks_avx: .cfi_startproc + endbranch mov 20($ctx),%r8d # is_base2_26 cmp \$128,$len jae .Lblocks_avx @@ -1384,6 +1387,7 @@ $code.=<<___; .align 32 poly1305_emit_avx: .cfi_startproc + endbranch cmpl \$0,20($ctx) # is_base2_26? je .Lemit @@ -1448,6 +1452,7 @@ $code.=<<___; .align 32 poly1305_blocks_avx2: .cfi_startproc + endbranch mov 20($ctx),%r8d # is_base2_26 cmp \$128,$len jae .Lblocks_avx2 @@ -2144,6 +2149,7 @@ $code.=<<___; .align 32 poly1305_blocks_avx512: .cfi_startproc + endbranch .Lblocks_avx512: mov \$15,%eax kmovw %eax,%k2 @@ -3778,6 +3784,7 @@ poly1305_emit_base2_44: ___ } } } $code.=<<___; +.section .rodata align=64 .align 64 .Lconst: .Lmask24: @@ -3809,6 +3816,7 @@ $code.=<<___; .Lx_mask42: .quad 0x3ffffffffff,0x3ffffffffff,0x3ffffffffff,0x3ffffffffff .quad 0x3ffffffffff,0x3ffffffffff,0x3ffffffffff,0x3ffffffffff +.previous ___ } $code.=<<___; diff --git a/crypto/openssl/crypto/rc4/asm/rc4-x86_64.pl b/crypto/openssl/crypto/rc4/asm/rc4-x86_64.pl index 5a903f5739e1..83a1d13635fd 100755 --- a/crypto/openssl/crypto/rc4/asm/rc4-x86_64.pl +++ b/crypto/openssl/crypto/rc4/asm/rc4-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2005-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2005-2023 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -23,7 +23,7 @@ # Presumably it has everything to do with AMD cache architecture and # RAW or whatever penalties. Once again! The module *requires* config # line *without* RC4_CHAR! As for coding "secret," I bet on partial -# register arithmetics. For example instead of 'inc %r8; and $255,%r8' +# register arithmetic. For example instead of 'inc %r8; and $255,%r8' # I simply 'inc %r8b'. Even though optimization manual discourages # to operate on partial registers, it turned out to be the best bet. # At least for AMD... How IA32E would perform remains to be seen... diff --git a/crypto/openssl/crypto/sha/asm/keccak1600-armv4.pl b/crypto/openssl/crypto/sha/asm/keccak1600-armv4.pl index eaad86d39ddf..f9a8cd1f0046 100755 --- a/crypto/openssl/crypto/sha/asm/keccak1600-armv4.pl +++ b/crypto/openssl/crypto/sha/asm/keccak1600-armv4.pl @@ -932,7 +932,15 @@ SHA3_absorb: .size SHA3_absorb,.-SHA3_absorb ___ } -{ my ($out,$len,$A_flat,$bsz) = map("r$_", (4,5,10,12)); + +{ my ($out,$len,$A_flat,$bsz,$next) = map("r$_", (4,5,10,12,0)); + + +# void SHA3_squeeze(uint64_t A[5][5], +# unsigned char *out, size_t len, size_t r, int next) +# +# The first 4 parameters are passed in via r0..r3, +# next is passed on the stack [sp, #0] $code.=<<___; .global SHA3_squeeze @@ -945,6 +953,7 @@ SHA3_squeeze: mov $out,r1 mov $len,r2 mov $bsz,r3 + ldr $next, [sp, #40] @ next is after the 10 pushed registers (10*4) #ifdef __thumb2__ mov r9,#0x00ff00ff @@ -966,6 +975,8 @@ SHA3_squeeze: stmdb sp!,{r6-r9} mov r14,$A_flat + cmp $next, #1 + beq .Lnext_block b .Loop_squeeze .align 4 @@ -1037,7 +1048,7 @@ SHA3_squeeze: subs $bsz,$bsz,#8 @ bsz -= 8 bhi .Loop_squeeze - +.Lnext_block: mov r0,r14 @ original $A_flat bl KeccakF1600 diff --git a/crypto/openssl/crypto/sha/asm/keccak1600-armv8.pl b/crypto/openssl/crypto/sha/asm/keccak1600-armv8.pl index 1d6623fa5494..c28a2c554f35 100755 --- a/crypto/openssl/crypto/sha/asm/keccak1600-armv8.pl +++ b/crypto/openssl/crypto/sha/asm/keccak1600-armv8.pl @@ -1,5 +1,5 @@ #!/usr/bin/env perl -# Copyright 2017-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2017-2025 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -82,7 +82,7 @@ my @rhotates = ([ 0, 1, 62, 28, 27 ], $code.=<<___; #include "arm_arch.h" -.text +.rodata .align 8 // strategic alignment and padding that allows to use // address value as loop termination condition... @@ -124,13 +124,16 @@ my @C = map("x$_", (26,27,28,30)); my @P = map("PTR($_)", (26,27,28,30)); $code.=<<___; +.text + #define KeccakF1600_int_FRAMESZ (16 + 2*PTR_WIDTH) .type KeccakF1600_int,%function .align 5 KeccakF1600_int: AARCH64_SIGN_LINK_REGISTER - adr @P[2],iotas + adrp @P[2],iotas + add @P[2],@P[2],:lo12:iotas stp @P[2],PTR(30),[PTRN(sp),#16] // 32/48 bytes on top are mine b .Loop .align 4 @@ -487,6 +490,8 @@ SHA3_squeeze: mov $out,PTR(1) mov $len,x2 mov $bsz,x3 + cmp w4, #0 // w4 = 'next' argument + bne .Lnext_block .Loop_squeeze: ldr x4,[PTR(0)],#8 @@ -501,7 +506,7 @@ SHA3_squeeze: subs x3,x3,#8 bhi .Loop_squeeze - +.Lnext_block: mov PTR(0),$A_flat bl KeccakF1600 mov PTR(0),$A_flat @@ -558,7 +563,8 @@ $code.=<<___; .align 5 KeccakF1600_ce: mov x9,#24 - adr PTR(10),iotas + adrp PTR(10),iotas + add PTR(10),PTR(10),:lo12:iotas b .Loop_ce .align 4 .Loop_ce: diff --git a/crypto/openssl/crypto/sha/asm/keccak1600-ppc64.pl b/crypto/openssl/crypto/sha/asm/keccak1600-ppc64.pl index 83f8d8ef337b..54f32e8c9291 100755 --- a/crypto/openssl/crypto/sha/asm/keccak1600-ppc64.pl +++ b/crypto/openssl/crypto/sha/asm/keccak1600-ppc64.pl @@ -51,6 +51,16 @@ if ($flavour =~ /64/) { $PUSH ="std"; } else { die "nonsense $flavour"; } +$LITTLE_ENDIAN = ($flavour=~/le$/) ? 1 : 0; + +if ($LITTLE_ENDIAN) { + $DWORD_LE_LOAD = "ldu r0,8(r3)"; + $LE_LOAD_SIZE = "8"; +} else { + $DWORD_LE_LOAD = "bl dword_le_load"; + $LE_LOAD_SIZE = "1"; +} + $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or @@ -384,7 +394,9 @@ KeccakF1600: .byte 0,12,4,1,0x80,18,1,0 .long 0 .size KeccakF1600,.-KeccakF1600 - +___ +if (!$LITTLE_ENDIAN) { +$code.=<<___; .type dword_le_load,\@function .align 5 dword_le_load: @@ -408,7 +420,10 @@ dword_le_load: .byte 0,12,0x14,0,0,0,1,0 .long 0 .size dword_le_load,.-dword_le_load +___ +} +$code.=<<___; .globl SHA3_absorb .type SHA3_absorb,\@function .align 5 @@ -436,7 +451,7 @@ SHA3_absorb: $PUSH r0,`$FRAME+$LRSAVE`($sp) bl PICmeup - subi r4,r4,1 ; prepare for lbzu + subi r4,r4,$LE_LOAD_SIZE ; prepare for ldu or lbzu subi r12,r12,8 ; prepare for ldu $PUSH r3,`$LOCALS+0*$SIZE_T`($sp) ; save A[][] @@ -487,79 +502,79 @@ SHA3_absorb: srwi r5,r5,3 $PUSH r4,`$LOCALS+2*$SIZE_T`($sp) ; save len mtctr r5 - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[0][0],$A[0][0],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[0][1],$A[0][1],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[0][2],$A[0][2],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[0][3],$A[0][3],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[0][4],$A[0][4],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[1][0],$A[1][0],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[1][1],$A[1][1],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[1][2],$A[1][2],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[1][3],$A[1][3],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[1][4],$A[1][4],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[2][0],$A[2][0],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[2][1],$A[2][1],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[2][2],$A[2][2],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[2][3],$A[2][3],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[2][4],$A[2][4],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[3][0],$A[3][0],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[3][1],$A[3][1],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[3][2],$A[3][2],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[3][3],$A[3][3],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[3][4],$A[3][4],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[4][0],$A[4][0],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[4][1],$A[4][1],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[4][2],$A[4][2],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[4][3],$A[4][3],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[4][4],$A[4][4],r0 .Lprocess_block: @@ -653,6 +668,8 @@ SHA3_squeeze: subi $out,r4,1 ; prepare for stbu mr $len,r5 mr $bsz,r6 + cmplwi r7,0 ; r7 = 'next' argument + bne .Lnext_block b .Loop_squeeze .align 4 @@ -683,6 +700,7 @@ SHA3_squeeze: subic. r6,r6,8 bgt .Loop_squeeze +.Lnext_block: mr r3,$A_flat bl KeccakF1600 subi r3,$A_flat,8 ; prepare for ldu diff --git a/crypto/openssl/crypto/sha/asm/keccak1600-s390x.pl b/crypto/openssl/crypto/sha/asm/keccak1600-s390x.pl index 86233c7e38f2..7d5ebde11719 100755 --- a/crypto/openssl/crypto/sha/asm/keccak1600-s390x.pl +++ b/crypto/openssl/crypto/sha/asm/keccak1600-s390x.pl @@ -472,7 +472,7 @@ SHA3_absorb: .size SHA3_absorb,.-SHA3_absorb ___ } -{ my ($A_flat,$out,$len,$bsz) = map("%r$_",(2..5)); +{ my ($A_flat,$out,$len,$bsz,$next) = map("%r$_",(2..6)); $code.=<<___; .globl SHA3_squeeze @@ -484,6 +484,7 @@ SHA3_squeeze: lghi %r14,8 st${g} $bsz,5*$SIZE_T($sp) la %r1,0($A_flat) + cijne $next,0,.Lnext_block j .Loop_squeeze @@ -501,6 +502,7 @@ SHA3_squeeze: brct $bsz,.Loop_squeeze # bsz-- +.Lnext_block: stm${g} $out,$len,3*$SIZE_T($sp) bras %r14,.LKeccakF1600 lm${g} $out,$bsz,3*$SIZE_T($sp) diff --git a/crypto/openssl/crypto/sha/asm/keccak1600-x86_64.pl b/crypto/openssl/crypto/sha/asm/keccak1600-x86_64.pl index 02f0116014d6..3dbf17731e4d 100755 --- a/crypto/openssl/crypto/sha/asm/keccak1600-x86_64.pl +++ b/crypto/openssl/crypto/sha/asm/keccak1600-x86_64.pl @@ -1,5 +1,5 @@ #!/usr/bin/env perl -# Copyright 2017-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2017-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -503,12 +503,12 @@ SHA3_absorb: .size SHA3_absorb,.-SHA3_absorb ___ } -{ my ($A_flat,$out,$len,$bsz) = ("%rdi","%rsi","%rdx","%rcx"); +{ my ($A_flat,$out,$len,$bsz,$next) = ("%rdi","%rsi","%rdx","%rcx","%r8"); ($out,$len,$bsz) = ("%r12","%r13","%r14"); $code.=<<___; .globl SHA3_squeeze -.type SHA3_squeeze,\@function,4 +.type SHA3_squeeze,\@function,5 .align 32 SHA3_squeeze: .cfi_startproc @@ -520,10 +520,12 @@ SHA3_squeeze: .cfi_push %r14 shr \$3,%rcx - mov $A_flat,%r8 + mov $A_flat,%r9 mov %rsi,$out mov %rdx,$len mov %rcx,$bsz + bt \$0,${next}d + jc .Lnext_block jmp .Loop_squeeze .align 32 @@ -531,8 +533,8 @@ SHA3_squeeze: cmp \$8,$len jb .Ltail_squeeze - mov (%r8),%rax - lea 8(%r8),%r8 + mov (%r9),%rax + lea 8(%r9),%r9 mov %rax,($out) lea 8($out),$out sub \$8,$len # len -= 8 @@ -540,14 +542,14 @@ SHA3_squeeze: sub \$1,%rcx # bsz-- jnz .Loop_squeeze - +.Lnext_block: call KeccakF1600 - mov $A_flat,%r8 + mov $A_flat,%r9 mov $bsz,%rcx jmp .Loop_squeeze .Ltail_squeeze: - mov %r8, %rsi + mov %r9, %rsi mov $out,%rdi mov $len,%rcx .byte 0xf3,0xa4 # rep movsb @@ -565,6 +567,7 @@ SHA3_squeeze: ___ } $code.=<<___; +.section .rodata align=256 .align 256 .quad 0,0,0,0,0,0,0,0 .type iotas,\@object diff --git a/crypto/openssl/crypto/sha/asm/sha1-armv4-large.pl b/crypto/openssl/crypto/sha/asm/sha1-armv4-large.pl index 6a2e12bf17d7..b3a8ec694145 100755 --- a/crypto/openssl/crypto/sha/asm/sha1-armv4-large.pl +++ b/crypto/openssl/crypto/sha/asm/sha1-armv4-large.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2007-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2007-2023 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -707,7 +707,8 @@ ___ }}} $code.=<<___; #if __ARM_MAX_ARCH__>=7 -.comm OPENSSL_armcap_P,4,4 +.extern OPENSSL_armcap_P +.hidden OPENSSL_armcap_P #endif ___ diff --git a/crypto/openssl/crypto/sha/asm/sha1-armv8.pl b/crypto/openssl/crypto/sha/asm/sha1-armv8.pl index 0ffc9766b9f9..3a3c1d1806a8 100755 --- a/crypto/openssl/crypto/sha/asm/sha1-armv8.pl +++ b/crypto/openssl/crypto/sha/asm/sha1-armv8.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2014-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2014-2025 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -265,7 +265,8 @@ sha1_block_armv8: stp PTR(29),PTR(30),[PTRN(sp),#-(2*PTR_WIDTH)]! add PTR(29),PTRN(sp),#0 - adr PTR(4),.Lconst + adrp PTR(4),.Lconst + add PTR(4),PTR(4),:lo12:.Lconst eor $E,$E,$E ld1.32 {$ABCD},[$ctx],#16 ld1.32 {$E}[0],[$ctx] @@ -325,6 +326,9 @@ $code.=<<___; ldr PTR(29),[PTRN(sp)],#(2*PTR_WIDTH) ret .size sha1_block_armv8,.-sha1_block_armv8 + +.rodata + .align 6 .Lconst: .long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 //K_00_19 diff --git a/crypto/openssl/crypto/sha/asm/sha1-mb-x86_64.pl b/crypto/openssl/crypto/sha/asm/sha1-mb-x86_64.pl index 67faba136d05..d9d1630d16db 100755 --- a/crypto/openssl/crypto/sha/asm/sha1-mb-x86_64.pl +++ b/crypto/openssl/crypto/sha/asm/sha1-mb-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2013-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2013-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -1374,7 +1374,7 @@ $code.=<<___; ___ } }}} $code.=<<___; - +.section .rodata align=256 .align 256 .long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 # K_00_19 .long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 # K_00_19 @@ -1389,6 +1389,7 @@ K_XX_XX: .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f # pbswap .byte 0xf,0xe,0xd,0xc,0xb,0xa,0x9,0x8,0x7,0x6,0x5,0x4,0x3,0x2,0x1,0x0 .asciz "SHA1 multi-block transform for x86_64, CRYPTOGAMS by " +.previous ___ if ($win64) { diff --git a/crypto/openssl/crypto/sha/asm/sha1-x86_64.pl b/crypto/openssl/crypto/sha/asm/sha1-x86_64.pl index c66b8fca875e..30c545cf419a 100755 --- a/crypto/openssl/crypto/sha/asm/sha1-x86_64.pl +++ b/crypto/openssl/crypto/sha/asm/sha1-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2006-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2006-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -1832,6 +1832,7 @@ ___ } } $code.=<<___; +.section .rodata align=64 .align 64 K_XX_XX: .long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 # K_00_19 @@ -1845,6 +1846,7 @@ K_XX_XX: .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f # pbswap mask .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f # pbswap mask .byte 0xf,0xe,0xd,0xc,0xb,0xa,0x9,0x8,0x7,0x6,0x5,0x4,0x3,0x2,0x1,0x0 +.previous ___ }}} $code.=<<___; diff --git a/crypto/openssl/crypto/sha/asm/sha256-armv4.pl b/crypto/openssl/crypto/sha/asm/sha256-armv4.pl index e8af05c2981f..feb1f26109a9 100755 --- a/crypto/openssl/crypto/sha/asm/sha256-armv4.pl +++ b/crypto/openssl/crypto/sha/asm/sha256-armv4.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2007-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2007-2023 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -693,7 +693,8 @@ $code.=<<___; .asciz "SHA256 block transform for ARMv4/NEON/ARMv8, CRYPTOGAMS by " .align 2 #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) -.comm OPENSSL_armcap_P,4,4 +.extern OPENSSL_armcap_P +.hidden OPENSSL_armcap_P #endif ___ diff --git a/crypto/openssl/crypto/sha/asm/sha256-mb-x86_64.pl b/crypto/openssl/crypto/sha/asm/sha256-mb-x86_64.pl index fa2ca86046cc..9398b7954a7f 100755 --- a/crypto/openssl/crypto/sha/asm/sha256-mb-x86_64.pl +++ b/crypto/openssl/crypto/sha/asm/sha256-mb-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2013-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2013-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -1340,6 +1340,7 @@ $code.=<<___; ___ } }}} $code.=<<___; +.section .rodata align=256 .align 256 K256: ___ @@ -1389,6 +1390,7 @@ K256_shaext: .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 .asciz "SHA256 multi-block transform for x86_64, CRYPTOGAMS by " +.previous ___ if ($win64) { diff --git a/crypto/openssl/crypto/sha/asm/sha512-armv4.pl b/crypto/openssl/crypto/sha/asm/sha512-armv4.pl index 1751c671e6d8..9aa310ffd05f 100755 --- a/crypto/openssl/crypto/sha/asm/sha512-armv4.pl +++ b/crypto/openssl/crypto/sha/asm/sha512-armv4.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2007-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2007-2023 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -660,7 +660,8 @@ $code.=<<___; .asciz "SHA512 block transform for ARMv4/NEON, CRYPTOGAMS by " .align 2 #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) -.comm OPENSSL_armcap_P,4,4 +.extern OPENSSL_armcap_P +.hidden OPENSSL_armcap_P #endif ___ diff --git a/crypto/openssl/crypto/sha/asm/sha512-armv8.pl b/crypto/openssl/crypto/sha/asm/sha512-armv8.pl index 7358b4b51767..a3bbac159971 100755 --- a/crypto/openssl/crypto/sha/asm/sha512-armv8.pl +++ b/crypto/openssl/crypto/sha/asm/sha512-armv8.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2014-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2014-2025 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -247,7 +247,8 @@ $code.=<<___; add $num,$inp,$num,lsl#`log(16*$SZ)/log(2)` // end of input #endif ldp $G,$H,[$ctx,#6*$SZ] - adr $Ktbl,.LK$BITS + adrp $Ktbl,.LK$BITS + add $Ktbl,$Ktbl,:lo12:.LK$BITS stp $ctx,$num,[PTR(29),#(12*PTR_WIDTH)] .Loop: @@ -297,6 +298,8 @@ $code.=<<___; ret .size $func,.-$func +.rodata + .align 6 .type .LK$BITS,%object .LK$BITS: @@ -367,6 +370,8 @@ $code.=<<___; .size .LK$BITS,.-.LK$BITS .asciz "SHA$BITS block transform for ARMv8, CRYPTOGAMS by " .align 2 + +.text ___ if ($SZ==4) { @@ -388,7 +393,8 @@ sha256_block_armv8: add PTR(29),PTRN(sp),#0 ld1.32 {$ABCD,$EFGH},[$ctx] - adr $Ktbl,.LK256 + adrp $Ktbl,.LK256 + add $Ktbl,$Ktbl,:lo12:.LK256 .Loop_hw: ld1 {@MSG[0]-@MSG[3]},[$inp],#64 @@ -654,7 +660,8 @@ sha256_block_neon: mov PTR(29), PTRN(sp) sub PTRN(sp),PTRN(sp),#16*4 - adr $Ktbl,.LK256 + adrp $Ktbl,.LK256 + add $Ktbl,$Ktbl,:lo12:.LK256 #ifdef __CHERI_PURE_CAPABILITY__ lsl $temp,$numx,#6 add $num,$inp,$temp // len to point at the end of inp @@ -781,7 +788,8 @@ sha512_block_armv8: ld1 {@MSG[4]-@MSG[7]},[$inp],#64 ld1.64 {@H[0]-@H[3]},[$ctx] // load context - adr $Ktbl,.LK512 + adrp $Ktbl,.LK512 + add $Ktbl,$Ktbl,:lo12:.LK512 rev64 @MSG[0],@MSG[0] rev64 @MSG[1],@MSG[1] diff --git a/crypto/openssl/crypto/sha/asm/sha512-x86_64.pl b/crypto/openssl/crypto/sha/asm/sha512-x86_64.pl index 1bd0256954b3..b37058ae03fa 100755 --- a/crypto/openssl/crypto/sha/asm/sha512-x86_64.pl +++ b/crypto/openssl/crypto/sha/asm/sha512-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2005-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2005-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -418,6 +418,7 @@ ___ if ($SZ==4) { $code.=<<___; +.section .rodata align=64 .align 64 .type $TABLE,\@object $TABLE: @@ -461,9 +462,11 @@ $TABLE: .long 0xffffffff,0xffffffff,0x03020100,0x0b0a0908 .long 0xffffffff,0xffffffff,0x03020100,0x0b0a0908 .asciz "SHA256 block transform for x86_64, CRYPTOGAMS by " +.previous ___ } else { $code.=<<___; +.section .rodata align=64 .align 64 .type $TABLE,\@object $TABLE: @@ -551,6 +554,7 @@ $TABLE: .quad 0x0001020304050607,0x08090a0b0c0d0e0f .quad 0x0001020304050607,0x08090a0b0c0d0e0f .asciz "SHA512 block transform for x86_64, CRYPTOGAMS by " +.previous ___ } diff --git a/crypto/openssl/crypto/whrlpool/asm/wp-x86_64.pl b/crypto/openssl/crypto/whrlpool/asm/wp-x86_64.pl index f94152af9769..e3421dcbdf18 100755 --- a/crypto/openssl/crypto/whrlpool/asm/wp-x86_64.pl +++ b/crypto/openssl/crypto/whrlpool/asm/wp-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2005-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2005-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -236,6 +236,7 @@ $code.=<<___; .cfi_endproc .size $func,.-$func +.section .rodata align=64 .align 64 .type $table,\@object $table: