commit 8b0b142b31397c0d179b1d20642e9f7f49c229b3 Merge: 3119cac49725 e7be843b4a16 Author: Pierre Pronchery Date: Thu Aug 7 15:50:32 2025 +0200 Merge commit '1095efe41feed8ea5a6fe5ca123c347ae0914801' Approved by: philip (mentor) Sponsored by: Alpha-Omega Beach Cleaning Project Sponsored by: The FreeBSD Foundation diff --git a/crypto/openssl/crypto/aes/asm/aes-riscv32-zkn.pl b/crypto/openssl/crypto/aes/asm/aes-riscv32-zkn.pl new file mode 100644 index 000000000000..6fac45184620 --- /dev/null +++ b/crypto/openssl/crypto/aes/asm/aes-riscv32-zkn.pl @@ -0,0 +1,1089 @@ +#! /usr/bin/env perl +# This file is dual-licensed, meaning that you can use it under your +# choice of either of the following two licenses: +# +# Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You can obtain +# a copy in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html +# +# or +# +# Copyright (c) 2022, Hongren (Zenithal) Zheng +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$output and open STDOUT,">$output"; + +################################################################################ +# Utility functions to help with keeping track of which registers to stack/ +# unstack when entering / exiting routines. +################################################################################ +{ + # Callee-saved registers + my @callee_saved = map("x$_",(2,8,9,18..27)); + # Caller-saved registers + my @caller_saved = map("x$_",(1,5..7,10..17,28..31)); + my @must_save; + sub use_reg { + my $reg = shift; + if (grep(/^$reg$/, @callee_saved)) { + push(@must_save, $reg); + } elsif (!grep(/^$reg$/, @caller_saved)) { + # Register is not usable! + die("Unusable register ".$reg); + } + return $reg; + } + sub use_regs { + return map(use_reg("x$_"), @_); + } + sub save_regs { + my $ret = ''; + my $stack_reservation = ($#must_save + 1) * 8; + my $stack_offset = $stack_reservation; + if ($stack_reservation % 16) { + $stack_reservation += 8; + } + $ret.=" addi sp,sp,-$stack_reservation\n"; + foreach (@must_save) { + $stack_offset -= 8; + $ret.=" sw $_,$stack_offset(sp)\n"; + } + return $ret; + } + sub load_regs { + my $ret = ''; + my $stack_reservation = ($#must_save + 1) * 8; + my $stack_offset = $stack_reservation; + if ($stack_reservation % 16) { + $stack_reservation += 8; + } + foreach (@must_save) { + $stack_offset -= 8; + $ret.=" lw $_,$stack_offset(sp)\n"; + } + $ret.=" addi sp,sp,$stack_reservation\n"; + return $ret; + } + sub clear_regs { + @must_save = (); + } +} + +################################################################################ +# util for encoding scalar crypto extension instructions +################################################################################ + +my @regs = map("x$_",(0..31)); +my %reglookup; +@reglookup{@regs} = @regs; + +# Takes a register name, possibly an alias, and converts it to a register index +# from 0 to 31 +sub read_reg { + my $reg = lc shift; + if (!exists($reglookup{$reg})) { + die("Unknown register ".$reg); + } + my $regstr = $reglookup{$reg}; + if (!($regstr =~ /^x([0-9]+)$/)) { + die("Could not process register ".$reg); + } + return $1; +} + +sub aes32dsi { + # Encoding for aes32dsi rd, rs1, rs2, bs instruction on RV32 + # bs_XXXXX_ rs2 _ rs1 _XXX_ rd _XXXXXXX + my $template = 0b00_10101_00000_00000_000_00000_0110011; + my $rd = read_reg shift; + my $rs1 = read_reg shift; + my $rs2 = read_reg shift; + my $bs = shift; + + return ".word ".($template | ($bs << 30) | ($rs2 << 20) | ($rs1 << 15) | ($rd << 7)); +} + +sub aes32dsmi { + # Encoding for aes32dsmi rd, rs1, rs2, bs instruction on RV32 + # bs_XXXXX_ rs2 _ rs1 _XXX_ rd _XXXXXXX + my $template = 0b00_10111_00000_00000_000_00000_0110011; + my $rd = read_reg shift; + my $rs1 = read_reg shift; + my $rs2 = read_reg shift; + my $bs = shift; + + return ".word ".($template | ($bs << 30) | ($rs2 << 20) | ($rs1 << 15) | ($rd << 7)); +} + +sub aes32esi { + # Encoding for aes32esi rd, rs1, rs2, bs instruction on RV32 + # bs_XXXXX_ rs2 _ rs1 _XXX_ rd _XXXXXXX + my $template = 0b00_10001_00000_00000_000_00000_0110011; + my $rd = read_reg shift; + my $rs1 = read_reg shift; + my $rs2 = read_reg shift; + my $bs = shift; + + return ".word ".($template | ($bs << 30) | ($rs2 << 20) | ($rs1 << 15) | ($rd << 7)); +} + +sub aes32esmi { + # Encoding for aes32esmi rd, rs1, rs2, bs instruction on RV32 + # bs_XXXXX_ rs2 _ rs1 _XXX_ rd _XXXXXXX + my $template = 0b00_10011_00000_00000_000_00000_0110011; + my $rd = read_reg shift; + my $rs1 = read_reg shift; + my $rs2 = read_reg shift; + my $bs = shift; + + return ".word ".($template | ($bs << 30) | ($rs2 << 20) | ($rs1 << 15) | ($rd << 7)); +} + +sub rori { + # Encoding for ror rd, rs1, imm instruction on RV64 + # XXXXXXX_shamt_ rs1 _XXX_ rd _XXXXXXX + my $template = 0b0110000_00000_00000_101_00000_0010011; + my $rd = read_reg shift; + my $rs1 = read_reg shift; + my $shamt = shift; + + return ".word ".($template | ($shamt << 20) | ($rs1 << 15) | ($rd << 7)); +} + +################################################################################ +# Register assignment for rv32i_zkne_encrypt and rv32i_zknd_decrypt +################################################################################ + +# Registers initially to hold AES state (called s0-s3 or y0-y3 elsewhere) +my ($Q0,$Q1,$Q2,$Q3) = use_regs(6..9); + +# Function arguments (x10-x12 are a0-a2 in the ABI) +# Input block pointer, output block pointer, key pointer +my ($INP,$OUTP,$KEYP) = use_regs(10..12); + +# Registers initially to hold Key +my ($T0,$T1,$T2,$T3) = use_regs(13..16); + +# Loop counter +my ($loopcntr) = use_regs(30); + +################################################################################ +# Utility for rv32i_zkne_encrypt and rv32i_zknd_decrypt +################################################################################ + +# outer product of whole state into one column of key +sub outer { + my $inst = shift; + my $key = shift; + # state 0 to 3 + my $s0 = shift; + my $s1 = shift; + my $s2 = shift; + my $s3 = shift; + my $ret = ''; +$ret .= <<___; + @{[$inst->($key,$key,$s0,0)]} + @{[$inst->($key,$key,$s1,1)]} + @{[$inst->($key,$key,$s2,2)]} + @{[$inst->($key,$key,$s3,3)]} +___ + return $ret; +} + +sub aes32esmi4 { + return outer(\&aes32esmi, @_) +} + +sub aes32esi4 { + return outer(\&aes32esi, @_) +} + +sub aes32dsmi4 { + return outer(\&aes32dsmi, @_) +} + +sub aes32dsi4 { + return outer(\&aes32dsi, @_) +} + +################################################################################ +# void rv32i_zkne_encrypt(const unsigned char *in, unsigned char *out, +# const AES_KEY *key); +################################################################################ +my $code .= <<___; +.text +.balign 16 +.globl rv32i_zkne_encrypt +.type rv32i_zkne_encrypt,\@function +rv32i_zkne_encrypt: +___ + +$code .= save_regs(); + +$code .= <<___; + # Load input to block cipher + lw $Q0,0($INP) + lw $Q1,4($INP) + lw $Q2,8($INP) + lw $Q3,12($INP) + + # Load key + lw $T0,0($KEYP) + lw $T1,4($KEYP) + lw $T2,8($KEYP) + lw $T3,12($KEYP) + + # Load number of rounds + lw $loopcntr,240($KEYP) + + # initial transformation + xor $Q0,$Q0,$T0 + xor $Q1,$Q1,$T1 + xor $Q2,$Q2,$T2 + xor $Q3,$Q3,$T3 + + # The main loop only executes the first N-2 rounds, each loop consumes two rounds + add $loopcntr,$loopcntr,-2 + srli $loopcntr,$loopcntr,1 +1: + # Grab next key in schedule + add $KEYP,$KEYP,16 + lw $T0,0($KEYP) + lw $T1,4($KEYP) + lw $T2,8($KEYP) + lw $T3,12($KEYP) + + @{[aes32esmi4 $T0,$Q0,$Q1,$Q2,$Q3]} + @{[aes32esmi4 $T1,$Q1,$Q2,$Q3,$Q0]} + @{[aes32esmi4 $T2,$Q2,$Q3,$Q0,$Q1]} + @{[aes32esmi4 $T3,$Q3,$Q0,$Q1,$Q2]} + # now T0~T3 hold the new state + + # Grab next key in schedule + add $KEYP,$KEYP,16 + lw $Q0,0($KEYP) + lw $Q1,4($KEYP) + lw $Q2,8($KEYP) + lw $Q3,12($KEYP) + + @{[aes32esmi4 $Q0,$T0,$T1,$T2,$T3]} + @{[aes32esmi4 $Q1,$T1,$T2,$T3,$T0]} + @{[aes32esmi4 $Q2,$T2,$T3,$T0,$T1]} + @{[aes32esmi4 $Q3,$T3,$T0,$T1,$T2]} + # now Q0~Q3 hold the new state + + add $loopcntr,$loopcntr,-1 + bgtz $loopcntr,1b + +# final two rounds + # Grab next key in schedule + add $KEYP,$KEYP,16 + lw $T0,0($KEYP) + lw $T1,4($KEYP) + lw $T2,8($KEYP) + lw $T3,12($KEYP) + + @{[aes32esmi4 $T0,$Q0,$Q1,$Q2,$Q3]} + @{[aes32esmi4 $T1,$Q1,$Q2,$Q3,$Q0]} + @{[aes32esmi4 $T2,$Q2,$Q3,$Q0,$Q1]} + @{[aes32esmi4 $T3,$Q3,$Q0,$Q1,$Q2]} + # now T0~T3 hold the new state + + # Grab next key in schedule + add $KEYP,$KEYP,16 + lw $Q0,0($KEYP) + lw $Q1,4($KEYP) + lw $Q2,8($KEYP) + lw $Q3,12($KEYP) + + # no mix column now + @{[aes32esi4 $Q0,$T0,$T1,$T2,$T3]} + @{[aes32esi4 $Q1,$T1,$T2,$T3,$T0]} + @{[aes32esi4 $Q2,$T2,$T3,$T0,$T1]} + @{[aes32esi4 $Q3,$T3,$T0,$T1,$T2]} + # now Q0~Q3 hold the new state + + sw $Q0,0($OUTP) + sw $Q1,4($OUTP) + sw $Q2,8($OUTP) + sw $Q3,12($OUTP) + + # Pop registers and return +___ + +$code .= load_regs(); + +$code .= <<___; + ret +___ + +################################################################################ +# void rv32i_zknd_decrypt(const unsigned char *in, unsigned char *out, +# const AES_KEY *key); +################################################################################ +$code .= <<___; +.text +.balign 16 +.globl rv32i_zknd_decrypt +.type rv32i_zknd_decrypt,\@function +rv32i_zknd_decrypt: +___ + +$code .= save_regs(); + +$code .= <<___; + # Load input to block cipher + lw $Q0,0($INP) + lw $Q1,4($INP) + lw $Q2,8($INP) + lw $Q3,12($INP) + + # Load number of rounds + lw $loopcntr,240($KEYP) + + # Load the last key + # use T0 as temporary now + slli $T0,$loopcntr,4 + add $KEYP,$KEYP,$T0 + # Load key + lw $T0,0($KEYP) + lw $T1,4($KEYP) + lw $T2,8($KEYP) + lw $T3,12($KEYP) + + # initial transformation + xor $Q0,$Q0,$T0 + xor $Q1,$Q1,$T1 + xor $Q2,$Q2,$T2 + xor $Q3,$Q3,$T3 + + # The main loop only executes the first N-2 rounds, each loop consumes two rounds + add $loopcntr,$loopcntr,-2 + srli $loopcntr,$loopcntr,1 +1: + # Grab next key in schedule + add $KEYP,$KEYP,-16 + lw $T0,0($KEYP) + lw $T1,4($KEYP) + lw $T2,8($KEYP) + lw $T3,12($KEYP) + + @{[aes32dsmi4 $T0,$Q0,$Q3,$Q2,$Q1]} + @{[aes32dsmi4 $T1,$Q1,$Q0,$Q3,$Q2]} + @{[aes32dsmi4 $T2,$Q2,$Q1,$Q0,$Q3]} + @{[aes32dsmi4 $T3,$Q3,$Q2,$Q1,$Q0]} + # now T0~T3 hold the new state + + # Grab next key in schedule + add $KEYP,$KEYP,-16 + lw $Q0,0($KEYP) + lw $Q1,4($KEYP) + lw $Q2,8($KEYP) + lw $Q3,12($KEYP) + + @{[aes32dsmi4 $Q0,$T0,$T3,$T2,$T1]} + @{[aes32dsmi4 $Q1,$T1,$T0,$T3,$T2]} + @{[aes32dsmi4 $Q2,$T2,$T1,$T0,$T3]} + @{[aes32dsmi4 $Q3,$T3,$T2,$T1,$T0]} + # now Q0~Q3 hold the new state + + add $loopcntr,$loopcntr,-1 + bgtz $loopcntr,1b + +# final two rounds + # Grab next key in schedule + add $KEYP,$KEYP,-16 + lw $T0,0($KEYP) + lw $T1,4($KEYP) + lw $T2,8($KEYP) + lw $T3,12($KEYP) + + @{[aes32dsmi4 $T0,$Q0,$Q3,$Q2,$Q1]} + @{[aes32dsmi4 $T1,$Q1,$Q0,$Q3,$Q2]} + @{[aes32dsmi4 $T2,$Q2,$Q1,$Q0,$Q3]} + @{[aes32dsmi4 $T3,$Q3,$Q2,$Q1,$Q0]} + # now T0~T3 hold the new state + + # Grab next key in schedule + add $KEYP,$KEYP,-16 + lw $Q0,0($KEYP) + lw $Q1,4($KEYP) + lw $Q2,8($KEYP) + lw $Q3,12($KEYP) + + # no mix column now + @{[aes32dsi4 $Q0,$T0,$T3,$T2,$T1]} + @{[aes32dsi4 $Q1,$T1,$T0,$T3,$T2]} + @{[aes32dsi4 $Q2,$T2,$T1,$T0,$T3]} + @{[aes32dsi4 $Q3,$T3,$T2,$T1,$T0]} + # now Q0~Q3 hold the new state + + sw $Q0,0($OUTP) + sw $Q1,4($OUTP) + sw $Q2,8($OUTP) + sw $Q3,12($OUTP) + + # Pop registers and return +___ + +$code .= load_regs(); + +$code .= <<___; + ret +___ + +clear_regs(); + +################################################################################ +# Register assignment for rv32i_zkn[e/d]_set_[en/de]crypt +################################################################################ + +# Function arguments (x10-x12 are a0-a2 in the ABI) +# Pointer to user key, number of bits in key, key pointer +my ($UKEY,$BITS,$KEYP) = use_regs(10..12); + +# Temporaries +my ($T0,$T1,$T2,$T3,$T4,$T5,$T6,$T7,$T8) = use_regs(13..17,28..31); + +################################################################################ +# utility functions for rv32i_zkne_set_encrypt_key +################################################################################ + +my @rcon = (0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36); + +# do 4 sbox on 4 bytes of rs, (possibly mix), then xor with rd +sub sbox4 { + my $inst = shift; + my $rd = shift; + my $rs = shift; + my $ret = <<___; + @{[$inst->($rd,$rd,$rs,0)]} + @{[$inst->($rd,$rd,$rs,1)]} + @{[$inst->($rd,$rd,$rs,2)]} + @{[$inst->($rd,$rd,$rs,3)]} +___ + return $ret; +} + +sub fwdsbox4 { + return sbox4(\&aes32esi, @_); +} + +sub ke128enc { + my $zbkb = shift; + my $rnum = 0; + my $ret = ''; +$ret .= <<___; + lw $T0,0($UKEY) + lw $T1,4($UKEY) + lw $T2,8($UKEY) + lw $T3,12($UKEY) + + sw $T0,0($KEYP) + sw $T1,4($KEYP) + sw $T2,8($KEYP) + sw $T3,12($KEYP) +___ + while($rnum < 10) { +$ret .= <<___; + # use T4 to store rcon + li $T4,$rcon[$rnum] + # as xor is associative and commutative + # we fist xor T0 with RCON, then use T0 to + # xor the result of each SBOX result of T3 + xor $T0,$T0,$T4 + # use T4 to store rotated T3 +___ + # right rotate by 8 + if ($zbkb) { +$ret .= <<___; + @{[rori $T4,$T3,8]} +___ + } else { +$ret .= <<___; + srli $T4,$T3,8 + slli $T5,$T3,24 + or $T4,$T4,$T5 +___ + } +$ret .= <<___; + # update T0 + @{[fwdsbox4 $T0,$T4]} + + # update new T1~T3 + xor $T1,$T1,$T0 + xor $T2,$T2,$T1 + xor $T3,$T3,$T2 + + add $KEYP,$KEYP,16 + sw $T0,0($KEYP) + sw $T1,4($KEYP) + sw $T2,8($KEYP) + sw $T3,12($KEYP) +___ + $rnum++; + } + return $ret; +} + +sub ke192enc { + my $zbkb = shift; + my $rnum = 0; + my $ret = ''; +$ret .= <<___; + lw $T0,0($UKEY) + lw $T1,4($UKEY) + lw $T2,8($UKEY) + lw $T3,12($UKEY) + lw $T4,16($UKEY) + lw $T5,20($UKEY) + + sw $T0,0($KEYP) + sw $T1,4($KEYP) + sw $T2,8($KEYP) + sw $T3,12($KEYP) + sw $T4,16($KEYP) + sw $T5,20($KEYP) +___ + while($rnum < 8) { +$ret .= <<___; + # see the comment in ke128enc + li $T6,$rcon[$rnum] + xor $T0,$T0,$T6 +___ + # right rotate by 8 + if ($zbkb) { +$ret .= <<___; + @{[rori $T6,$T5,8]} +___ + } else { +$ret .= <<___; + srli $T6,$T5,8 + slli $T7,$T5,24 + or $T6,$T6,$T7 +___ + } +$ret .= <<___; + @{[fwdsbox4 $T0,$T6]} + xor $T1,$T1,$T0 + xor $T2,$T2,$T1 + xor $T3,$T3,$T2 +___ + if ($rnum != 7) { + # note that (8+1)*24 = 216, (12+1)*16 = 208 + # thus the last 8 bytes can be dropped +$ret .= <<___; + xor $T4,$T4,$T3 + xor $T5,$T5,$T4 +___ + } +$ret .= <<___; + add $KEYP,$KEYP,24 + sw $T0,0($KEYP) + sw $T1,4($KEYP) + sw $T2,8($KEYP) + sw $T3,12($KEYP) +___ + if ($rnum != 7) { +$ret .= <<___; + sw $T4,16($KEYP) + sw $T5,20($KEYP) +___ + } + $rnum++; + } + return $ret; +} + +sub ke256enc { + my $zbkb = shift; + my $rnum = 0; + my $ret = ''; +$ret .= <<___; + lw $T0,0($UKEY) + lw $T1,4($UKEY) + lw $T2,8($UKEY) + lw $T3,12($UKEY) + lw $T4,16($UKEY) + lw $T5,20($UKEY) + lw $T6,24($UKEY) + lw $T7,28($UKEY) + + sw $T0,0($KEYP) + sw $T1,4($KEYP) + sw $T2,8($KEYP) + sw $T3,12($KEYP) + sw $T4,16($KEYP) + sw $T5,20($KEYP) + sw $T6,24($KEYP) + sw $T7,28($KEYP) +___ + while($rnum < 7) { +$ret .= <<___; + # see the comment in ke128enc + li $T8,$rcon[$rnum] + xor $T0,$T0,$T8 +___ + # right rotate by 8 + if ($zbkb) { +$ret .= <<___; + @{[rori $T8,$T7,8]} +___ + } else { +$ret .= <<___; + srli $T8,$T7,8 + slli $BITS,$T7,24 + or $T8,$T8,$BITS +___ + } +$ret .= <<___; + @{[fwdsbox4 $T0,$T8]} + xor $T1,$T1,$T0 + xor $T2,$T2,$T1 + xor $T3,$T3,$T2 + + add $KEYP,$KEYP,32 + sw $T0,0($KEYP) + sw $T1,4($KEYP) + sw $T2,8($KEYP) + sw $T3,12($KEYP) +___ + if ($rnum != 6) { + # note that (7+1)*32 = 256, (14+1)*16 = 240 + # thus the last 16 bytes can be dropped +$ret .= <<___; + # for aes256, T3->T4 needs 4sbox but no rotate/rcon + @{[fwdsbox4 $T4,$T3]} + xor $T5,$T5,$T4 + xor $T6,$T6,$T5 + xor $T7,$T7,$T6 + sw $T4,16($KEYP) + sw $T5,20($KEYP) + sw $T6,24($KEYP) + sw $T7,28($KEYP) +___ + } + $rnum++; + } + return $ret; +} + +################################################################################ +# void rv32i_zkne_set_encrypt_key(const unsigned char *userKey, const int bits, +# AES_KEY *key) +################################################################################ +sub AES_set_common { + my ($ke128, $ke192, $ke256) = @_; + my $ret = ''; +$ret .= <<___; + bnez $UKEY,1f # if (!userKey || !key) return -1; + bnez $KEYP,1f + li a0,-1 + ret +1: + # Determine number of rounds from key size in bits + li $T0,128 + bne $BITS,$T0,1f + li $T1,10 # key->rounds = 10 if bits == 128 + sw $T1,240($KEYP) # store key->rounds +$ke128 + j 4f +1: + li $T0,192 + bne $BITS,$T0,2f + li $T1,12 # key->rounds = 12 if bits == 192 + sw $T1,240($KEYP) # store key->rounds +$ke192 + j 4f +2: + li $T1,14 # key->rounds = 14 if bits == 256 + li $T0,256 + beq $BITS,$T0,3f + li a0,-2 # If bits != 128, 192, or 256, return -2 + j 5f +3: + sw $T1,240($KEYP) # store key->rounds +$ke256 +4: # return 0 + li a0,0 +5: # return a0 +___ + return $ret; +} +$code .= <<___; +.text +.balign 16 +.globl rv32i_zkne_set_encrypt_key +.type rv32i_zkne_set_encrypt_key,\@function +rv32i_zkne_set_encrypt_key: +___ + +$code .= save_regs(); +$code .= AES_set_common(ke128enc(0), ke192enc(0),ke256enc(0)); +$code .= load_regs(); +$code .= <<___; + ret +___ + +################################################################################ +# void rv32i_zbkb_zkne_set_encrypt_key(const unsigned char *userKey, +# const int bits, AES_KEY *key) +################################################################################ +$code .= <<___; +.text +.balign 16 +.globl rv32i_zbkb_zkne_set_encrypt_key +.type rv32i_zbkb_zkne_set_encrypt_key,\@function +rv32i_zbkb_zkne_set_encrypt_key: +___ + +$code .= save_regs(); +$code .= AES_set_common(ke128enc(1), ke192enc(1),ke256enc(1)); +$code .= load_regs(); +$code .= <<___; + ret +___ + +################################################################################ +# utility functions for rv32i_zknd_zkne_set_decrypt_key +################################################################################ + +sub invm4 { + # fwd sbox then inv sbox then mix column + # the result is only mix column + # this simulates aes64im T0 + my $rd = shift; + my $tmp = shift; + my $rs = shift; + my $ret = <<___; + li $tmp,0 + li $rd,0 + @{[fwdsbox4 $tmp,$rs]} + @{[sbox4(\&aes32dsmi, $rd,$tmp)]} +___ + return $ret; +} + +sub ke128dec { + my $zbkb = shift; + my $rnum = 0; + my $ret = ''; +$ret .= <<___; + lw $T0,0($UKEY) + lw $T1,4($UKEY) + lw $T2,8($UKEY) + lw $T3,12($UKEY) + + sw $T0,0($KEYP) + sw $T1,4($KEYP) + sw $T2,8($KEYP) + sw $T3,12($KEYP) +___ + while($rnum < 10) { +$ret .= <<___; + # see comments in ke128enc + li $T4,$rcon[$rnum] + xor $T0,$T0,$T4 +___ + # right rotate by 8 + if ($zbkb) { +$ret .= <<___; + @{[rori $T4,$T3,8]} +___ + } else { +$ret .= <<___; + srli $T4,$T3,8 + slli $T5,$T3,24 + or $T4,$T4,$T5 +___ + } +$ret .= <<___; + @{[fwdsbox4 $T0,$T4]} + xor $T1,$T1,$T0 + xor $T2,$T2,$T1 + xor $T3,$T3,$T2 + add $KEYP,$KEYP,16 +___ + # need to mixcolumn only for [1:N-1] round keys + # this is from the fact that aes32dsmi subwords first then mix column + # intuitively decryption needs to first mix column then subwords + # however, for merging datapaths (encryption first subwords then mix column) + # aes32dsmi chooses to inverse the order of them, thus + # transform should then be done on the round key + if ($rnum < 9) { +$ret .= <<___; + # T4 and T5 are temp variables + @{[invm4 $T5,$T4,$T0]} + sw $T5,0($KEYP) + @{[invm4 $T5,$T4,$T1]} + sw $T5,4($KEYP) + @{[invm4 $T5,$T4,$T2]} + sw $T5,8($KEYP) + @{[invm4 $T5,$T4,$T3]} + sw $T5,12($KEYP) +___ + } else { +$ret .= <<___; + sw $T0,0($KEYP) + sw $T1,4($KEYP) + sw $T2,8($KEYP) + sw $T3,12($KEYP) +___ + } + $rnum++; + } + return $ret; +} + +sub ke192dec { + my $zbkb = shift; + my $rnum = 0; + my $ret = ''; +$ret .= <<___; + lw $T0,0($UKEY) + lw $T1,4($UKEY) + lw $T2,8($UKEY) + lw $T3,12($UKEY) + lw $T4,16($UKEY) + lw $T5,20($UKEY) + + sw $T0,0($KEYP) + sw $T1,4($KEYP) + sw $T2,8($KEYP) + sw $T3,12($KEYP) + # see the comment in ke128dec + # T7 and T6 are temp variables + @{[invm4 $T7,$T6,$T4]} + sw $T7,16($KEYP) + @{[invm4 $T7,$T6,$T5]} + sw $T7,20($KEYP) +___ + while($rnum < 8) { +$ret .= <<___; + # see the comment in ke128enc + li $T6,$rcon[$rnum] + xor $T0,$T0,$T6 +___ + # right rotate by 8 + if ($zbkb) { +$ret .= <<___; + @{[rori $T6,$T5,8]} +___ + } else { +$ret .= <<___; + srli $T6,$T5,8 + slli $T7,$T5,24 + or $T6,$T6,$T7 +___ + } +$ret .= <<___; + @{[fwdsbox4 $T0,$T6]} + xor $T1,$T1,$T0 + xor $T2,$T2,$T1 + xor $T3,$T3,$T2 + + add $KEYP,$KEYP,24 +___ + if ($rnum < 7) { +$ret .= <<___; + xor $T4,$T4,$T3 + xor $T5,$T5,$T4 + + # see the comment in ke128dec + # T7 and T6 are temp variables + @{[invm4 $T7,$T6,$T0]} + sw $T7,0($KEYP) + @{[invm4 $T7,$T6,$T1]} + sw $T7,4($KEYP) + @{[invm4 $T7,$T6,$T2]} + sw $T7,8($KEYP) + @{[invm4 $T7,$T6,$T3]} + sw $T7,12($KEYP) + @{[invm4 $T7,$T6,$T4]} + sw $T7,16($KEYP) + @{[invm4 $T7,$T6,$T5]} + sw $T7,20($KEYP) +___ + } else { # rnum == 7 +$ret .= <<___; + # the reason for dropping T4/T5 is in ke192enc + # the reason for not invm4 is in ke128dec + sw $T0,0($KEYP) + sw $T1,4($KEYP) + sw $T2,8($KEYP) + sw $T3,12($KEYP) +___ + } + $rnum++; + } + return $ret; +} + +sub ke256dec { + my $zbkb = shift; + my $rnum = 0; + my $ret = ''; +$ret .= <<___; + lw $T0,0($UKEY) + lw $T1,4($UKEY) + lw $T2,8($UKEY) + lw $T3,12($UKEY) + lw $T4,16($UKEY) + lw $T5,20($UKEY) + lw $T6,24($UKEY) + lw $T7,28($UKEY) + + sw $T0,0($KEYP) + sw $T1,4($KEYP) + sw $T2,8($KEYP) + sw $T3,12($KEYP) + # see the comment in ke128dec + # BITS and T8 are temp variables + # BITS are not used anymore + @{[invm4 $T8,$BITS,$T4]} + sw $T8,16($KEYP) + @{[invm4 $T8,$BITS,$T5]} + sw $T8,20($KEYP) + @{[invm4 $T8,$BITS,$T6]} + sw $T8,24($KEYP) + @{[invm4 $T8,$BITS,$T7]} + sw $T8,28($KEYP) +___ + while($rnum < 7) { +$ret .= <<___; + # see the comment in ke128enc + li $T8,$rcon[$rnum] + xor $T0,$T0,$T8 +___ + # right rotate by 8 + if ($zbkb) { +$ret .= <<___; + @{[rori $T8,$T7,8]} +___ + } else { +$ret .= <<___; + srli $T8,$T7,8 + slli $BITS,$T7,24 + or $T8,$T8,$BITS +___ + } +$ret .= <<___; + @{[fwdsbox4 $T0,$T8]} + xor $T1,$T1,$T0 + xor $T2,$T2,$T1 + xor $T3,$T3,$T2 + + add $KEYP,$KEYP,32 +___ + if ($rnum < 6) { +$ret .= <<___; + # for aes256, T3->T4 needs 4sbox but no rotate/rcon + @{[fwdsbox4 $T4,$T3]} + xor $T5,$T5,$T4 + xor $T6,$T6,$T5 + xor $T7,$T7,$T6 + + # see the comment in ke128dec + # T8 and BITS are temp variables + @{[invm4 $T8,$BITS,$T0]} + sw $T8,0($KEYP) + @{[invm4 $T8,$BITS,$T1]} + sw $T8,4($KEYP) + @{[invm4 $T8,$BITS,$T2]} + sw $T8,8($KEYP) + @{[invm4 $T8,$BITS,$T3]} + sw $T8,12($KEYP) + @{[invm4 $T8,$BITS,$T4]} + sw $T8,16($KEYP) + @{[invm4 $T8,$BITS,$T5]} + sw $T8,20($KEYP) + @{[invm4 $T8,$BITS,$T6]} + sw $T8,24($KEYP) + @{[invm4 $T8,$BITS,$T7]} + sw $T8,28($KEYP) +___ + } else { +$ret .= <<___; + sw $T0,0($KEYP) + sw $T1,4($KEYP) + sw $T2,8($KEYP) + sw $T3,12($KEYP) + # last 16 bytes are dropped + # see the comment in ke256enc +___ + } + $rnum++; + } + return $ret; +} + +################################################################################ +# void rv32i_zknd_zkne_set_decrypt_key(const unsigned char *userKey, const int bits, +# AES_KEY *key) +################################################################################ +# a note on naming: set_decrypt_key needs aes32esi thus add zkne on name +$code .= <<___; +.text +.balign 16 +.globl rv32i_zknd_zkne_set_decrypt_key +.type rv32i_zknd_zkne_set_decrypt_key,\@function +rv32i_zknd_zkne_set_decrypt_key: +___ +$code .= save_regs(); +$code .= AES_set_common(ke128dec(0), ke192dec(0),ke256dec(0)); +$code .= load_regs(); +$code .= <<___; + ret +___ + +################################################################################ +# void rv32i_zbkb_zknd_zkne_set_decrypt_key(const unsigned char *userKey, +# const int bits, AES_KEY *key) +################################################################################ +$code .= <<___; +.text +.balign 16 +.globl rv32i_zbkb_zknd_zkne_set_decrypt_key +.type rv32i_zbkb_zknd_zkne_set_decrypt_key,\@function +rv32i_zbkb_zknd_zkne_set_decrypt_key: +___ + +$code .= save_regs(); +$code .= AES_set_common(ke128dec(1), ke192dec(1),ke256dec(1)); +$code .= load_regs(); +$code .= <<___; + ret +___ + + + +print $code; +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/aes/asm/aes-riscv64-zkn.pl b/crypto/openssl/crypto/aes/asm/aes-riscv64-zkn.pl new file mode 100644 index 000000000000..0e8a1540c438 --- /dev/null +++ b/crypto/openssl/crypto/aes/asm/aes-riscv64-zkn.pl @@ -0,0 +1,594 @@ +#! /usr/bin/env perl +# This file is dual-licensed, meaning that you can use it under your +# choice of either of the following two licenses: +# +# Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You can obtain +# a copy in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html +# +# or +# +# Copyright (c) 2022, Hongren (Zenithal) Zheng +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +use strict; +use warnings; + +use FindBin qw($Bin); +use lib "$Bin"; +use lib "$Bin/../../perlasm"; +use riscv; + +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$output and open STDOUT,">$output"; + +################################################################################ +# Utility functions to help with keeping track of which registers to stack/ +# unstack when entering / exiting routines. +################################################################################ +{ + # Callee-saved registers + my @callee_saved = map("x$_",(2,8,9,18..27)); + # Caller-saved registers + my @caller_saved = map("x$_",(1,5..7,10..17,28..31)); + my @must_save; + sub use_reg { + my $reg = shift; + if (grep(/^$reg$/, @callee_saved)) { + push(@must_save, $reg); + } elsif (!grep(/^$reg$/, @caller_saved)) { + # Register is not usable! + die("Unusable register ".$reg); + } + return $reg; + } + sub use_regs { + return map(use_reg("x$_"), @_); + } + sub save_regs { + my $ret = ''; + my $stack_reservation = ($#must_save + 1) * 8; + my $stack_offset = $stack_reservation; + if ($stack_reservation % 16) { + $stack_reservation += 8; + } + $ret.=" addi sp,sp,-$stack_reservation\n"; + foreach (@must_save) { + $stack_offset -= 8; + $ret.=" sd $_,$stack_offset(sp)\n"; + } + return $ret; + } + sub load_regs { + my $ret = ''; + my $stack_reservation = ($#must_save + 1) * 8; + my $stack_offset = $stack_reservation; + if ($stack_reservation % 16) { + $stack_reservation += 8; + } + foreach (@must_save) { + $stack_offset -= 8; + $ret.=" ld $_,$stack_offset(sp)\n"; + } + $ret.=" addi sp,sp,$stack_reservation\n"; + return $ret; + } + sub clear_regs { + @must_save = (); + } +} + +################################################################################ +# Register assignment for rv64i_zkne_encrypt and rv64i_zknd_decrypt +################################################################################ + +# Registers to hold AES state (called s0-s3 or y0-y3 elsewhere) +my ($Q0,$Q1,$Q2,$Q3) = use_regs(6..9); + +# Function arguments (x10-x12 are a0-a2 in the ABI) +# Input block pointer, output block pointer, key pointer +my ($INP,$OUTP,$KEYP) = use_regs(10..12); + +# Temporaries +my ($T0,$T1) = use_regs(13..14); + +# Loop counter +my ($loopcntr) = use_regs(30); + +################################################################################ +# void rv64i_zkne_encrypt(const unsigned char *in, unsigned char *out, +# const AES_KEY *key); +################################################################################ +my $code .= <<___; +.text +.balign 16 +.globl rv64i_zkne_encrypt +.type rv64i_zkne_encrypt,\@function +rv64i_zkne_encrypt: +___ + +$code .= save_regs(); + +$code .= <<___; + + # Load input to block cipher + ld $Q0,0($INP) + ld $Q1,8($INP) + + # Load key + ld $T0,0($KEYP) + ld $T1,8($KEYP) + + # Load number of rounds + lwu $loopcntr,240($KEYP) + + # initial transformation + xor $Q0,$Q0,$T0 + xor $Q1,$Q1,$T1 + + # The main loop only executes the first N-1 rounds. + add $loopcntr,$loopcntr,-1 + + # Do Nr - 1 rounds (final round is special) +1: + @{[aes64esm $Q2,$Q0,$Q1]} + @{[aes64esm $Q3,$Q1,$Q0]} + + # Update key ptr to point to next key in schedule + add $KEYP,$KEYP,16 + + # Grab next key in schedule + ld $T0,0($KEYP) + ld $T1,8($KEYP) + xor $Q0,$Q2,$T0 + xor $Q1,$Q3,$T1 + + add $loopcntr,$loopcntr,-1 + bgtz $loopcntr,1b + + # final round + @{[aes64es $Q2,$Q0,$Q1]} + @{[aes64es $Q3,$Q1,$Q0]} + + # since not added 16 before + ld $T0,16($KEYP) + ld $T1,24($KEYP) + xor $Q0,$Q2,$T0 + xor $Q1,$Q3,$T1 + + sd $Q0,0($OUTP) + sd $Q1,8($OUTP) + + # Pop registers and return +___ + +$code .= load_regs(); + +$code .= <<___; + ret +___ + +################################################################################ +# void rv64i_zknd_decrypt(const unsigned char *in, unsigned char *out, +# const AES_KEY *key); +################################################################################ +$code .= <<___; +.text +.balign 16 +.globl rv64i_zknd_decrypt +.type rv64i_zknd_decrypt,\@function +rv64i_zknd_decrypt: +___ + +$code .= save_regs(); + +$code .= <<___; + + # Load input to block cipher + ld $Q0,0($INP) + ld $Q1,8($INP) + + # Load number of rounds + lwu $loopcntr,240($KEYP) + + # Load the last key + slli $T0,$loopcntr,4 + add $KEYP,$KEYP,$T0 + ld $T0,0($KEYP) + ld $T1,8($KEYP) + + xor $Q0,$Q0,$T0 + xor $Q1,$Q1,$T1 + + # The main loop only executes the first N-1 rounds. + add $loopcntr,$loopcntr,-1 + + # Do Nr - 1 rounds (final round is special) +1: + @{[aes64dsm $Q2,$Q0,$Q1]} + @{[aes64dsm $Q3,$Q1,$Q0]} + + # Update key ptr to point to next key in schedule + add $KEYP,$KEYP,-16 + + # Grab next key in schedule + ld $T0,0($KEYP) + ld $T1,8($KEYP) + xor $Q0,$Q2,$T0 + xor $Q1,$Q3,$T1 + + add $loopcntr,$loopcntr,-1 + bgtz $loopcntr,1b + + # final round + @{[aes64ds $Q2,$Q0,$Q1]} + @{[aes64ds $Q3,$Q1,$Q0]} + + add $KEYP,$KEYP,-16 + ld $T0,0($KEYP) + ld $T1,8($KEYP) + xor $Q0,$Q2,$T0 + xor $Q1,$Q3,$T1 + + sd $Q0,0($OUTP) + sd $Q1,8($OUTP) + # Pop registers and return +___ + +$code .= load_regs(); + +$code .= <<___; + ret +___ + +clear_regs(); + +################################################################################ +# Register assignment for rv64i_zkn[e/d]_set_[en/de]crypt_key +################################################################################ + +# Function arguments (x10-x12 are a0-a2 in the ABI) +# Pointer to user key, number of bits in key, key pointer +my ($UKEY,$BITS,$KEYP) = use_regs(10..12); + +# Temporaries +my ($T0,$T1,$T2,$T3,$T4) = use_regs(6..8,13..14); + +################################################################################ +# utility functions for rv64i_zkne_set_encrypt_key +################################################################################ +sub ke128enc { + my $rnum = 0; + my $ret = ''; +$ret .= <<___; + ld $T0,0($UKEY) + ld $T1,8($UKEY) + sd $T0,0($KEYP) + sd $T1,8($KEYP) +___ + while($rnum < 10) { +$ret .= <<___; + @{[aes64ks1i $T2,$T1,$rnum]} + @{[aes64ks2 $T0,$T2,$T0]} + @{[aes64ks2 $T1,$T0,$T1]} + add $KEYP,$KEYP,16 + sd $T0,0($KEYP) + sd $T1,8($KEYP) +___ + $rnum++; + } + return $ret; +} + +sub ke192enc { + my $rnum = 0; + my $ret = ''; +$ret .= <<___; + ld $T0,0($UKEY) + ld $T1,8($UKEY) + ld $T2,16($UKEY) + sd $T0,0($KEYP) + sd $T1,8($KEYP) + sd $T2,16($KEYP) +___ + while($rnum < 8) { +$ret .= <<___; + @{[aes64ks1i $T3,$T2,$rnum]} + @{[aes64ks2 $T0,$T3,$T0]} + @{[aes64ks2 $T1,$T0,$T1]} +___ + if ($rnum != 7) { + # note that (8+1)*24 = 216, (12+1)*16 = 208 + # thus the last 8 bytes can be dropped +$ret .= <<___; + @{[aes64ks2 $T2,$T1,$T2]} +___ + } +$ret .= <<___; + add $KEYP,$KEYP,24 + sd $T0,0($KEYP) + sd $T1,8($KEYP) +___ + if ($rnum != 7) { +$ret .= <<___; + sd $T2,16($KEYP) +___ + } + $rnum++; + } + return $ret; +} + +sub ke256enc { + my $rnum = 0; + my $ret = ''; +$ret .= <<___; + ld $T0,0($UKEY) + ld $T1,8($UKEY) + ld $T2,16($UKEY) + ld $T3,24($UKEY) + sd $T0,0($KEYP) + sd $T1,8($KEYP) + sd $T2,16($KEYP) + sd $T3,24($KEYP) +___ + while($rnum < 7) { +$ret .= <<___; + @{[aes64ks1i $T4,$T3,$rnum]} + @{[aes64ks2 $T0,$T4,$T0]} + @{[aes64ks2 $T1,$T0,$T1]} + add $KEYP,$KEYP,32 + sd $T0,0($KEYP) + sd $T1,8($KEYP) +___ + if ($rnum != 6) { + # note that (7+1)*32 = 256, (14+1)*16 = 240 + # thus the last 16 bytes can be dropped +$ret .= <<___; + @{[aes64ks1i $T4,$T1,0xA]} + @{[aes64ks2 $T2,$T4,$T2]} + @{[aes64ks2 $T3,$T2,$T3]} + sd $T2,16($KEYP) + sd $T3,24($KEYP) +___ + } + $rnum++; + } + return $ret; +} + +################################################################################ +# void rv64i_zkne_set_encrypt_key(const unsigned char *userKey, const int bits, +# AES_KEY *key) +################################################################################ +sub AES_set_common { + my ($ke128, $ke192, $ke256) = @_; + my $ret = ''; +$ret .= <<___; + bnez $UKEY,1f # if (!userKey || !key) return -1; + bnez $KEYP,1f + li a0,-1 + ret +1: + # Determine number of rounds from key size in bits + li $T0,128 + bne $BITS,$T0,1f + li $T1,10 # key->rounds = 10 if bits == 128 + sw $T1,240($KEYP) # store key->rounds +$ke128 + j 4f +1: + li $T0,192 + bne $BITS,$T0,2f + li $T1,12 # key->rounds = 12 if bits == 192 + sw $T1,240($KEYP) # store key->rounds +$ke192 + j 4f +2: + li $T1,14 # key->rounds = 14 if bits == 256 + li $T0,256 + beq $BITS,$T0,3f + li a0,-2 # If bits != 128, 192, or 256, return -2 + j 5f +3: + sw $T1,240($KEYP) # store key->rounds +$ke256 +4: # return 0 + li a0,0 +5: # return a0 +___ + return $ret; +} +$code .= <<___; +.text +.balign 16 +.globl rv64i_zkne_set_encrypt_key +.type rv64i_zkne_set_encrypt_key,\@function +rv64i_zkne_set_encrypt_key: +___ +$code .= save_regs(); +$code .= AES_set_common(ke128enc(), ke192enc(),ke256enc()); +$code .= load_regs(); +$code .= <<___; + ret +___ + +################################################################################ +# utility functions for rv64i_zknd_set_decrypt_key +################################################################################ +sub ke128dec { + my $rnum = 0; + my $ret = ''; +$ret .= <<___; + ld $T0,0($UKEY) + ld $T1,8($UKEY) + sd $T0,0($KEYP) + sd $T1,8($KEYP) +___ + while($rnum < 10) { +$ret .= <<___; + @{[aes64ks1i $T2,$T1,$rnum]} + @{[aes64ks2 $T0,$T2,$T0]} + @{[aes64ks2 $T1,$T0,$T1]} + add $KEYP,$KEYP,16 +___ + # need to aes64im for [1:N-1] round keys + # this is from the fact that aes64dsm subwords first then mix column + # intuitively decryption needs to first mix column then subwords + # however, for merging datapaths (encryption first subwords then mix column) + # aes64dsm chooses to inverse the order of them, thus + # transform should then be done on the round key + if ($rnum < 9) { +$ret .= <<___; + @{[aes64im $T2,$T0]} + sd $T2,0($KEYP) + @{[aes64im $T2,$T1]} + sd $T2,8($KEYP) +___ + } else { +$ret .= <<___; + sd $T0,0($KEYP) + sd $T1,8($KEYP) +___ + } + $rnum++; + } + return $ret; +} + +sub ke192dec { + my $rnum = 0; + my $ret = ''; +$ret .= <<___; + ld $T0,0($UKEY) + ld $T1,8($UKEY) + ld $T2,16($UKEY) + sd $T0,0($KEYP) + sd $T1,8($KEYP) + @{[aes64im $T3,$T2]} + sd $T3,16($KEYP) +___ + while($rnum < 8) { +$ret .= <<___; + @{[aes64ks1i $T3,$T2,$rnum]} + @{[aes64ks2 $T0,$T3,$T0]} + @{[aes64ks2 $T1,$T0,$T1]} + add $KEYP,$KEYP,24 +___ + if ($rnum < 7) { +$ret .= <<___; + @{[aes64im $T3,$T0]} + sd $T3,0($KEYP) + @{[aes64im $T3,$T1]} + sd $T3,8($KEYP) + # the reason is in ke192enc + @{[aes64ks2 $T2,$T1,$T2]} + @{[aes64im $T3,$T2]} + sd $T3,16($KEYP) +___ + } else { # rnum == 7 +$ret .= <<___; + sd $T0,0($KEYP) + sd $T1,8($KEYP) +___ + } + $rnum++; + } + return $ret; +} + +sub ke256dec { + my $rnum = 0; + my $ret = ''; +$ret .= <<___; + ld $T0,0($UKEY) + ld $T1,8($UKEY) + ld $T2,16($UKEY) + ld $T3,24($UKEY) + sd $T0,0($KEYP) + sd $T1,8($KEYP) + @{[aes64im $T4,$T2]} + sd $T4,16($KEYP) + @{[aes64im $T4,$T3]} + sd $T4,24($KEYP) +___ + while($rnum < 7) { +$ret .= <<___; + @{[aes64ks1i $T4,$T3,$rnum]} + @{[aes64ks2 $T0,$T4,$T0]} + @{[aes64ks2 $T1,$T0,$T1]} + add $KEYP,$KEYP,32 +___ + if ($rnum < 6) { +$ret .= <<___; + @{[aes64ks1i $T4,$T1,0xA]} + @{[aes64ks2 $T2,$T4,$T2]} + @{[aes64ks2 $T3,$T2,$T3]} + @{[aes64im $T4,$T0]} + sd $T4,0($KEYP) + @{[aes64im $T4,$T1]} + sd $T4,8($KEYP) + @{[aes64im $T4,$T2]} + sd $T4,16($KEYP) + @{[aes64im $T4,$T3]} + sd $T4,24($KEYP) +___ + } else { +$ret .= <<___; + sd $T0,0($KEYP) + sd $T1,8($KEYP) + # last two one dropped +___ + } + $rnum++; + } + return $ret; +} + +################################################################################ +# void rv64i_zknd_set_decrypt_key(const unsigned char *userKey, const int bits, +# AES_KEY *key) +################################################################################ +$code .= <<___; +.text +.balign 16 +.globl rv64i_zknd_set_decrypt_key +.type rv64i_zknd_set_decrypt_key,\@function +rv64i_zknd_set_decrypt_key: +___ +$code .= save_regs(); +$code .= AES_set_common(ke128dec(), ke192dec(),ke256dec()); +$code .= load_regs(); +$code .= <<___; + ret +___ + +print $code; +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/aes/asm/aes-riscv64-zvbb-zvkg-zvkned.pl b/crypto/openssl/crypto/aes/asm/aes-riscv64-zvbb-zvkg-zvkned.pl new file mode 100644 index 000000000000..5fb2cc33f34c --- /dev/null +++ b/crypto/openssl/crypto/aes/asm/aes-riscv64-zvbb-zvkg-zvkned.pl @@ -0,0 +1,710 @@ +#! /usr/bin/env perl +# This file is dual-licensed, meaning that you can use it under your +# choice of either of the following two licenses: +# +# Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You can obtain +# a copy in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html +# +# or +# +# Copyright (c) 2023, Jerry Shih +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# - RV64I +# - RISC-V Vector ('V') with VLEN >= 128 +# - RISC-V Vector Bit-manipulation extension ('Zvbb') +# - RISC-V Vector GCM/GMAC extension ('Zvkg') +# - RISC-V Vector AES block cipher extension ('Zvkned') +# - RISC-V Zicclsm(Main memory supports misaligned loads/stores) + +use strict; +use warnings; + +use FindBin qw($Bin); +use lib "$Bin"; +use lib "$Bin/../../perlasm"; +use riscv; + +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$output and open STDOUT,">$output"; + +my $code=<<___; +.text +___ + +{ +################################################################################ +# void rv64i_zvbb_zvkg_zvkned_aes_xts_encrypt(const unsigned char *in, +# unsigned char *out, size_t length, +# const AES_KEY *key1, +# const AES_KEY *key2, +# const unsigned char iv[16]) +my ($INPUT, $OUTPUT, $LENGTH, $KEY1, $KEY2, $IV) = ("a0", "a1", "a2", "a3", "a4", "a5"); +my ($TAIL_LENGTH) = ("a6"); +my ($VL) = ("a7"); +my ($T0, $T1, $T2) = ("t0", "t1", "t2"); +my ($STORE_LEN32) = ("t3"); +my ($LEN32) = ("t4"); +my ($V0, $V1, $V2, $V3, $V4, $V5, $V6, $V7, + $V8, $V9, $V10, $V11, $V12, $V13, $V14, $V15, + $V16, $V17, $V18, $V19, $V20, $V21, $V22, $V23, + $V24, $V25, $V26, $V27, $V28, $V29, $V30, $V31, +) = map("v$_",(0..31)); + +sub compute_xts_iv0 { + my $code=<<___; + # Load number of rounds + lwu $T0, 240($KEY2) + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vle32_v $V28, $IV]} + @{[vle32_v $V29, $KEY2]} + @{[vaesz_vs $V28, $V29]} + addi $T0, $T0, -1 + addi $KEY2, $KEY2, 16 +1: + @{[vle32_v $V29, $KEY2]} + @{[vaesem_vs $V28, $V29]} + addi $T0, $T0, -1 + addi $KEY2, $KEY2, 16 + bnez $T0, 1b + @{[vle32_v $V29, $KEY2]} + @{[vaesef_vs $V28, $V29]} +___ + + return $code; +} + +# prepare input data(v24), iv(v28), bit-reversed-iv(v16), bit-reversed-iv-multiplier(v20) +sub init_first_round { + my $code=<<___; + # load input + @{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "ma"]} + @{[vle32_v $V24, $INPUT]} + + li $T0, 5 + # We could simplify the initialization steps if we have `block<=1`. + blt $LEN32, $T0, 1f + + # Note: We use `vgmul` for GF(2^128) multiplication. The `vgmul` uses + # different order of coefficients. We should use`vbrev8` to reverse the + # data when we use `vgmul`. + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vbrev8_v $V0, $V28]} + @{[vsetvli "zero", $LEN32, "e32", "m4", "ta", "ma"]} + @{[vmv_v_i $V16, 0]} + # v16: [r-IV0, r-IV0, ...] + @{[vaesz_vs $V16, $V0]} + + # Prepare GF(2^128) multiplier [1, x, x^2, x^3, ...] in v8. + slli $T0, $LEN32, 2 + @{[vsetvli "zero", $T0, "e32", "m1", "ta", "ma"]} + # v2: [`1`, `1`, `1`, `1`, ...] + @{[vmv_v_i $V2, 1]} + # v3: [`0`, `1`, `2`, `3`, ...] + @{[vid_v $V3]} + @{[vsetvli "zero", $T0, "e64", "m2", "ta", "ma"]} + # v4: [`1`, 0, `1`, 0, `1`, 0, `1`, 0, ...] + @{[vzext_vf2 $V4, $V2]} + # v6: [`0`, 0, `1`, 0, `2`, 0, `3`, 0, ...] + @{[vzext_vf2 $V6, $V3]} + slli $T0, $LEN32, 1 + @{[vsetvli "zero", $T0, "e32", "m2", "ta", "ma"]} + # v8: [1<<0=1, 0, 0, 0, 1<<1=x, 0, 0, 0, 1<<2=x^2, 0, 0, 0, ...] + @{[vwsll_vv $V8, $V4, $V6]} + + # Compute [r-IV0*1, r-IV0*x, r-IV0*x^2, r-IV0*x^3, ...] in v16 + @{[vsetvli "zero", $LEN32, "e32", "m4", "ta", "ma"]} + @{[vbrev8_v $V8, $V8]} + @{[vgmul_vv $V16, $V8]} + + # Compute [IV0*1, IV0*x, IV0*x^2, IV0*x^3, ...] in v28. + # Reverse the bits order back. + @{[vbrev8_v $V28, $V16]} + + # Prepare the x^n multiplier in v20. The `n` is the aes-xts block number + # in a LMUL=4 register group. + # n = ((VLEN*LMUL)/(32*4)) = ((VLEN*4)/(32*4)) + # = (VLEN/32) + # We could use vsetvli with `e32, m1` to compute the `n` number. + @{[vsetvli $T0, "zero", "e32", "m1", "ta", "ma"]} + li $T1, 1 + sll $T0, $T1, $T0 + @{[vsetivli "zero", 2, "e64", "m1", "ta", "ma"]} + @{[vmv_v_i $V0, 0]} + @{[vsetivli "zero", 1, "e64", "m1", "tu", "ma"]} + @{[vmv_v_x $V0, $T0]} + @{[vsetivli "zero", 2, "e64", "m1", "ta", "ma"]} + @{[vbrev8_v $V0, $V0]} + @{[vsetvli "zero", $LEN32, "e32", "m4", "ta", "ma"]} + @{[vmv_v_i $V20, 0]} + @{[vaesz_vs $V20, $V0]} + + j 2f +1: + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vbrev8_v $V16, $V28]} +2: +___ + + return $code; +} + +# prepare xts enc last block's input(v24) and iv(v28) +sub handle_xts_enc_last_block { + my $code=<<___; + bnez $TAIL_LENGTH, 1f + ret +1: + # slidedown second to last block + addi $VL, $VL, -4 + @{[vsetivli "zero", 4, "e32", "m4", "ta", "ma"]} + # ciphertext + @{[vslidedown_vx $V24, $V24, $VL]} + # multiplier + @{[vslidedown_vx $V16, $V16, $VL]} + + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vmv_v_v $V25, $V24]} + + # load last block into v24 + # note: We should load the last block before store the second to last block + # for in-place operation. + @{[vsetvli "zero", $TAIL_LENGTH, "e8", "m1", "tu", "ma"]} + @{[vle8_v $V24, $INPUT]} + + # setup `x` multiplier with byte-reversed order + # 0b00000010 => 0b01000000 (0x40) + li $T0, 0x40 + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vmv_v_i $V28, 0]} + @{[vsetivli "zero", 1, "e8", "m1", "tu", "ma"]} + @{[vmv_v_x $V28, $T0]} + + # compute IV for last block + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vgmul_vv $V16, $V28]} + @{[vbrev8_v $V28, $V16]} + + # store second to last block + @{[vsetvli "zero", $TAIL_LENGTH, "e8", "m1", "ta", "ma"]} + @{[vse8_v $V25, $OUTPUT]} +___ + + return $code; +} + +# prepare xts dec second to last block's input(v24) and iv(v29) and +# last block's and iv(v28) +sub handle_xts_dec_last_block { + my $code=<<___; + bnez $TAIL_LENGTH, 1f + ret +1: + # load second to last block's ciphertext + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vle32_v $V24, $INPUT]} + addi $INPUT, $INPUT, 16 + + # setup `x` multiplier with byte-reversed order + # 0b00000010 => 0b01000000 (0x40) + li $T0, 0x40 + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vmv_v_i $V20, 0]} + @{[vsetivli "zero", 1, "e8", "m1", "tu", "ma"]} + @{[vmv_v_x $V20, $T0]} + + beqz $LENGTH, 1f + # slidedown third to last block + addi $VL, $VL, -4 + @{[vsetivli "zero", 4, "e32", "m4", "ta", "ma"]} + # multiplier + @{[vslidedown_vx $V16, $V16, $VL]} + + # compute IV for last block + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vgmul_vv $V16, $V20]} + @{[vbrev8_v $V28, $V16]} + + # compute IV for second to last block + @{[vgmul_vv $V16, $V20]} + @{[vbrev8_v $V29, $V16]} + j 2f +1: + # compute IV for second to last block + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vgmul_vv $V16, $V20]} + @{[vbrev8_v $V29, $V16]} +2: +___ + + return $code; +} + +# Load all 11 round keys to v1-v11 registers. +sub aes_128_load_key { + my $code=<<___; + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vle32_v $V1, $KEY1]} + addi $KEY1, $KEY1, 16 + @{[vle32_v $V2, $KEY1]} + addi $KEY1, $KEY1, 16 + @{[vle32_v $V3, $KEY1]} + addi $KEY1, $KEY1, 16 + @{[vle32_v $V4, $KEY1]} + addi $KEY1, $KEY1, 16 + @{[vle32_v $V5, $KEY1]} + addi $KEY1, $KEY1, 16 + @{[vle32_v $V6, $KEY1]} + addi $KEY1, $KEY1, 16 + @{[vle32_v $V7, $KEY1]} + addi $KEY1, $KEY1, 16 + @{[vle32_v $V8, $KEY1]} + addi $KEY1, $KEY1, 16 + @{[vle32_v $V9, $KEY1]} + addi $KEY1, $KEY1, 16 + @{[vle32_v $V10, $KEY1]} + addi $KEY1, $KEY1, 16 + @{[vle32_v $V11, $KEY1]} +___ + + return $code; +} + +# Load all 15 round keys to v1-v15 registers. +sub aes_256_load_key { + my $code=<<___; + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vle32_v $V1, $KEY1]} + addi $KEY1, $KEY1, 16 + @{[vle32_v $V2, $KEY1]} + addi $KEY1, $KEY1, 16 + @{[vle32_v $V3, $KEY1]} + addi $KEY1, $KEY1, 16 + @{[vle32_v $V4, $KEY1]} + addi $KEY1, $KEY1, 16 + @{[vle32_v $V5, $KEY1]} + addi $KEY1, $KEY1, 16 + @{[vle32_v $V6, $KEY1]} + addi $KEY1, $KEY1, 16 + @{[vle32_v $V7, $KEY1]} + addi $KEY1, $KEY1, 16 + @{[vle32_v $V8, $KEY1]} + addi $KEY1, $KEY1, 16 + @{[vle32_v $V9, $KEY1]} + addi $KEY1, $KEY1, 16 + @{[vle32_v $V10, $KEY1]} + addi $KEY1, $KEY1, 16 + @{[vle32_v $V11, $KEY1]} + addi $KEY1, $KEY1, 16 + @{[vle32_v $V12, $KEY1]} + addi $KEY1, $KEY1, 16 + @{[vle32_v $V13, $KEY1]} + addi $KEY1, $KEY1, 16 + @{[vle32_v $V14, $KEY1]} + addi $KEY1, $KEY1, 16 + @{[vle32_v $V15, $KEY1]} +___ + + return $code; +} + +# aes-128 enc with round keys v1-v11 +sub aes_128_enc { + my $code=<<___; + @{[vaesz_vs $V24, $V1]} + @{[vaesem_vs $V24, $V2]} + @{[vaesem_vs $V24, $V3]} + @{[vaesem_vs $V24, $V4]} + @{[vaesem_vs $V24, $V5]} + @{[vaesem_vs $V24, $V6]} + @{[vaesem_vs $V24, $V7]} + @{[vaesem_vs $V24, $V8]} + @{[vaesem_vs $V24, $V9]} + @{[vaesem_vs $V24, $V10]} + @{[vaesef_vs $V24, $V11]} +___ + + return $code; +} + +# aes-128 dec with round keys v1-v11 +sub aes_128_dec { + my $code=<<___; + @{[vaesz_vs $V24, $V11]} + @{[vaesdm_vs $V24, $V10]} + @{[vaesdm_vs $V24, $V9]} + @{[vaesdm_vs $V24, $V8]} + @{[vaesdm_vs $V24, $V7]} + @{[vaesdm_vs $V24, $V6]} + @{[vaesdm_vs $V24, $V5]} + @{[vaesdm_vs $V24, $V4]} + @{[vaesdm_vs $V24, $V3]} + @{[vaesdm_vs $V24, $V2]} + @{[vaesdf_vs $V24, $V1]} +___ + + return $code; +} + +# aes-256 enc with round keys v1-v15 +sub aes_256_enc { + my $code=<<___; + @{[vaesz_vs $V24, $V1]} + @{[vaesem_vs $V24, $V2]} + @{[vaesem_vs $V24, $V3]} + @{[vaesem_vs $V24, $V4]} + @{[vaesem_vs $V24, $V5]} + @{[vaesem_vs $V24, $V6]} + @{[vaesem_vs $V24, $V7]} + @{[vaesem_vs $V24, $V8]} + @{[vaesem_vs $V24, $V9]} + @{[vaesem_vs $V24, $V10]} + @{[vaesem_vs $V24, $V11]} + @{[vaesem_vs $V24, $V12]} + @{[vaesem_vs $V24, $V13]} + @{[vaesem_vs $V24, $V14]} + @{[vaesef_vs $V24, $V15]} +___ + + return $code; +} + +# aes-256 dec with round keys v1-v15 +sub aes_256_dec { + my $code=<<___; + @{[vaesz_vs $V24, $V15]} + @{[vaesdm_vs $V24, $V14]} + @{[vaesdm_vs $V24, $V13]} + @{[vaesdm_vs $V24, $V12]} + @{[vaesdm_vs $V24, $V11]} + @{[vaesdm_vs $V24, $V10]} + @{[vaesdm_vs $V24, $V9]} + @{[vaesdm_vs $V24, $V8]} + @{[vaesdm_vs $V24, $V7]} + @{[vaesdm_vs $V24, $V6]} + @{[vaesdm_vs $V24, $V5]} + @{[vaesdm_vs $V24, $V4]} + @{[vaesdm_vs $V24, $V3]} + @{[vaesdm_vs $V24, $V2]} + @{[vaesdf_vs $V24, $V1]} +___ + + return $code; +} + +$code .= <<___; +.p2align 3 +.globl rv64i_zvbb_zvkg_zvkned_aes_xts_encrypt +.type rv64i_zvbb_zvkg_zvkned_aes_xts_encrypt,\@function +rv64i_zvbb_zvkg_zvkned_aes_xts_encrypt: + @{[compute_xts_iv0]} + + # aes block size is 16 + andi $TAIL_LENGTH, $LENGTH, 15 + mv $STORE_LEN32, $LENGTH + beqz $TAIL_LENGTH, 1f + sub $LENGTH, $LENGTH, $TAIL_LENGTH + addi $STORE_LEN32, $LENGTH, -16 +1: + # We make the `LENGTH` become e32 length here. + srli $LEN32, $LENGTH, 2 + srli $STORE_LEN32, $STORE_LEN32, 2 + + # Load number of rounds + lwu $T0, 240($KEY1) + li $T1, 14 + li $T2, 10 + beq $T0, $T1, aes_xts_enc_256 + beq $T0, $T2, aes_xts_enc_128 +.size rv64i_zvbb_zvkg_zvkned_aes_xts_encrypt,.-rv64i_zvbb_zvkg_zvkned_aes_xts_encrypt +___ + +$code .= <<___; +.p2align 3 +aes_xts_enc_128: + @{[init_first_round]} + @{[aes_128_load_key]} + + @{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "ma"]} + j 1f + +.Lenc_blocks_128: + @{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "ma"]} + # load plaintext into v24 + @{[vle32_v $V24, $INPUT]} + # update iv + @{[vgmul_vv $V16, $V20]} + # reverse the iv's bits order back + @{[vbrev8_v $V28, $V16]} +1: + @{[vxor_vv $V24, $V24, $V28]} + slli $T0, $VL, 2 + sub $LEN32, $LEN32, $VL + add $INPUT, $INPUT, $T0 + @{[aes_128_enc]} + @{[vxor_vv $V24, $V24, $V28]} + + # store ciphertext + @{[vsetvli "zero", $STORE_LEN32, "e32", "m4", "ta", "ma"]} + @{[vse32_v $V24, $OUTPUT]} + add $OUTPUT, $OUTPUT, $T0 + sub $STORE_LEN32, $STORE_LEN32, $VL + + bnez $LEN32, .Lenc_blocks_128 + + @{[handle_xts_enc_last_block]} + + # xts last block + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vxor_vv $V24, $V24, $V28]} + @{[aes_128_enc]} + @{[vxor_vv $V24, $V24, $V28]} + + # store last block ciphertext + addi $OUTPUT, $OUTPUT, -16 + @{[vse32_v $V24, $OUTPUT]} + + ret +.size aes_xts_enc_128,.-aes_xts_enc_128 +___ + +$code .= <<___; +.p2align 3 +aes_xts_enc_256: + @{[init_first_round]} + @{[aes_256_load_key]} + + @{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "ma"]} + j 1f + +.Lenc_blocks_256: + @{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "ma"]} + # load plaintext into v24 + @{[vle32_v $V24, $INPUT]} + # update iv + @{[vgmul_vv $V16, $V20]} + # reverse the iv's bits order back + @{[vbrev8_v $V28, $V16]} +1: + @{[vxor_vv $V24, $V24, $V28]} + slli $T0, $VL, 2 + sub $LEN32, $LEN32, $VL + add $INPUT, $INPUT, $T0 + @{[aes_256_enc]} + @{[vxor_vv $V24, $V24, $V28]} + + # store ciphertext + @{[vsetvli "zero", $STORE_LEN32, "e32", "m4", "ta", "ma"]} + @{[vse32_v $V24, $OUTPUT]} + add $OUTPUT, $OUTPUT, $T0 + sub $STORE_LEN32, $STORE_LEN32, $VL + + bnez $LEN32, .Lenc_blocks_256 + + @{[handle_xts_enc_last_block]} + + # xts last block + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vxor_vv $V24, $V24, $V28]} + @{[aes_256_enc]} + @{[vxor_vv $V24, $V24, $V28]} + + # store last block ciphertext + addi $OUTPUT, $OUTPUT, -16 + @{[vse32_v $V24, $OUTPUT]} + + ret +.size aes_xts_enc_256,.-aes_xts_enc_256 +___ + +################################################################################ +# void rv64i_zvbb_zvkg_zvkned_aes_xts_decrypt(const unsigned char *in, +# unsigned char *out, size_t length, +# const AES_KEY *key1, +# const AES_KEY *key2, +# const unsigned char iv[16]) +$code .= <<___; +.p2align 3 +.globl rv64i_zvbb_zvkg_zvkned_aes_xts_decrypt +.type rv64i_zvbb_zvkg_zvkned_aes_xts_decrypt,\@function +rv64i_zvbb_zvkg_zvkned_aes_xts_decrypt: + @{[compute_xts_iv0]} + + # aes block size is 16 + andi $TAIL_LENGTH, $LENGTH, 15 + beqz $TAIL_LENGTH, 1f + sub $LENGTH, $LENGTH, $TAIL_LENGTH + addi $LENGTH, $LENGTH, -16 +1: + # We make the `LENGTH` become e32 length here. + srli $LEN32, $LENGTH, 2 + + # Load number of rounds + lwu $T0, 240($KEY1) + li $T1, 14 + li $T2, 10 + beq $T0, $T1, aes_xts_dec_256 + beq $T0, $T2, aes_xts_dec_128 +.size rv64i_zvbb_zvkg_zvkned_aes_xts_decrypt,.-rv64i_zvbb_zvkg_zvkned_aes_xts_decrypt +___ + +$code .= <<___; +.p2align 3 +aes_xts_dec_128: + @{[init_first_round]} + @{[aes_128_load_key]} + + beqz $LEN32, 2f + + @{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "ma"]} + j 1f + +.Ldec_blocks_128: + @{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "ma"]} + # load ciphertext into v24 + @{[vle32_v $V24, $INPUT]} + # update iv + @{[vgmul_vv $V16, $V20]} + # reverse the iv's bits order back + @{[vbrev8_v $V28, $V16]} +1: + @{[vxor_vv $V24, $V24, $V28]} + slli $T0, $VL, 2 + sub $LEN32, $LEN32, $VL + add $INPUT, $INPUT, $T0 + @{[aes_128_dec]} + @{[vxor_vv $V24, $V24, $V28]} + + # store plaintext + @{[vse32_v $V24, $OUTPUT]} + add $OUTPUT, $OUTPUT, $T0 + + bnez $LEN32, .Ldec_blocks_128 + +2: + @{[handle_xts_dec_last_block]} + + ## xts second to last block + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vxor_vv $V24, $V24, $V29]} + @{[aes_128_dec]} + @{[vxor_vv $V24, $V24, $V29]} + @{[vmv_v_v $V25, $V24]} + + # load last block ciphertext + @{[vsetvli "zero", $TAIL_LENGTH, "e8", "m1", "tu", "ma"]} + @{[vle8_v $V24, $INPUT]} + + # store second to last block plaintext + addi $T0, $OUTPUT, 16 + @{[vse8_v $V25, $T0]} + + ## xts last block + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vxor_vv $V24, $V24, $V28]} + @{[aes_128_dec]} + @{[vxor_vv $V24, $V24, $V28]} + + # store second to last block plaintext + @{[vse32_v $V24, $OUTPUT]} + + ret +.size aes_xts_dec_128,.-aes_xts_dec_128 +___ + +$code .= <<___; +.p2align 3 +aes_xts_dec_256: + @{[init_first_round]} + @{[aes_256_load_key]} + + beqz $LEN32, 2f + + @{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "ma"]} + j 1f + +.Ldec_blocks_256: + @{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "ma"]} + # load ciphertext into v24 + @{[vle32_v $V24, $INPUT]} + # update iv + @{[vgmul_vv $V16, $V20]} + # reverse the iv's bits order back + @{[vbrev8_v $V28, $V16]} +1: + @{[vxor_vv $V24, $V24, $V28]} + slli $T0, $VL, 2 + sub $LEN32, $LEN32, $VL + add $INPUT, $INPUT, $T0 + @{[aes_256_dec]} + @{[vxor_vv $V24, $V24, $V28]} + + # store plaintext + @{[vse32_v $V24, $OUTPUT]} + add $OUTPUT, $OUTPUT, $T0 + + bnez $LEN32, .Ldec_blocks_256 + +2: + @{[handle_xts_dec_last_block]} + + ## xts second to last block + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vxor_vv $V24, $V24, $V29]} + @{[aes_256_dec]} + @{[vxor_vv $V24, $V24, $V29]} + @{[vmv_v_v $V25, $V24]} + + # load last block ciphertext + @{[vsetvli "zero", $TAIL_LENGTH, "e8", "m1", "tu", "ma"]} + @{[vle8_v $V24, $INPUT]} + + # store second to last block plaintext + addi $T0, $OUTPUT, 16 + @{[vse8_v $V25, $T0]} + + ## xts last block + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vxor_vv $V24, $V24, $V28]} + @{[aes_256_dec]} + @{[vxor_vv $V24, $V24, $V28]} + + # store second to last block plaintext + @{[vse32_v $V24, $OUTPUT]} + + ret +.size aes_xts_dec_256,.-aes_xts_dec_256 +___ +} + +print $code; + +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/aes/asm/aes-riscv64-zvkb-zvkned.pl b/crypto/openssl/crypto/aes/asm/aes-riscv64-zvkb-zvkned.pl new file mode 100644 index 000000000000..8b87b21e4c84 --- /dev/null +++ b/crypto/openssl/crypto/aes/asm/aes-riscv64-zvkb-zvkned.pl @@ -0,0 +1,376 @@ +#! /usr/bin/env perl +# This file is dual-licensed, meaning that you can use it under your +# choice of either of the following two licenses: +# +# Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You can obtain +# a copy in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html +# +# or +# +# Copyright (c) 2023, Jerry Shih +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# - RV64I +# - RISC-V Vector ('V') with VLEN >= 128 +# - RISC-V Vector Cryptography Bit-manipulation extension ('Zvkb') +# - RISC-V Vector AES block cipher extension ('Zvkned') +# - RISC-V Zicclsm(Main memory supports misaligned loads/stores) + +use strict; +use warnings; + +use FindBin qw($Bin); +use lib "$Bin"; +use lib "$Bin/../../perlasm"; +use riscv; + +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$output and open STDOUT,">$output"; + +my $code=<<___; +.text +___ + +################################################################################ +# void rv64i_zvkb_zvkned_ctr32_encrypt_blocks(const unsigned char *in, +# unsigned char *out, size_t blocks, +# const void *key, +# const unsigned char ivec[16]); +{ +my ($INP, $OUTP, $BLOCK_NUM, $KEYP, $IVP) = ("a0", "a1", "a2", "a3", "a4"); +my ($T0, $T1, $T2, $T3) = ("t0", "t1", "t2", "t3"); +my ($VL) = ("t4"); +my ($LEN32) = ("t5"); +my ($CTR) = ("t6"); +my ($MASK) = ("v0"); +my ($V0, $V1, $V2, $V3, $V4, $V5, $V6, $V7, + $V8, $V9, $V10, $V11, $V12, $V13, $V14, $V15, + $V16, $V17, $V18, $V19, $V20, $V21, $V22, $V23, + $V24, $V25, $V26, $V27, $V28, $V29, $V30, $V31, +) = map("v$_",(0..31)); + +# Prepare the AES ctr input data into v16. +sub init_aes_ctr_input { + my $code=<<___; + # Setup mask into v0 + # The mask pattern for 4*N-th elements + # mask v0: [000100010001....] + # Note: + # We could setup the mask just for the maximum element length instead of + # the VLMAX. + li $T0, 0b10001000 + @{[vsetvli $T2, "zero", "e8", "m1", "ta", "ma"]} + @{[vmv_v_x $MASK, $T0]} + # Load IV. + # v31:[IV0, IV1, IV2, big-endian count] + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vle32_v $V31, $IVP]} + # Convert the big-endian counter into little-endian. + @{[vsetivli "zero", 4, "e32", "m1", "ta", "mu"]} + @{[vrev8_v $V31, $V31, $MASK]} + # Splat the IV to v16 + @{[vsetvli "zero", $LEN32, "e32", "m4", "ta", "ma"]} + @{[vmv_v_i $V16, 0]} + @{[vaesz_vs $V16, $V31]} + # Prepare the ctr pattern into v20 + # v20: [x, x, x, 0, x, x, x, 1, x, x, x, 2, ...] + @{[viota_m $V20, $MASK, $MASK]} + # v16:[IV0, IV1, IV2, count+0, IV0, IV1, IV2, count+1, ...] + @{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "mu"]} + @{[vadd_vv $V16, $V16, $V20, $MASK]} +___ + + return $code; +} + +$code .= <<___; +.p2align 3 +.globl rv64i_zvkb_zvkned_ctr32_encrypt_blocks +.type rv64i_zvkb_zvkned_ctr32_encrypt_blocks,\@function +rv64i_zvkb_zvkned_ctr32_encrypt_blocks: + beqz $BLOCK_NUM, 1f + + # Load number of rounds + lwu $T0, 240($KEYP) + li $T1, 14 + li $T2, 12 + li $T3, 10 + + slli $LEN32, $BLOCK_NUM, 2 + + beq $T0, $T1, ctr32_encrypt_blocks_256 + beq $T0, $T2, ctr32_encrypt_blocks_192 + beq $T0, $T3, ctr32_encrypt_blocks_128 + +1: + ret + +.size rv64i_zvkb_zvkned_ctr32_encrypt_blocks,.-rv64i_zvkb_zvkned_ctr32_encrypt_blocks +___ + +$code .= <<___; +.p2align 3 +ctr32_encrypt_blocks_128: + # Load all 11 round keys to v1-v11 registers. + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vle32_v $V1, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V2, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V3, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V4, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V5, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V6, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V7, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V8, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V9, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V10, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V11, $KEYP]} + + @{[init_aes_ctr_input]} + + ##### AES body + j 2f +1: + @{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "mu"]} + # Increase ctr in v16. + @{[vadd_vx $V16, $V16, $CTR, $MASK]} +2: + # Load plaintext into v20 + @{[vle32_v $V20, $INP]} + slli $T0, $VL, 2 + srli $CTR, $VL, 2 + sub $LEN32, $LEN32, $VL + add $INP, $INP, $T0 + # Prepare the AES ctr input into v24. + # The ctr data uses big-endian form. + @{[vmv_v_v $V24, $V16]} + @{[vrev8_v $V24, $V24, $MASK]} + + @{[vaesz_vs $V24, $V1]} + @{[vaesem_vs $V24, $V2]} + @{[vaesem_vs $V24, $V3]} + @{[vaesem_vs $V24, $V4]} + @{[vaesem_vs $V24, $V5]} + @{[vaesem_vs $V24, $V6]} + @{[vaesem_vs $V24, $V7]} + @{[vaesem_vs $V24, $V8]} + @{[vaesem_vs $V24, $V9]} + @{[vaesem_vs $V24, $V10]} + @{[vaesef_vs $V24, $V11]} + + # ciphertext + @{[vxor_vv $V24, $V24, $V20]} + + # Store the ciphertext. + @{[vse32_v $V24, $OUTP]} + add $OUTP, $OUTP, $T0 + + bnez $LEN32, 1b + + ret +.size ctr32_encrypt_blocks_128,.-ctr32_encrypt_blocks_128 +___ + +$code .= <<___; +.p2align 3 +ctr32_encrypt_blocks_192: + # Load all 13 round keys to v1-v13 registers. + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vle32_v $V1, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V2, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V3, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V4, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V5, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V6, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V7, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V8, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V9, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V10, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V11, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V12, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V13, $KEYP]} + + @{[init_aes_ctr_input]} + + ##### AES body + j 2f +1: + @{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "mu"]} + # Increase ctr in v16. + @{[vadd_vx $V16, $V16, $CTR, $MASK]} +2: + # Load plaintext into v20 + @{[vle32_v $V20, $INP]} + slli $T0, $VL, 2 + srli $CTR, $VL, 2 + sub $LEN32, $LEN32, $VL + add $INP, $INP, $T0 + # Prepare the AES ctr input into v24. + # The ctr data uses big-endian form. + @{[vmv_v_v $V24, $V16]} + @{[vrev8_v $V24, $V24, $MASK]} + + @{[vaesz_vs $V24, $V1]} + @{[vaesem_vs $V24, $V2]} + @{[vaesem_vs $V24, $V3]} + @{[vaesem_vs $V24, $V4]} + @{[vaesem_vs $V24, $V5]} + @{[vaesem_vs $V24, $V6]} + @{[vaesem_vs $V24, $V7]} + @{[vaesem_vs $V24, $V8]} + @{[vaesem_vs $V24, $V9]} + @{[vaesem_vs $V24, $V10]} + @{[vaesem_vs $V24, $V11]} + @{[vaesem_vs $V24, $V12]} + @{[vaesef_vs $V24, $V13]} + + # ciphertext + @{[vxor_vv $V24, $V24, $V20]} + + # Store the ciphertext. + @{[vse32_v $V24, $OUTP]} + add $OUTP, $OUTP, $T0 + + bnez $LEN32, 1b + + ret +.size ctr32_encrypt_blocks_192,.-ctr32_encrypt_blocks_192 +___ + +$code .= <<___; +.p2align 3 +ctr32_encrypt_blocks_256: + # Load all 15 round keys to v1-v15 registers. + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vle32_v $V1, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V2, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V3, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V4, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V5, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V6, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V7, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V8, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V9, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V10, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V11, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V12, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V13, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V14, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V15, $KEYP]} + + @{[init_aes_ctr_input]} + + ##### AES body + j 2f +1: + @{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "mu"]} + # Increase ctr in v16. + @{[vadd_vx $V16, $V16, $CTR, $MASK]} +2: + # Load plaintext into v20 + @{[vle32_v $V20, $INP]} + slli $T0, $VL, 2 + srli $CTR, $VL, 2 + sub $LEN32, $LEN32, $VL + add $INP, $INP, $T0 + # Prepare the AES ctr input into v24. + # The ctr data uses big-endian form. + @{[vmv_v_v $V24, $V16]} + @{[vrev8_v $V24, $V24, $MASK]} + + @{[vaesz_vs $V24, $V1]} + @{[vaesem_vs $V24, $V2]} + @{[vaesem_vs $V24, $V3]} + @{[vaesem_vs $V24, $V4]} + @{[vaesem_vs $V24, $V5]} + @{[vaesem_vs $V24, $V6]} + @{[vaesem_vs $V24, $V7]} + @{[vaesem_vs $V24, $V8]} + @{[vaesem_vs $V24, $V9]} + @{[vaesem_vs $V24, $V10]} + @{[vaesem_vs $V24, $V11]} + @{[vaesem_vs $V24, $V12]} + @{[vaesem_vs $V24, $V13]} + @{[vaesem_vs $V24, $V14]} + @{[vaesef_vs $V24, $V15]} + + # ciphertext + @{[vxor_vv $V24, $V24, $V20]} + + # Store the ciphertext. + @{[vse32_v $V24, $OUTP]} + add $OUTP, $OUTP, $T0 + + bnez $LEN32, 1b + + ret +.size ctr32_encrypt_blocks_256,.-ctr32_encrypt_blocks_256 +___ +} + +print $code; + +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/aes/asm/aes-riscv64-zvkned.pl b/crypto/openssl/crypto/aes/asm/aes-riscv64-zvkned.pl new file mode 100644 index 000000000000..4c0292781d9f --- /dev/null +++ b/crypto/openssl/crypto/aes/asm/aes-riscv64-zvkned.pl @@ -0,0 +1,1376 @@ +#! /usr/bin/env perl +# This file is dual-licensed, meaning that you can use it under your +# choice of either of the following two licenses: +# +# Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You can obtain +# a copy in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html +# +# or +# +# Copyright (c) 2023, Christoph Müllner +# Copyright (c) 2023, Phoebe Chen +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# - RV64I +# - RISC-V Vector ('V') with VLEN >= 128 +# - RISC-V Vector AES block cipher extension ('Zvkned') + +use strict; +use warnings; + +use FindBin qw($Bin); +use lib "$Bin"; +use lib "$Bin/../../perlasm"; +use riscv; + +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$output and open STDOUT,">$output"; + +my $code=<<___; +.text +___ + +my ($V0, $V1, $V2, $V3, $V4, $V5, $V6, $V7, + $V8, $V9, $V10, $V11, $V12, $V13, $V14, $V15, + $V16, $V17, $V18, $V19, $V20, $V21, $V22, $V23, + $V24, $V25, $V26, $V27, $V28, $V29, $V30, $V31, +) = map("v$_",(0..31)); + +# Load all 11 round keys to v1-v11 registers. +sub aes_128_load_key { + my $KEYP = shift; + + my $code=<<___; + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vle32_v $V1, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V2, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V3, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V4, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V5, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V6, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V7, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V8, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V9, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V10, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V11, $KEYP]} +___ + + return $code; +} + +# Load all 13 round keys to v1-v13 registers. +sub aes_192_load_key { + my $KEYP = shift; + + my $code=<<___; + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vle32_v $V1, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V2, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V3, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V4, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V5, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V6, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V7, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V8, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V9, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V10, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V11, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V12, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V13, $KEYP]} +___ + + return $code; +} + +# Load all 15 round keys to v1-v15 registers. +sub aes_256_load_key { + my $KEYP = shift; + + my $code=<<___; + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vle32_v $V1, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V2, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V3, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V4, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V5, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V6, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V7, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V8, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V9, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V10, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V11, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V12, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V13, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V14, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V15, $KEYP]} +___ + + return $code; +} + +# aes-128 encryption with round keys v1-v11 +sub aes_128_encrypt { + my $code=<<___; + @{[vaesz_vs $V24, $V1]} # with round key w[ 0, 3] + @{[vaesem_vs $V24, $V2]} # with round key w[ 4, 7] + @{[vaesem_vs $V24, $V3]} # with round key w[ 8,11] + @{[vaesem_vs $V24, $V4]} # with round key w[12,15] + @{[vaesem_vs $V24, $V5]} # with round key w[16,19] + @{[vaesem_vs $V24, $V6]} # with round key w[20,23] + @{[vaesem_vs $V24, $V7]} # with round key w[24,27] + @{[vaesem_vs $V24, $V8]} # with round key w[28,31] + @{[vaesem_vs $V24, $V9]} # with round key w[32,35] + @{[vaesem_vs $V24, $V10]} # with round key w[36,39] + @{[vaesef_vs $V24, $V11]} # with round key w[40,43] +___ + + return $code; +} + +# aes-128 decryption with round keys v1-v11 +sub aes_128_decrypt { + my $code=<<___; + @{[vaesz_vs $V24, $V11]} # with round key w[40,43] + @{[vaesdm_vs $V24, $V10]} # with round key w[36,39] + @{[vaesdm_vs $V24, $V9]} # with round key w[32,35] + @{[vaesdm_vs $V24, $V8]} # with round key w[28,31] + @{[vaesdm_vs $V24, $V7]} # with round key w[24,27] + @{[vaesdm_vs $V24, $V6]} # with round key w[20,23] + @{[vaesdm_vs $V24, $V5]} # with round key w[16,19] + @{[vaesdm_vs $V24, $V4]} # with round key w[12,15] + @{[vaesdm_vs $V24, $V3]} # with round key w[ 8,11] + @{[vaesdm_vs $V24, $V2]} # with round key w[ 4, 7] + @{[vaesdf_vs $V24, $V1]} # with round key w[ 0, 3] +___ + + return $code; +} + +# aes-192 encryption with round keys v1-v13 +sub aes_192_encrypt { + my $code=<<___; + @{[vaesz_vs $V24, $V1]} # with round key w[ 0, 3] + @{[vaesem_vs $V24, $V2]} # with round key w[ 4, 7] + @{[vaesem_vs $V24, $V3]} # with round key w[ 8,11] + @{[vaesem_vs $V24, $V4]} # with round key w[12,15] + @{[vaesem_vs $V24, $V5]} # with round key w[16,19] + @{[vaesem_vs $V24, $V6]} # with round key w[20,23] + @{[vaesem_vs $V24, $V7]} # with round key w[24,27] + @{[vaesem_vs $V24, $V8]} # with round key w[28,31] + @{[vaesem_vs $V24, $V9]} # with round key w[32,35] + @{[vaesem_vs $V24, $V10]} # with round key w[36,39] + @{[vaesem_vs $V24, $V11]} # with round key w[40,43] + @{[vaesem_vs $V24, $V12]} # with round key w[44,47] + @{[vaesef_vs $V24, $V13]} # with round key w[48,51] +___ + + return $code; +} + +# aes-192 decryption with round keys v1-v13 +sub aes_192_decrypt { + my $code=<<___; + @{[vaesz_vs $V24, $V13]} # with round key w[48,51] + @{[vaesdm_vs $V24, $V12]} # with round key w[44,47] + @{[vaesdm_vs $V24, $V11]} # with round key w[40,43] + @{[vaesdm_vs $V24, $V10]} # with round key w[36,39] + @{[vaesdm_vs $V24, $V9]} # with round key w[32,35] + @{[vaesdm_vs $V24, $V8]} # with round key w[28,31] + @{[vaesdm_vs $V24, $V7]} # with round key w[24,27] + @{[vaesdm_vs $V24, $V6]} # with round key w[20,23] + @{[vaesdm_vs $V24, $V5]} # with round key w[16,19] + @{[vaesdm_vs $V24, $V4]} # with round key w[12,15] + @{[vaesdm_vs $V24, $V3]} # with round key w[ 8,11] + @{[vaesdm_vs $V24, $V2]} # with round key w[ 4, 7] + @{[vaesdf_vs $V24, $V1]} # with round key w[ 0, 3] +___ + + return $code; +} + +# aes-256 encryption with round keys v1-v15 +sub aes_256_encrypt { + my $code=<<___; + @{[vaesz_vs $V24, $V1]} # with round key w[ 0, 3] + @{[vaesem_vs $V24, $V2]} # with round key w[ 4, 7] + @{[vaesem_vs $V24, $V3]} # with round key w[ 8,11] + @{[vaesem_vs $V24, $V4]} # with round key w[12,15] + @{[vaesem_vs $V24, $V5]} # with round key w[16,19] + @{[vaesem_vs $V24, $V6]} # with round key w[20,23] + @{[vaesem_vs $V24, $V7]} # with round key w[24,27] + @{[vaesem_vs $V24, $V8]} # with round key w[28,31] + @{[vaesem_vs $V24, $V9]} # with round key w[32,35] + @{[vaesem_vs $V24, $V10]} # with round key w[36,39] + @{[vaesem_vs $V24, $V11]} # with round key w[40,43] + @{[vaesem_vs $V24, $V12]} # with round key w[44,47] + @{[vaesem_vs $V24, $V13]} # with round key w[48,51] + @{[vaesem_vs $V24, $V14]} # with round key w[52,55] + @{[vaesef_vs $V24, $V15]} # with round key w[56,59] +___ + + return $code; +} + +# aes-256 decryption with round keys v1-v15 +sub aes_256_decrypt { + my $code=<<___; + @{[vaesz_vs $V24, $V15]} # with round key w[56,59] + @{[vaesdm_vs $V24, $V14]} # with round key w[52,55] + @{[vaesdm_vs $V24, $V13]} # with round key w[48,51] + @{[vaesdm_vs $V24, $V12]} # with round key w[44,47] + @{[vaesdm_vs $V24, $V11]} # with round key w[40,43] + @{[vaesdm_vs $V24, $V10]} # with round key w[36,39] + @{[vaesdm_vs $V24, $V9]} # with round key w[32,35] + @{[vaesdm_vs $V24, $V8]} # with round key w[28,31] + @{[vaesdm_vs $V24, $V7]} # with round key w[24,27] + @{[vaesdm_vs $V24, $V6]} # with round key w[20,23] + @{[vaesdm_vs $V24, $V5]} # with round key w[16,19] + @{[vaesdm_vs $V24, $V4]} # with round key w[12,15] + @{[vaesdm_vs $V24, $V3]} # with round key w[ 8,11] + @{[vaesdm_vs $V24, $V2]} # with round key w[ 4, 7] + @{[vaesdf_vs $V24, $V1]} # with round key w[ 0, 3] +___ + + return $code; +} + +{ +############################################################################### +# void rv64i_zvkned_cbc_encrypt(const unsigned char *in, unsigned char *out, +# size_t length, const AES_KEY *key, +# unsigned char *ivec, const int enc); +my ($INP, $OUTP, $LEN, $KEYP, $IVP, $ENC) = ("a0", "a1", "a2", "a3", "a4", "a5"); +my ($T0, $T1, $ROUNDS) = ("t0", "t1", "t2"); + +$code .= <<___; +.p2align 3 +.globl rv64i_zvkned_cbc_encrypt +.type rv64i_zvkned_cbc_encrypt,\@function +rv64i_zvkned_cbc_encrypt: + # check whether the length is a multiple of 16 and >= 16 + li $T1, 16 + blt $LEN, $T1, L_end + andi $T1, $LEN, 15 + bnez $T1, L_end + + # Load number of rounds + lwu $ROUNDS, 240($KEYP) + + # Get proper routine for key size + li $T0, 10 + beq $ROUNDS, $T0, L_cbc_enc_128 + + li $T0, 12 + beq $ROUNDS, $T0, L_cbc_enc_192 + + li $T0, 14 + beq $ROUNDS, $T0, L_cbc_enc_256 + + ret +.size rv64i_zvkned_cbc_encrypt,.-rv64i_zvkned_cbc_encrypt +___ + +$code .= <<___; +.p2align 3 +L_cbc_enc_128: + # Load all 11 round keys to v1-v11 registers. + @{[aes_128_load_key $KEYP]} + + # Load IV. + @{[vle32_v $V16, $IVP]} + + @{[vle32_v $V24, $INP]} + @{[vxor_vv $V24, $V24, $V16]} + j 2f + +1: + @{[vle32_v $V17, $INP]} + @{[vxor_vv $V24, $V24, $V17]} + +2: + # AES body + @{[aes_128_encrypt]} + + @{[vse32_v $V24, $OUTP]} + + addi $INP, $INP, 16 + addi $OUTP, $OUTP, 16 + addi $LEN, $LEN, -16 + + bnez $LEN, 1b + + @{[vse32_v $V24, $IVP]} + + ret +.size L_cbc_enc_128,.-L_cbc_enc_128 +___ + +$code .= <<___; +.p2align 3 +L_cbc_enc_192: + # Load all 13 round keys to v1-v13 registers. + @{[aes_192_load_key $KEYP]} + + # Load IV. + @{[vle32_v $V16, $IVP]} + + @{[vle32_v $V24, $INP]} + @{[vxor_vv $V24, $V24, $V16]} + j 2f + +1: + @{[vle32_v $V17, $INP]} + @{[vxor_vv $V24, $V24, $V17]} + +2: + # AES body + @{[aes_192_encrypt]} + + @{[vse32_v $V24, $OUTP]} + + addi $INP, $INP, 16 + addi $OUTP, $OUTP, 16 + addi $LEN, $LEN, -16 + + bnez $LEN, 1b + + @{[vse32_v $V24, $IVP]} + + ret +.size L_cbc_enc_192,.-L_cbc_enc_192 +___ + +$code .= <<___; +.p2align 3 +L_cbc_enc_256: + # Load all 15 round keys to v1-v15 registers. + @{[aes_256_load_key $KEYP]} + + # Load IV. + @{[vle32_v $V16, $IVP]} + + @{[vle32_v $V24, $INP]} + @{[vxor_vv $V24, $V24, $V16]} + j 2f + +1: + @{[vle32_v $V17, $INP]} + @{[vxor_vv $V24, $V24, $V17]} + +2: + # AES body + @{[aes_256_encrypt]} + + @{[vse32_v $V24, $OUTP]} + + addi $INP, $INP, 16 + addi $OUTP, $OUTP, 16 + addi $LEN, $LEN, -16 + + bnez $LEN, 1b + + @{[vse32_v $V24, $IVP]} + + ret +.size L_cbc_enc_256,.-L_cbc_enc_256 +___ + +############################################################################### +# void rv64i_zvkned_cbc_decrypt(const unsigned char *in, unsigned char *out, +# size_t length, const AES_KEY *key, +# unsigned char *ivec, const int enc); + +$code .= <<___; +.p2align 3 +.globl rv64i_zvkned_cbc_decrypt +.type rv64i_zvkned_cbc_decrypt,\@function +rv64i_zvkned_cbc_decrypt: + # check whether the length is a multiple of 16 and >= 16 + li $T1, 16 + blt $LEN, $T1, L_end + andi $T1, $LEN, 15 + bnez $T1, L_end + + # Load number of rounds + lwu $ROUNDS, 240($KEYP) + + # Get proper routine for key size + li $T0, 10 + beq $ROUNDS, $T0, L_cbc_dec_128 + + li $T0, 12 + beq $ROUNDS, $T0, L_cbc_dec_192 + + li $T0, 14 + beq $ROUNDS, $T0, L_cbc_dec_256 + + ret +.size rv64i_zvkned_cbc_decrypt,.-rv64i_zvkned_cbc_decrypt +___ + +$code .= <<___; +.p2align 3 +L_cbc_dec_128: + # Load all 11 round keys to v1-v11 registers. + @{[aes_128_load_key $KEYP]} + + # Load IV. + @{[vle32_v $V16, $IVP]} + + @{[vle32_v $V24, $INP]} + @{[vmv_v_v $V17, $V24]} + j 2f + +1: + @{[vle32_v $V24, $INP]} + @{[vmv_v_v $V17, $V24]} + addi $OUTP, $OUTP, 16 + +2: + # AES body + @{[aes_128_decrypt]} + + @{[vxor_vv $V24, $V24, $V16]} + @{[vse32_v $V24, $OUTP]} + @{[vmv_v_v $V16, $V17]} + + addi $LEN, $LEN, -16 + addi $INP, $INP, 16 + + bnez $LEN, 1b + + @{[vse32_v $V16, $IVP]} + + ret +.size L_cbc_dec_128,.-L_cbc_dec_128 +___ + +$code .= <<___; +.p2align 3 +L_cbc_dec_192: + # Load all 13 round keys to v1-v13 registers. + @{[aes_192_load_key $KEYP]} + + # Load IV. + @{[vle32_v $V16, $IVP]} + + @{[vle32_v $V24, $INP]} + @{[vmv_v_v $V17, $V24]} + j 2f + +1: + @{[vle32_v $V24, $INP]} + @{[vmv_v_v $V17, $V24]} + addi $OUTP, $OUTP, 16 + +2: + # AES body + @{[aes_192_decrypt]} + + @{[vxor_vv $V24, $V24, $V16]} + @{[vse32_v $V24, $OUTP]} + @{[vmv_v_v $V16, $V17]} + + addi $LEN, $LEN, -16 + addi $INP, $INP, 16 + + bnez $LEN, 1b + + @{[vse32_v $V16, $IVP]} + + ret +.size L_cbc_dec_192,.-L_cbc_dec_192 +___ + +$code .= <<___; +.p2align 3 +L_cbc_dec_256: + # Load all 15 round keys to v1-v15 registers. + @{[aes_256_load_key $KEYP]} + + # Load IV. + @{[vle32_v $V16, $IVP]} + + @{[vle32_v $V24, $INP]} + @{[vmv_v_v $V17, $V24]} + j 2f + +1: + @{[vle32_v $V24, $INP]} + @{[vmv_v_v $V17, $V24]} + addi $OUTP, $OUTP, 16 + +2: + # AES body + @{[aes_256_decrypt]} + + @{[vxor_vv $V24, $V24, $V16]} + @{[vse32_v $V24, $OUTP]} + @{[vmv_v_v $V16, $V17]} + + addi $LEN, $LEN, -16 + addi $INP, $INP, 16 + + bnez $LEN, 1b + + @{[vse32_v $V16, $IVP]} + + ret +.size L_cbc_dec_256,.-L_cbc_dec_256 +___ +} + +{ +############################################################################### +# void rv64i_zvkned_ecb_encrypt(const unsigned char *in, unsigned char *out, +# size_t length, const AES_KEY *key, +# const int enc); +my ($INP, $OUTP, $LEN, $KEYP, $ENC) = ("a0", "a1", "a2", "a3", "a4"); +my ($REMAIN_LEN) = ("a5"); +my ($VL) = ("a6"); +my ($T0, $T1, $ROUNDS) = ("t0", "t1", "t2"); +my ($LEN32) = ("t3"); + +$code .= <<___; +.p2align 3 +.globl rv64i_zvkned_ecb_encrypt +.type rv64i_zvkned_ecb_encrypt,\@function +rv64i_zvkned_ecb_encrypt: + # Make the LEN become e32 length. + srli $LEN32, $LEN, 2 + + # Load number of rounds + lwu $ROUNDS, 240($KEYP) + + # Get proper routine for key size + li $T0, 10 + beq $ROUNDS, $T0, L_ecb_enc_128 + + li $T0, 12 + beq $ROUNDS, $T0, L_ecb_enc_192 + + li $T0, 14 + beq $ROUNDS, $T0, L_ecb_enc_256 + + ret +.size rv64i_zvkned_ecb_encrypt,.-rv64i_zvkned_ecb_encrypt +___ + +$code .= <<___; +.p2align 3 +L_ecb_enc_128: + # Load all 11 round keys to v1-v11 registers. + @{[aes_128_load_key $KEYP]} + +1: + @{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "ma"]} + slli $T0, $VL, 2 + sub $LEN32, $LEN32, $VL + + @{[vle32_v $V24, $INP]} + + # AES body + @{[aes_128_encrypt]} + + @{[vse32_v $V24, $OUTP]} + + add $INP, $INP, $T0 + add $OUTP, $OUTP, $T0 + + bnez $LEN32, 1b + + ret +.size L_ecb_enc_128,.-L_ecb_enc_128 +___ + +$code .= <<___; +.p2align 3 +L_ecb_enc_192: + # Load all 13 round keys to v1-v13 registers. + @{[aes_192_load_key $KEYP]} + +1: + @{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "ma"]} + slli $T0, $VL, 2 + sub $LEN32, $LEN32, $VL + + @{[vle32_v $V24, $INP]} + + # AES body + @{[aes_192_encrypt]} + + @{[vse32_v $V24, $OUTP]} + + add $INP, $INP, $T0 + add $OUTP, $OUTP, $T0 + + bnez $LEN32, 1b + + ret +.size L_ecb_enc_192,.-L_ecb_enc_192 +___ + +$code .= <<___; +.p2align 3 +L_ecb_enc_256: + # Load all 15 round keys to v1-v15 registers. + @{[aes_256_load_key $KEYP]} + +1: + @{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "ma"]} + slli $T0, $VL, 2 + sub $LEN32, $LEN32, $VL + + @{[vle32_v $V24, $INP]} + + # AES body + @{[aes_256_encrypt]} + + @{[vse32_v $V24, $OUTP]} + + add $INP, $INP, $T0 + add $OUTP, $OUTP, $T0 + + bnez $LEN32, 1b + + ret +.size L_ecb_enc_256,.-L_ecb_enc_256 +___ + +############################################################################### +# void rv64i_zvkned_ecb_decrypt(const unsigned char *in, unsigned char *out, +# size_t length, const AES_KEY *key, +# const int enc); + +$code .= <<___; +.p2align 3 +.globl rv64i_zvkned_ecb_decrypt +.type rv64i_zvkned_ecb_decrypt,\@function +rv64i_zvkned_ecb_decrypt: + # Make the LEN become e32 length. + srli $LEN32, $LEN, 2 + + # Load number of rounds + lwu $ROUNDS, 240($KEYP) + + # Get proper routine for key size + li $T0, 10 + beq $ROUNDS, $T0, L_ecb_dec_128 + + li $T0, 12 + beq $ROUNDS, $T0, L_ecb_dec_192 + + li $T0, 14 + beq $ROUNDS, $T0, L_ecb_dec_256 + + ret +.size rv64i_zvkned_ecb_decrypt,.-rv64i_zvkned_ecb_decrypt +___ + +$code .= <<___; +.p2align 3 +L_ecb_dec_128: + # Load all 11 round keys to v1-v11 registers. + @{[aes_128_load_key $KEYP]} + +1: + @{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "ma"]} + slli $T0, $VL, 2 + sub $LEN32, $LEN32, $VL + + @{[vle32_v $V24, $INP]} + + # AES body + @{[aes_128_decrypt]} + + @{[vse32_v $V24, $OUTP]} + + add $INP, $INP, $T0 + add $OUTP, $OUTP, $T0 + + bnez $LEN32, 1b + + ret +.size L_ecb_dec_128,.-L_ecb_dec_128 +___ + +$code .= <<___; +.p2align 3 +L_ecb_dec_192: + # Load all 13 round keys to v1-v13 registers. + @{[aes_192_load_key $KEYP]} + +1: + @{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "ma"]} + slli $T0, $VL, 2 + sub $LEN32, $LEN32, $VL + + @{[vle32_v $V24, $INP]} + + # AES body + @{[aes_192_decrypt]} + + @{[vse32_v $V24, $OUTP]} + + add $INP, $INP, $T0 + add $OUTP, $OUTP, $T0 + + bnez $LEN32, 1b + + ret +.size L_ecb_dec_192,.-L_ecb_dec_192 +___ + +$code .= <<___; +.p2align 3 +L_ecb_dec_256: + # Load all 15 round keys to v1-v15 registers. + @{[aes_256_load_key $KEYP]} + +1: + @{[vsetvli $VL, $LEN32, "e32", "m4", "ta", "ma"]} + slli $T0, $VL, 2 + sub $LEN32, $LEN32, $VL + + @{[vle32_v $V24, $INP]} + + # AES body + @{[aes_256_decrypt]} + + @{[vse32_v $V24, $OUTP]} + + add $INP, $INP, $T0 + add $OUTP, $OUTP, $T0 + + bnez $LEN32, 1b + + ret +.size L_ecb_dec_256,.-L_ecb_dec_256 +___ + +} + +{ +################################################################################ +# int rv64i_zvkned_set_encrypt_key(const unsigned char *userKey, const int bits, +# AES_KEY *key) +# int rv64i_zvkned_set_decrypt_key(const unsigned char *userKey, const int bits, +# AES_KEY *key) +my ($UKEY,$BITS,$KEYP) = ("a0", "a1", "a2"); +my ($T0,$T1,$T4) = ("t1", "t2", "t4"); + +$code .= <<___; +.p2align 3 +.globl rv64i_zvkned_set_encrypt_key +.type rv64i_zvkned_set_encrypt_key,\@function +rv64i_zvkned_set_encrypt_key: + beqz $UKEY, L_fail_m1 + beqz $KEYP, L_fail_m1 + + # Get proper routine for key size + li $T0, 256 + beq $BITS, $T0, L_set_key_256 + li $T0, 128 + beq $BITS, $T0, L_set_key_128 + + j L_fail_m2 + +.size rv64i_zvkned_set_encrypt_key,.-rv64i_zvkned_set_encrypt_key +___ + +$code .= <<___; +.p2align 3 +.globl rv64i_zvkned_set_decrypt_key +.type rv64i_zvkned_set_decrypt_key,\@function +rv64i_zvkned_set_decrypt_key: + beqz $UKEY, L_fail_m1 + beqz $KEYP, L_fail_m1 + + # Get proper routine for key size + li $T0, 256 + beq $BITS, $T0, L_set_key_256 + li $T0, 128 + beq $BITS, $T0, L_set_key_128 + + j L_fail_m2 + +.size rv64i_zvkned_set_decrypt_key,.-rv64i_zvkned_set_decrypt_key +___ + +$code .= <<___; +.p2align 3 +L_set_key_128: + # Store the number of rounds + li $T1, 10 + sw $T1, 240($KEYP) + + @{[vsetivli__x0_4_e32_m1_tu_mu]} + + # Load the key + @{[vle32_v $V10, ($UKEY)]} + + # Generate keys for round 2-11 into registers v11-v20. + @{[vaeskf1_vi $V11, $V10, 1]} # v11 <- rk2 (w[ 4, 7]) + @{[vaeskf1_vi $V12, $V11, 2]} # v12 <- rk3 (w[ 8,11]) + @{[vaeskf1_vi $V13, $V12, 3]} # v13 <- rk4 (w[12,15]) + @{[vaeskf1_vi $V14, $V13, 4]} # v14 <- rk5 (w[16,19]) + @{[vaeskf1_vi $V15, $V14, 5]} # v15 <- rk6 (w[20,23]) + @{[vaeskf1_vi $V16, $V15, 6]} # v16 <- rk7 (w[24,27]) + @{[vaeskf1_vi $V17, $V16, 7]} # v17 <- rk8 (w[28,31]) + @{[vaeskf1_vi $V18, $V17, 8]} # v18 <- rk9 (w[32,35]) + @{[vaeskf1_vi $V19, $V18, 9]} # v19 <- rk10 (w[36,39]) + @{[vaeskf1_vi $V20, $V19, 10]} # v20 <- rk11 (w[40,43]) + + # Store the round keys + @{[vse32_v $V10, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vse32_v $V11, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vse32_v $V12, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vse32_v $V13, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vse32_v $V14, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vse32_v $V15, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vse32_v $V16, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vse32_v $V17, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vse32_v $V18, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vse32_v $V19, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vse32_v $V20, $KEYP]} + + li a0, 1 + ret +.size L_set_key_128,.-L_set_key_128 +___ + +$code .= <<___; +.p2align 3 +L_set_key_256: + # Store the number of rounds + li $T1, 14 + sw $T1, 240($KEYP) + + @{[vsetivli__x0_4_e32_m1_tu_mu]} + + # Load the key + @{[vle32_v $V10, ($UKEY)]} + addi $UKEY, $UKEY, 16 + @{[vle32_v $V11, ($UKEY)]} + + @{[vmv_v_v $V12, $V10]} + @{[vaeskf2_vi $V12, $V11, 2]} + @{[vmv_v_v $V13, $V11]} + @{[vaeskf2_vi $V13, $V12, 3]} + @{[vmv_v_v $V14, $V12]} + @{[vaeskf2_vi $V14, $V13, 4]} + @{[vmv_v_v $V15, $V13]} + @{[vaeskf2_vi $V15, $V14, 5]} + @{[vmv_v_v $V16, $V14]} + @{[vaeskf2_vi $V16, $V15, 6]} + @{[vmv_v_v $V17, $V15]} + @{[vaeskf2_vi $V17, $V16, 7]} + @{[vmv_v_v $V18, $V16]} + @{[vaeskf2_vi $V18, $V17, 8]} + @{[vmv_v_v $V19, $V17]} + @{[vaeskf2_vi $V19, $V18, 9]} + @{[vmv_v_v $V20, $V18]} + @{[vaeskf2_vi $V20, $V19, 10]} + @{[vmv_v_v $V21, $V19]} + @{[vaeskf2_vi $V21, $V20, 11]} + @{[vmv_v_v $V22, $V20]} + @{[vaeskf2_vi $V22, $V21, 12]} + @{[vmv_v_v $V23, $V21]} + @{[vaeskf2_vi $V23, $V22, 13]} + @{[vmv_v_v $V24, $V22]} + @{[vaeskf2_vi $V24, $V23, 14]} + + @{[vse32_v $V10, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vse32_v $V11, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vse32_v $V12, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vse32_v $V13, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vse32_v $V14, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vse32_v $V15, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vse32_v $V16, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vse32_v $V17, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vse32_v $V18, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vse32_v $V19, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vse32_v $V20, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vse32_v $V21, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vse32_v $V22, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vse32_v $V23, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vse32_v $V24, $KEYP]} + + li a0, 1 + ret +.size L_set_key_256,.-L_set_key_256 +___ +} + +{ +################################################################################ +# void rv64i_zvkned_encrypt(const unsigned char *in, unsigned char *out, +# const AES_KEY *key); +my ($INP,$OUTP,$KEYP) = ("a0", "a1", "a2"); +my ($T0,$T1, $ROUNDS, $T6) = ("a3", "a4", "t5", "t6"); + +$code .= <<___; +.p2align 3 +.globl rv64i_zvkned_encrypt +.type rv64i_zvkned_encrypt,\@function +rv64i_zvkned_encrypt: + # Load number of rounds + lwu $ROUNDS, 240($KEYP) + + # Get proper routine for key size + li $T6, 14 + beq $ROUNDS, $T6, L_enc_256 + li $T6, 10 + beq $ROUNDS, $T6, L_enc_128 + li $T6, 12 + beq $ROUNDS, $T6, L_enc_192 + + j L_fail_m2 +.size rv64i_zvkned_encrypt,.-rv64i_zvkned_encrypt +___ + +$code .= <<___; +.p2align 3 +L_enc_128: + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + + @{[vle32_v $V1, $INP]} + + @{[vle32_v $V10, $KEYP]} + @{[vaesz_vs $V1, $V10]} # with round key w[ 0, 3] + addi $KEYP, $KEYP, 16 + @{[vle32_v $V11, $KEYP]} + @{[vaesem_vs $V1, $V11]} # with round key w[ 4, 7] + addi $KEYP, $KEYP, 16 + @{[vle32_v $V12, $KEYP]} + @{[vaesem_vs $V1, $V12]} # with round key w[ 8,11] + addi $KEYP, $KEYP, 16 + @{[vle32_v $V13, $KEYP]} + @{[vaesem_vs $V1, $V13]} # with round key w[12,15] + addi $KEYP, $KEYP, 16 + @{[vle32_v $V14, $KEYP]} + @{[vaesem_vs $V1, $V14]} # with round key w[16,19] + addi $KEYP, $KEYP, 16 + @{[vle32_v $V15, $KEYP]} + @{[vaesem_vs $V1, $V15]} # with round key w[20,23] + addi $KEYP, $KEYP, 16 + @{[vle32_v $V16, $KEYP]} + @{[vaesem_vs $V1, $V16]} # with round key w[24,27] + addi $KEYP, $KEYP, 16 + @{[vle32_v $V17, $KEYP]} + @{[vaesem_vs $V1, $V17]} # with round key w[28,31] + addi $KEYP, $KEYP, 16 + @{[vle32_v $V18, $KEYP]} + @{[vaesem_vs $V1, $V18]} # with round key w[32,35] + addi $KEYP, $KEYP, 16 + @{[vle32_v $V19, $KEYP]} + @{[vaesem_vs $V1, $V19]} # with round key w[36,39] + addi $KEYP, $KEYP, 16 + @{[vle32_v $V20, $KEYP]} + @{[vaesef_vs $V1, $V20]} # with round key w[40,43] + + @{[vse32_v $V1, $OUTP]} + + ret +.size L_enc_128,.-L_enc_128 +___ + +$code .= <<___; +.p2align 3 +L_enc_192: + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + + @{[vle32_v $V1, $INP]} + + @{[vle32_v $V10, $KEYP]} + @{[vaesz_vs $V1, $V10]} # with round key w[ 0, 3] + addi $KEYP, $KEYP, 16 + @{[vle32_v $V11, $KEYP]} + @{[vaesem_vs $V1, $V11]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V12, $KEYP]} + @{[vaesem_vs $V1, $V12]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V13, $KEYP]} + @{[vaesem_vs $V1, $V13]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V14, $KEYP]} + @{[vaesem_vs $V1, $V14]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V15, $KEYP]} + @{[vaesem_vs $V1, $V15]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V16, $KEYP]} + @{[vaesem_vs $V1, $V16]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V17, $KEYP]} + @{[vaesem_vs $V1, $V17]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V18, $KEYP]} + @{[vaesem_vs $V1, $V18]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V19, $KEYP]} + @{[vaesem_vs $V1, $V19]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V20, $KEYP]} + @{[vaesem_vs $V1, $V20]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V21, $KEYP]} + @{[vaesem_vs $V1, $V21]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V22, $KEYP]} + @{[vaesef_vs $V1, $V22]} + + @{[vse32_v $V1, $OUTP]} + ret +.size L_enc_192,.-L_enc_192 +___ + +$code .= <<___; +.p2align 3 +L_enc_256: + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + + @{[vle32_v $V1, $INP]} + + @{[vle32_v $V10, $KEYP]} + @{[vaesz_vs $V1, $V10]} # with round key w[ 0, 3] + addi $KEYP, $KEYP, 16 + @{[vle32_v $V11, $KEYP]} + @{[vaesem_vs $V1, $V11]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V12, $KEYP]} + @{[vaesem_vs $V1, $V12]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V13, $KEYP]} + @{[vaesem_vs $V1, $V13]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V14, $KEYP]} + @{[vaesem_vs $V1, $V14]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V15, $KEYP]} + @{[vaesem_vs $V1, $V15]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V16, $KEYP]} + @{[vaesem_vs $V1, $V16]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V17, $KEYP]} + @{[vaesem_vs $V1, $V17]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V18, $KEYP]} + @{[vaesem_vs $V1, $V18]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V19, $KEYP]} + @{[vaesem_vs $V1, $V19]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V20, $KEYP]} + @{[vaesem_vs $V1, $V20]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V21, $KEYP]} + @{[vaesem_vs $V1, $V21]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V22, $KEYP]} + @{[vaesem_vs $V1, $V22]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V23, $KEYP]} + @{[vaesem_vs $V1, $V23]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V24, $KEYP]} + @{[vaesef_vs $V1, $V24]} + + @{[vse32_v $V1, $OUTP]} + ret +.size L_enc_256,.-L_enc_256 +___ + +################################################################################ +# void rv64i_zvkned_decrypt(const unsigned char *in, unsigned char *out, +# const AES_KEY *key); + +$code .= <<___; +.p2align 3 +.globl rv64i_zvkned_decrypt +.type rv64i_zvkned_decrypt,\@function +rv64i_zvkned_decrypt: + # Load number of rounds + lwu $ROUNDS, 240($KEYP) + + # Get proper routine for key size + li $T6, 14 + beq $ROUNDS, $T6, L_dec_256 + li $T6, 10 + beq $ROUNDS, $T6, L_dec_128 + li $T6, 12 + beq $ROUNDS, $T6, L_dec_192 + + j L_fail_m2 +.size rv64i_zvkned_decrypt,.-rv64i_zvkned_decrypt +___ + +$code .= <<___; +.p2align 3 +L_dec_128: + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + + @{[vle32_v $V1, $INP]} + + addi $KEYP, $KEYP, 160 + @{[vle32_v $V20, $KEYP]} + @{[vaesz_vs $V1, $V20]} # with round key w[40,43] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V19, $KEYP]} + @{[vaesdm_vs $V1, $V19]} # with round key w[36,39] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V18, $KEYP]} + @{[vaesdm_vs $V1, $V18]} # with round key w[32,35] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V17, $KEYP]} + @{[vaesdm_vs $V1, $V17]} # with round key w[28,31] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V16, $KEYP]} + @{[vaesdm_vs $V1, $V16]} # with round key w[24,27] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V15, $KEYP]} + @{[vaesdm_vs $V1, $V15]} # with round key w[20,23] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V14, $KEYP]} + @{[vaesdm_vs $V1, $V14]} # with round key w[16,19] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V13, $KEYP]} + @{[vaesdm_vs $V1, $V13]} # with round key w[12,15] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V12, $KEYP]} + @{[vaesdm_vs $V1, $V12]} # with round key w[ 8,11] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V11, $KEYP]} + @{[vaesdm_vs $V1, $V11]} # with round key w[ 4, 7] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V10, $KEYP]} + @{[vaesdf_vs $V1, $V10]} # with round key w[ 0, 3] + + @{[vse32_v $V1, $OUTP]} + + ret +.size L_dec_128,.-L_dec_128 +___ + +$code .= <<___; +.p2align 3 +L_dec_192: + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + + @{[vle32_v $V1, $INP]} + + addi $KEYP, $KEYP, 192 + @{[vle32_v $V22, $KEYP]} + @{[vaesz_vs $V1, $V22]} # with round key w[48,51] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V21, $KEYP]} + @{[vaesdm_vs $V1, $V21]} # with round key w[44,47] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V20, $KEYP]} + @{[vaesdm_vs $V1, $V20]} # with round key w[40,43] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V19, $KEYP]} + @{[vaesdm_vs $V1, $V19]} # with round key w[36,39] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V18, $KEYP]} + @{[vaesdm_vs $V1, $V18]} # with round key w[32,35] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V17, $KEYP]} + @{[vaesdm_vs $V1, $V17]} # with round key w[28,31] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V16, $KEYP]} + @{[vaesdm_vs $V1, $V16]} # with round key w[24,27] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V15, $KEYP]} + @{[vaesdm_vs $V1, $V15]} # with round key w[20,23] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V14, $KEYP]} + @{[vaesdm_vs $V1, $V14]} # with round key w[16,19] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V13, $KEYP]} + @{[vaesdm_vs $V1, $V13]} # with round key w[12,15] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V12, $KEYP]} + @{[vaesdm_vs $V1, $V12]} # with round key w[ 8,11] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V11, $KEYP]} + @{[vaesdm_vs $V1, $V11]} # with round key w[ 4, 7] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V10, $KEYP]} + @{[vaesdf_vs $V1, $V10]} # with round key w[ 0, 3] + + @{[vse32_v $V1, $OUTP]} + + ret +.size L_dec_192,.-L_dec_192 +___ + +$code .= <<___; +.p2align 3 +L_dec_256: + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + + @{[vle32_v $V1, $INP]} + + addi $KEYP, $KEYP, 224 + @{[vle32_v $V24, $KEYP]} + @{[vaesz_vs $V1, $V24]} # with round key w[56,59] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V23, $KEYP]} + @{[vaesdm_vs $V1, $V23]} # with round key w[52,55] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V22, $KEYP]} + @{[vaesdm_vs $V1, $V22]} # with round key w[48,51] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V21, $KEYP]} + @{[vaesdm_vs $V1, $V21]} # with round key w[44,47] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V20, $KEYP]} + @{[vaesdm_vs $V1, $V20]} # with round key w[40,43] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V19, $KEYP]} + @{[vaesdm_vs $V1, $V19]} # with round key w[36,39] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V18, $KEYP]} + @{[vaesdm_vs $V1, $V18]} # with round key w[32,35] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V17, $KEYP]} + @{[vaesdm_vs $V1, $V17]} # with round key w[28,31] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V16, $KEYP]} + @{[vaesdm_vs $V1, $V16]} # with round key w[24,27] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V15, $KEYP]} + @{[vaesdm_vs $V1, $V15]} # with round key w[20,23] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V14, $KEYP]} + @{[vaesdm_vs $V1, $V14]} # with round key w[16,19] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V13, $KEYP]} + @{[vaesdm_vs $V1, $V13]} # with round key w[12,15] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V12, $KEYP]} + @{[vaesdm_vs $V1, $V12]} # with round key w[ 8,11] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V11, $KEYP]} + @{[vaesdm_vs $V1, $V11]} # with round key w[ 4, 7] + addi $KEYP, $KEYP, -16 + @{[vle32_v $V10, $KEYP]} + @{[vaesdf_vs $V1, $V10]} # with round key w[ 0, 3] + + @{[vse32_v $V1, $OUTP]} + + ret +.size L_dec_256,.-L_dec_256 +___ +} + +$code .= <<___; +L_fail_m1: + li a0, -1 + ret +.size L_fail_m1,.-L_fail_m1 + +L_fail_m2: + li a0, -2 + ret +.size L_fail_m2,.-L_fail_m2 + +L_end: + ret +.size L_end,.-L_end +___ + +print $code; + +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/aes/asm/aes-riscv64.pl b/crypto/openssl/crypto/aes/asm/aes-riscv64.pl new file mode 100644 index 000000000000..525eba4b4638 --- /dev/null +++ b/crypto/openssl/crypto/aes/asm/aes-riscv64.pl @@ -0,0 +1,1709 @@ +#! /usr/bin/env perl +# Copyright 2022 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$output and open STDOUT,">$output"; + +################################################################################ +# Utility functions to help with keeping track of which registers to stack/ +# unstack when entering / exiting routines. +################################################################################ +{ + # Callee-saved registers + my @callee_saved = map("x$_",(2,8,9,18..27)); + # Caller-saved registers + my @caller_saved = map("x$_",(1,5..7,10..17,28..31)); + my @must_save; + sub use_reg { + my $reg = shift; + if (grep(/^$reg$/, @callee_saved)) { + push(@must_save, $reg); + } elsif (!grep(/^$reg$/, @caller_saved)) { + # Register is not usable! + die("Unusable register ".$reg); + } + return $reg; + } + sub use_regs { + return map(use_reg("x$_"), @_); + } + sub save_regs { + my $ret = ''; + my $stack_reservation = ($#must_save + 1) * 8; + my $stack_offset = $stack_reservation; + if ($stack_reservation % 16) { + $stack_reservation += 8; + } + $ret.=" addi sp,sp,-$stack_reservation\n"; + foreach (@must_save) { + $stack_offset -= 8; + $ret.=" sd $_,$stack_offset(sp)\n"; + } + return $ret; + } + sub load_regs { + my $ret = ''; + my $stack_reservation = ($#must_save + 1) * 8; + my $stack_offset = $stack_reservation; + if ($stack_reservation % 16) { + $stack_reservation += 8; + } + foreach (@must_save) { + $stack_offset -= 8; + $ret.=" ld $_,$stack_offset(sp)\n"; + } + $ret.=" addi sp,sp,$stack_reservation\n"; + return $ret; + } + sub clear_regs { + @must_save = (); + } +} + +################################################################################ +# Register assignment for AES_encrypt and AES_decrypt +################################################################################ + +# Registers to hold AES state (called s0-s3 or y0-y3 elsewhere) +my ($Q0,$Q1,$Q2,$Q3) = use_regs(6..9); + +# Function arguments (x10-x12 are a0-a2 in the ABI) +# Input block pointer, output block pointer, key pointer +my ($INP,$OUTP,$KEYP) = use_regs(10..12); + +# Temporaries +my ($T0,$T1,$T2,$T3) = use_regs(13..16); +my ($T4,$T5,$T6,$T7,$T8,$T9,$T10,$T11) = use_regs(17..24); +my ($T12,$T13,$T14,$T15) = use_regs(25..28); + +# Register to hold table offset +my ($I0) = use_regs(29); + +# Loop counter +my ($loopcntr) = use_regs(30); + +# Lookup table address register +my ($TBL) = use_regs(31); + +# Lookup table mask register +my ($MSK) = use_regs(5); + +# Aliases for readability +my $K0 = $loopcntr; +my $K1 = $KEYP; + +################################################################################ +# Table lookup utility functions for AES_encrypt and AES_decrypt +################################################################################ + +# do_lookup([destination regs], [state regs], [temporary regs], shamt) +# do_lookup loads four entries from an AES encryption/decryption table +# and stores the result in the specified destination register set +# Ds->[0] = Table[Qs->[0] >> shamt] +# Ds->[1] = Table[Qs->[1] >> shamt] +# Ds->[2] = Table[Qs->[2] >> shamt] +# Ds->[3] = Table[Qs->[3] >> shamt] +# Four temporary regs are used to generate these lookups. The temporary regs +# can be equal to the destination regs, but only if they appear in the same +# order. I.e. do_lookup([A,B,C,D],[...],[A,B,C,D],...) is OK +sub do_lookup { + # (destination regs, state regs, temporary regs, shift amount) + my ($Ds, $Qs, $Ts, $shamt) = @_; + + my $ret = ''; + + # AES encryption/decryption table entries have word-sized (4-byte) entries. + # To convert the table index into a byte offset, we compute + # ((Qs->[i] >> shamt) & 0xFF) << 2 + # However, to save work, we compute the equivalent expression + # (Qs->[i] >> (shamt-2)) & 0x3FC + if ($shamt < 2) { +$ret .= <<___; + + slli $Ts->[0],$Qs->[0],$shamt+2 + slli $Ts->[1],$Qs->[1],$shamt+2 + slli $Ts->[2],$Qs->[2],$shamt+2 + slli $Ts->[3],$Qs->[3],$shamt+2 +___ + } else { +$ret .= <<___; + + srli $Ts->[0],$Qs->[0],$shamt-2 + srli $Ts->[1],$Qs->[1],$shamt-2 + srli $Ts->[2],$Qs->[2],$shamt-2 + srli $Ts->[3],$Qs->[3],$shamt-2 +___ + } + +$ret .= <<___; + + andi $Ts->[0],$Ts->[0],0x3FC + andi $Ts->[1],$Ts->[1],0x3FC + andi $Ts->[2],$Ts->[2],0x3FC + andi $Ts->[3],$Ts->[3],0x3FC + + # Index into table. + add $I0,$TBL,$Ts->[0] + lwu $Ds->[0],0($I0) + add $I0,$TBL,$Ts->[1] + lwu $Ds->[1],0($I0) + add $I0,$TBL,$Ts->[2] + lwu $Ds->[2],0($I0) + add $I0,$TBL,$Ts->[3] + lwu $Ds->[3],0($I0) + +___ + + return $ret; +} + +# Identical to do_lookup(), but loads only a single byte into each destination +# register (replaces lwu with lbu). Used in the final round of AES_encrypt. +sub do_lookup_byte { + my $ret = do_lookup(@_); + $ret =~ s/lwu/lbu/g; + return $ret; +} + +# do_lookup_Td4([destination regs], [state regs], [temporary regs]) +# Used in final phase of AES_decrypt +# Ds->[0] = Table[(Qs->[0]) &0xFF] +# Ds->[1] = Table[(Qs->[1] >> 8 )&0xFF] +# Ds->[2] = Table[(Qs->[2] >> 16)&0xFF] +# Ds->[3] = Table[(Qs->[3] >> 24)&0xFF] +# Four temporary regs are used to generate these lookups. The temporary regs +# can be equal to the destination regs, but only if they appear in the same +# order. I.e. do_lookup([A,B,C,D],[...],[A,B,C,D],...) is OK +sub do_lookup_Td4 { + my ($Ds, $Qs, $Ts) = @_; + + my $ret = ''; + +$ret .= <<___; + srli $Ts->[1],$Qs->[1],8 + srli $Ts->[2],$Qs->[2],16 + srli $Ts->[3],$Qs->[3],24 + + andi $Ts->[0],$Qs->[0],0xFF + andi $Ts->[1],$Ts->[1],0xFF + andi $Ts->[2],$Ts->[2],0xFF + andi $Ts->[3],$Ts->[3],0xFF + + add $I0,$TBL,$Ts->[0] + lbu $Ds->[0],0($I0) + add $I0,$TBL,$Ts->[1] + lbu $Ds->[1],0($I0) + add $I0,$TBL,$Ts->[2] + lbu $Ds->[2],0($I0) + add $I0,$TBL,$Ts->[3] + lbu $Ds->[3],0($I0) + +___ + + return $ret; +} + +################################################################################ +# void AES_encrypt(const unsigned char *in, unsigned char *out, +# const AES_KEY *key); +################################################################################ +my $code .= <<___; +.text +.balign 16 +.globl AES_encrypt +.type AES_encrypt,\@function +AES_encrypt: +___ + +$code .= save_regs(); + +$code .= <<___; + + # Load input to block cipher + ld $Q0,0($INP) + ld $Q2,8($INP) + + + # Load key + ld $T0,0($KEYP) + ld $T2,8($KEYP) + + + # Load number of rounds + lwu $loopcntr,240($KEYP) + + # Load address of substitution table and wrap-around mask + la $TBL,AES_Te0 + li $MSK,~0xFFF + + # y = n xor k, stored in Q0-Q3 + + xor $Q0,$Q0,$T0 + xor $Q2,$Q2,$T2 + srli $Q1,$Q0,32 + srli $Q3,$Q2,32 + + # The main loop only executes the first N-1 rounds. + add $loopcntr,$loopcntr,-1 + + # Do Nr - 1 rounds (final round is special) + +1: +___ + +# Lookup in table Te0 +$code .= do_lookup( + [$T4,$T5,$T6,$T7], # Destination registers + [$Q0,$Q1,$Q2,$Q3], # State registers + [$T0,$T1,$T2,$T3], # Temporaries + 0 # Shift amount +); + +$code .= <<___; + add $TBL,$TBL,1024 +___ + +# Lookup in table Te1 +$code .= do_lookup( + [$T8,$T9,$T10,$T11], + [$Q1,$Q2,$Q3,$Q0], + [$T0,$T1,$T2,$T3], + 8 +); + +$code .= <<___; + add $TBL,$TBL,1024 +___ + +# Lookup in table Te2 +$code .= do_lookup( + [$T12,$T13,$T14,$T15], + [$Q2,$Q3,$Q0,$Q1], + [$T0,$T1,$T2,$T3], + 16 +); + +$code .= <<___; + add $TBL,$TBL,1024 +___ + +# Lookup in table Te3 +$code .= do_lookup( + [$T0,$T1,$T2,$T3], + [$Q3,$Q0,$Q1,$Q2], + [$T0,$T1,$T2,$T3], + 24 +); + +$code .= <<___; + + # Combine table lookups + xor $T4,$T4,$T8 + xor $T5,$T5,$T9 + xor $T6,$T6,$T10 + xor $T7,$T7,$T11 + + xor $T4,$T4,$T12 + xor $T5,$T5,$T13 + xor $T6,$T6,$T14 + xor $T7,$T7,$T15 + + xor $T0,$T0,$T4 + xor $T1,$T1,$T5 + xor $T2,$T2,$T6 + xor $T3,$T3,$T7 + + # Update key ptr to point to next key in schedule + add $KEYP,$KEYP,16 + + # Grab next key in schedule + ld $T4,0($KEYP) + ld $T6,8($KEYP) + + # Round TBL back to 4k boundary + and $TBL,$TBL,$MSK + + add $loopcntr,$loopcntr,-1 + + xor $Q0,$T0,$T4 + xor $Q2,$T2,$T6 + srli $T5,$T4,32 + xor $Q1,$T1,$T5 + srli $T7,$T6,32 + xor $Q3,$T3,$T7 + + bgtz $loopcntr,1b + +#================================FINAL ROUND==================================== + +# In the final round, all lookup table accesses would appear as follows: +# +# ... compute index I0 +# add I0,TBL,T0 +# lbu T0,1(I0) +# +# Instead of indexing with a 1 offset, we can add 1 to the TBL pointer, and use +# a 0 offset when indexing in the following code. This enables some instruction +# fusion opportunities. + + add $TBL,$TBL,1 + + ld $K0,16($KEYP) + ld $K1,24($KEYP) +___ + +$code .= do_lookup_byte( + [$T4,$T5,$T6,$T7], + [$Q0,$Q1,$Q2,$Q3], + [$T0,$T1,$T2,$T3], + 0 +); + +$code .= do_lookup_byte( + [$T8,$T9,$T10,$T11], + [$Q1,$Q2,$Q3,$Q0], + [$T0,$T1,$T2,$T3], + 8 +); + +$code .= do_lookup_byte( + [$T12,$T13,$T14,$T15], + [$Q2,$Q3,$Q0,$Q1], + [$T0,$T1,$T2,$T3], + 16 +); + +$code .= do_lookup_byte( + [$T0,$T1,$T2,$T3], + [$Q3,$Q0,$Q1,$Q2], + [$T0,$T1,$T2,$T3], + 24 +); + +$code .= <<___; + + # Combine table lookups into T0 and T2 + + slli $T5,$T5,32 + slli $T7,$T7,32 + slli $T8,$T8,8 + slli $T9,$T9,8+32 + slli $T10,$T10,8 + slli $T11,$T11,8+32 + slli $T12,$T12,16 + slli $T13,$T13,16+32 + slli $T14,$T14,16 + slli $T15,$T15,16+32 + + slli $T0,$T0,24 + slli $T1,$T1,24+32 + slli $T2,$T2,24 + slli $T3,$T3,24+32 + + xor $T4,$T4,$T0 + xor $T5,$T5,$T1 + xor $T6,$T6,$T2 + xor $T7,$T7,$T3 + + xor $T8,$T8,$T12 + xor $T9,$T9,$T13 + xor $T10,$T10,$T14 + xor $T11,$T11,$T15 + + xor $T0,$T4,$T8 + xor $T1,$T5,$T9 + xor $T2,$T6,$T10 + xor $T3,$T7,$T11 + + + xor $T0,$T0,$T1 + # T0 = [T1 T13 T9 T5 T0 T12 T8 T4] + xor $T0,$T0,$K0 # XOR in key + + xor $T2,$T2,$T3 + # T2 = [T3 T15 T11 T7 T2 T14 T10 T6] + xor $T2,$T2,$K1 # XOR in key + + sd $T0,0($OUTP) + sd $T2,8($OUTP) + + # Pop registers and return +2: +___ + +$code .= load_regs(); + +$code .= <<___; + ret +___ + +################################################################################ +# void AES_decrypt(const unsigned char *in, unsigned char *out, +# const AES_KEY *key); +################################################################################ +$code .= <<___; +.text +.balign 16 +.globl AES_decrypt +.type AES_decrypt,\@function +AES_decrypt: +___ + +$code .= save_regs(); + +$code .= <<___; + + # Load input to block cipher + ld $Q0,0($INP) + ld $Q2,8($INP) + + # Load key + # Note that key is assumed in BE byte order + # (This routine was written against a key scheduling implementation that + # placed keys in BE byte order.) + ld $T0,0($KEYP) + ld $T2,8($KEYP) + + # Load number of rounds + lwu $loopcntr,240($KEYP) + + # Load address of substitution table and wrap-around mask + la $TBL,AES_Td0 + li $MSK,~0xFFF + + xor $Q0,$Q0,$T0 + xor $Q2,$Q2,$T2 + srli $Q1,$Q0,32 + srli $Q3,$Q2,32 + + # The main loop only executes the first N-1 rounds. + add $loopcntr,$loopcntr,-1 + + # Do Nr - 1 rounds (final round is special) +1: +___ + +# Lookup in Td0 +$code .= do_lookup( + [$T4,$T5,$T6,$T7], # Destination registers + [$Q0,$Q1,$Q2,$Q3], # State registers + [$T0,$T1,$T2,$T3], # Temporaries + 0 # Shift amount +); + +$code .= <<___; + add $TBL,$TBL,1024 +___ + +# Lookup in Td1 +$code .= do_lookup( + [$T8,$T9,$T10,$T11], + [$Q3,$Q0,$Q1,$Q2], + [$T0,$T1,$T2,$T3], + 8 +); + +$code .= <<___; + add $TBL,$TBL,1024 +___ + +# Lookup in Td2 +$code .= do_lookup( + [$T12,$T13,$T14,$T15], + [$Q2,$Q3,$Q0,$Q1], + [$T0,$T1,$T2,$T3], + 16 +); + +$code .= <<___; + add $TBL,$TBL,1024 +___ + +# Lookup in Td3 +$code .= do_lookup( + [$T0,$T1,$T2,$T3], + [$Q1,$Q2,$Q3,$Q0], + [$T0,$T1,$T2,$T3], + 24 +); + +$code .= <<___; + xor $T4,$T4,$T8 + xor $T5,$T5,$T9 + xor $T6,$T6,$T10 + xor $T7,$T7,$T11 + + xor $T4,$T4,$T12 + xor $T5,$T5,$T13 + xor $T6,$T6,$T14 + xor $T7,$T7,$T15 + + xor $T0,$T0,$T4 + xor $T1,$T1,$T5 + xor $T2,$T2,$T6 + xor $T3,$T3,$T7 + + # Update key ptr to point to next key in schedule + add $KEYP,$KEYP,16 + + # Grab next key in schedule + ld $T4,0($KEYP) + ld $T6,8($KEYP) + + # Round TBL back to 4k boundary + and $TBL,$TBL,$MSK + + add $loopcntr,$loopcntr,-1 + + xor $Q0,$T0,$T4 + xor $Q2,$T2,$T6 + srli $T5,$T4,32 + xor $Q1,$T1,$T5 + srli $T7,$T6,32 + xor $Q3,$T3,$T7 + + bgtz $loopcntr,1b + +#================================FINAL ROUND==================================== + + la $TBL,AES_Td4 + + # K0,K1 are aliases for loopcntr,KEYP + # As these registers will no longer be used after these loads, reuse them + # to store the final key in the schedule. + ld $K0,16($KEYP) + ld $K1,24($KEYP) +___ + +$code .= do_lookup_Td4( + [$T4,$T5,$T6,$T7], + [$Q0,$Q3,$Q2,$Q1], + [$T0,$T1,$T2,$T3] +); + +$code .= do_lookup_Td4( + [$T8,$T9,$T10,$T11], + [$Q1,$Q0,$Q3,$Q2], + [$T0,$T1,$T2,$T3] +); + +$code .= do_lookup_Td4( + [$T12,$T13,$T14,$T15], + [$Q2,$Q1,$Q0,$Q3], + [$T0,$T1,$T2,$T3] +); + +$code .= do_lookup_Td4( + [$T0,$T1,$T2,$T3], + [$Q3,$Q2,$Q1,$Q0], + [$T0,$T1,$T2,$T3] +); + +$code .= <<___; + + # T0-T15 now contain the decrypted block, minus xoring with the final round + # key. We pack T0-T15 into the two 64-bit registers T0 and T4, then xor + # in the key and store. + + slli $T5,$T5,8 + slli $T6,$T6,16 + slli $T7,$T7,24 + slli $T8,$T8,32 + slli $T9,$T9,8+32 + slli $T10,$T10,16+32 + slli $T11,$T11,32+24 + slli $T13,$T13,8 + slli $T14,$T14,16 + slli $T15,$T15,24 + slli $T0,$T0,32 + slli $T1,$T1,8+32 + slli $T2,$T2,16+32 + slli $T3,$T3,24+32 + + xor $T4,$T4,$T5 + xor $T6,$T6,$T7 + xor $T8,$T8,$T9 + xor $T10,$T10,$T11 + + xor $T12,$T12,$T13 + xor $T14,$T14,$T15 + xor $T0,$T0,$T1 + xor $T2,$T2,$T3 + + xor $T4,$T4,$T6 + xor $T8,$T8,$T10 + xor $T12,$T12,$T14 + xor $T0,$T0,$T2 + + xor $T4,$T4,$T8 + # T4 = [T11 T10 T9 T8 T7 T6 T5 T4] + xor $T4,$T4,$K0 # xor in key + + xor $T0,$T0,$T12 + # T0 = [T3 T2 T1 T0 T15 T14 T13 T12] + xor $T0,$T0,$K1 # xor in key + + sd $T4,0($OUTP) + sd $T0,8($OUTP) + + # Pop registers and return +___ + +$code .= load_regs(); + +$code .= <<___; + ret +___ + +clear_regs(); + +################################################################################ +# Register assignment for AES_set_encrypt_key +################################################################################ + +# Function arguments (x10-x12 are a0-a2 in the ABI) +# Pointer to user key, number of bits in key, key pointer +my ($UKEY,$BITS,$KEYP) = use_regs(10..12); + +# Temporaries +my ($T0,$T1,$T2,$T3) = use_regs(6..8,13); +my ($T4,$T5,$T6,$T7,$T8,$T9,$T10,$T11) = use_regs(14..17,28..31); + +# Pointer into rcon table +my ($RCON) = use_regs(9); + +# Register to hold table offset and used as a temporary +my ($I0) = use_regs(18); + +# Loop counter +my ($loopcntr) = use_regs(19); + +# Lookup table address register +my ($TBL) = use_regs(20); + +# Calculates dest = [ +# S[(in>>shifts[3])&0xFF], +# S[(in>>shifts[2])&0xFF], +# S[(in>>shifts[1])&0xFF], +# S[(in>>shifts[0])&0xFF] +# ] +# This routine spreads accesses across Te0-Te3 to help bring those tables +# into cache, in anticipation of running AES_[en/de]crypt. +sub do_enc_lookup { + # (destination reg, input reg, shifts array, temporary regs) + my ($dest, $in, $shifts, $Ts) = @_; + + my $ret = ''; + +$ret .= <<___; + + # Round TBL back to 4k boundary + srli $TBL,$TBL,12 + slli $TBL,$TBL,12 + + # Offset by 1 byte, since Te0[x] = S[x].[03, 01, 01, 02] + # So that, later on, a 0-offset lbu yields S[x].01 == S[x] + addi $TBL,$TBL,1 +___ + + for ($i = 0; $i < 4; $i++) { + if ($shifts->[$i] < 2) { + $ret .= " slli $Ts->[$i],$in,2-$shifts->[$i]\n"; + } else { + $ret .= " srli $Ts->[$i],$in,$shifts->[$i]-2\n"; + } + } + +$ret .= <<___; + + andi $Ts->[0],$Ts->[0],0x3FC + andi $Ts->[1],$Ts->[1],0x3FC + andi $Ts->[2],$Ts->[2],0x3FC + andi $Ts->[3],$Ts->[3],0x3FC + + # Index into tables Te0-Te3 (spread access across tables to help bring + # them into cache for later) + + add $I0,$TBL,$Ts->[0] + lbu $Ts->[0],0($I0) + + add $TBL,$TBL,1025 # yes, 1025 + add $I0,$TBL,$Ts->[1] + lbu $Ts->[1],0($I0) + + add $TBL,$TBL,1025 + add $I0,$TBL,$Ts->[2] + lbu $Ts->[2],0($I0) + + add $TBL,$TBL,1022 + add $I0,$TBL,$Ts->[3] + lbu $Ts->[3],0($I0) + + slli $Ts->[1],$Ts->[1],8 + slli $Ts->[2],$Ts->[2],16 + slli $Ts->[3],$Ts->[3],24 + + xor $Ts->[0],$Ts->[0],$Ts->[1] + xor $Ts->[2],$Ts->[2],$Ts->[3] + xor $dest,$Ts->[0],$Ts->[2] +___ + + return $ret; +} + +################################################################################ +# void AES_set_encrypt_key(const unsigned char *userKey, const int bits, +# AES_KEY *key) +################################################################################ +$code .= <<___; +.text +.balign 16 +.globl AES_set_encrypt_key +.type AES_set_encrypt_key,\@function +AES_set_encrypt_key: +___ +$code .= save_regs(); +$code .= <<___; + bnez $UKEY,1f # if (!userKey || !key) return -1; + bnez $KEYP,1f + li a0,-1 + ret +1: + la $RCON,AES_rcon + la $TBL,AES_Te0 + li $T8,128 + li $T9,192 + li $T10,256 + + # Determine number of rounds from key size in bits + bne $BITS,$T8,1f + li $T3,10 # key->rounds = 10 if bits == 128 + j 3f +1: + bne $BITS,$T9,2f + li $T3,12 # key->rounds = 12 if bits == 192 + j 3f +2: + li $T3,14 # key->rounds = 14 if bits == 256 + beq $BITS,$T10,3f + li a0,-2 # If bits != 128, 192, or 256, return -2 + j 5f +3: + ld $T0,0($UKEY) + ld $T2,8($UKEY) + + sw $T3,240($KEYP) + + li $loopcntr,0 # == i*4 + + srli $T1,$T0,32 + srli $T3,$T2,32 + + sd $T0,0($KEYP) + sd $T2,8($KEYP) + + # if bits == 128 + # jump into loop + beq $BITS,$T8,1f + + ld $T4,16($UKEY) + srli $T5,$T4,32 + sd $T4,16($KEYP) + + # if bits == 192 + # jump into loop + beq $BITS,$T9,2f + + ld $T6,24($UKEY) + srli $T7,$T6,32 + sd $T6,24($KEYP) + + # bits == 256 + j 3f +___ + +$code .= <<___; +1: + addi $KEYP,$KEYP,16 +1: +___ +$code .= do_enc_lookup($T4,$T3,[8,16,24,0],[$T4,$T5,$T6,$T7]); + +$code .= <<___; + add $T5,$RCON,$loopcntr # rcon[i] (i increments by 4 so it can double as + # a word offset) + lwu $T5,0($T5) + + addi $loopcntr,$loopcntr,4 + li $I0,10*4 + + xor $T0,$T0,$T4 + xor $T0,$T0,$T5 + xor $T1,$T1,$T0 + xor $T2,$T2,$T1 + xor $T3,$T3,$T2 + + sw $T0,0($KEYP) + sw $T1,4($KEYP) + sw $T2,8($KEYP) + sw $T3,12($KEYP) + + addi $KEYP,$KEYP,16 + + + bne $loopcntr,$I0,1b + j 4f +___ +$code .= <<___; +2: + addi $KEYP,$KEYP,24 +2: +___ +$code .= do_enc_lookup($T6,$T5,[8,16,24,0],[$T6,$T7,$T8,$T9]); + +$code .= <<___; + add $T7,$RCON,$loopcntr # rcon[i] (i increments by 4 so it can double as + # a word offset) + lwu $T7,0($T7) + + addi $loopcntr,$loopcntr,4 + li $I0,8*4 + + xor $T0,$T0,$T6 + xor $T0,$T0,$T7 + xor $T1,$T1,$T0 + xor $T2,$T2,$T1 + xor $T3,$T3,$T2 + + sw $T0,0($KEYP) + sw $T1,4($KEYP) + sw $T2,8($KEYP) + sw $T3,12($KEYP) + + beq $loopcntr,$I0,4f + + xor $T4,$T4,$T3 + xor $T5,$T5,$T4 + sw $T4,16($KEYP) + sw $T5,20($KEYP) + + addi $KEYP,$KEYP,24 + j 2b +___ +$code .= <<___; +3: + addi $KEYP,$KEYP,32 +3: +___ +$code .= do_enc_lookup($T8,$T7,[8,16,24,0],[$T8,$T9,$T10,$T11]); + +$code .= <<___; + add $T9,$RCON,$loopcntr # rcon[i] (i increments by 4 so it can double as + # a word offset) + lwu $T9,0($T9) + + addi $loopcntr,$loopcntr,4 + li $I0,7*4 + + xor $T0,$T0,$T8 + xor $T0,$T0,$T9 + xor $T1,$T1,$T0 + xor $T2,$T2,$T1 + xor $T3,$T3,$T2 + + sw $T0,0($KEYP) + sw $T1,4($KEYP) + sw $T2,8($KEYP) + sw $T3,12($KEYP) + + beq $loopcntr,$I0,4f +___ +$code .= do_enc_lookup($T8,$T3,[0,8,16,24],[$T8,$T9,$T10,$T11]); +$code .= <<___; + xor $T4,$T4,$T8 + xor $T5,$T5,$T4 + xor $T6,$T6,$T5 + xor $T7,$T7,$T6 + sw $T4,16($KEYP) + sw $T5,20($KEYP) + sw $T6,24($KEYP) + sw $T7,28($KEYP) + + addi $KEYP,$KEYP,32 + j 3b + +4: # return 0 + li a0,0 +5: # return a0 +___ +$code .= load_regs(); +$code .= <<___; + ret +___ + +clear_regs(); + +################################################################################ +# Register assignment for AES_set_decrypt_key +################################################################################ + +# Function arguments (x10-x12 are a0-a2 in the ABI) +# Pointer to user key, number of bits in key, key pointer +my ($UKEY,$BITS,$KEYP) = use_regs(10..12); + +# Temporaries +my ($T0,$T1,$T2,$T3) = use_regs(6..8,9); +my ($T4,$T5,$T6,$T7,$T8) = use_regs(13..17); + +my ($I1) = use_regs(18); + +# Register to hold table offset and used as a temporary +my ($I0) = use_regs(19); + +# Loop counter +my ($loopcntr) = use_regs(20); + +# Lookup table address register +my ($TBL) = use_regs(21); + +# Calculates dest = [ +# Td0[Te1[(in >> 24) & 0xff] & 0xff] ^ +# Td1[Te1[(in >> 16) & 0xff] & 0xff] ^ +# Td2[Te1[(in >> 8) & 0xff] & 0xff] ^ +# Td3[Te1[(in ) & 0xff] & 0xff] +# ] +sub do_dec_lookup { + # (destination reg, input reg, temporary regs) + my ($dest, $in, $Ts) = @_; + + my $ret = ''; + +$ret .= <<___; + + la $TBL,AES_Te2 + + slli $Ts->[0],$in,2 + srli $Ts->[1],$in,8-2 + srli $Ts->[2],$in,16-2 + srli $Ts->[3],$in,24-2 + + andi $Ts->[0],$Ts->[0],0x3FC + andi $Ts->[1],$Ts->[1],0x3FC + andi $Ts->[2],$Ts->[2],0x3FC + andi $Ts->[3],$Ts->[3],0x3FC + + # Index into table Te2 + + add $I0,$TBL,$Ts->[0] + lwu $Ts->[0],0($I0) + + add $I0,$TBL,$Ts->[1] + lwu $Ts->[1],0($I0) + + add $I0,$TBL,$Ts->[2] + lwu $Ts->[2],0($I0) + + add $I0,$TBL,$Ts->[3] + lwu $Ts->[3],0($I0) + + andi $Ts->[0],$Ts->[0],0xFF + andi $Ts->[1],$Ts->[1],0xFF + andi $Ts->[2],$Ts->[2],0xFF + andi $Ts->[3],$Ts->[3],0xFF + + slli $Ts->[0],$Ts->[0],2 + slli $Ts->[1],$Ts->[1],2 + slli $Ts->[2],$Ts->[2],2 + slli $Ts->[3],$Ts->[3],2 + + la $TBL,AES_Td0 + + # Lookup in Td0-Td3 + + add $I0,$TBL,$Ts->[0] + lwu $Ts->[0],0($I0) + + add $TBL,$TBL,1024 + add $I0,$TBL,$Ts->[1] + lwu $Ts->[1],0($I0) + + add $TBL,$TBL,1024 + add $I0,$TBL,$Ts->[2] + lwu $Ts->[2],0($I0) + + add $TBL,$TBL,1024 + add $I0,$TBL,$Ts->[3] + lwu $Ts->[3],0($I0) + + xor $Ts->[0],$Ts->[0],$Ts->[1] + xor $Ts->[2],$Ts->[2],$Ts->[3] + xor $dest,$Ts->[0],$Ts->[2] +___ + + return $ret; +} + +################################################################################ +# void AES_set_decrypt_key(const unsigned char *userKey, const int bits, +# AES_KEY *key) +################################################################################ +$code .= <<___; +.text +.balign 16 +.globl AES_set_decrypt_key +.type AES_set_decrypt_key,\@function +AES_set_decrypt_key: + # Call AES_set_encrypt_key first + addi sp,sp,-16 + sd $KEYP,0(sp) # We need to hold onto this! + sd ra,8(sp) + jal ra,AES_set_encrypt_key + ld $KEYP,0(sp) + ld ra,8(sp) + addi sp,sp,16 + bgez a0,1f # If error, return error + ret +1: +___ +$code .= save_regs(); +$code .= <<___; + + li $T4,0 + lwu $T8,240($KEYP) + slli $T5,$T8,4 + # Invert order of round keys +1: + add $I0,$KEYP,$T4 + ld $T0,0($I0) + ld $T1,8($I0) + add $I1,$KEYP,$T5 + ld $T2,0($I1) + ld $T3,8($I1) + addi $T4,$T4,16 + addi $T5,$T5,-16 + sd $T0,0($I1) + sd $T1,8($I1) + sd $T2,0($I0) + sd $T3,8($I0) + blt $T4,$T5,1b + + li $loopcntr,1 + +1: + addi $KEYP,$KEYP,16 + lwu $T0,0($KEYP) + lwu $T1,4($KEYP) + lwu $T2,8($KEYP) + lwu $T3,12($KEYP) +___ +$code .= do_dec_lookup($T0,$T0,[$T4,$T5,$T6,$T7]); +$code .= do_dec_lookup($T1,$T1,[$T4,$T5,$T6,$T7]); +$code .= do_dec_lookup($T2,$T2,[$T4,$T5,$T6,$T7]); +$code .= do_dec_lookup($T3,$T3,[$T4,$T5,$T6,$T7]); +$code .= <<___; + sw $T0,0($KEYP) + sw $T1,4($KEYP) + sw $T2,8($KEYP) + sw $T3,12($KEYP) + addi $loopcntr,$loopcntr,1 + blt $loopcntr,$T8,1b +___ +$code .= load_regs(); +$code .= <<___; + li a0,0 + ret +___ +$code .= <<___; + +.section .rodata +.p2align 12 +.type AES_Te0,\@object +AES_Te0: +.word 0xa56363c6U, 0x847c7cf8U, 0x997777eeU, 0x8d7b7bf6U +.word 0x0df2f2ffU, 0xbd6b6bd6U, 0xb16f6fdeU, 0x54c5c591U +.word 0x50303060U, 0x03010102U, 0xa96767ceU, 0x7d2b2b56U +.word 0x19fefee7U, 0x62d7d7b5U, 0xe6abab4dU, 0x9a7676ecU +.word 0x45caca8fU, 0x9d82821fU, 0x40c9c989U, 0x877d7dfaU +.word 0x15fafaefU, 0xeb5959b2U, 0xc947478eU, 0x0bf0f0fbU +.word 0xecadad41U, 0x67d4d4b3U, 0xfda2a25fU, 0xeaafaf45U +.word 0xbf9c9c23U, 0xf7a4a453U, 0x967272e4U, 0x5bc0c09bU +.word 0xc2b7b775U, 0x1cfdfde1U, 0xae93933dU, 0x6a26264cU +.word 0x5a36366cU, 0x413f3f7eU, 0x02f7f7f5U, 0x4fcccc83U +.word 0x5c343468U, 0xf4a5a551U, 0x34e5e5d1U, 0x08f1f1f9U +.word 0x937171e2U, 0x73d8d8abU, 0x53313162U, 0x3f15152aU +.word 0x0c040408U, 0x52c7c795U, 0x65232346U, 0x5ec3c39dU +.word 0x28181830U, 0xa1969637U, 0x0f05050aU, 0xb59a9a2fU +.word 0x0907070eU, 0x36121224U, 0x9b80801bU, 0x3de2e2dfU +.word 0x26ebebcdU, 0x6927274eU, 0xcdb2b27fU, 0x9f7575eaU +.word 0x1b090912U, 0x9e83831dU, 0x742c2c58U, 0x2e1a1a34U +.word 0x2d1b1b36U, 0xb26e6edcU, 0xee5a5ab4U, 0xfba0a05bU +.word 0xf65252a4U, 0x4d3b3b76U, 0x61d6d6b7U, 0xceb3b37dU +.word 0x7b292952U, 0x3ee3e3ddU, 0x712f2f5eU, 0x97848413U +.word 0xf55353a6U, 0x68d1d1b9U, 0x00000000U, 0x2cededc1U +.word 0x60202040U, 0x1ffcfce3U, 0xc8b1b179U, 0xed5b5bb6U +.word 0xbe6a6ad4U, 0x46cbcb8dU, 0xd9bebe67U, 0x4b393972U +.word 0xde4a4a94U, 0xd44c4c98U, 0xe85858b0U, 0x4acfcf85U +.word 0x6bd0d0bbU, 0x2aefefc5U, 0xe5aaaa4fU, 0x16fbfbedU +.word 0xc5434386U, 0xd74d4d9aU, 0x55333366U, 0x94858511U +.word 0xcf45458aU, 0x10f9f9e9U, 0x06020204U, 0x817f7ffeU +.word 0xf05050a0U, 0x443c3c78U, 0xba9f9f25U, 0xe3a8a84bU +.word 0xf35151a2U, 0xfea3a35dU, 0xc0404080U, 0x8a8f8f05U +.word 0xad92923fU, 0xbc9d9d21U, 0x48383870U, 0x04f5f5f1U +.word 0xdfbcbc63U, 0xc1b6b677U, 0x75dadaafU, 0x63212142U +.word 0x30101020U, 0x1affffe5U, 0x0ef3f3fdU, 0x6dd2d2bfU +.word 0x4ccdcd81U, 0x140c0c18U, 0x35131326U, 0x2fececc3U +.word 0xe15f5fbeU, 0xa2979735U, 0xcc444488U, 0x3917172eU +.word 0x57c4c493U, 0xf2a7a755U, 0x827e7efcU, 0x473d3d7aU +.word 0xac6464c8U, 0xe75d5dbaU, 0x2b191932U, 0x957373e6U +.word 0xa06060c0U, 0x98818119U, 0xd14f4f9eU, 0x7fdcdca3U +.word 0x66222244U, 0x7e2a2a54U, 0xab90903bU, 0x8388880bU +.word 0xca46468cU, 0x29eeeec7U, 0xd3b8b86bU, 0x3c141428U +.word 0x79dedea7U, 0xe25e5ebcU, 0x1d0b0b16U, 0x76dbdbadU +.word 0x3be0e0dbU, 0x56323264U, 0x4e3a3a74U, 0x1e0a0a14U +.word 0xdb494992U, 0x0a06060cU, 0x6c242448U, 0xe45c5cb8U +.word 0x5dc2c29fU, 0x6ed3d3bdU, 0xefacac43U, 0xa66262c4U +.word 0xa8919139U, 0xa4959531U, 0x37e4e4d3U, 0x8b7979f2U +.word 0x32e7e7d5U, 0x43c8c88bU, 0x5937376eU, 0xb76d6ddaU +.word 0x8c8d8d01U, 0x64d5d5b1U, 0xd24e4e9cU, 0xe0a9a949U +.word 0xb46c6cd8U, 0xfa5656acU, 0x07f4f4f3U, 0x25eaeacfU +.word 0xaf6565caU, 0x8e7a7af4U, 0xe9aeae47U, 0x18080810U +.word 0xd5baba6fU, 0x887878f0U, 0x6f25254aU, 0x722e2e5cU +.word 0x241c1c38U, 0xf1a6a657U, 0xc7b4b473U, 0x51c6c697U +.word 0x23e8e8cbU, 0x7cdddda1U, 0x9c7474e8U, 0x211f1f3eU +.word 0xdd4b4b96U, 0xdcbdbd61U, 0x868b8b0dU, 0x858a8a0fU +.word 0x907070e0U, 0x423e3e7cU, 0xc4b5b571U, 0xaa6666ccU +.word 0xd8484890U, 0x05030306U, 0x01f6f6f7U, 0x120e0e1cU +.word 0xa36161c2U, 0x5f35356aU, 0xf95757aeU, 0xd0b9b969U +.word 0x91868617U, 0x58c1c199U, 0x271d1d3aU, 0xb99e9e27U +.word 0x38e1e1d9U, 0x13f8f8ebU, 0xb398982bU, 0x33111122U +.word 0xbb6969d2U, 0x70d9d9a9U, 0x898e8e07U, 0xa7949433U +.word 0xb69b9b2dU, 0x221e1e3cU, 0x92878715U, 0x20e9e9c9U +.word 0x49cece87U, 0xff5555aaU, 0x78282850U, 0x7adfdfa5U +.word 0x8f8c8c03U, 0xf8a1a159U, 0x80898909U, 0x170d0d1aU +.word 0xdabfbf65U, 0x31e6e6d7U, 0xc6424284U, 0xb86868d0U +.word 0xc3414182U, 0xb0999929U, 0x772d2d5aU, 0x110f0f1eU +.word 0xcbb0b07bU, 0xfc5454a8U, 0xd6bbbb6dU, 0x3a16162cU + +.type AES_Te1,\@object +AES_Te1: +.word 0x6363c6a5U, 0x7c7cf884U, 0x7777ee99U, 0x7b7bf68dU +.word 0xf2f2ff0dU, 0x6b6bd6bdU, 0x6f6fdeb1U, 0xc5c59154U +.word 0x30306050U, 0x01010203U, 0x6767cea9U, 0x2b2b567dU +.word 0xfefee719U, 0xd7d7b562U, 0xabab4de6U, 0x7676ec9aU +.word 0xcaca8f45U, 0x82821f9dU, 0xc9c98940U, 0x7d7dfa87U +.word 0xfafaef15U, 0x5959b2ebU, 0x47478ec9U, 0xf0f0fb0bU +.word 0xadad41ecU, 0xd4d4b367U, 0xa2a25ffdU, 0xafaf45eaU +.word 0x9c9c23bfU, 0xa4a453f7U, 0x7272e496U, 0xc0c09b5bU +.word 0xb7b775c2U, 0xfdfde11cU, 0x93933daeU, 0x26264c6aU +.word 0x36366c5aU, 0x3f3f7e41U, 0xf7f7f502U, 0xcccc834fU +.word 0x3434685cU, 0xa5a551f4U, 0xe5e5d134U, 0xf1f1f908U +.word 0x7171e293U, 0xd8d8ab73U, 0x31316253U, 0x15152a3fU +.word 0x0404080cU, 0xc7c79552U, 0x23234665U, 0xc3c39d5eU +.word 0x18183028U, 0x969637a1U, 0x05050a0fU, 0x9a9a2fb5U +.word 0x07070e09U, 0x12122436U, 0x80801b9bU, 0xe2e2df3dU +.word 0xebebcd26U, 0x27274e69U, 0xb2b27fcdU, 0x7575ea9fU +.word 0x0909121bU, 0x83831d9eU, 0x2c2c5874U, 0x1a1a342eU +.word 0x1b1b362dU, 0x6e6edcb2U, 0x5a5ab4eeU, 0xa0a05bfbU +.word 0x5252a4f6U, 0x3b3b764dU, 0xd6d6b761U, 0xb3b37dceU +.word 0x2929527bU, 0xe3e3dd3eU, 0x2f2f5e71U, 0x84841397U +.word 0x5353a6f5U, 0xd1d1b968U, 0x00000000U, 0xededc12cU +.word 0x20204060U, 0xfcfce31fU, 0xb1b179c8U, 0x5b5bb6edU +.word 0x6a6ad4beU, 0xcbcb8d46U, 0xbebe67d9U, 0x3939724bU +.word 0x4a4a94deU, 0x4c4c98d4U, 0x5858b0e8U, 0xcfcf854aU +.word 0xd0d0bb6bU, 0xefefc52aU, 0xaaaa4fe5U, 0xfbfbed16U +.word 0x434386c5U, 0x4d4d9ad7U, 0x33336655U, 0x85851194U +.word 0x45458acfU, 0xf9f9e910U, 0x02020406U, 0x7f7ffe81U +.word 0x5050a0f0U, 0x3c3c7844U, 0x9f9f25baU, 0xa8a84be3U +.word 0x5151a2f3U, 0xa3a35dfeU, 0x404080c0U, 0x8f8f058aU +.word 0x92923fadU, 0x9d9d21bcU, 0x38387048U, 0xf5f5f104U +.word 0xbcbc63dfU, 0xb6b677c1U, 0xdadaaf75U, 0x21214263U +.word 0x10102030U, 0xffffe51aU, 0xf3f3fd0eU, 0xd2d2bf6dU +.word 0xcdcd814cU, 0x0c0c1814U, 0x13132635U, 0xececc32fU +.word 0x5f5fbee1U, 0x979735a2U, 0x444488ccU, 0x17172e39U +.word 0xc4c49357U, 0xa7a755f2U, 0x7e7efc82U, 0x3d3d7a47U +.word 0x6464c8acU, 0x5d5dbae7U, 0x1919322bU, 0x7373e695U +.word 0x6060c0a0U, 0x81811998U, 0x4f4f9ed1U, 0xdcdca37fU +.word 0x22224466U, 0x2a2a547eU, 0x90903babU, 0x88880b83U +.word 0x46468ccaU, 0xeeeec729U, 0xb8b86bd3U, 0x1414283cU +.word 0xdedea779U, 0x5e5ebce2U, 0x0b0b161dU, 0xdbdbad76U +.word 0xe0e0db3bU, 0x32326456U, 0x3a3a744eU, 0x0a0a141eU +.word 0x494992dbU, 0x06060c0aU, 0x2424486cU, 0x5c5cb8e4U +.word 0xc2c29f5dU, 0xd3d3bd6eU, 0xacac43efU, 0x6262c4a6U +.word 0x919139a8U, 0x959531a4U, 0xe4e4d337U, 0x7979f28bU +.word 0xe7e7d532U, 0xc8c88b43U, 0x37376e59U, 0x6d6ddab7U +.word 0x8d8d018cU, 0xd5d5b164U, 0x4e4e9cd2U, 0xa9a949e0U +.word 0x6c6cd8b4U, 0x5656acfaU, 0xf4f4f307U, 0xeaeacf25U +.word 0x6565caafU, 0x7a7af48eU, 0xaeae47e9U, 0x08081018U +.word 0xbaba6fd5U, 0x7878f088U, 0x25254a6fU, 0x2e2e5c72U +.word 0x1c1c3824U, 0xa6a657f1U, 0xb4b473c7U, 0xc6c69751U +.word 0xe8e8cb23U, 0xdddda17cU, 0x7474e89cU, 0x1f1f3e21U +.word 0x4b4b96ddU, 0xbdbd61dcU, 0x8b8b0d86U, 0x8a8a0f85U +.word 0x7070e090U, 0x3e3e7c42U, 0xb5b571c4U, 0x6666ccaaU +.word 0x484890d8U, 0x03030605U, 0xf6f6f701U, 0x0e0e1c12U +.word 0x6161c2a3U, 0x35356a5fU, 0x5757aef9U, 0xb9b969d0U +.word 0x86861791U, 0xc1c19958U, 0x1d1d3a27U, 0x9e9e27b9U +.word 0xe1e1d938U, 0xf8f8eb13U, 0x98982bb3U, 0x11112233U +.word 0x6969d2bbU, 0xd9d9a970U, 0x8e8e0789U, 0x949433a7U +.word 0x9b9b2db6U, 0x1e1e3c22U, 0x87871592U, 0xe9e9c920U +.word 0xcece8749U, 0x5555aaffU, 0x28285078U, 0xdfdfa57aU +.word 0x8c8c038fU, 0xa1a159f8U, 0x89890980U, 0x0d0d1a17U +.word 0xbfbf65daU, 0xe6e6d731U, 0x424284c6U, 0x6868d0b8U +.word 0x414182c3U, 0x999929b0U, 0x2d2d5a77U, 0x0f0f1e11U +.word 0xb0b07bcbU, 0x5454a8fcU, 0xbbbb6dd6U, 0x16162c3aU + +.type AES_Te2,\@object +AES_Te2: +.word 0x63c6a563U, 0x7cf8847cU, 0x77ee9977U, 0x7bf68d7bU +.word 0xf2ff0df2U, 0x6bd6bd6bU, 0x6fdeb16fU, 0xc59154c5U +.word 0x30605030U, 0x01020301U, 0x67cea967U, 0x2b567d2bU +.word 0xfee719feU, 0xd7b562d7U, 0xab4de6abU, 0x76ec9a76U +.word 0xca8f45caU, 0x821f9d82U, 0xc98940c9U, 0x7dfa877dU +.word 0xfaef15faU, 0x59b2eb59U, 0x478ec947U, 0xf0fb0bf0U +.word 0xad41ecadU, 0xd4b367d4U, 0xa25ffda2U, 0xaf45eaafU +.word 0x9c23bf9cU, 0xa453f7a4U, 0x72e49672U, 0xc09b5bc0U +.word 0xb775c2b7U, 0xfde11cfdU, 0x933dae93U, 0x264c6a26U +.word 0x366c5a36U, 0x3f7e413fU, 0xf7f502f7U, 0xcc834fccU +.word 0x34685c34U, 0xa551f4a5U, 0xe5d134e5U, 0xf1f908f1U +.word 0x71e29371U, 0xd8ab73d8U, 0x31625331U, 0x152a3f15U +.word 0x04080c04U, 0xc79552c7U, 0x23466523U, 0xc39d5ec3U +.word 0x18302818U, 0x9637a196U, 0x050a0f05U, 0x9a2fb59aU +.word 0x070e0907U, 0x12243612U, 0x801b9b80U, 0xe2df3de2U +.word 0xebcd26ebU, 0x274e6927U, 0xb27fcdb2U, 0x75ea9f75U +.word 0x09121b09U, 0x831d9e83U, 0x2c58742cU, 0x1a342e1aU +.word 0x1b362d1bU, 0x6edcb26eU, 0x5ab4ee5aU, 0xa05bfba0U +.word 0x52a4f652U, 0x3b764d3bU, 0xd6b761d6U, 0xb37dceb3U +.word 0x29527b29U, 0xe3dd3ee3U, 0x2f5e712fU, 0x84139784U +.word 0x53a6f553U, 0xd1b968d1U, 0x00000000U, 0xedc12cedU +.word 0x20406020U, 0xfce31ffcU, 0xb179c8b1U, 0x5bb6ed5bU +.word 0x6ad4be6aU, 0xcb8d46cbU, 0xbe67d9beU, 0x39724b39U +.word 0x4a94de4aU, 0x4c98d44cU, 0x58b0e858U, 0xcf854acfU +.word 0xd0bb6bd0U, 0xefc52aefU, 0xaa4fe5aaU, 0xfbed16fbU +.word 0x4386c543U, 0x4d9ad74dU, 0x33665533U, 0x85119485U +.word 0x458acf45U, 0xf9e910f9U, 0x02040602U, 0x7ffe817fU +.word 0x50a0f050U, 0x3c78443cU, 0x9f25ba9fU, 0xa84be3a8U +.word 0x51a2f351U, 0xa35dfea3U, 0x4080c040U, 0x8f058a8fU +.word 0x923fad92U, 0x9d21bc9dU, 0x38704838U, 0xf5f104f5U +.word 0xbc63dfbcU, 0xb677c1b6U, 0xdaaf75daU, 0x21426321U +.word 0x10203010U, 0xffe51affU, 0xf3fd0ef3U, 0xd2bf6dd2U +.word 0xcd814ccdU, 0x0c18140cU, 0x13263513U, 0xecc32fecU +.word 0x5fbee15fU, 0x9735a297U, 0x4488cc44U, 0x172e3917U +.word 0xc49357c4U, 0xa755f2a7U, 0x7efc827eU, 0x3d7a473dU +.word 0x64c8ac64U, 0x5dbae75dU, 0x19322b19U, 0x73e69573U +.word 0x60c0a060U, 0x81199881U, 0x4f9ed14fU, 0xdca37fdcU +.word 0x22446622U, 0x2a547e2aU, 0x903bab90U, 0x880b8388U +.word 0x468cca46U, 0xeec729eeU, 0xb86bd3b8U, 0x14283c14U +.word 0xdea779deU, 0x5ebce25eU, 0x0b161d0bU, 0xdbad76dbU +.word 0xe0db3be0U, 0x32645632U, 0x3a744e3aU, 0x0a141e0aU +.word 0x4992db49U, 0x060c0a06U, 0x24486c24U, 0x5cb8e45cU +.word 0xc29f5dc2U, 0xd3bd6ed3U, 0xac43efacU, 0x62c4a662U +.word 0x9139a891U, 0x9531a495U, 0xe4d337e4U, 0x79f28b79U +.word 0xe7d532e7U, 0xc88b43c8U, 0x376e5937U, 0x6ddab76dU +.word 0x8d018c8dU, 0xd5b164d5U, 0x4e9cd24eU, 0xa949e0a9U +.word 0x6cd8b46cU, 0x56acfa56U, 0xf4f307f4U, 0xeacf25eaU +.word 0x65caaf65U, 0x7af48e7aU, 0xae47e9aeU, 0x08101808U +.word 0xba6fd5baU, 0x78f08878U, 0x254a6f25U, 0x2e5c722eU +.word 0x1c38241cU, 0xa657f1a6U, 0xb473c7b4U, 0xc69751c6U +.word 0xe8cb23e8U, 0xdda17cddU, 0x74e89c74U, 0x1f3e211fU +.word 0x4b96dd4bU, 0xbd61dcbdU, 0x8b0d868bU, 0x8a0f858aU +.word 0x70e09070U, 0x3e7c423eU, 0xb571c4b5U, 0x66ccaa66U +.word 0x4890d848U, 0x03060503U, 0xf6f701f6U, 0x0e1c120eU +.word 0x61c2a361U, 0x356a5f35U, 0x57aef957U, 0xb969d0b9U +.word 0x86179186U, 0xc19958c1U, 0x1d3a271dU, 0x9e27b99eU +.word 0xe1d938e1U, 0xf8eb13f8U, 0x982bb398U, 0x11223311U +.word 0x69d2bb69U, 0xd9a970d9U, 0x8e07898eU, 0x9433a794U +.word 0x9b2db69bU, 0x1e3c221eU, 0x87159287U, 0xe9c920e9U +.word 0xce8749ceU, 0x55aaff55U, 0x28507828U, 0xdfa57adfU +.word 0x8c038f8cU, 0xa159f8a1U, 0x89098089U, 0x0d1a170dU +.word 0xbf65dabfU, 0xe6d731e6U, 0x4284c642U, 0x68d0b868U +.word 0x4182c341U, 0x9929b099U, 0x2d5a772dU, 0x0f1e110fU +.word 0xb07bcbb0U, 0x54a8fc54U, 0xbb6dd6bbU, 0x162c3a16U + +.type AES_Te3,\@object +AES_Te3: +.word 0xc6a56363U, 0xf8847c7cU, 0xee997777U, 0xf68d7b7bU +.word 0xff0df2f2U, 0xd6bd6b6bU, 0xdeb16f6fU, 0x9154c5c5U +.word 0x60503030U, 0x02030101U, 0xcea96767U, 0x567d2b2bU +.word 0xe719fefeU, 0xb562d7d7U, 0x4de6ababU, 0xec9a7676U +.word 0x8f45cacaU, 0x1f9d8282U, 0x8940c9c9U, 0xfa877d7dU +.word 0xef15fafaU, 0xb2eb5959U, 0x8ec94747U, 0xfb0bf0f0U +.word 0x41ecadadU, 0xb367d4d4U, 0x5ffda2a2U, 0x45eaafafU +.word 0x23bf9c9cU, 0x53f7a4a4U, 0xe4967272U, 0x9b5bc0c0U +.word 0x75c2b7b7U, 0xe11cfdfdU, 0x3dae9393U, 0x4c6a2626U +.word 0x6c5a3636U, 0x7e413f3fU, 0xf502f7f7U, 0x834fccccU +.word 0x685c3434U, 0x51f4a5a5U, 0xd134e5e5U, 0xf908f1f1U +.word 0xe2937171U, 0xab73d8d8U, 0x62533131U, 0x2a3f1515U +.word 0x080c0404U, 0x9552c7c7U, 0x46652323U, 0x9d5ec3c3U +.word 0x30281818U, 0x37a19696U, 0x0a0f0505U, 0x2fb59a9aU +.word 0x0e090707U, 0x24361212U, 0x1b9b8080U, 0xdf3de2e2U +.word 0xcd26ebebU, 0x4e692727U, 0x7fcdb2b2U, 0xea9f7575U +.word 0x121b0909U, 0x1d9e8383U, 0x58742c2cU, 0x342e1a1aU +.word 0x362d1b1bU, 0xdcb26e6eU, 0xb4ee5a5aU, 0x5bfba0a0U +.word 0xa4f65252U, 0x764d3b3bU, 0xb761d6d6U, 0x7dceb3b3U +.word 0x527b2929U, 0xdd3ee3e3U, 0x5e712f2fU, 0x13978484U +.word 0xa6f55353U, 0xb968d1d1U, 0x00000000U, 0xc12cededU +.word 0x40602020U, 0xe31ffcfcU, 0x79c8b1b1U, 0xb6ed5b5bU +.word 0xd4be6a6aU, 0x8d46cbcbU, 0x67d9bebeU, 0x724b3939U +.word 0x94de4a4aU, 0x98d44c4cU, 0xb0e85858U, 0x854acfcfU +.word 0xbb6bd0d0U, 0xc52aefefU, 0x4fe5aaaaU, 0xed16fbfbU +.word 0x86c54343U, 0x9ad74d4dU, 0x66553333U, 0x11948585U +.word 0x8acf4545U, 0xe910f9f9U, 0x04060202U, 0xfe817f7fU +.word 0xa0f05050U, 0x78443c3cU, 0x25ba9f9fU, 0x4be3a8a8U +.word 0xa2f35151U, 0x5dfea3a3U, 0x80c04040U, 0x058a8f8fU +.word 0x3fad9292U, 0x21bc9d9dU, 0x70483838U, 0xf104f5f5U +.word 0x63dfbcbcU, 0x77c1b6b6U, 0xaf75dadaU, 0x42632121U +.word 0x20301010U, 0xe51affffU, 0xfd0ef3f3U, 0xbf6dd2d2U +.word 0x814ccdcdU, 0x18140c0cU, 0x26351313U, 0xc32fececU +.word 0xbee15f5fU, 0x35a29797U, 0x88cc4444U, 0x2e391717U +.word 0x9357c4c4U, 0x55f2a7a7U, 0xfc827e7eU, 0x7a473d3dU +.word 0xc8ac6464U, 0xbae75d5dU, 0x322b1919U, 0xe6957373U +.word 0xc0a06060U, 0x19988181U, 0x9ed14f4fU, 0xa37fdcdcU +.word 0x44662222U, 0x547e2a2aU, 0x3bab9090U, 0x0b838888U +.word 0x8cca4646U, 0xc729eeeeU, 0x6bd3b8b8U, 0x283c1414U +.word 0xa779dedeU, 0xbce25e5eU, 0x161d0b0bU, 0xad76dbdbU +.word 0xdb3be0e0U, 0x64563232U, 0x744e3a3aU, 0x141e0a0aU +.word 0x92db4949U, 0x0c0a0606U, 0x486c2424U, 0xb8e45c5cU +.word 0x9f5dc2c2U, 0xbd6ed3d3U, 0x43efacacU, 0xc4a66262U +.word 0x39a89191U, 0x31a49595U, 0xd337e4e4U, 0xf28b7979U +.word 0xd532e7e7U, 0x8b43c8c8U, 0x6e593737U, 0xdab76d6dU +.word 0x018c8d8dU, 0xb164d5d5U, 0x9cd24e4eU, 0x49e0a9a9U +.word 0xd8b46c6cU, 0xacfa5656U, 0xf307f4f4U, 0xcf25eaeaU +.word 0xcaaf6565U, 0xf48e7a7aU, 0x47e9aeaeU, 0x10180808U +.word 0x6fd5babaU, 0xf0887878U, 0x4a6f2525U, 0x5c722e2eU +.word 0x38241c1cU, 0x57f1a6a6U, 0x73c7b4b4U, 0x9751c6c6U +.word 0xcb23e8e8U, 0xa17cddddU, 0xe89c7474U, 0x3e211f1fU +.word 0x96dd4b4bU, 0x61dcbdbdU, 0x0d868b8bU, 0x0f858a8aU +.word 0xe0907070U, 0x7c423e3eU, 0x71c4b5b5U, 0xccaa6666U +.word 0x90d84848U, 0x06050303U, 0xf701f6f6U, 0x1c120e0eU +.word 0xc2a36161U, 0x6a5f3535U, 0xaef95757U, 0x69d0b9b9U +.word 0x17918686U, 0x9958c1c1U, 0x3a271d1dU, 0x27b99e9eU +.word 0xd938e1e1U, 0xeb13f8f8U, 0x2bb39898U, 0x22331111U +.word 0xd2bb6969U, 0xa970d9d9U, 0x07898e8eU, 0x33a79494U +.word 0x2db69b9bU, 0x3c221e1eU, 0x15928787U, 0xc920e9e9U +.word 0x8749ceceU, 0xaaff5555U, 0x50782828U, 0xa57adfdfU +.word 0x038f8c8cU, 0x59f8a1a1U, 0x09808989U, 0x1a170d0dU +.word 0x65dabfbfU, 0xd731e6e6U, 0x84c64242U, 0xd0b86868U +.word 0x82c34141U, 0x29b09999U, 0x5a772d2dU, 0x1e110f0fU +.word 0x7bcbb0b0U, 0xa8fc5454U, 0x6dd6bbbbU, 0x2c3a1616U + +.p2align 12 +.type AES_Td0,\@object +AES_Td0: +.word 0x50a7f451U, 0x5365417eU, 0xc3a4171aU, 0x965e273aU +.word 0xcb6bab3bU, 0xf1459d1fU, 0xab58faacU, 0x9303e34bU +.word 0x55fa3020U, 0xf66d76adU, 0x9176cc88U, 0x254c02f5U +.word 0xfcd7e54fU, 0xd7cb2ac5U, 0x80443526U, 0x8fa362b5U +.word 0x495ab1deU, 0x671bba25U, 0x980eea45U, 0xe1c0fe5dU +.word 0x02752fc3U, 0x12f04c81U, 0xa397468dU, 0xc6f9d36bU +.word 0xe75f8f03U, 0x959c9215U, 0xeb7a6dbfU, 0xda595295U +.word 0x2d83bed4U, 0xd3217458U, 0x2969e049U, 0x44c8c98eU +.word 0x6a89c275U, 0x78798ef4U, 0x6b3e5899U, 0xdd71b927U +.word 0xb64fe1beU, 0x17ad88f0U, 0x66ac20c9U, 0xb43ace7dU +.word 0x184adf63U, 0x82311ae5U, 0x60335197U, 0x457f5362U +.word 0xe07764b1U, 0x84ae6bbbU, 0x1ca081feU, 0x942b08f9U +.word 0x58684870U, 0x19fd458fU, 0x876cde94U, 0xb7f87b52U +.word 0x23d373abU, 0xe2024b72U, 0x578f1fe3U, 0x2aab5566U +.word 0x0728ebb2U, 0x03c2b52fU, 0x9a7bc586U, 0xa50837d3U +.word 0xf2872830U, 0xb2a5bf23U, 0xba6a0302U, 0x5c8216edU +.word 0x2b1ccf8aU, 0x92b479a7U, 0xf0f207f3U, 0xa1e2694eU +.word 0xcdf4da65U, 0xd5be0506U, 0x1f6234d1U, 0x8afea6c4U +.word 0x9d532e34U, 0xa055f3a2U, 0x32e18a05U, 0x75ebf6a4U +.word 0x39ec830bU, 0xaaef6040U, 0x069f715eU, 0x51106ebdU +.word 0xf98a213eU, 0x3d06dd96U, 0xae053eddU, 0x46bde64dU +.word 0xb58d5491U, 0x055dc471U, 0x6fd40604U, 0xff155060U +.word 0x24fb9819U, 0x97e9bdd6U, 0xcc434089U, 0x779ed967U +.word 0xbd42e8b0U, 0x888b8907U, 0x385b19e7U, 0xdbeec879U +.word 0x470a7ca1U, 0xe90f427cU, 0xc91e84f8U, 0x00000000U +.word 0x83868009U, 0x48ed2b32U, 0xac70111eU, 0x4e725a6cU +.word 0xfbff0efdU, 0x5638850fU, 0x1ed5ae3dU, 0x27392d36U +.word 0x64d90f0aU, 0x21a65c68U, 0xd1545b9bU, 0x3a2e3624U +.word 0xb1670a0cU, 0x0fe75793U, 0xd296eeb4U, 0x9e919b1bU +.word 0x4fc5c080U, 0xa220dc61U, 0x694b775aU, 0x161a121cU +.word 0x0aba93e2U, 0xe52aa0c0U, 0x43e0223cU, 0x1d171b12U +.word 0x0b0d090eU, 0xadc78bf2U, 0xb9a8b62dU, 0xc8a91e14U +.word 0x8519f157U, 0x4c0775afU, 0xbbdd99eeU, 0xfd607fa3U +.word 0x9f2601f7U, 0xbcf5725cU, 0xc53b6644U, 0x347efb5bU +.word 0x7629438bU, 0xdcc623cbU, 0x68fcedb6U, 0x63f1e4b8U +.word 0xcadc31d7U, 0x10856342U, 0x40229713U, 0x2011c684U +.word 0x7d244a85U, 0xf83dbbd2U, 0x1132f9aeU, 0x6da129c7U +.word 0x4b2f9e1dU, 0xf330b2dcU, 0xec52860dU, 0xd0e3c177U +.word 0x6c16b32bU, 0x99b970a9U, 0xfa489411U, 0x2264e947U +.word 0xc48cfca8U, 0x1a3ff0a0U, 0xd82c7d56U, 0xef903322U +.word 0xc74e4987U, 0xc1d138d9U, 0xfea2ca8cU, 0x360bd498U +.word 0xcf81f5a6U, 0x28de7aa5U, 0x268eb7daU, 0xa4bfad3fU +.word 0xe49d3a2cU, 0x0d927850U, 0x9bcc5f6aU, 0x62467e54U +.word 0xc2138df6U, 0xe8b8d890U, 0x5ef7392eU, 0xf5afc382U +.word 0xbe805d9fU, 0x7c93d069U, 0xa92dd56fU, 0xb31225cfU +.word 0x3b99acc8U, 0xa77d1810U, 0x6e639ce8U, 0x7bbb3bdbU +.word 0x097826cdU, 0xf418596eU, 0x01b79aecU, 0xa89a4f83U +.word 0x656e95e6U, 0x7ee6ffaaU, 0x08cfbc21U, 0xe6e815efU +.word 0xd99be7baU, 0xce366f4aU, 0xd4099feaU, 0xd67cb029U +.word 0xafb2a431U, 0x31233f2aU, 0x3094a5c6U, 0xc066a235U +.word 0x37bc4e74U, 0xa6ca82fcU, 0xb0d090e0U, 0x15d8a733U +.word 0x4a9804f1U, 0xf7daec41U, 0x0e50cd7fU, 0x2ff69117U +.word 0x8dd64d76U, 0x4db0ef43U, 0x544daaccU, 0xdf0496e4U +.word 0xe3b5d19eU, 0x1b886a4cU, 0xb81f2cc1U, 0x7f516546U +.word 0x04ea5e9dU, 0x5d358c01U, 0x737487faU, 0x2e410bfbU +.word 0x5a1d67b3U, 0x52d2db92U, 0x335610e9U, 0x1347d66dU +.word 0x8c61d79aU, 0x7a0ca137U, 0x8e14f859U, 0x893c13ebU +.word 0xee27a9ceU, 0x35c961b7U, 0xede51ce1U, 0x3cb1477aU +.word 0x59dfd29cU, 0x3f73f255U, 0x79ce1418U, 0xbf37c773U +.word 0xeacdf753U, 0x5baafd5fU, 0x146f3ddfU, 0x86db4478U +.word 0x81f3afcaU, 0x3ec468b9U, 0x2c342438U, 0x5f40a3c2U +.word 0x72c31d16U, 0x0c25e2bcU, 0x8b493c28U, 0x41950dffU +.word 0x7101a839U, 0xdeb30c08U, 0x9ce4b4d8U, 0x90c15664U +.word 0x6184cb7bU, 0x70b632d5U, 0x745c6c48U, 0x4257b8d0U + +.type AES_Td1,\@object +AES_Td1: +.word 0xa7f45150U, 0x65417e53U, 0xa4171ac3U, 0x5e273a96U +.word 0x6bab3bcbU, 0x459d1ff1U, 0x58faacabU, 0x03e34b93U +.word 0xfa302055U, 0x6d76adf6U, 0x76cc8891U, 0x4c02f525U +.word 0xd7e54ffcU, 0xcb2ac5d7U, 0x44352680U, 0xa362b58fU +.word 0x5ab1de49U, 0x1bba2567U, 0x0eea4598U, 0xc0fe5de1U +.word 0x752fc302U, 0xf04c8112U, 0x97468da3U, 0xf9d36bc6U +.word 0x5f8f03e7U, 0x9c921595U, 0x7a6dbfebU, 0x595295daU +.word 0x83bed42dU, 0x217458d3U, 0x69e04929U, 0xc8c98e44U +.word 0x89c2756aU, 0x798ef478U, 0x3e58996bU, 0x71b927ddU +.word 0x4fe1beb6U, 0xad88f017U, 0xac20c966U, 0x3ace7db4U +.word 0x4adf6318U, 0x311ae582U, 0x33519760U, 0x7f536245U +.word 0x7764b1e0U, 0xae6bbb84U, 0xa081fe1cU, 0x2b08f994U +.word 0x68487058U, 0xfd458f19U, 0x6cde9487U, 0xf87b52b7U +.word 0xd373ab23U, 0x024b72e2U, 0x8f1fe357U, 0xab55662aU +.word 0x28ebb207U, 0xc2b52f03U, 0x7bc5869aU, 0x0837d3a5U +.word 0x872830f2U, 0xa5bf23b2U, 0x6a0302baU, 0x8216ed5cU +.word 0x1ccf8a2bU, 0xb479a792U, 0xf207f3f0U, 0xe2694ea1U +.word 0xf4da65cdU, 0xbe0506d5U, 0x6234d11fU, 0xfea6c48aU +.word 0x532e349dU, 0x55f3a2a0U, 0xe18a0532U, 0xebf6a475U +.word 0xec830b39U, 0xef6040aaU, 0x9f715e06U, 0x106ebd51U +.word 0x8a213ef9U, 0x06dd963dU, 0x053eddaeU, 0xbde64d46U +.word 0x8d5491b5U, 0x5dc47105U, 0xd406046fU, 0x155060ffU +.word 0xfb981924U, 0xe9bdd697U, 0x434089ccU, 0x9ed96777U +.word 0x42e8b0bdU, 0x8b890788U, 0x5b19e738U, 0xeec879dbU +.word 0x0a7ca147U, 0x0f427ce9U, 0x1e84f8c9U, 0x00000000U +.word 0x86800983U, 0xed2b3248U, 0x70111eacU, 0x725a6c4eU +.word 0xff0efdfbU, 0x38850f56U, 0xd5ae3d1eU, 0x392d3627U +.word 0xd90f0a64U, 0xa65c6821U, 0x545b9bd1U, 0x2e36243aU +.word 0x670a0cb1U, 0xe757930fU, 0x96eeb4d2U, 0x919b1b9eU +.word 0xc5c0804fU, 0x20dc61a2U, 0x4b775a69U, 0x1a121c16U +.word 0xba93e20aU, 0x2aa0c0e5U, 0xe0223c43U, 0x171b121dU +.word 0x0d090e0bU, 0xc78bf2adU, 0xa8b62db9U, 0xa91e14c8U +.word 0x19f15785U, 0x0775af4cU, 0xdd99eebbU, 0x607fa3fdU +.word 0x2601f79fU, 0xf5725cbcU, 0x3b6644c5U, 0x7efb5b34U +.word 0x29438b76U, 0xc623cbdcU, 0xfcedb668U, 0xf1e4b863U +.word 0xdc31d7caU, 0x85634210U, 0x22971340U, 0x11c68420U +.word 0x244a857dU, 0x3dbbd2f8U, 0x32f9ae11U, 0xa129c76dU +.word 0x2f9e1d4bU, 0x30b2dcf3U, 0x52860decU, 0xe3c177d0U +.word 0x16b32b6cU, 0xb970a999U, 0x489411faU, 0x64e94722U +.word 0x8cfca8c4U, 0x3ff0a01aU, 0x2c7d56d8U, 0x903322efU +.word 0x4e4987c7U, 0xd138d9c1U, 0xa2ca8cfeU, 0x0bd49836U +.word 0x81f5a6cfU, 0xde7aa528U, 0x8eb7da26U, 0xbfad3fa4U +.word 0x9d3a2ce4U, 0x9278500dU, 0xcc5f6a9bU, 0x467e5462U +.word 0x138df6c2U, 0xb8d890e8U, 0xf7392e5eU, 0xafc382f5U +.word 0x805d9fbeU, 0x93d0697cU, 0x2dd56fa9U, 0x1225cfb3U +.word 0x99acc83bU, 0x7d1810a7U, 0x639ce86eU, 0xbb3bdb7bU +.word 0x7826cd09U, 0x18596ef4U, 0xb79aec01U, 0x9a4f83a8U +.word 0x6e95e665U, 0xe6ffaa7eU, 0xcfbc2108U, 0xe815efe6U +.word 0x9be7bad9U, 0x366f4aceU, 0x099fead4U, 0x7cb029d6U +.word 0xb2a431afU, 0x233f2a31U, 0x94a5c630U, 0x66a235c0U +.word 0xbc4e7437U, 0xca82fca6U, 0xd090e0b0U, 0xd8a73315U +.word 0x9804f14aU, 0xdaec41f7U, 0x50cd7f0eU, 0xf691172fU +.word 0xd64d768dU, 0xb0ef434dU, 0x4daacc54U, 0x0496e4dfU +.word 0xb5d19ee3U, 0x886a4c1bU, 0x1f2cc1b8U, 0x5165467fU +.word 0xea5e9d04U, 0x358c015dU, 0x7487fa73U, 0x410bfb2eU +.word 0x1d67b35aU, 0xd2db9252U, 0x5610e933U, 0x47d66d13U +.word 0x61d79a8cU, 0x0ca1377aU, 0x14f8598eU, 0x3c13eb89U +.word 0x27a9ceeeU, 0xc961b735U, 0xe51ce1edU, 0xb1477a3cU +.word 0xdfd29c59U, 0x73f2553fU, 0xce141879U, 0x37c773bfU +.word 0xcdf753eaU, 0xaafd5f5bU, 0x6f3ddf14U, 0xdb447886U +.word 0xf3afca81U, 0xc468b93eU, 0x3424382cU, 0x40a3c25fU +.word 0xc31d1672U, 0x25e2bc0cU, 0x493c288bU, 0x950dff41U +.word 0x01a83971U, 0xb30c08deU, 0xe4b4d89cU, 0xc1566490U +.word 0x84cb7b61U, 0xb632d570U, 0x5c6c4874U, 0x57b8d042U + +.type AES_Td2,\@object +AES_Td2: +.word 0xf45150a7U, 0x417e5365U, 0x171ac3a4U, 0x273a965eU +.word 0xab3bcb6bU, 0x9d1ff145U, 0xfaacab58U, 0xe34b9303U +.word 0x302055faU, 0x76adf66dU, 0xcc889176U, 0x02f5254cU +.word 0xe54ffcd7U, 0x2ac5d7cbU, 0x35268044U, 0x62b58fa3U +.word 0xb1de495aU, 0xba25671bU, 0xea45980eU, 0xfe5de1c0U +.word 0x2fc30275U, 0x4c8112f0U, 0x468da397U, 0xd36bc6f9U +.word 0x8f03e75fU, 0x9215959cU, 0x6dbfeb7aU, 0x5295da59U +.word 0xbed42d83U, 0x7458d321U, 0xe0492969U, 0xc98e44c8U +.word 0xc2756a89U, 0x8ef47879U, 0x58996b3eU, 0xb927dd71U +.word 0xe1beb64fU, 0x88f017adU, 0x20c966acU, 0xce7db43aU +.word 0xdf63184aU, 0x1ae58231U, 0x51976033U, 0x5362457fU +.word 0x64b1e077U, 0x6bbb84aeU, 0x81fe1ca0U, 0x08f9942bU +.word 0x48705868U, 0x458f19fdU, 0xde94876cU, 0x7b52b7f8U +.word 0x73ab23d3U, 0x4b72e202U, 0x1fe3578fU, 0x55662aabU +.word 0xebb20728U, 0xb52f03c2U, 0xc5869a7bU, 0x37d3a508U +.word 0x2830f287U, 0xbf23b2a5U, 0x0302ba6aU, 0x16ed5c82U +.word 0xcf8a2b1cU, 0x79a792b4U, 0x07f3f0f2U, 0x694ea1e2U +.word 0xda65cdf4U, 0x0506d5beU, 0x34d11f62U, 0xa6c48afeU +.word 0x2e349d53U, 0xf3a2a055U, 0x8a0532e1U, 0xf6a475ebU +.word 0x830b39ecU, 0x6040aaefU, 0x715e069fU, 0x6ebd5110U +.word 0x213ef98aU, 0xdd963d06U, 0x3eddae05U, 0xe64d46bdU +.word 0x5491b58dU, 0xc471055dU, 0x06046fd4U, 0x5060ff15U +.word 0x981924fbU, 0xbdd697e9U, 0x4089cc43U, 0xd967779eU +.word 0xe8b0bd42U, 0x8907888bU, 0x19e7385bU, 0xc879dbeeU +.word 0x7ca1470aU, 0x427ce90fU, 0x84f8c91eU, 0x00000000U +.word 0x80098386U, 0x2b3248edU, 0x111eac70U, 0x5a6c4e72U +.word 0x0efdfbffU, 0x850f5638U, 0xae3d1ed5U, 0x2d362739U +.word 0x0f0a64d9U, 0x5c6821a6U, 0x5b9bd154U, 0x36243a2eU +.word 0x0a0cb167U, 0x57930fe7U, 0xeeb4d296U, 0x9b1b9e91U +.word 0xc0804fc5U, 0xdc61a220U, 0x775a694bU, 0x121c161aU +.word 0x93e20abaU, 0xa0c0e52aU, 0x223c43e0U, 0x1b121d17U +.word 0x090e0b0dU, 0x8bf2adc7U, 0xb62db9a8U, 0x1e14c8a9U +.word 0xf1578519U, 0x75af4c07U, 0x99eebbddU, 0x7fa3fd60U +.word 0x01f79f26U, 0x725cbcf5U, 0x6644c53bU, 0xfb5b347eU +.word 0x438b7629U, 0x23cbdcc6U, 0xedb668fcU, 0xe4b863f1U +.word 0x31d7cadcU, 0x63421085U, 0x97134022U, 0xc6842011U +.word 0x4a857d24U, 0xbbd2f83dU, 0xf9ae1132U, 0x29c76da1U +.word 0x9e1d4b2fU, 0xb2dcf330U, 0x860dec52U, 0xc177d0e3U +.word 0xb32b6c16U, 0x70a999b9U, 0x9411fa48U, 0xe9472264U +.word 0xfca8c48cU, 0xf0a01a3fU, 0x7d56d82cU, 0x3322ef90U +.word 0x4987c74eU, 0x38d9c1d1U, 0xca8cfea2U, 0xd498360bU +.word 0xf5a6cf81U, 0x7aa528deU, 0xb7da268eU, 0xad3fa4bfU +.word 0x3a2ce49dU, 0x78500d92U, 0x5f6a9bccU, 0x7e546246U +.word 0x8df6c213U, 0xd890e8b8U, 0x392e5ef7U, 0xc382f5afU +.word 0x5d9fbe80U, 0xd0697c93U, 0xd56fa92dU, 0x25cfb312U +.word 0xacc83b99U, 0x1810a77dU, 0x9ce86e63U, 0x3bdb7bbbU +.word 0x26cd0978U, 0x596ef418U, 0x9aec01b7U, 0x4f83a89aU +.word 0x95e6656eU, 0xffaa7ee6U, 0xbc2108cfU, 0x15efe6e8U +.word 0xe7bad99bU, 0x6f4ace36U, 0x9fead409U, 0xb029d67cU +.word 0xa431afb2U, 0x3f2a3123U, 0xa5c63094U, 0xa235c066U +.word 0x4e7437bcU, 0x82fca6caU, 0x90e0b0d0U, 0xa73315d8U +.word 0x04f14a98U, 0xec41f7daU, 0xcd7f0e50U, 0x91172ff6U +.word 0x4d768dd6U, 0xef434db0U, 0xaacc544dU, 0x96e4df04U +.word 0xd19ee3b5U, 0x6a4c1b88U, 0x2cc1b81fU, 0x65467f51U +.word 0x5e9d04eaU, 0x8c015d35U, 0x87fa7374U, 0x0bfb2e41U +.word 0x67b35a1dU, 0xdb9252d2U, 0x10e93356U, 0xd66d1347U +.word 0xd79a8c61U, 0xa1377a0cU, 0xf8598e14U, 0x13eb893cU +.word 0xa9ceee27U, 0x61b735c9U, 0x1ce1ede5U, 0x477a3cb1U +.word 0xd29c59dfU, 0xf2553f73U, 0x141879ceU, 0xc773bf37U +.word 0xf753eacdU, 0xfd5f5baaU, 0x3ddf146fU, 0x447886dbU +.word 0xafca81f3U, 0x68b93ec4U, 0x24382c34U, 0xa3c25f40U +.word 0x1d1672c3U, 0xe2bc0c25U, 0x3c288b49U, 0x0dff4195U +.word 0xa8397101U, 0x0c08deb3U, 0xb4d89ce4U, 0x566490c1U +.word 0xcb7b6184U, 0x32d570b6U, 0x6c48745cU, 0xb8d04257U + +.type AES_Td3,\@object +AES_Td3: +.word 0x5150a7f4U, 0x7e536541U, 0x1ac3a417U, 0x3a965e27U +.word 0x3bcb6babU, 0x1ff1459dU, 0xacab58faU, 0x4b9303e3U +.word 0x2055fa30U, 0xadf66d76U, 0x889176ccU, 0xf5254c02U +.word 0x4ffcd7e5U, 0xc5d7cb2aU, 0x26804435U, 0xb58fa362U +.word 0xde495ab1U, 0x25671bbaU, 0x45980eeaU, 0x5de1c0feU +.word 0xc302752fU, 0x8112f04cU, 0x8da39746U, 0x6bc6f9d3U +.word 0x03e75f8fU, 0x15959c92U, 0xbfeb7a6dU, 0x95da5952U +.word 0xd42d83beU, 0x58d32174U, 0x492969e0U, 0x8e44c8c9U +.word 0x756a89c2U, 0xf478798eU, 0x996b3e58U, 0x27dd71b9U +.word 0xbeb64fe1U, 0xf017ad88U, 0xc966ac20U, 0x7db43aceU +.word 0x63184adfU, 0xe582311aU, 0x97603351U, 0x62457f53U +.word 0xb1e07764U, 0xbb84ae6bU, 0xfe1ca081U, 0xf9942b08U +.word 0x70586848U, 0x8f19fd45U, 0x94876cdeU, 0x52b7f87bU +.word 0xab23d373U, 0x72e2024bU, 0xe3578f1fU, 0x662aab55U +.word 0xb20728ebU, 0x2f03c2b5U, 0x869a7bc5U, 0xd3a50837U +.word 0x30f28728U, 0x23b2a5bfU, 0x02ba6a03U, 0xed5c8216U +.word 0x8a2b1ccfU, 0xa792b479U, 0xf3f0f207U, 0x4ea1e269U +.word 0x65cdf4daU, 0x06d5be05U, 0xd11f6234U, 0xc48afea6U +.word 0x349d532eU, 0xa2a055f3U, 0x0532e18aU, 0xa475ebf6U +.word 0x0b39ec83U, 0x40aaef60U, 0x5e069f71U, 0xbd51106eU +.word 0x3ef98a21U, 0x963d06ddU, 0xddae053eU, 0x4d46bde6U +.word 0x91b58d54U, 0x71055dc4U, 0x046fd406U, 0x60ff1550U +.word 0x1924fb98U, 0xd697e9bdU, 0x89cc4340U, 0x67779ed9U +.word 0xb0bd42e8U, 0x07888b89U, 0xe7385b19U, 0x79dbeec8U +.word 0xa1470a7cU, 0x7ce90f42U, 0xf8c91e84U, 0x00000000U +.word 0x09838680U, 0x3248ed2bU, 0x1eac7011U, 0x6c4e725aU +.word 0xfdfbff0eU, 0x0f563885U, 0x3d1ed5aeU, 0x3627392dU +.word 0x0a64d90fU, 0x6821a65cU, 0x9bd1545bU, 0x243a2e36U +.word 0x0cb1670aU, 0x930fe757U, 0xb4d296eeU, 0x1b9e919bU +.word 0x804fc5c0U, 0x61a220dcU, 0x5a694b77U, 0x1c161a12U +.word 0xe20aba93U, 0xc0e52aa0U, 0x3c43e022U, 0x121d171bU +.word 0x0e0b0d09U, 0xf2adc78bU, 0x2db9a8b6U, 0x14c8a91eU +.word 0x578519f1U, 0xaf4c0775U, 0xeebbdd99U, 0xa3fd607fU +.word 0xf79f2601U, 0x5cbcf572U, 0x44c53b66U, 0x5b347efbU +.word 0x8b762943U, 0xcbdcc623U, 0xb668fcedU, 0xb863f1e4U +.word 0xd7cadc31U, 0x42108563U, 0x13402297U, 0x842011c6U +.word 0x857d244aU, 0xd2f83dbbU, 0xae1132f9U, 0xc76da129U +.word 0x1d4b2f9eU, 0xdcf330b2U, 0x0dec5286U, 0x77d0e3c1U +.word 0x2b6c16b3U, 0xa999b970U, 0x11fa4894U, 0x472264e9U +.word 0xa8c48cfcU, 0xa01a3ff0U, 0x56d82c7dU, 0x22ef9033U +.word 0x87c74e49U, 0xd9c1d138U, 0x8cfea2caU, 0x98360bd4U +.word 0xa6cf81f5U, 0xa528de7aU, 0xda268eb7U, 0x3fa4bfadU +.word 0x2ce49d3aU, 0x500d9278U, 0x6a9bcc5fU, 0x5462467eU +.word 0xf6c2138dU, 0x90e8b8d8U, 0x2e5ef739U, 0x82f5afc3U +.word 0x9fbe805dU, 0x697c93d0U, 0x6fa92dd5U, 0xcfb31225U +.word 0xc83b99acU, 0x10a77d18U, 0xe86e639cU, 0xdb7bbb3bU +.word 0xcd097826U, 0x6ef41859U, 0xec01b79aU, 0x83a89a4fU +.word 0xe6656e95U, 0xaa7ee6ffU, 0x2108cfbcU, 0xefe6e815U +.word 0xbad99be7U, 0x4ace366fU, 0xead4099fU, 0x29d67cb0U +.word 0x31afb2a4U, 0x2a31233fU, 0xc63094a5U, 0x35c066a2U +.word 0x7437bc4eU, 0xfca6ca82U, 0xe0b0d090U, 0x3315d8a7U +.word 0xf14a9804U, 0x41f7daecU, 0x7f0e50cdU, 0x172ff691U +.word 0x768dd64dU, 0x434db0efU, 0xcc544daaU, 0xe4df0496U +.word 0x9ee3b5d1U, 0x4c1b886aU, 0xc1b81f2cU, 0x467f5165U +.word 0x9d04ea5eU, 0x015d358cU, 0xfa737487U, 0xfb2e410bU +.word 0xb35a1d67U, 0x9252d2dbU, 0xe9335610U, 0x6d1347d6U +.word 0x9a8c61d7U, 0x377a0ca1U, 0x598e14f8U, 0xeb893c13U +.word 0xceee27a9U, 0xb735c961U, 0xe1ede51cU, 0x7a3cb147U +.word 0x9c59dfd2U, 0x553f73f2U, 0x1879ce14U, 0x73bf37c7U +.word 0x53eacdf7U, 0x5f5baafdU, 0xdf146f3dU, 0x7886db44U +.word 0xca81f3afU, 0xb93ec468U, 0x382c3424U, 0xc25f40a3U +.word 0x1672c31dU, 0xbc0c25e2U, 0x288b493cU, 0xff41950dU +.word 0x397101a8U, 0x08deb30cU, 0xd89ce4b4U, 0x6490c156U +.word 0x7b6184cbU, 0xd570b632U, 0x48745c6cU, 0xd04257b8U + +.type AES_Td4,\@object +AES_Td4: +.byte 0x52U, 0x09U, 0x6aU, 0xd5U, 0x30U, 0x36U, 0xa5U, 0x38U +.byte 0xbfU, 0x40U, 0xa3U, 0x9eU, 0x81U, 0xf3U, 0xd7U, 0xfbU +.byte 0x7cU, 0xe3U, 0x39U, 0x82U, 0x9bU, 0x2fU, 0xffU, 0x87U +.byte 0x34U, 0x8eU, 0x43U, 0x44U, 0xc4U, 0xdeU, 0xe9U, 0xcbU +.byte 0x54U, 0x7bU, 0x94U, 0x32U, 0xa6U, 0xc2U, 0x23U, 0x3dU +.byte 0xeeU, 0x4cU, 0x95U, 0x0bU, 0x42U, 0xfaU, 0xc3U, 0x4eU +.byte 0x08U, 0x2eU, 0xa1U, 0x66U, 0x28U, 0xd9U, 0x24U, 0xb2U +.byte 0x76U, 0x5bU, 0xa2U, 0x49U, 0x6dU, 0x8bU, 0xd1U, 0x25U +.byte 0x72U, 0xf8U, 0xf6U, 0x64U, 0x86U, 0x68U, 0x98U, 0x16U +.byte 0xd4U, 0xa4U, 0x5cU, 0xccU, 0x5dU, 0x65U, 0xb6U, 0x92U +.byte 0x6cU, 0x70U, 0x48U, 0x50U, 0xfdU, 0xedU, 0xb9U, 0xdaU +.byte 0x5eU, 0x15U, 0x46U, 0x57U, 0xa7U, 0x8dU, 0x9dU, 0x84U +.byte 0x90U, 0xd8U, 0xabU, 0x00U, 0x8cU, 0xbcU, 0xd3U, 0x0aU +.byte 0xf7U, 0xe4U, 0x58U, 0x05U, 0xb8U, 0xb3U, 0x45U, 0x06U +.byte 0xd0U, 0x2cU, 0x1eU, 0x8fU, 0xcaU, 0x3fU, 0x0fU, 0x02U +.byte 0xc1U, 0xafU, 0xbdU, 0x03U, 0x01U, 0x13U, 0x8aU, 0x6bU +.byte 0x3aU, 0x91U, 0x11U, 0x41U, 0x4fU, 0x67U, 0xdcU, 0xeaU +.byte 0x97U, 0xf2U, 0xcfU, 0xceU, 0xf0U, 0xb4U, 0xe6U, 0x73U +.byte 0x96U, 0xacU, 0x74U, 0x22U, 0xe7U, 0xadU, 0x35U, 0x85U +.byte 0xe2U, 0xf9U, 0x37U, 0xe8U, 0x1cU, 0x75U, 0xdfU, 0x6eU +.byte 0x47U, 0xf1U, 0x1aU, 0x71U, 0x1dU, 0x29U, 0xc5U, 0x89U +.byte 0x6fU, 0xb7U, 0x62U, 0x0eU, 0xaaU, 0x18U, 0xbeU, 0x1bU +.byte 0xfcU, 0x56U, 0x3eU, 0x4bU, 0xc6U, 0xd2U, 0x79U, 0x20U +.byte 0x9aU, 0xdbU, 0xc0U, 0xfeU, 0x78U, 0xcdU, 0x5aU, 0xf4U +.byte 0x1fU, 0xddU, 0xa8U, 0x33U, 0x88U, 0x07U, 0xc7U, 0x31U +.byte 0xb1U, 0x12U, 0x10U, 0x59U, 0x27U, 0x80U, 0xecU, 0x5fU +.byte 0x60U, 0x51U, 0x7fU, 0xa9U, 0x19U, 0xb5U, 0x4aU, 0x0dU +.byte 0x2dU, 0xe5U, 0x7aU, 0x9fU, 0x93U, 0xc9U, 0x9cU, 0xefU +.byte 0xa0U, 0xe0U, 0x3bU, 0x4dU, 0xaeU, 0x2aU, 0xf5U, 0xb0U +.byte 0xc8U, 0xebU, 0xbbU, 0x3cU, 0x83U, 0x53U, 0x99U, 0x61U +.byte 0x17U, 0x2bU, 0x04U, 0x7eU, 0xbaU, 0x77U, 0xd6U, 0x26U +.byte 0xe1U, 0x69U, 0x14U, 0x63U, 0x55U, 0x21U, 0x0cU, 0x7dU + +.type AES_rcon,\@object +AES_rcon: +.word 0x00000001U, 0x00000002U, 0x00000004U, 0x00000008U +.word 0x00000010U, 0x00000020U, 0x00000040U, 0x00000080U +.word 0x0000001BU, 0x00000036U +___ + +print $code; +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/aes/asm/aes-x86_64.pl b/crypto/openssl/crypto/aes/asm/aes-x86_64.pl index 25f7ded947ed..8c7b205a0512 100755 --- a/crypto/openssl/crypto/aes/asm/aes-x86_64.pl +++ b/crypto/openssl/crypto/aes/asm/aes-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2005-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2005-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -2221,6 +2221,7 @@ ___ } $code.=<<___; +.section .rodata align=64 .align 64 .LAES_Te: ___ @@ -2643,6 +2644,7 @@ $code.=<<___; .long 0x1b1b1b1b, 0x1b1b1b1b, 0, 0 .asciz "AES for x86_64, CRYPTOGAMS by " .align 64 +.previous ___ # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, diff --git a/crypto/openssl/crypto/aes/asm/aesni-sha1-x86_64.pl b/crypto/openssl/crypto/aes/asm/aesni-sha1-x86_64.pl index dbe33a3f1a02..4e8fa1d753d7 100755 --- a/crypto/openssl/crypto/aes/asm/aesni-sha1-x86_64.pl +++ b/crypto/openssl/crypto/aes/asm/aesni-sha1-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2011-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2011-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -1738,6 +1738,7 @@ ___ }}} } $code.=<<___; +.section .rodata align=64 .align 64 K_XX_XX: .long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 # K_00_19 @@ -1749,6 +1750,7 @@ K_XX_XX: .asciz "AESNI-CBC+SHA1 stitch for x86_64, CRYPTOGAMS by " .align 64 +.previous ___ if ($shaext) {{{ ($in0,$out,$len,$key,$ivp,$ctx,$inp)=("%rdi","%rsi","%rdx","%rcx","%r8","%r9","%r10"); diff --git a/crypto/openssl/crypto/aes/asm/aesni-sha256-x86_64.pl b/crypto/openssl/crypto/aes/asm/aesni-sha256-x86_64.pl index 5521766a6a7d..39d29ddbb022 100755 --- a/crypto/openssl/crypto/aes/asm/aesni-sha256-x86_64.pl +++ b/crypto/openssl/crypto/aes/asm/aesni-sha256-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2013-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2013-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -168,6 +168,7 @@ $code.=<<___; .cfi_endproc .size $func,.-$func +.section .rodata align=64 .align 64 .type $TABLE,\@object $TABLE: @@ -210,6 +211,7 @@ $TABLE: .long 0,0,0,0, 0,0,0,0 .asciz "AESNI-CBC+SHA256 stitch for x86_64, CRYPTOGAMS by " .align 64 +.previous ___ ###################################################################### diff --git a/crypto/openssl/crypto/aes/asm/aesni-x86.pl b/crypto/openssl/crypto/aes/asm/aesni-x86.pl index d8cc378b5f9b..7cf838db170b 100755 --- a/crypto/openssl/crypto/aes/asm/aesni-x86.pl +++ b/crypto/openssl/crypto/aes/asm/aesni-x86.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2009-2022 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2009-2020 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy diff --git a/crypto/openssl/crypto/aes/asm/aesni-x86_64.pl b/crypto/openssl/crypto/aes/asm/aesni-x86_64.pl index 09c8f78890ed..bff33f2b8dcd 100755 --- a/crypto/openssl/crypto/aes/asm/aesni-x86_64.pl +++ b/crypto/openssl/crypto/aes/asm/aesni-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2009-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2009-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -4743,6 +4743,7 @@ ___ } $code.=<<___; +.section .rodata align=64 .align 64 .Lbswap_mask: .byte 15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0 @@ -4765,6 +4766,7 @@ $code.=<<___; .asciz "AES for Intel AES-NI, CRYPTOGAMS by " .align 64 +.previous ___ # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, diff --git a/crypto/openssl/crypto/aes/asm/aesni-xts-avx512.pl b/crypto/openssl/crypto/aes/asm/aesni-xts-avx512.pl new file mode 100644 index 000000000000..0dc69589838b --- /dev/null +++ b/crypto/openssl/crypto/aes/asm/aesni-xts-avx512.pl @@ -0,0 +1,2872 @@ +#! /usr/bin/env perl +# Copyright (C) 2023 Intel Corporation +# +# Licensed under the OpenSSL license (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +# This implementation is based on the AES-XTS code (AVX512VAES + VPCLMULQDQ) +# from Intel(R) Intelligent Storage Acceleration Library Crypto Version +# (https://github.com/intel/isa-l_crypto). +# +###################################################################### +# The main building block of the loop is code that encrypts/decrypts +# 8/16 blocks of data stitching with generation of tweak for the next +# 8/16 blocks, utilizing VAES and VPCLMULQDQ instructions with full width +# of ZMM registers. The main loop is selected based on the input length. +# main_loop_run_16 encrypts/decrypts 16 blocks in parallel and it's selected +# when input length >= 256 bytes (16 blocks) +# main_loop_run_8 encrypts/decrypts 8 blocks in parallel and it's selected +# when 128 bytes <= input length < 256 bytes (8-15 blocks) +# Input length < 128 bytes (8 blocks) is handled by do_n_blocks. +# +# This implementation mainly uses vpshrdq from AVX-512-VBMI2 family and vaesenc, +# vaesdec, vpclmulqdq from AVX-512F family. +$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); +$avx512vaes=0; + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or +die "can't locate x86_64-xlate.pl"; + +if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` + =~ /GNU assembler version ([2-9]\.[0-9]+)/) { + $avx512vaes = ($1>=2.30); +} + +if (!$avx512vaes && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && + `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?/) { + $avx512vaes = ($1==2.11 && $2>=8) + ($1>=2.12); +} + +if (!$avx512vaes && `$ENV{CC} -v 2>&1` + =~ /(Apple)?\s*((?:clang|LLVM) version|.*based on LLVM) ([0-9]+)\.([0-9]+)\.([0-9]+)?/) { + my $ver = $3 + $4/100.0 + $5/10000.0; # 3.1.0->3.01, 3.10.1->3.1001 + if ($1) { + # Apple conditions, they use a different version series, see + # https://en.wikipedia.org/wiki/Xcode#Xcode_7.0_-_10.x_(since_Free_On-Device_Development)_2 + # clang 7.0.0 is Apple clang 10.0.1 + $avx512vaes = ($ver>=10.0001) + } else { + $avx512vaes = ($ver>=7.0); + } +} + +open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"" + or die "can't call $xlate: $!"; +*STDOUT=*OUT; + +#====================================================================== + +if ($avx512vaes) { + + my $GP_STORAGE = $win64 ? (16 * 18) : (16 * 8); # store rbx + my $XMM_STORAGE = $win64 ? (16 * 8) : 0; # store xmm6:xmm15 + my $VARIABLE_OFFSET = $win64 ? (16*8 + 16*10 + 8*3) : + (16*8 + 8*1); + + # right now, >= 0x80 (128) is used for expanded keys. all usages of + # rsp should be invoked via $TW, not shadowed by any other name or + # used directly. + my $TW = "%rsp"; + my $TEMPHIGH = "%rbx"; + my $TEMPLOW = "%rax"; + my $ZPOLY = "%zmm25"; + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;;; Function arguments abstraction + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + my ($key2, $key1, $tweak, $length, $input, $output); + + +$input = "%rdi"; +$output = "%rsi"; +$length = "%rdx"; +$key1 = "%rcx"; +$key2 = "%r8"; +$tweak = "%r9"; + + # arguments for temp parameters + my ($tmp1, $gf_poly_8b, $gf_poly_8b_temp); + $tmp1 = "%r8"; + $gf_poly_8b = "%r10"; + $gf_poly_8b_temp = "%r11"; + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;;; Helper functions + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + # Generates "random" local labels + sub random_string() { + my @chars = ('a' .. 'z', 'A' .. 'Z', '0' .. '9', '_'); + my $length = 15; + my $str; + map { $str .= $chars[rand(33)] } 1 .. $length; + return $str; + } + + # ; Seed the RNG so the labels are generated deterministically + srand(12345); + + sub encrypt_tweak { + my $state_tweak = $_[0]; + my $is_128 = $_[1]; + + $code.=<<___; + vpxor ($key2), $state_tweak, $state_tweak + vaesenc 0x10($key2), $state_tweak, $state_tweak + vaesenc 0x20($key2), $state_tweak, $state_tweak + vaesenc 0x30($key2), $state_tweak, $state_tweak + vaesenc 0x40($key2), $state_tweak, $state_tweak + vaesenc 0x50($key2), $state_tweak, $state_tweak + vaesenc 0x60($key2), $state_tweak, $state_tweak + vaesenc 0x70($key2), $state_tweak, $state_tweak + vaesenc 0x80($key2), $state_tweak, $state_tweak + vaesenc 0x90($key2), $state_tweak, $state_tweak +___ + + if ($is_128) { + $code .= "vaesenclast 0xa0($key2), $state_tweak, $state_tweak\n"; + } else { + $code .= "vaesenc 0xa0($key2), $state_tweak, $state_tweak\n"; + $code .= "vaesenc 0xb0($key2), $state_tweak, $state_tweak\n"; + $code .= "vaesenc 0xc0($key2), $state_tweak, $state_tweak\n"; + $code .= "vaesenc 0xd0($key2), $state_tweak, $state_tweak\n"; + $code .= "vaesenclast 0xe0($key2), $state_tweak, $state_tweak\n"; + } + $code .= "vmovdqa $state_tweak, ($TW)\n"; + } + + sub encrypt_final { + my $st = $_[0]; + my $tw = $_[1]; + my $is_128 = $_[2]; + + # xor Tweak value + $code .= "vpxor $tw, $st, $st\n"; + $code .= "vpxor ($key1), $st, $st\n"; + + my $rounds = $is_128 ? 10 : 14; + for (my $i = 1; $i < $rounds; $i++) { + $code .= "vaesenc 16*$i($key1), $st, $st\n"; + } + + $code .=<<___; + vaesenclast 16*$rounds($key1), $st, $st + vpxor $tw, $st, $st +___ + } + + # decrypt initial blocks of AES + # 1, 2, 3, 4, 5, 6 or 7 blocks are encrypted + # next 8 Tweak values are generated + sub decrypt_initial { + my @st; + $st[0] = $_[0]; + $st[1] = $_[1]; + $st[2] = $_[2]; + $st[3] = $_[3]; + $st[4] = $_[4]; + $st[5] = $_[5]; + $st[6] = $_[6]; + $st[7] = $_[7]; + + my @tw; + $tw[0] = $_[8]; + $tw[1] = $_[9]; + $tw[2] = $_[10]; + $tw[3] = $_[11]; + $tw[4] = $_[12]; + $tw[5] = $_[13]; + $tw[6] = $_[14]; + my $t0 = $_[15]; + my $num_blocks = $_[16]; + my $lt128 = $_[17]; + my $is_128 = $_[18]; + + # num_blocks blocks encrypted + # num_blocks can be 1, 2, 3, 4, 5, 6, 7 + + # xor Tweak value + for (my $i = 0; $i < $num_blocks; $i++) { + $code .= "vpxor $tw[$i], $st[$i], $st[$i]\n"; + } + + $code .= "vmovdqu ($key1), $t0\n"; + + for (my $i = 0; $i < $num_blocks; $i++) { + $code .= "vpxor $t0, $st[$i], $st[$i]\n"; + } + + if (0 == $lt128) { + $code .= <<___; + xor $gf_poly_8b_temp, $gf_poly_8b_temp + shl \$1, $TEMPLOW + adc $TEMPHIGH, $TEMPHIGH +___ + } + # round 1 + $code .= "vmovdqu 0x10($key1), $t0\n"; + + for (my $i = 0; $i < $num_blocks; $i++) { + $code .= "vaesdec $t0, $st[$i], $st[$i]\n"; + } + + if (0 == $lt128) { + $code .= <<___; + cmovc $gf_poly_8b, $gf_poly_8b_temp + xor $gf_poly_8b_temp, $TEMPLOW + mov $TEMPLOW, ($TW) # next Tweak1 generated + mov $TEMPLOW, 0x08($TW) + xor $gf_poly_8b_temp, $gf_poly_8b_temp +___ + } + + # round 2 + $code .= "vmovdqu 0x20($key1), $t0\n"; + + for (my $i = 0; $i < $num_blocks; $i++) { + $code .= "vaesdec $t0, $st[$i], $st[$i]\n"; + } + + if (0 == $lt128) { + $code .= <<___; + shl \$1, $TEMPLOW + adc $TEMPHIGH, $TEMPHIGH + cmovc $gf_poly_8b, $gf_poly_8b_temp + xor $gf_poly_8b_temp, $TEMPLOW + mov $TEMPLOW, 0x10($TW) # next Tweak2 generated +___ + } + + # round 3 + $code .= "vmovdqu 0x30($key1), $t0\n"; + + for (my $i = 0; $i < $num_blocks; $i++) { + $code .= "vaesdec $t0, $st[$i], $st[$i]\n"; + } + + if (0 == $lt128) { + $code .= <<___; + mov $TEMPHIGH, 0x18($TW) + xor $gf_poly_8b_temp, $gf_poly_8b_temp + shl \$1, $TEMPLOW + adc $TEMPHIGH, $TEMPHIGH + cmovc $gf_poly_8b, $gf_poly_8b_temp +___ + } + + # round 4 + $code .= "vmovdqu 0x40($key1), $t0\n"; + + for (my $i = 0; $i < $num_blocks; $i++) { + $code .= "vaesdec $t0, $st[$i], $st[$i]\n"; + } + + if (0 == $lt128) { + $code .= <<___; + xor $gf_poly_8b_temp, $TEMPLOW + mov $TEMPLOW, 0x20($TW) # next Tweak3 generated + mov $TEMPHIGH, 0x28($TW) + xor $gf_poly_8b_temp, $gf_poly_8b_temp + shl \$1, $TEMPLOW +___ + } + + # round 5 + $code .= "vmovdqu 0x50($key1), $t0\n"; + + for (my $i = 0; $i < $num_blocks; $i++) { + $code .= "vaesdec $t0, $st[$i], $st[$i]\n"; + } + + if (0 == $lt128) { + $code .= <<___; + adc $TEMPHIGH, $TEMPHIGH + cmovc $gf_poly_8b, $gf_poly_8b_temp + xor $gf_poly_8b_temp, $TEMPLOW + mov $TEMPLOW, 0x30($TW) # next Tweak4 generated + mov $TEMPHIGH, 0x38($TW) +___ + } + + # round 6 + $code .= "vmovdqu 0x60($key1), $t0\n"; + + for (my $i = 0; $i < $num_blocks; $i++) { + $code .= "vaesdec $t0, $st[$i], $st[$i]\n"; + } + + if (0 == $lt128) { + $code .= <<___; + xor $gf_poly_8b_temp, $gf_poly_8b_temp + shl \$1, $TEMPLOW + adc $TEMPHIGH, $TEMPHIGH + cmovc $gf_poly_8b, $gf_poly_8b_temp + xor $gf_poly_8b_temp, $TEMPLOW + mov $TEMPLOW, 0x40($TW) # next Tweak5 generated + mov $TEMPHIGH, 0x48($TW) +___ + } + + # round 7 + $code .= "vmovdqu 0x70($key1), $t0\n"; + + for (my $i = 0; $i < $num_blocks; $i++) { + $code .= "vaesdec $t0, $st[$i], $st[$i]\n"; + } + + if (0 == $lt128) { + $code .= <<___; + xor $gf_poly_8b_temp, $gf_poly_8b_temp + shl \$1, $TEMPLOW + adc $TEMPHIGH, $TEMPHIGH + cmovc $gf_poly_8b, $gf_poly_8b_temp + xor $gf_poly_8b_temp, $TEMPLOW + mov $TEMPLOW, 0x50($TW) # next Tweak6 generated + mov $TEMPHIGH, 0x58($TW) +___ + } + + # round 8 + $code .= "vmovdqu 0x80($key1), $t0\n"; + + for (my $i = 0; $i < $num_blocks; $i++) { + $code .= "vaesdec $t0, $st[$i], $st[$i]\n"; + } + + if (0 == $lt128) { + $code .= <<___; + xor $gf_poly_8b_temp, $gf_poly_8b_temp + shl \$1, $TEMPLOW + adc $TEMPHIGH, $TEMPHIGH + cmovc $gf_poly_8b, $gf_poly_8b_temp + xor $gf_poly_8b_temp, $TEMPLOW + mov $TEMPLOW, 0x60($TW) # next Tweak7 generated + mov $TEMPHIGH, 0x68($TW) +___ + } + + # round 9 + $code .= "vmovdqu 0x90($key1), $t0\n"; + + for (my $i = 0; $i < $num_blocks; $i++) { + $code .= "vaesdec $t0, $st[$i], $st[$i]\n"; + } + + if (0 == $lt128) { + $code .= <<___; + xor $gf_poly_8b_temp, $gf_poly_8b_temp + shl \$1, $TEMPLOW + adc $TEMPHIGH, $TEMPHIGH + cmovc $gf_poly_8b, $gf_poly_8b_temp + xor $gf_poly_8b_temp, $TEMPLOW + mov $TEMPLOW, 0x70($TW) # next Tweak8 generated + mov $TEMPHIGH, 0x78($TW) +___ + } + + if ($is_128) { + # round 10 + $code .= "vmovdqu 0xa0($key1), $t0\n"; + for (my $i = 0; $i < $num_blocks; $i++) { + $code .= "vaesdeclast $t0, $st[$i], $st[$i]\n"; + } + } else { + # round 10 + $code .= "vmovdqu 0xa0($key1), $t0\n"; + for (my $i = 0; $i < $num_blocks; $i++) { + $code .= "vaesdec $t0, $st[$i], $st[$i]\n"; + } + + # round 11 + $code .= "vmovdqu 0xb0($key1), $t0\n"; + for (my $i = 0; $i < $num_blocks; $i++) { + $code .= "vaesdec $t0, $st[$i], $st[$i]\n"; + } + + # round 12 + $code .= "vmovdqu 0xc0($key1), $t0\n"; + for (my $i = 0; $i < $num_blocks; $i++) { + $code .= "vaesdec $t0, $st[$i], $st[$i]\n"; + } + + # round 13 + $code .= "vmovdqu 0xd0($key1), $t0\n"; + for (my $i = 0; $i < $num_blocks; $i++) { + $code .= "vaesdec $t0, $st[$i], $st[$i]\n"; + } + + # round 14 + $code .= "vmovdqu 0xe0($key1), $t0\n"; + for (my $i = 0; $i < $num_blocks; $i++) { + $code .= "vaesdeclast $t0, $st[$i], $st[$i]\n"; + } + } + + # xor Tweak values + for (my $i = 0; $i < $num_blocks; $i++) { + $code .= "vpxor $tw[$i], $st[$i], $st[$i]\n"; + } + + if (0 == $lt128) { + # load next Tweak values + $code .= <<___; + vmovdqa ($TW), $tw1 + vmovdqa 0x10($TW), $tw2 + vmovdqa 0x20($TW), $tw3 + vmovdqa 0x30($TW), $tw4 + vmovdqa 0x40($TW), $tw5 + vmovdqa 0x50($TW), $tw6 + vmovdqa 0x60($TW), $tw7 +___ + } + } + + sub initialize { + my @st; + $st[0] = $_[0]; + $st[1] = $_[1]; + $st[2] = $_[2]; + $st[3] = $_[3]; + $st[4] = $_[4]; + $st[5] = $_[5]; + $st[6] = $_[6]; + $st[7] = $_[7]; + + my @tw; + $tw[0] = $_[8]; + $tw[1] = $_[9]; + $tw[2] = $_[10]; + $tw[3] = $_[11]; + $tw[4] = $_[12]; + $tw[5] = $_[13]; + $tw[6] = $_[14]; + my $num_initial_blocks = $_[15]; + + $code .= <<___; + vmovdqa 0x0($TW), $tw[0] + mov 0x0($TW), $TEMPLOW + mov 0x08($TW), $TEMPHIGH + vmovdqu 0x0($input), $st[0] +___ + + if ($num_initial_blocks >= 2) { + for (my $i = 1; $i < $num_initial_blocks; $i++) { + $code .= "xor $gf_poly_8b_temp, $gf_poly_8b_temp\n"; + $code .= "shl \$1, $TEMPLOW\n"; + $code .= "adc $TEMPHIGH, $TEMPHIGH\n"; + $code .= "cmovc $gf_poly_8b, $gf_poly_8b_temp\n"; + $code .= "xor $gf_poly_8b_temp, $TEMPLOW\n"; + my $offset = $i * 16; + $code .= "mov $TEMPLOW, $offset($TW)\n"; + $code .= "mov $TEMPHIGH, $offset + 8($TW)\n"; + $code .= "vmovdqa $offset($TW), $tw[$i]\n"; + $code .= "vmovdqu $offset($input), $st[$i]\n"; + } + } + } + + # Encrypt 4 blocks in parallel + sub encrypt_by_four { + my $st1 = $_[0]; # state 1 + my $tw1 = $_[1]; # tweak 1 + my $tmp = $_[2]; + my $is_128 = $_[3]; + + $code .= "vbroadcasti32x4 ($key1), $tmp\n"; + $code .= "vpternlogq \$0x96, $tmp, $tw1, $st1\n"; + + my $rounds = $is_128 ? 10 : 14; + for (my $i = 1; $i < $rounds; $i++) { + $code .= "vbroadcasti32x4 16*$i($key1), $tmp\n"; + $code .= "vaesenc $tmp, $st1, $st1\n"; + } + + $code .= "vbroadcasti32x4 16*$rounds($key1), $tmp\n"; + $code .= "vaesenclast $tmp, $st1, $st1\n"; + + $code .= "vpxorq $tw1, $st1, $st1\n"; + } + + # Encrypt 8 blocks in parallel + # generate next 8 tweak values + sub encrypt_by_eight_zmm { + my $st1 = $_[0]; + my $st2 = $_[1]; + my $tw1 = $_[2]; + my $tw2 = $_[3]; + my $t0 = $_[4]; + my $last_eight = $_[5]; + my $is_128 = $_[6]; + + $code .= <<___; + vbroadcasti32x4 ($key1), $t0 + vpternlogq \$0x96, $t0, $tw1, $st1 + vpternlogq \$0x96, $t0, $tw2, $st2 +___ + + if (0 == $last_eight) { + $code .= <<___; + vpsrldq \$0xf, $tw1, %zmm13 + vpclmulqdq \$0x0, $ZPOLY, %zmm13, %zmm14 + vpslldq \$0x1, $tw1, %zmm15 + vpxord %zmm14, %zmm15, %zmm15 +___ + } + # round 1 + $code .= <<___; + vbroadcasti32x4 0x10($key1), $t0 + vaesenc $t0, $st1, $st1 + vaesenc $t0, $st2, $st2 + + # round 2 + vbroadcasti32x4 0x20($key1), $t0 + vaesenc $t0, $st1, $st1 + vaesenc $t0, $st2, $st2 + + # round 3 + vbroadcasti32x4 0x30($key1), $t0 + vaesenc $t0, $st1, $st1 + vaesenc $t0, $st2, $st2 +___ + + if (0 == $last_eight) { + $code .= <<___; + vpsrldq \$0xf, $tw2, %zmm13 + vpclmulqdq \$0x0, $ZPOLY, %zmm13, %zmm14 + vpslldq \$0x1, $tw2, %zmm16 + vpxord %zmm14, %zmm16, %zmm16 +___ + } + + $code .= <<___; + # round 4 + vbroadcasti32x4 0x40($key1), $t0 + vaesenc $t0, $st1, $st1 + vaesenc $t0, $st2, $st2 + + # round 5 + vbroadcasti32x4 0x50($key1), $t0 + vaesenc $t0, $st1, $st1 + vaesenc $t0, $st2, $st2 + + # round 6 + vbroadcasti32x4 0x60($key1), $t0 + vaesenc $t0, $st1, $st1 + vaesenc $t0, $st2, $st2 + + # round 7 + vbroadcasti32x4 0x70($key1), $t0 + vaesenc $t0, $st1, $st1 + vaesenc $t0, $st2, $st2 + + # round 8 + vbroadcasti32x4 0x80($key1), $t0 + vaesenc $t0, $st1, $st1 + vaesenc $t0, $st2, $st2 + + # round 9 + vbroadcasti32x4 0x90($key1), $t0 + vaesenc $t0, $st1, $st1 + vaesenc $t0, $st2, $st2 +___ + + if ($is_128) { + $code .= <<___; + # round 10 + vbroadcasti32x4 0xa0($key1), $t0 + vaesenclast $t0, $st1, $st1 + vaesenclast $t0, $st2, $st2 +___ + } else { + $code .= <<___; + # round 10 + vbroadcasti32x4 0xa0($key1), $t0 + vaesenc $t0, $st1, $st1 + vaesenc $t0, $st2, $st2 + + # round 11 + vbroadcasti32x4 0xb0($key1), $t0 + vaesenc $t0, $st1, $st1 + vaesenc $t0, $st2, $st2 + + # round 12 + vbroadcasti32x4 0xc0($key1), $t0 + vaesenc $t0, $st1, $st1 + vaesenc $t0, $st2, $st2 + + # round 13 + vbroadcasti32x4 0xd0($key1), $t0 + vaesenc $t0, $st1, $st1 + vaesenc $t0, $st2, $st2 + + # round 14 + vbroadcasti32x4 0xe0($key1), $t0 + vaesenclast $t0, $st1, $st1 + vaesenclast $t0, $st2, $st2 +___ + } + + # xor Tweak values + $code .= "vpxorq $tw1, $st1, $st1\n"; + $code .= "vpxorq $tw2, $st2, $st2\n"; + + if (0 == $last_eight) { + # load next Tweak values + $code .= <<___; + vmovdqa32 %zmm15, $tw1 + vmovdqa32 %zmm16, $tw2 +___ + } + } + + # Decrypt 8 blocks in parallel + # generate next 8 tweak values + sub decrypt_by_eight_zmm { + my $st1 = $_[0]; + my $st2 = $_[1]; + my $tw1 = $_[2]; + my $tw2 = $_[3]; + my $t0 = $_[4]; + my $last_eight = $_[5]; + my $is_128 = $_[6]; + + $code .= <<___; + # xor Tweak values + vpxorq $tw1, $st1, $st1 + vpxorq $tw2, $st2, $st2 + + # ARK + vbroadcasti32x4 ($key1), $t0 + vpxorq $t0, $st1, $st1 + vpxorq $t0, $st2, $st2 +___ + + if (0 == $last_eight) { + $code .= <<___; + vpsrldq \$0xf, $tw1, %zmm13 + vpclmulqdq \$0x0,$ZPOLY, %zmm13, %zmm14 + vpslldq \$0x1, $tw1, %zmm15 + vpxord %zmm14, %zmm15, %zmm15 +___ + } + # round 1 + $code .= <<___; + vbroadcasti32x4 0x10($key1), $t0 + vaesdec $t0, $st1, $st1 + vaesdec $t0, $st2, $st2 + + # round 2 + vbroadcasti32x4 0x20($key1), $t0 + vaesdec $t0, $st1, $st1 + vaesdec $t0, $st2, $st2 + + # round 3 + vbroadcasti32x4 0x30($key1), $t0 + vaesdec $t0, $st1, $st1 + vaesdec $t0, $st2, $st2 +___ + + if (0 == $last_eight) { + $code .= <<___; + vpsrldq \$0xf, $tw2, %zmm13 + vpclmulqdq \$0x0,$ZPOLY, %zmm13, %zmm14 + vpslldq \$0x1, $tw2, %zmm16 + vpxord %zmm14, %zmm16, %zmm16 +___ + } + + $code .= <<___; + # round 4 + vbroadcasti32x4 0x40($key1), $t0 + vaesdec $t0, $st1, $st1 + vaesdec $t0, $st2, $st2 + + # round 5 + vbroadcasti32x4 0x50($key1), $t0 + vaesdec $t0, $st1, $st1 + vaesdec $t0, $st2, $st2 + + # round 6 + vbroadcasti32x4 0x60($key1), $t0 + vaesdec $t0, $st1, $st1 + vaesdec $t0, $st2, $st2 + + # round 7 + vbroadcasti32x4 0x70($key1), $t0 + vaesdec $t0, $st1, $st1 + vaesdec $t0, $st2, $st2 + + # round 8 + vbroadcasti32x4 0x80($key1), $t0 + vaesdec $t0, $st1, $st1 + vaesdec $t0, $st2, $st2 + + # round 9 + vbroadcasti32x4 0x90($key1), $t0 + vaesdec $t0, $st1, $st1 + vaesdec $t0, $st2, $st2 + +___ + if ($is_128) { + $code .= <<___; + # round 10 + vbroadcasti32x4 0xa0($key1), $t0 + vaesdeclast $t0, $st1, $st1 + vaesdeclast $t0, $st2, $st2 +___ + } else { + $code .= <<___; + # round 10 + vbroadcasti32x4 0xa0($key1), $t0 + vaesdec $t0, $st1, $st1 + vaesdec $t0, $st2, $st2 + + # round 11 + vbroadcasti32x4 0xb0($key1), $t0 + vaesdec $t0, $st1, $st1 + vaesdec $t0, $st2, $st2 + + # round 12 + vbroadcasti32x4 0xc0($key1), $t0 + vaesdec $t0, $st1, $st1 + vaesdec $t0, $st2, $st2 + + # round 13 + vbroadcasti32x4 0xd0($key1), $t0 + vaesdec $t0, $st1, $st1 + vaesdec $t0, $st2, $st2 + + # round 14 + vbroadcasti32x4 0xe0($key1), $t0 + vaesdeclast $t0, $st1, $st1 + vaesdeclast $t0, $st2, $st2 +___ + } + + $code .= <<___; + # xor Tweak values + vpxorq $tw1, $st1, $st1 + vpxorq $tw2, $st2, $st2 + + # load next Tweak values + vmovdqa32 %zmm15, $tw1 + vmovdqa32 %zmm16, $tw2 +___ + } + + # Encrypt 16 blocks in parallel + # generate next 16 tweak values + sub encrypt_by_16_zmm { + my @st; + $st[0] = $_[0]; + $st[1] = $_[1]; + $st[2] = $_[2]; + $st[3] = $_[3]; + + my @tw; + $tw[0] = $_[4]; + $tw[1] = $_[5]; + $tw[2] = $_[6]; + $tw[3] = $_[7]; + + my $t0 = $_[8]; + my $last_eight = $_[9]; + my $is_128 = $_[10]; + + # xor Tweak values + for (my $i = 0; $i < 4; $i++) { + $code .= "vpxorq $tw[$i], $st[$i], $st[$i]\n"; + } + + # ARK + $code .= "vbroadcasti32x4 ($key1), $t0\n"; + for (my $i = 0; $i < 4; $i++) { + $code .= "vpxorq $t0, $st[$i], $st[$i]\n"; + } + + if (0 == $last_eight) { + $code .= <<___; + vpsrldq \$0xf, $tw[2], %zmm13 + vpclmulqdq \$0x0,$ZPOLY, %zmm13, %zmm14 + vpslldq \$0x1, $tw[2], %zmm15 + vpxord %zmm14, %zmm15, %zmm15 +___ + } + + # round 1 + $code .= "vbroadcasti32x4 0x10($key1), $t0\n"; + for (my $i = 0; $i < 4; $i++) { + $code .= "vaesenc $t0, $st[$i], $st[$i]\n"; + } + + # round 2 + $code .= "vbroadcasti32x4 0x20($key1), $t0\n"; + for (my $i = 0; $i < 4; $i++) { + $code .= "vaesenc $t0, $st[$i], $st[$i]\n"; + } + + # round 3 + $code .= "vbroadcasti32x4 0x30($key1), $t0\n"; + for (my $i = 0; $i < 4; $i++) { + $code .= "vaesenc $t0, $st[$i], $st[$i]\n"; + } + + if (0 == $last_eight) { + $code .= <<___; + vpsrldq \$0xf, $tw[3], %zmm13 + vpclmulqdq \$0x0,$ZPOLY, %zmm13, %zmm14 + vpslldq \$0x1, $tw[3], %zmm16 + vpxord %zmm14, %zmm16, %zmm16 +___ + } + # round 4 + $code .= "vbroadcasti32x4 0x40($key1), $t0\n"; + for (my $i = 0; $i < 4; $i++) { + $code .= "vaesenc $t0, $st[$i], $st[$i]\n"; + } + + # round 5 + $code .= "vbroadcasti32x4 0x50($key1), $t0\n"; + for (my $i = 0; $i < 4; $i++) { + $code .= "vaesenc $t0, $st[$i], $st[$i]\n"; + } + + # round 6 + $code .= "vbroadcasti32x4 0x60($key1), $t0\n"; + for (my $i = 0; $i < 4; $i++) { + $code .= "vaesenc $t0, $st[$i], $st[$i]\n"; + } + + if (0 == $last_eight) { + $code .= <<___; + vpsrldq \$0xf, %zmm15, %zmm13 + vpclmulqdq \$0x0,$ZPOLY, %zmm13, %zmm14 + vpslldq \$0x1, %zmm15, %zmm17 + vpxord %zmm14, %zmm17, %zmm17 +___ + } + # round 7 + $code .= "vbroadcasti32x4 0x70($key1), $t0\n"; + for (my $i = 0; $i < 4; $i++) { + $code .= "vaesenc $t0, $st[$i], $st[$i]\n"; + } + + # round 8 + $code .= "vbroadcasti32x4 0x80($key1), $t0\n"; + for (my $i = 0; $i < 4; $i++) { + $code .= "vaesenc $t0, $st[$i], $st[$i]\n"; + } + + # round 9 + $code .= "vbroadcasti32x4 0x90($key1), $t0\n"; + for (my $i = 0; $i < 4; $i++) { + $code .= "vaesenc $t0, $st[$i], $st[$i]\n"; + } + + if (0 == $last_eight) { + $code .= <<___; + vpsrldq \$0xf, %zmm16, %zmm13 + vpclmulqdq \$0x0,$ZPOLY, %zmm13, %zmm14 + vpslldq \$0x1, %zmm16, %zmm18 + vpxord %zmm14, %zmm18, %zmm18 +___ + } + if ($is_128) { + # round 10 + $code .= "vbroadcasti32x4 0xa0($key1), $t0\n"; + for (my $i = 0; $i < 4; $i++) { + $code .= "vaesenclast $t0, $st[$i], $st[$i]\n"; + } + } else { + # round 10 + $code .= "vbroadcasti32x4 0xa0($key1), $t0\n"; + for (my $i = 0; $i < 4; $i++) { + $code .= "vaesenc $t0, $st[$i], $st[$i]\n"; + } + # round 11 + $code .= "vbroadcasti32x4 0xb0($key1), $t0\n"; + for (my $i = 0; $i < 4; $i++) { + $code .= "vaesenc $t0, $st[$i], $st[$i]\n"; + } + # round 12 + $code .= "vbroadcasti32x4 0xc0($key1), $t0\n"; + for (my $i = 0; $i < 4; $i++) { + $code .= "vaesenc $t0, $st[$i], $st[$i]\n"; + } + # round 13 + $code .= "vbroadcasti32x4 0xd0($key1), $t0\n"; + for (my $i = 0; $i < 4; $i++) { + $code .= "vaesenc $t0, $st[$i], $st[$i]\n"; + } + # round 14 + $code .= "vbroadcasti32x4 0xe0($key1), $t0\n"; + for (my $i = 0; $i < 4; $i++) { + $code .= "vaesenclast $t0, $st[$i], $st[$i]\n"; + } + } + + # xor Tweak values + for (my $i = 0; $i < 4; $i++) { + $code .= "vpxorq $tw[$i], $st[$i], $st[$i]\n"; + } + + $code .= <<___; + # load next Tweak values + vmovdqa32 %zmm15, $tw[0] + vmovdqa32 %zmm16, $tw[1] + vmovdqa32 %zmm17, $tw[2] + vmovdqa32 %zmm18, $tw[3] +___ + } + + # Decrypt 16 blocks in parallel + # generate next 8 tweak values + sub decrypt_by_16_zmm { + my @st; + $st[0] = $_[0]; + $st[1] = $_[1]; + $st[2] = $_[2]; + $st[3] = $_[3]; + + my @tw; + $tw[0] = $_[4]; + $tw[1] = $_[5]; + $tw[2] = $_[6]; + $tw[3] = $_[7]; + + my $t0 = $_[8]; + my $last_eight = $_[9]; + my $is_128 = $_[10]; + + # xor Tweak values + for (my $i = 0; $i < 4; $i++) { + $code .= "vpxorq $tw[$i], $st[$i], $st[$i]\n"; + } + + # ARK + $code .= "vbroadcasti32x4 ($key1), $t0\n"; + for (my $i = 0; $i < 4; $i++) { + $code .= "vpxorq $t0, $st[$i], $st[$i]\n"; + } + + if (0 == $last_eight) { + $code .= <<___; + vpsrldq \$0xf, $tw[2], %zmm13 + vpclmulqdq \$0x0,$ZPOLY, %zmm13, %zmm14 + vpslldq \$0x1, $tw[2], %zmm15 + vpxord %zmm14, %zmm15, %zmm15 +___ + } + + # round 1 + $code .= "vbroadcasti32x4 0x10($key1), $t0\n"; + for (my $i = 0; $i < 4; $i++) { + $code .= "vaesdec $t0, $st[$i], $st[$i]\n"; + } + + # round 2 + $code .= "vbroadcasti32x4 0x20($key1), $t0\n"; + for (my $i = 0; $i < 4; $i++) { + $code .= "vaesdec $t0, $st[$i], $st[$i]\n"; + } + + # round 3 + $code .= "vbroadcasti32x4 0x30($key1), $t0\n"; + for (my $i = 0; $i < 4; $i++) { + $code .= "vaesdec $t0, $st[$i], $st[$i]\n"; + } + + if (0 == $last_eight) { + $code .= <<___; + vpsrldq \$0xf, $tw[3], %zmm13 + vpclmulqdq \$0x0,$ZPOLY, %zmm13, %zmm14 + vpslldq \$0x1, $tw[3], %zmm16 + vpxord %zmm14, %zmm16, %zmm16 +___ + } + # round 4 + $code .= "vbroadcasti32x4 0x40($key1), $t0\n"; + for (my $i = 0; $i < 4; $i++) { + $code .= "vaesdec $t0, $st[$i], $st[$i]\n"; + } + + # round 5 + $code .= "vbroadcasti32x4 0x50($key1), $t0\n"; + for (my $i = 0; $i < 4; $i++) { + $code .= "vaesdec $t0, $st[$i], $st[$i]\n"; + } + + # round 6 + $code .= "vbroadcasti32x4 0x60($key1), $t0\n"; + for (my $i = 0; $i < 4; $i++) { + $code .= "vaesdec $t0, $st[$i], $st[$i]\n"; + } + + if (0 == $last_eight) { + $code .= <<___; + vpsrldq \$0xf, %zmm15, %zmm13 + vpclmulqdq \$0x0,$ZPOLY, %zmm13, %zmm14 + vpslldq \$0x1, %zmm15, %zmm17 + vpxord %zmm14, %zmm17, %zmm17 +___ + } + # round 7 + $code .= "vbroadcasti32x4 0x70($key1), $t0\n"; + for (my $i = 0; $i < 4; $i++) { + $code .= "vaesdec $t0, $st[$i], $st[$i]\n"; + } + + # round 8 + $code .= "vbroadcasti32x4 0x80($key1), $t0\n"; + for (my $i = 0; $i < 4; $i++) { + $code .= "vaesdec $t0, $st[$i], $st[$i]\n"; + } + + # round 9 + $code .= "vbroadcasti32x4 0x90($key1), $t0\n"; + for (my $i = 0; $i < 4; $i++) { + $code .= "vaesdec $t0, $st[$i], $st[$i]\n"; + } + + if (0 == $last_eight) { + $code .= <<___; + vpsrldq \$0xf, %zmm16, %zmm13 + vpclmulqdq \$0x0,$ZPOLY, %zmm13, %zmm14 + vpslldq \$0x1, %zmm16, %zmm18 + vpxord %zmm14, %zmm18, %zmm18 +___ + } + if ($is_128) { + # round 10 + $code .= "vbroadcasti32x4 0xa0($key1), $t0\n"; + for (my $i = 0; $i < 4; $i++) { + $code .= "vaesdeclast $t0, $st[$i], $st[$i]\n"; + } + } else { + # round 10 + $code .= "vbroadcasti32x4 0xa0($key1), $t0\n"; + for (my $i = 0; $i < 4; $i++) { + $code .= "vaesdec $t0, $st[$i], $st[$i]\n"; + } + + # round 11 + $code .= "vbroadcasti32x4 0xb0($key1), $t0\n"; + for (my $i = 0; $i < 4; $i++) { + $code .= "vaesdec $t0, $st[$i], $st[$i]\n"; + } + + # round 12 + $code .= "vbroadcasti32x4 0xc0($key1), $t0\n"; + for (my $i = 0; $i < 4; $i++) { + $code .= "vaesdec $t0, $st[$i], $st[$i]\n"; + } + + # round 13 + $code .= "vbroadcasti32x4 0xd0($key1), $t0\n"; + for (my $i = 0; $i < 4; $i++) { + $code .= "vaesdec $t0, $st[$i], $st[$i]\n"; + } + + # round 14 + $code .= "vbroadcasti32x4 0xe0($key1), $t0\n"; + for (my $i = 0; $i < 4; $i++) { + $code .= "vaesdeclast $t0, $st[$i], $st[$i]\n"; + } + } + + # xor Tweak values + for (my $i = 0; $i < 4; $i++) { + $code .= "vpxorq $tw[$i], $st[$i], $st[$i]\n"; + } + + $code .= <<___; + # load next Tweak values + vmovdqa32 %zmm15, $tw[0] + vmovdqa32 %zmm16, $tw[1] + vmovdqa32 %zmm17, $tw[2] + vmovdqa32 %zmm18, $tw[3] +___ + } + + $code .= ".text\n"; + + { + $code.=<<"___"; + .extern OPENSSL_ia32cap_P + .globl aesni_xts_avx512_eligible + .type aesni_xts_avx512_eligible,\@abi-omnipotent + .align 32 + aesni_xts_avx512_eligible: + mov OPENSSL_ia32cap_P+8(%rip), %ecx + xor %eax,%eax + # 1<<31|1<<30|1<<17|1<<16 avx512vl + avx512bw + avx512dq + avx512f + and \$0xc0030000, %ecx + cmp \$0xc0030000, %ecx + jne .L_done + mov OPENSSL_ia32cap_P+12(%rip), %ecx + # 1<<10|1<<9|1<<6 vaes + vpclmulqdq + vbmi2 + and \$0x640, %ecx + cmp \$0x640, %ecx + cmove %ecx,%eax + .L_done: + ret + .size aesni_xts_avx512_eligible, .-aesni_xts_avx512_eligible +___ + } + + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;void aesni_xts_[128|256]_encrypt_avx512( + # ; const uint8_t *in, // input data + # ; uint8_t *out, // output data + # ; size_t length, // sector size, in bytes + # ; const AES_KEY *key1, // key used for "ECB" encryption + # ; const AES_KEY *key2, // key used for tweaking + # ; const uint8_t iv[16]) // initial tweak value, 16 bytes + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + sub enc { + my $is_128 = $_[0]; + my $rndsuffix = &random_string(); + + if ($is_128) { + $code.=<<___; + .globl aesni_xts_128_encrypt_avx512 + .hidden aesni_xts_128_encrypt_avx512 + .type aesni_xts_128_encrypt_avx512,\@function,6 + .align 32 + aesni_xts_128_encrypt_avx512: + .cfi_startproc + endbranch +___ + } else { + $code.=<<___; + .globl aesni_xts_256_encrypt_avx512 + .hidden aesni_xts_256_encrypt_avx512 + .type aesni_xts_256_encrypt_avx512,\@function,6 + .align 32 + aesni_xts_256_encrypt_avx512: + .cfi_startproc + endbranch +___ + } + $code .= "push %rbp\n"; + $code .= "mov $TW,%rbp\n"; + $code .= "sub \$$VARIABLE_OFFSET,$TW\n"; + $code .= "and \$0xffffffffffffffc0,$TW\n"; + $code .= "mov %rbx,$GP_STORAGE($TW)\n"; + + if ($win64) { + $code .= "mov %rdi,$GP_STORAGE + 8*1($TW)\n"; + $code .= "mov %rsi,$GP_STORAGE + 8*2($TW)\n"; + $code .= "vmovdqa %xmm6, $XMM_STORAGE + 16*0($TW)\n"; + $code .= "vmovdqa %xmm7, $XMM_STORAGE + 16*1($TW)\n"; + $code .= "vmovdqa %xmm8, $XMM_STORAGE + 16*2($TW)\n"; + $code .= "vmovdqa %xmm9, $XMM_STORAGE + 16*3($TW)\n"; + $code .= "vmovdqa %xmm10, $XMM_STORAGE + 16*4($TW)\n"; + $code .= "vmovdqa %xmm11, $XMM_STORAGE + 16*5($TW)\n"; + $code .= "vmovdqa %xmm12, $XMM_STORAGE + 16*6($TW)\n"; + $code .= "vmovdqa %xmm13, $XMM_STORAGE + 16*7($TW)\n"; + $code .= "vmovdqa %xmm14, $XMM_STORAGE + 16*8($TW)\n"; + $code .= "vmovdqa %xmm15, $XMM_STORAGE + 16*9($TW)\n"; + } + + $code .= "mov \$0x87, $gf_poly_8b\n"; + $code .= "vmovdqu ($tweak),%xmm1\n"; # read initial tweak values + + encrypt_tweak("%xmm1", $is_128); + + if ($win64) { + $code .= "mov $input, 8 + 8*5(%rbp)\n"; # ciphertext pointer + $code .= "mov $output, 8 + 8*6(%rbp)\n"; # plaintext pointer + } + + { + $code.=<<___; + + cmp \$0x80,$length + jl .L_less_than_128_bytes_${rndsuffix} + vpbroadcastq $gf_poly_8b,$ZPOLY + cmp \$0x100,$length + jge .L_start_by16_${rndsuffix} + cmp \$0x80,$length + jge .L_start_by8_${rndsuffix} + + .L_do_n_blocks_${rndsuffix}: + cmp \$0x0,$length + je .L_ret_${rndsuffix} + cmp \$0x70,$length + jge .L_remaining_num_blocks_is_7_${rndsuffix} + cmp \$0x60,$length + jge .L_remaining_num_blocks_is_6_${rndsuffix} + cmp \$0x50,$length + jge .L_remaining_num_blocks_is_5_${rndsuffix} + cmp \$0x40,$length + jge .L_remaining_num_blocks_is_4_${rndsuffix} + cmp \$0x30,$length + jge .L_remaining_num_blocks_is_3_${rndsuffix} + cmp \$0x20,$length + jge .L_remaining_num_blocks_is_2_${rndsuffix} + cmp \$0x10,$length + jge .L_remaining_num_blocks_is_1_${rndsuffix} + vmovdqa %xmm0,%xmm8 + vmovdqa %xmm9,%xmm0 + jmp .L_steal_cipher_${rndsuffix} + + .L_remaining_num_blocks_is_7_${rndsuffix}: + mov \$0x0000ffffffffffff,$tmp1 + kmovq $tmp1,%k1 + vmovdqu8 ($input),%zmm1 + vmovdqu8 0x40($input),%zmm2{%k1} + add \$0x70,$input +___ + } + + encrypt_by_eight_zmm("%zmm1", "%zmm2", "%zmm9", "%zmm10", "%zmm0", 1, $is_128); + + { + $code .= <<___; + vmovdqu8 %zmm1,($output) + vmovdqu8 %zmm2,0x40($output){%k1} + add \$0x70,$output + vextracti32x4 \$0x2,%zmm2,%xmm8 + vextracti32x4 \$0x3,%zmm10,%xmm0 + and \$0xf,$length + je .L_ret_${rndsuffix} + jmp .L_steal_cipher_${rndsuffix} + + .L_remaining_num_blocks_is_6_${rndsuffix}: + vmovdqu8 ($input),%zmm1 + vmovdqu8 0x40($input),%ymm2 + add \$0x60,$input +___ + } + + encrypt_by_eight_zmm("%zmm1", "%zmm2", "%zmm9", "%zmm10", "%zmm0", 1, $is_128); + + { + $code .= <<___; + vmovdqu8 %zmm1,($output) + vmovdqu8 %ymm2,0x40($output) + add \$0x60,$output + vextracti32x4 \$0x1,%zmm2,%xmm8 + vextracti32x4 \$0x2,%zmm10,%xmm0 + and \$0xf,$length + je .L_ret_${rndsuffix} + jmp .L_steal_cipher_${rndsuffix} + + .L_remaining_num_blocks_is_5_${rndsuffix}: + vmovdqu8 ($input),%zmm1 + vmovdqu 0x40($input),%xmm2 + add \$0x50,$input +___ + } + + encrypt_by_eight_zmm("%zmm1", "%zmm2", "%zmm9", "%zmm10", "%zmm0", 1, $is_128); + + { + $code .= <<___; + vmovdqu8 %zmm1,($output) + vmovdqu %xmm2,0x40($output) + add \$0x50,$output + vmovdqa %xmm2,%xmm8 + vextracti32x4 \$0x1,%zmm10,%xmm0 + and \$0xf,$length + je .L_ret_${rndsuffix} + jmp .L_steal_cipher_${rndsuffix} + + .L_remaining_num_blocks_is_4_${rndsuffix}: + vmovdqu8 ($input),%zmm1 + add \$0x40,$input +___ + } + + encrypt_by_four("%zmm1", "%zmm9", "%zmm0", $is_128); + + { + $code .= <<___; + vmovdqu8 %zmm1,($output) + add \$0x40,$output + vextracti32x4 \$0x3,%zmm1,%xmm8 + vmovdqa64 %xmm10, %xmm0 + and \$0xf,$length + je .L_ret_${rndsuffix} + jmp .L_steal_cipher_${rndsuffix} +___ + } + + { + $code .= <<___; + .L_remaining_num_blocks_is_3_${rndsuffix}: + mov \$-1, $tmp1 + shr \$0x10, $tmp1 + kmovq $tmp1, %k1 + vmovdqu8 ($input), %zmm1{%k1} + add \$0x30, $input +___ + } + + encrypt_by_four("%zmm1", "%zmm9", "%zmm0", $is_128); + + { + $code .= <<___; + vmovdqu8 %zmm1, ($output){%k1} + add \$0x30, $output + vextracti32x4 \$0x2, %zmm1, %xmm8 + vextracti32x4 \$0x3, %zmm9, %xmm0 + and \$0xf, $length + je .L_ret_${rndsuffix} + jmp .L_steal_cipher_${rndsuffix} +___ + } + + { + $code .= <<___; + .L_remaining_num_blocks_is_2_${rndsuffix}: + vmovdqu8 ($input), %ymm1 + add \$0x20, $input +___ + } + + encrypt_by_four("%ymm1", "%ymm9", "%ymm0", $is_128); + + { + $code .= <<___; + vmovdqu %ymm1,($output) + add \$0x20,$output + vextracti32x4 \$0x1, %zmm1, %xmm8 + vextracti32x4 \$0x2,%zmm9,%xmm0 + and \$0xf,$length + je .L_ret_${rndsuffix} + jmp .L_steal_cipher_${rndsuffix} +___ + } + + { + $code .= <<___; + .L_remaining_num_blocks_is_1_${rndsuffix}: + vmovdqu ($input),%xmm1 + add \$0x10,$input +___ + } + + encrypt_final("%xmm1", "%xmm9", $is_128); + + { + $code .= <<___; + vmovdqu %xmm1,($output) + add \$0x10,$output + vmovdqa %xmm1,%xmm8 + vextracti32x4 \$0x1,%zmm9,%xmm0 + and \$0xf,$length + je .L_ret_${rndsuffix} + jmp .L_steal_cipher_${rndsuffix} + + + .L_start_by16_${rndsuffix}: + vbroadcasti32x4 ($TW),%zmm0 + vbroadcasti32x4 shufb_15_7(%rip),%zmm8 + mov \$0xaa,$tmp1 + kmovq $tmp1,%k2 + vpshufb %zmm8,%zmm0,%zmm1 + vpsllvq const_dq3210(%rip),%zmm0,%zmm4 + vpsrlvq const_dq5678(%rip),%zmm1,%zmm2 + vpclmulqdq \$0x0,%zmm25,%zmm2,%zmm3 + vpxorq %zmm2,%zmm4,%zmm4{%k2} + vpxord %zmm4,%zmm3,%zmm9 + vpsllvq const_dq7654(%rip),%zmm0,%zmm5 + vpsrlvq const_dq1234(%rip),%zmm1,%zmm6 + vpclmulqdq \$0x0,%zmm25,%zmm6,%zmm7 + vpxorq %zmm6,%zmm5,%zmm5{%k2} + vpxord %zmm5,%zmm7,%zmm10 + vpsrldq \$0xf,%zmm9,%zmm13 + vpclmulqdq \$0x0,%zmm25,%zmm13,%zmm14 + vpslldq \$0x1,%zmm9,%zmm11 + vpxord %zmm14,%zmm11,%zmm11 + vpsrldq \$0xf,%zmm10,%zmm15 + vpclmulqdq \$0x0,%zmm25,%zmm15,%zmm16 + vpslldq \$0x1,%zmm10,%zmm12 + vpxord %zmm16,%zmm12,%zmm12 + + .L_main_loop_run_16_${rndsuffix}: + vmovdqu8 ($input),%zmm1 + vmovdqu8 0x40($input),%zmm2 + vmovdqu8 0x80($input),%zmm3 + vmovdqu8 0xc0($input),%zmm4 + add \$0x100,$input +___ + } + + encrypt_by_16_zmm("%zmm1", "%zmm2", "%zmm3", "%zmm4", "%zmm9", + "%zmm10", "%zmm11", "%zmm12", "%zmm0", 0, $is_128); + + { + $code .= <<___; + vmovdqu8 %zmm1,($output) + vmovdqu8 %zmm2,0x40($output) + vmovdqu8 %zmm3,0x80($output) + vmovdqu8 %zmm4,0xc0($output) + add \$0x100,$output + sub \$0x100,$length + cmp \$0x100,$length + jae .L_main_loop_run_16_${rndsuffix} + cmp \$0x80,$length + jae .L_main_loop_run_8_${rndsuffix} + vextracti32x4 \$0x3,%zmm4,%xmm0 + jmp .L_do_n_blocks_${rndsuffix} + + .L_start_by8_${rndsuffix}: + vbroadcasti32x4 ($TW),%zmm0 + vbroadcasti32x4 shufb_15_7(%rip),%zmm8 + mov \$0xaa,$tmp1 + kmovq $tmp1,%k2 + vpshufb %zmm8,%zmm0,%zmm1 + vpsllvq const_dq3210(%rip),%zmm0,%zmm4 + vpsrlvq const_dq5678(%rip),%zmm1,%zmm2 + vpclmulqdq \$0x0,%zmm25,%zmm2,%zmm3 + vpxorq %zmm2,%zmm4,%zmm4{%k2} + vpxord %zmm4,%zmm3,%zmm9 + vpsllvq const_dq7654(%rip),%zmm0,%zmm5 + vpsrlvq const_dq1234(%rip),%zmm1,%zmm6 + vpclmulqdq \$0x0,%zmm25,%zmm6,%zmm7 + vpxorq %zmm6,%zmm5,%zmm5{%k2} + vpxord %zmm5,%zmm7,%zmm10 + + .L_main_loop_run_8_${rndsuffix}: + vmovdqu8 ($input),%zmm1 + vmovdqu8 0x40($input),%zmm2 + add \$0x80,$input +___ + } + + encrypt_by_eight_zmm("%zmm1", "%zmm2", "%zmm9", "%zmm10", "%zmm0", 0, $is_128); + + { + $code .= <<___; + vmovdqu8 %zmm1,($output) + vmovdqu8 %zmm2,0x40($output) + add \$0x80,$output + sub \$0x80,$length + cmp \$0x80,$length + jae .L_main_loop_run_8_${rndsuffix} + vextracti32x4 \$0x3,%zmm2,%xmm0 + jmp .L_do_n_blocks_${rndsuffix} + + .L_steal_cipher_${rndsuffix}: + vmovdqa %xmm8,%xmm2 + lea vpshufb_shf_table(%rip),$TEMPLOW + vmovdqu ($TEMPLOW,$length,1),%xmm10 + vpshufb %xmm10,%xmm8,%xmm8 + vmovdqu -0x10($input,$length,1),%xmm3 + vmovdqu %xmm8,-0x10($output,$length,1) + lea vpshufb_shf_table(%rip),$TEMPLOW + add \$16, $TEMPLOW + sub $length,$TEMPLOW + vmovdqu ($TEMPLOW),%xmm10 + vpxor mask1(%rip),%xmm10,%xmm10 + vpshufb %xmm10,%xmm3,%xmm3 + vpblendvb %xmm10,%xmm2,%xmm3,%xmm3 + vpxor %xmm0,%xmm3,%xmm8 + vpxor ($key1),%xmm8,%xmm8 + vaesenc 0x10($key1),%xmm8,%xmm8 + vaesenc 0x20($key1),%xmm8,%xmm8 + vaesenc 0x30($key1),%xmm8,%xmm8 + vaesenc 0x40($key1),%xmm8,%xmm8 + vaesenc 0x50($key1),%xmm8,%xmm8 + vaesenc 0x60($key1),%xmm8,%xmm8 + vaesenc 0x70($key1),%xmm8,%xmm8 + vaesenc 0x80($key1),%xmm8,%xmm8 + vaesenc 0x90($key1),%xmm8,%xmm8 +___ + if ($is_128) { + $code .= "vaesenclast 0xa0($key1),%xmm8,%xmm8\n"; + } else { + $code .= <<___ + vaesenc 0xa0($key1),%xmm8,%xmm8 + vaesenc 0xb0($key1),%xmm8,%xmm8 + vaesenc 0xc0($key1),%xmm8,%xmm8 + vaesenc 0xd0($key1),%xmm8,%xmm8 + vaesenclast 0xe0($key1),%xmm8,%xmm8 +___ + } + $code .= "vpxor %xmm0,%xmm8,%xmm8\n"; + $code .= "vmovdqu %xmm8,-0x10($output)\n"; + } + + { + $code .= <<___; + .L_ret_${rndsuffix}: + mov $GP_STORAGE($TW),%rbx + xor $tmp1,$tmp1 + mov $tmp1,$GP_STORAGE($TW) + # Zero-out the whole of `%zmm0`. + vpxorq %zmm0,%zmm0,%zmm0 +___ + } + + if ($win64) { + $code .= <<___; + mov $GP_STORAGE + 8*1($TW),%rdi + mov $tmp1,$GP_STORAGE + 8*1($TW) + mov $GP_STORAGE + 8*2($TW),%rsi + mov $tmp1,$GP_STORAGE + 8*2($TW) + + vmovdqa $XMM_STORAGE + 16 * 0($TW), %xmm6 + vmovdqa $XMM_STORAGE + 16 * 1($TW), %xmm7 + vmovdqa $XMM_STORAGE + 16 * 2($TW), %xmm8 + vmovdqa $XMM_STORAGE + 16 * 3($TW), %xmm9 + + # Zero the 64 bytes we just restored to the xmm registers. + vmovdqa64 %zmm0,$XMM_STORAGE($TW) + + vmovdqa $XMM_STORAGE + 16 * 4($TW), %xmm10 + vmovdqa $XMM_STORAGE + 16 * 5($TW), %xmm11 + vmovdqa $XMM_STORAGE + 16 * 6($TW), %xmm12 + vmovdqa $XMM_STORAGE + 16 * 7($TW), %xmm13 + + # And again. + vmovdqa64 %zmm0,$XMM_STORAGE + 16 * 4($TW) + + vmovdqa $XMM_STORAGE + 16 * 8($TW), %xmm14 + vmovdqa $XMM_STORAGE + 16 * 9($TW), %xmm15 + + # Last round is only 32 bytes (256-bits), so we use `%ymm` as the + # source operand. + vmovdqa %ymm0,$XMM_STORAGE + 16 * 8($TW) +___ + } + + { + $code .= <<___; + mov %rbp,$TW + pop %rbp + vzeroupper + ret + + .L_less_than_128_bytes_${rndsuffix}: + vpbroadcastq $gf_poly_8b, $ZPOLY + cmp \$0x10,$length + jb .L_ret_${rndsuffix} + vbroadcasti32x4 ($TW), %zmm0 + vbroadcasti32x4 shufb_15_7(%rip), %zmm8 + movl \$0xaa, %r8d + kmovq %r8, %k2 + mov $length,$tmp1 + and \$0x70,$tmp1 + cmp \$0x60,$tmp1 + je .L_num_blocks_is_6_${rndsuffix} + cmp \$0x50,$tmp1 + je .L_num_blocks_is_5_${rndsuffix} + cmp \$0x40,$tmp1 + je .L_num_blocks_is_4_${rndsuffix} + cmp \$0x30,$tmp1 + je .L_num_blocks_is_3_${rndsuffix} + cmp \$0x20,$tmp1 + je .L_num_blocks_is_2_${rndsuffix} + cmp \$0x10,$tmp1 + je .L_num_blocks_is_1_${rndsuffix} + + .L_num_blocks_is_7_${rndsuffix}: + vpshufb %zmm8, %zmm0, %zmm1 + vpsllvq const_dq3210(%rip), %zmm0, %zmm4 + vpsrlvq const_dq5678(%rip), %zmm1, %zmm2 + vpclmulqdq \$0x00, $ZPOLY, %zmm2, %zmm3 + vpxorq %zmm2, %zmm4, %zmm4{%k2} + vpxord %zmm4, %zmm3, %zmm9 + vpsllvq const_dq7654(%rip), %zmm0, %zmm5 + vpsrlvq const_dq1234(%rip), %zmm1, %zmm6 + vpclmulqdq \$0x00, $ZPOLY, %zmm6, %zmm7 + vpxorq %zmm6, %zmm5, %zmm5{%k2} + vpxord %zmm5, %zmm7, %zmm10 + mov \$0x0000ffffffffffff, $tmp1 + kmovq $tmp1, %k1 + vmovdqu8 16*0($input), %zmm1 + vmovdqu8 16*4($input), %zmm2{%k1} + + add \$0x70,$input +___ + } + + encrypt_by_eight_zmm("%zmm1", "%zmm2", "%zmm9", "%zmm10", "%zmm0", 1, $is_128); + + { + $code .= <<___; + vmovdqu8 %zmm1, 16*0($output) + vmovdqu8 %zmm2, 16*4($output){%k1} + add \$0x70,$output + vextracti32x4 \$0x2, %zmm2, %xmm8 + vextracti32x4 \$0x3, %zmm10, %xmm0 + and \$0xf,$length + je .L_ret_${rndsuffix} + jmp .L_steal_cipher_${rndsuffix} +___ + } + + { + $code .= <<___; + .L_num_blocks_is_6_${rndsuffix}: + vpshufb %zmm8, %zmm0, %zmm1 + vpsllvq const_dq3210(%rip), %zmm0, %zmm4 + vpsrlvq const_dq5678(%rip), %zmm1, %zmm2 + vpclmulqdq \$0x00, $ZPOLY, %zmm2, %zmm3 + vpxorq %zmm2, %zmm4, %zmm4{%k2} + vpxord %zmm4, %zmm3, %zmm9 + vpsllvq const_dq7654(%rip), %zmm0, %zmm5 + vpsrlvq const_dq1234(%rip), %zmm1, %zmm6 + vpclmulqdq \$0x00, $ZPOLY, %zmm6, %zmm7 + vpxorq %zmm6, %zmm5, %zmm5{%k2} + vpxord %zmm5, %zmm7, %zmm10 + vmovdqu8 16*0($input), %zmm1 + vmovdqu8 16*4($input), %ymm2 + add \$96, $input +___ + } + + encrypt_by_eight_zmm("%zmm1", "%zmm2", "%zmm9", "%zmm10", "%zmm0", 1, $is_128); + + { + $code .= <<___; + vmovdqu8 %zmm1, 16*0($output) + vmovdqu8 %ymm2, 16*4($output) + add \$96, $output + + vextracti32x4 \$0x1, %ymm2, %xmm8 + vextracti32x4 \$0x2, %zmm10, %xmm0 + and \$0xf,$length + je .L_ret_${rndsuffix} + jmp .L_steal_cipher_${rndsuffix} +___ + } + + { + $code .= <<___; + .L_num_blocks_is_5_${rndsuffix}: + vpshufb %zmm8, %zmm0, %zmm1 + vpsllvq const_dq3210(%rip), %zmm0, %zmm4 + vpsrlvq const_dq5678(%rip), %zmm1, %zmm2 + vpclmulqdq \$0x00, $ZPOLY, %zmm2, %zmm3 + vpxorq %zmm2, %zmm4, %zmm4{%k2} + vpxord %zmm4, %zmm3, %zmm9 + vpsllvq const_dq7654(%rip), %zmm0, %zmm5 + vpsrlvq const_dq1234(%rip), %zmm1, %zmm6 + vpclmulqdq \$0x00, $ZPOLY, %zmm6, %zmm7 + vpxorq %zmm6, %zmm5, %zmm5{%k2} + vpxord %zmm5, %zmm7, %zmm10 + vmovdqu8 16*0($input), %zmm1 + vmovdqu8 16*4($input), %xmm2 + add \$80, $input +___ + } + + encrypt_by_eight_zmm("%zmm1", "%zmm2", "%zmm9", "%zmm10", "%zmm0", 1, $is_128); + + { + $code .= <<___; + vmovdqu8 %zmm1, 16*0($output) + vmovdqu8 %xmm2, 16*4($output) + add \$80, $output + + vmovdqa %xmm2, %xmm8 + vextracti32x4 \$0x1, %zmm10, %xmm0 + and \$0xf,$length + je .L_ret_${rndsuffix} + jmp .L_steal_cipher_${rndsuffix} +___ + } + + { + $code .= <<___; + .L_num_blocks_is_4_${rndsuffix}: + vpshufb %zmm8, %zmm0, %zmm1 + vpsllvq const_dq3210(%rip), %zmm0, %zmm4 + vpsrlvq const_dq5678(%rip), %zmm1, %zmm2 + vpclmulqdq \$0x00, $ZPOLY, %zmm2, %zmm3 + vpxorq %zmm2, %zmm4, %zmm4{%k2} + vpxord %zmm4, %zmm3, %zmm9 + vpsllvq const_dq7654(%rip), %zmm0, %zmm5 + vpsrlvq const_dq1234(%rip), %zmm1, %zmm6 + vpclmulqdq \$0x00, $ZPOLY, %zmm6, %zmm7 + vpxorq %zmm6, %zmm5, %zmm5{%k2} + vpxord %zmm5, %zmm7, %zmm10 + vmovdqu8 16*0($input), %zmm1 + add \$64, $input +___ + } + + encrypt_by_four("%zmm1", "%zmm9", "%zmm0", $is_128); + + { + $code .= <<___; + vmovdqu8 %zmm1, 16*0($output) + add \$64, $output + vextracti32x4 \$0x3, %zmm1, %xmm8 + vmovdqa %xmm10, %xmm0 + and \$0xf,$length + je .L_ret_${rndsuffix} + jmp .L_steal_cipher_${rndsuffix} +___ + } + + { + $code .= <<___; + .L_num_blocks_is_3_${rndsuffix}: + vpshufb %zmm8, %zmm0, %zmm1 + vpsllvq const_dq3210(%rip), %zmm0, %zmm4 + vpsrlvq const_dq5678(%rip), %zmm1, %zmm2 + vpclmulqdq \$0x00, $ZPOLY, %zmm2, %zmm3 + vpxorq %zmm2, %zmm4, %zmm4{%k2} + vpxord %zmm4, %zmm3, %zmm9 + mov \$0x0000ffffffffffff, $tmp1 + kmovq $tmp1, %k1 + vmovdqu8 16*0($input), %zmm1{%k1} + add \$48, $input +___ + } + + encrypt_by_four("%zmm1", "%zmm9", "%zmm0", $is_128); + + { + $code .= <<___; + vmovdqu8 %zmm1, 16*0($output){%k1} + add \$48, $output + vextracti32x4 \$2, %zmm1, %xmm8 + vextracti32x4 \$3, %zmm9, %xmm0 + and \$0xf,$length + je .L_ret_${rndsuffix} + jmp .L_steal_cipher_${rndsuffix} +___ + } + + { + $code .= <<___; + .L_num_blocks_is_2_${rndsuffix}: + vpshufb %zmm8, %zmm0, %zmm1 + vpsllvq const_dq3210(%rip), %zmm0, %zmm4 + vpsrlvq const_dq5678(%rip), %zmm1, %zmm2 + vpclmulqdq \$0x00, $ZPOLY, %zmm2, %zmm3 + vpxorq %zmm2, %zmm4, %zmm4{%k2} + vpxord %zmm4, %zmm3, %zmm9 + + vmovdqu8 16*0($input), %ymm1 + add \$32, $input +___ + } + + encrypt_by_four("%ymm1", "%ymm9", "%ymm0", $is_128); + + { + $code .= <<___; + vmovdqu8 %ymm1, 16*0($output) + add \$32, $output + + vextracti32x4 \$1, %ymm1, %xmm8 + vextracti32x4 \$2, %zmm9, %xmm0 + and \$0xf,$length + je .L_ret_${rndsuffix} + jmp .L_steal_cipher_${rndsuffix} +___ + } + + { + $code .= <<___; + .L_num_blocks_is_1_${rndsuffix}: + vpshufb %zmm8, %zmm0, %zmm1 + vpsllvq const_dq3210(%rip), %zmm0, %zmm4 + vpsrlvq const_dq5678(%rip), %zmm1, %zmm2 + vpclmulqdq \$0x00, $ZPOLY, %zmm2, %zmm3 + vpxorq %zmm2, %zmm4, %zmm4{%k2} + vpxord %zmm4, %zmm3, %zmm9 + + vmovdqu8 16*0($input), %xmm1 + add \$16, $input +___ + } + + encrypt_by_four("%ymm1", "%ymm9", "%ymm0", $is_128); + + { + $code .= <<___; + vmovdqu8 %xmm1, 16*0($output) + add \$16, $output + + vmovdqa %xmm1, %xmm8 + vextracti32x4 \$1, %zmm9, %xmm0 + and \$0xf,$length + je .L_ret_${rndsuffix} + jmp .L_steal_cipher_${rndsuffix} + .cfi_endproc +___ + } + } + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;void aesni_xts_[128|256]_decrypt_avx512( + # ; const uint8_t *in, // input data + # ; uint8_t *out, // output data + # ; size_t length, // sector size, in bytes + # ; const AES_KEY *key1, // key used for "ECB" encryption, 16*2 bytes + # ; const AES_KEY *key2, // key used for tweaking, 16*2 bytes + # ; const uint8_t iv[16]) // initial tweak value, 16 bytes + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + sub dec { + my $is_128 = $_[0]; + my $rndsuffix = &random_string(); + + if ($is_128) { + $code.=<<___; + .globl aesni_xts_128_decrypt_avx512 + .hidden aesni_xts_128_decrypt_avx512 + .type aesni_xts_128_decrypt_avx512,\@function,6 + .align 32 + aesni_xts_128_decrypt_avx512: + .cfi_startproc + endbranch +___ + } else { + $code.=<<___; + .globl aesni_xts_256_decrypt_avx512 + .hidden aesni_xts_256_decrypt_avx512 + .type aesni_xts_256_decrypt_avx512,\@function,6 + .align 32 + aesni_xts_256_decrypt_avx512: + .cfi_startproc + endbranch +___ + } + $code .= "push %rbp\n"; + $code .= "mov $TW,%rbp\n"; + $code .= "sub \$$VARIABLE_OFFSET,$TW\n"; + $code .= "and \$0xffffffffffffffc0,$TW\n"; + $code .= "mov %rbx,$GP_STORAGE($TW)\n"; + + if ($win64) { + $code .= "mov %rdi,$GP_STORAGE + 8*1($TW)\n"; + $code .= "mov %rsi,$GP_STORAGE + 8*2($TW)\n"; + $code .= "vmovdqa %xmm6, $XMM_STORAGE + 16*0($TW)\n"; + $code .= "vmovdqa %xmm7, $XMM_STORAGE + 16*1($TW)\n"; + $code .= "vmovdqa %xmm8, $XMM_STORAGE + 16*2($TW)\n"; + $code .= "vmovdqa %xmm9, $XMM_STORAGE + 16*3($TW)\n"; + $code .= "vmovdqa %xmm10, $XMM_STORAGE + 16*4($TW)\n"; + $code .= "vmovdqa %xmm11, $XMM_STORAGE + 16*5($TW)\n"; + $code .= "vmovdqa %xmm12, $XMM_STORAGE + 16*6($TW)\n"; + $code .= "vmovdqa %xmm13, $XMM_STORAGE + 16*7($TW)\n"; + $code .= "vmovdqa %xmm14, $XMM_STORAGE + 16*8($TW)\n"; + $code .= "vmovdqa %xmm15, $XMM_STORAGE + 16*9($TW)\n"; + } + + $code .= "mov \$0x87, $gf_poly_8b\n"; + $code .= "vmovdqu ($tweak),%xmm1\n"; # read initial tweak values + + encrypt_tweak("%xmm1", $is_128); + + if ($win64) { + $code .= "mov $input, 8 + 8*5(%rbp)\n"; # ciphertext pointer + $code .= "mov $output, 8 + 8*6(%rbp)\n"; # plaintext pointer + } + + { + $code.=<<___; + + cmp \$0x80,$length + jb .L_less_than_128_bytes_${rndsuffix} + vpbroadcastq $gf_poly_8b,$ZPOLY + cmp \$0x100,$length + jge .L_start_by16_${rndsuffix} + jmp .L_start_by8_${rndsuffix} + + .L_do_n_blocks_${rndsuffix}: + cmp \$0x0,$length + je .L_ret_${rndsuffix} + cmp \$0x70,$length + jge .L_remaining_num_blocks_is_7_${rndsuffix} + cmp \$0x60,$length + jge .L_remaining_num_blocks_is_6_${rndsuffix} + cmp \$0x50,$length + jge .L_remaining_num_blocks_is_5_${rndsuffix} + cmp \$0x40,$length + jge .L_remaining_num_blocks_is_4_${rndsuffix} + cmp \$0x30,$length + jge .L_remaining_num_blocks_is_3_${rndsuffix} + cmp \$0x20,$length + jge .L_remaining_num_blocks_is_2_${rndsuffix} + cmp \$0x10,$length + jge .L_remaining_num_blocks_is_1_${rndsuffix} + + # _remaining_num_blocks_is_0: + vmovdqu %xmm5, %xmm1 + # xmm5 contains last full block to decrypt with next teawk +___ + } + decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", + "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", + "%xmm13", "%xmm14", "%xmm15", "%xmm0", 1, 1, $is_128); + + { + $code .= <<___; + vmovdqu %xmm1, -0x10($output) + vmovdqa %xmm1, %xmm8 + + # Calc previous tweak + mov \$0x1,$tmp1 + kmovq $tmp1, %k1 + vpsllq \$0x3f,%xmm9,%xmm13 + vpsraq \$0x3f,%xmm13,%xmm14 + vpandq %xmm25,%xmm14,%xmm5 + vpxorq %xmm5,%xmm9,%xmm9{%k1} + vpsrldq \$0x8,%xmm9,%xmm10 + .byte 98, 211, 181, 8, 115, 194, 1 #vpshrdq \$0x1,%xmm10,%xmm9,%xmm0 + vpslldq \$0x8,%xmm13,%xmm13 + vpxorq %xmm13,%xmm0,%xmm0 + jmp .L_steal_cipher_${rndsuffix} + + .L_remaining_num_blocks_is_7_${rndsuffix}: + mov \$0xffffffffffffffff,$tmp1 + shr \$0x10,$tmp1 + kmovq $tmp1,%k1 + vmovdqu8 ($input),%zmm1 + vmovdqu8 0x40($input),%zmm2{%k1} + add \$0x70,$input + and \$0xf,$length + je .L_done_7_remain_${rndsuffix} + vextracti32x4 \$0x2,%zmm10,%xmm12 + vextracti32x4 \$0x3,%zmm10,%xmm13 + vinserti32x4 \$0x2,%xmm13,%zmm10,%zmm10 +___ + } + + decrypt_by_eight_zmm("%zmm1", "%zmm2", "%zmm9", "%zmm10", "%zmm0", 1, $is_128); + + { + $code .= <<___; + vmovdqu8 %zmm1, ($output) + vmovdqu8 %zmm2, 0x40($output){%k1} + add \$0x70, $output + vextracti32x4 \$0x2,%zmm2,%xmm8 + vmovdqa %xmm12,%xmm0 + jmp .L_steal_cipher_${rndsuffix} +___ + } + + $code .= "\n.L_done_7_remain_${rndsuffix}:\n"; + decrypt_by_eight_zmm("%zmm1", "%zmm2", "%zmm9", "%zmm10", "%zmm0", 1, $is_128); + + { + $code .= <<___; + vmovdqu8 %zmm1, ($output) + vmovdqu8 %zmm2, 0x40($output){%k1} + jmp .L_ret_${rndsuffix} + + .L_remaining_num_blocks_is_6_${rndsuffix}: + vmovdqu8 ($input),%zmm1 + vmovdqu8 0x40($input),%ymm2 + add \$0x60,$input + and \$0xf, $length + je .L_done_6_remain_${rndsuffix} + vextracti32x4 \$0x1,%zmm10,%xmm12 + vextracti32x4 \$0x2,%zmm10,%xmm13 + vinserti32x4 \$0x1,%xmm13,%zmm10,%zmm10 +___ + } + + decrypt_by_eight_zmm("%zmm1", "%zmm2", "%zmm9", "%zmm10", "%zmm0", 1, $is_128); + + { + $code .= <<___; + vmovdqu8 %zmm1, ($output) + vmovdqu8 %ymm2, 0x40($output) + add \$0x60,$output + vextracti32x4 \$0x1,%zmm2,%xmm8 + vmovdqa %xmm12,%xmm0 + jmp .L_steal_cipher_${rndsuffix} +___ + } + + $code .= "\n.L_done_6_remain_${rndsuffix}:\n"; + decrypt_by_eight_zmm("%zmm1", "%zmm2", "%zmm9", "%zmm10", "%zmm0", 1, $is_128); + + { + $code .= <<___; + vmovdqu8 %zmm1, ($output) + vmovdqu8 %ymm2,0x40($output) + jmp .L_ret_${rndsuffix} + + .L_remaining_num_blocks_is_5_${rndsuffix}: + vmovdqu8 ($input),%zmm1 + vmovdqu 0x40($input),%xmm2 + add \$0x50,$input + and \$0xf,$length + je .L_done_5_remain_${rndsuffix} + vmovdqa %xmm10,%xmm12 + vextracti32x4 \$0x1,%zmm10,%xmm10 +___ + } + + decrypt_by_eight_zmm("%zmm1", "%zmm2", "%zmm9", "%zmm10", "%zmm0", 1, $is_128); + + { + $code .= <<___; + vmovdqu8 %zmm1, ($output) + vmovdqu %xmm2, 0x40($output) + add \$0x50, $output + vmovdqa %xmm2,%xmm8 + vmovdqa %xmm12,%xmm0 + jmp .L_steal_cipher_${rndsuffix} +___ + } + + $code .= "\n.L_done_5_remain_${rndsuffix}:\n"; + decrypt_by_eight_zmm("%zmm1", "%zmm2", "%zmm9", "%zmm10", "%zmm0", 1, $is_128); + + { + $code .= <<___; + vmovdqu8 %zmm1, ($output) + vmovdqu8 %xmm2, 0x40($output) + jmp .L_ret_${rndsuffix} + + .L_remaining_num_blocks_is_4_${rndsuffix}: + vmovdqu8 ($input),%zmm1 + add \$0x40,$input + and \$0xf, $length + je .L_done_4_remain_${rndsuffix} + vextracti32x4 \$0x3,%zmm9,%xmm12 + vinserti32x4 \$0x3,%xmm10,%zmm9,%zmm9 +___ + } + + decrypt_by_eight_zmm("%zmm1", "%zmm2", "%zmm9", "%zmm10", "%zmm0", 1, $is_128); + + { + $code .= <<___; + vmovdqu8 %zmm1,($output) + add \$0x40,$output + vextracti32x4 \$0x3,%zmm1,%xmm8 + vmovdqa %xmm12,%xmm0 + jmp .L_steal_cipher_${rndsuffix} +___ + } + + $code .= "\n.L_done_4_remain_${rndsuffix}:\n"; + decrypt_by_eight_zmm("%zmm1", "%zmm2", "%zmm9", "%zmm10", "%zmm0", 1, $is_128); + + { + $code .= <<___; + vmovdqu8 %zmm1, ($output) + jmp .L_ret_${rndsuffix} + + .L_remaining_num_blocks_is_3_${rndsuffix}: + vmovdqu ($input),%xmm1 + vmovdqu 0x10($input),%xmm2 + vmovdqu 0x20($input),%xmm3 + add \$0x30,$input + and \$0xf,$length + je .L_done_3_remain_${rndsuffix} + vextracti32x4 \$0x2,%zmm9,%xmm13 + vextracti32x4 \$0x1,%zmm9,%xmm10 + vextracti32x4 \$0x3,%zmm9,%xmm11 +___ + } + + decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", + "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", + "%xmm13", "%xmm14", "%xmm15", "%xmm0", 3, 1, $is_128); + + { + $code .= <<___; + vmovdqu %xmm1,($output) + vmovdqu %xmm2,0x10($output) + vmovdqu %xmm3,0x20($output) + add \$0x30,$output + vmovdqa %xmm3,%xmm8 + vmovdqa %xmm13,%xmm0 + jmp .L_steal_cipher_${rndsuffix} +___ + } + $code .= "\n.L_done_3_remain_${rndsuffix}:\n"; + $code .= "vextracti32x4 \$0x1,%zmm9,%xmm10\n"; + $code .= "vextracti32x4 \$0x2,%zmm9,%xmm11\n"; + + decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", + "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", + "%xmm13", "%xmm14", "%xmm15", "%xmm0", 3, 1, $is_128); + + { + $code .= <<___; + vmovdqu %xmm1,($output) + vmovdqu %xmm2,0x10($output) + vmovdqu %xmm3,0x20($output) + jmp .L_ret_${rndsuffix} + + .L_remaining_num_blocks_is_2_${rndsuffix}: + vmovdqu ($input),%xmm1 + vmovdqu 0x10($input),%xmm2 + add \$0x20,$input + and \$0xf,$length + je .L_done_2_remain_${rndsuffix} + vextracti32x4 \$0x2,%zmm9,%xmm10 + vextracti32x4 \$0x1,%zmm9,%xmm12 +___ + } + + decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", + "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", + "%xmm13", "%xmm14", "%xmm15", "%xmm0", 2, 1, $is_128); + + { + $code .= <<___; + vmovdqu %xmm1,($output) + vmovdqu %xmm2,0x10($output) + add \$0x20,$output + vmovdqa %xmm2,%xmm8 + vmovdqa %xmm12,%xmm0 + jmp .L_steal_cipher_${rndsuffix} +___ + } + $code .= "\n.L_done_2_remain_${rndsuffix}:\n"; + $code .= "vextracti32x4 \$0x1,%zmm9,%xmm10\n"; + + decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", + "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", + "%xmm13", "%xmm14", "%xmm15", "%xmm0", 2, 1, $is_128); + + { + $code .= <<___; + vmovdqu %xmm1,($output) + vmovdqu %xmm2,0x10($output) + jmp .L_ret_${rndsuffix} + + .L_remaining_num_blocks_is_1_${rndsuffix}: + vmovdqu ($input),%xmm1 + add \$0x10,$input + and \$0xf,$length + je .L_done_1_remain_${rndsuffix} + vextracti32x4 \$0x1,%zmm9,%xmm11 +___ + } + + decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", + "%xmm7", "%xmm8", "%xmm11", "%xmm10", "%xmm9", "%xmm12", + "%xmm13", "%xmm14", "%xmm15", "%xmm0", 1, 1, $is_128); + { + $code .= <<___; + vmovdqu %xmm1,($output) + add \$0x10,$output + vmovdqa %xmm1,%xmm8 + vmovdqa %xmm9,%xmm0 + jmp .L_steal_cipher_${rndsuffix} +___ + } + + $code .= "\n.L_done_1_remain_${rndsuffix}:\n"; + + decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", + "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", + "%xmm13", "%xmm14", "%xmm15", "%xmm0", 1, 1, $is_128); + + { + $code .= <<___; + vmovdqu %xmm1, ($output) + jmp .L_ret_${rndsuffix} + + .L_start_by16_${rndsuffix}: + vbroadcasti32x4 ($TW),%zmm0 + vbroadcasti32x4 shufb_15_7(%rip),%zmm8 + mov \$0xaa,$tmp1 + kmovq $tmp1,%k2 + + # Mult tweak by 2^{3, 2, 1, 0} + vpshufb %zmm8,%zmm0,%zmm1 + vpsllvq const_dq3210(%rip),%zmm0,%zmm4 + vpsrlvq const_dq5678(%rip),%zmm1,%zmm2 + vpclmulqdq \$0x0,$ZPOLY,%zmm2,%zmm3 + vpxorq %zmm2,%zmm4,%zmm4{%k2} + vpxord %zmm4,%zmm3,%zmm9 + + # Mult tweak by 2^{7, 6, 5, 4} + vpsllvq const_dq7654(%rip),%zmm0,%zmm5 + vpsrlvq const_dq1234(%rip),%zmm1,%zmm6 + vpclmulqdq \$0x0,%zmm25,%zmm6,%zmm7 + vpxorq %zmm6,%zmm5,%zmm5{%k2} + vpxord %zmm5,%zmm7,%zmm10 + + # Make next 8 tweek values by all x 2^8 + vpsrldq \$0xf,%zmm9,%zmm13 + vpclmulqdq \$0x0,%zmm25,%zmm13,%zmm14 + vpslldq \$0x1,%zmm9,%zmm11 + vpxord %zmm14,%zmm11,%zmm11 + + vpsrldq \$0xf,%zmm10,%zmm15 + vpclmulqdq \$0x0,%zmm25,%zmm15,%zmm16 + vpslldq \$0x1,%zmm10,%zmm12 + vpxord %zmm16,%zmm12,%zmm12 + + .L_main_loop_run_16_${rndsuffix}: + vmovdqu8 ($input),%zmm1 + vmovdqu8 0x40($input),%zmm2 + vmovdqu8 0x80($input),%zmm3 + vmovdqu8 0xc0($input),%zmm4 + vmovdqu8 0xf0($input),%xmm5 + add \$0x100,$input +___ + } + + decrypt_by_16_zmm("%zmm1", "%zmm2", "%zmm3", "%zmm4", "%zmm9", + "%zmm10", "%zmm11", "%zmm12", "%zmm0", 0, $is_128); + + { + $code .= <<___; + vmovdqu8 %zmm1,($output) + vmovdqu8 %zmm2,0x40($output) + vmovdqu8 %zmm3,0x80($output) + vmovdqu8 %zmm4,0xc0($output) + add \$0x100,$output + sub \$0x100,$length + cmp \$0x100,$length + jge .L_main_loop_run_16_${rndsuffix} + + cmp \$0x80,$length + jge .L_main_loop_run_8_${rndsuffix} + jmp .L_do_n_blocks_${rndsuffix} + + .L_start_by8_${rndsuffix}: + # Make first 7 tweek values + vbroadcasti32x4 ($TW),%zmm0 + vbroadcasti32x4 shufb_15_7(%rip),%zmm8 + mov \$0xaa,$tmp1 + kmovq $tmp1,%k2 + + # Mult tweak by 2^{3, 2, 1, 0} + vpshufb %zmm8,%zmm0,%zmm1 + vpsllvq const_dq3210(%rip),%zmm0,%zmm4 + vpsrlvq const_dq5678(%rip),%zmm1,%zmm2 + vpclmulqdq \$0x0,%zmm25,%zmm2,%zmm3 + vpxorq %zmm2,%zmm4,%zmm4{%k2} + vpxord %zmm4,%zmm3,%zmm9 + + # Mult tweak by 2^{7, 6, 5, 4} + vpsllvq const_dq7654(%rip),%zmm0,%zmm5 + vpsrlvq const_dq1234(%rip),%zmm1,%zmm6 + vpclmulqdq \$0x0,%zmm25,%zmm6,%zmm7 + vpxorq %zmm6,%zmm5,%zmm5{%k2} + vpxord %zmm5,%zmm7,%zmm10 + + .L_main_loop_run_8_${rndsuffix}: + vmovdqu8 ($input),%zmm1 + vmovdqu8 0x40($input),%zmm2 + vmovdqu8 0x70($input),%xmm5 + add \$0x80,$input +___ + } + + + decrypt_by_eight_zmm("%zmm1", "%zmm2", "%zmm9", "%zmm10", "%zmm0", 0, $is_128); + + { + $code .= <<___; + vmovdqu8 %zmm1,($output) + vmovdqu8 %zmm2,0x40($output) + add \$0x80,$output + sub \$0x80,$length + cmp \$0x80,$length + jge .L_main_loop_run_8_${rndsuffix} + jmp .L_do_n_blocks_${rndsuffix} + + .L_steal_cipher_${rndsuffix}: + # start cipher stealing simplified: xmm8-last cipher block, xmm0-next tweak + vmovdqa %xmm8,%xmm2 + + # shift xmm8 to the left by 16-N_val bytes + lea vpshufb_shf_table(%rip),$TEMPLOW + vmovdqu ($TEMPLOW,$length,1),%xmm10 + vpshufb %xmm10,%xmm8,%xmm8 + + + vmovdqu -0x10($input,$length,1),%xmm3 + vmovdqu %xmm8,-0x10($output,$length,1) + + # shift xmm3 to the right by 16-N_val bytes + lea vpshufb_shf_table(%rip), $TEMPLOW + add \$16, $TEMPLOW + sub $length,$TEMPLOW + vmovdqu ($TEMPLOW),%xmm10 + vpxor mask1(%rip),%xmm10,%xmm10 + vpshufb %xmm10,%xmm3,%xmm3 + + vpblendvb %xmm10,%xmm2,%xmm3,%xmm3 + + # xor Tweak value + vpxor %xmm0,%xmm3,%xmm8 + + # decrypt last block with cipher stealing + vpxor ($key1),%xmm8,%xmm8 + vaesdec 0x10($key1),%xmm8,%xmm8 + vaesdec 0x20($key1),%xmm8,%xmm8 + vaesdec 0x30($key1),%xmm8,%xmm8 + vaesdec 0x40($key1),%xmm8,%xmm8 + vaesdec 0x50($key1),%xmm8,%xmm8 + vaesdec 0x60($key1),%xmm8,%xmm8 + vaesdec 0x70($key1),%xmm8,%xmm8 + vaesdec 0x80($key1),%xmm8,%xmm8 + vaesdec 0x90($key1),%xmm8,%xmm8 +___ + if ($is_128) { + $code .= "vaesdeclast 0xa0($key1),%xmm8,%xmm8\n"; + } else { + $code .= <<___; + vaesdec 0xa0($key1),%xmm8,%xmm8 + vaesdec 0xb0($key1),%xmm8,%xmm8 + vaesdec 0xc0($key1),%xmm8,%xmm8 + vaesdec 0xd0($key1),%xmm8,%xmm8 + vaesdeclast 0xe0($key1),%xmm8,%xmm8 +___ + } + $code .= <<___ + # xor Tweak value + vpxor %xmm0,%xmm8,%xmm8 + + .L_done_${rndsuffix}: + # store last ciphertext value + vmovdqu %xmm8,-0x10($output) +___ + } + + { + $code .= <<___; + .L_ret_${rndsuffix}: + mov $GP_STORAGE($TW),%rbx + xor $tmp1,$tmp1 + mov $tmp1,$GP_STORAGE($TW) + # Zero-out the whole of `%zmm0`. + vpxorq %zmm0,%zmm0,%zmm0 +___ + } + + if ($win64) { + $code .= <<___; + mov $GP_STORAGE + 8*1($TW),%rdi + mov $tmp1,$GP_STORAGE + 8*1($TW) + mov $GP_STORAGE + 8*2($TW),%rsi + mov $tmp1,$GP_STORAGE + 8*2($TW) + + vmovdqa $XMM_STORAGE + 16 * 0($TW), %xmm6 + vmovdqa $XMM_STORAGE + 16 * 1($TW), %xmm7 + vmovdqa $XMM_STORAGE + 16 * 2($TW), %xmm8 + vmovdqa $XMM_STORAGE + 16 * 3($TW), %xmm9 + + # Zero the 64 bytes we just restored to the xmm registers. + vmovdqa64 %zmm0,$XMM_STORAGE($TW) + + vmovdqa $XMM_STORAGE + 16 * 4($TW), %xmm10 + vmovdqa $XMM_STORAGE + 16 * 5($TW), %xmm11 + vmovdqa $XMM_STORAGE + 16 * 6($TW), %xmm12 + vmovdqa $XMM_STORAGE + 16 * 7($TW), %xmm13 + + # And again. + vmovdqa64 %zmm0,$XMM_STORAGE + 16 * 4($TW) + + vmovdqa $XMM_STORAGE + 16 * 8($TW), %xmm14 + vmovdqa $XMM_STORAGE + 16 * 9($TW), %xmm15 + + # Last round is only 32 bytes (256-bits), so we use `%ymm` as the + # source operand. + vmovdqa %ymm0,$XMM_STORAGE + 16 * 8($TW) +___ + } + + { + $code .= <<___; + mov %rbp,$TW + pop %rbp + vzeroupper + ret + + .L_less_than_128_bytes_${rndsuffix}: + cmp \$0x10,$length + jb .L_ret_${rndsuffix} + + mov $length,$tmp1 + and \$0x70,$tmp1 + cmp \$0x60,$tmp1 + je .L_num_blocks_is_6_${rndsuffix} + cmp \$0x50,$tmp1 + je .L_num_blocks_is_5_${rndsuffix} + cmp \$0x40,$tmp1 + je .L_num_blocks_is_4_${rndsuffix} + cmp \$0x30,$tmp1 + je .L_num_blocks_is_3_${rndsuffix} + cmp \$0x20,$tmp1 + je .L_num_blocks_is_2_${rndsuffix} + cmp \$0x10,$tmp1 + je .L_num_blocks_is_1_${rndsuffix} +___ + } + + $code .= "\n.L_num_blocks_is_7_${rndsuffix}:\n"; + initialize("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", + "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", + "%xmm13", "%xmm14", "%xmm15", 7); + + { + $code .= <<___; + add \$0x70,$input + and \$0xf,$length + je .L_done_7_${rndsuffix} + + .L_steal_cipher_7_${rndsuffix}: + xor $gf_poly_8b_temp, $gf_poly_8b_temp + shl \$1, $TEMPLOW + adc $TEMPHIGH, $TEMPHIGH + cmovc $gf_poly_8b, $gf_poly_8b_temp + xor $gf_poly_8b_temp, $TEMPLOW + mov $TEMPLOW,0x10($TW) + mov $TEMPHIGH,0x18($TW) + vmovdqa64 %xmm15,%xmm16 + vmovdqa 0x10($TW),%xmm15 +___ + } + + decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", + "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", + "%xmm13", "%xmm14", "%xmm15", "%xmm0", 7, 1, $is_128); + + { + $code .= <<___; + vmovdqu %xmm1,($output) + vmovdqu %xmm2,0x10($output) + vmovdqu %xmm3,0x20($output) + vmovdqu %xmm4,0x30($output) + vmovdqu %xmm5,0x40($output) + vmovdqu %xmm6,0x50($output) + add \$0x70,$output + vmovdqa64 %xmm16,%xmm0 + vmovdqa %xmm7,%xmm8 + jmp .L_steal_cipher_${rndsuffix} +___ + } + + $code .= "\n.L_done_7_${rndsuffix}:\n"; + decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", + "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", + "%xmm13", "%xmm14", "%xmm15", "%xmm0", 7, 1, $is_128); + + { + $code .= <<___; + vmovdqu %xmm1,($output) + vmovdqu %xmm2,0x10($output) + vmovdqu %xmm3,0x20($output) + vmovdqu %xmm4,0x30($output) + vmovdqu %xmm5,0x40($output) + vmovdqu %xmm6,0x50($output) + add \$0x70,$output + vmovdqa %xmm7,%xmm8 + jmp .L_done_${rndsuffix} +___ + } + + $code .= "\n.L_num_blocks_is_6_${rndsuffix}:\n"; + initialize("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", + "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", + "%xmm13", "%xmm14", "%xmm15", 6); + + { + $code .= <<___; + add \$0x60,$input + and \$0xf,$length + je .L_done_6_${rndsuffix} + + .L_steal_cipher_6_${rndsuffix}: + xor $gf_poly_8b_temp, $gf_poly_8b_temp + shl \$1, $TEMPLOW + adc $TEMPHIGH, $TEMPHIGH + cmovc $gf_poly_8b, $gf_poly_8b_temp + xor $gf_poly_8b_temp, $TEMPLOW + mov $TEMPLOW,0x10($TW) + mov $TEMPHIGH,0x18($TW) + vmovdqa64 %xmm14,%xmm15 + vmovdqa 0x10($TW),%xmm14 +___ + } + + decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", + "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", + "%xmm13", "%xmm14", "%xmm15", "%xmm0", 6, 1, $is_128); + + { + $code .= <<___; + vmovdqu %xmm1,($output) + vmovdqu %xmm2,0x10($output) + vmovdqu %xmm3,0x20($output) + vmovdqu %xmm4,0x30($output) + vmovdqu %xmm5,0x40($output) + add \$0x60,$output + vmovdqa %xmm15,%xmm0 + vmovdqa %xmm6,%xmm8 + jmp .L_steal_cipher_${rndsuffix} +___ + } + $code .= "\n.L_done_6_${rndsuffix}:\n"; + decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", + "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", + "%xmm13", "%xmm14", "%xmm15", "%xmm0", 6, 1, $is_128); + + { + $code .= <<___; + vmovdqu %xmm1,($output) + vmovdqu %xmm2,0x10($output) + vmovdqu %xmm3,0x20($output) + vmovdqu %xmm4,0x30($output) + vmovdqu %xmm5,0x40($output) + add \$0x60,$output + vmovdqa %xmm6,%xmm8 + jmp .L_done_${rndsuffix} +___ + } + + $code .= "\n.L_num_blocks_is_5_${rndsuffix}:\n"; + initialize("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", + "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", + "%xmm13", "%xmm14", "%xmm15", 5); + + { + $code .= <<___; + add \$0x50,$input + and \$0xf,$length + je .L_done_5_${rndsuffix} + + .L_steal_cipher_5_${rndsuffix}: + xor $gf_poly_8b_temp, $gf_poly_8b_temp + shl \$1, $TEMPLOW + adc $TEMPHIGH, $TEMPHIGH + cmovc $gf_poly_8b, $gf_poly_8b_temp + xor $gf_poly_8b_temp, $TEMPLOW + mov $TEMPLOW,0x10($TW) + mov $TEMPHIGH,0x18($TW) + vmovdqa64 %xmm13,%xmm14 + vmovdqa 0x10($TW),%xmm13 +___ + } + + decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", + "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", + "%xmm13", "%xmm14", "%xmm15", "%xmm0", 5, 1, $is_128); + + { + $code .= <<___; + vmovdqu %xmm1,($output) + vmovdqu %xmm2,0x10($output) + vmovdqu %xmm3,0x20($output) + vmovdqu %xmm4,0x30($output) + add \$0x50,$output + vmovdqa %xmm14,%xmm0 + vmovdqa %xmm5,%xmm8 + jmp .L_steal_cipher_${rndsuffix} +___ + } + + $code .= "\n.L_done_5_${rndsuffix}:\n"; + decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", + "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", + "%xmm13", "%xmm14", "%xmm15", "%xmm0", 5, 1, $is_128); + + { + $code .= <<___; + vmovdqu %xmm1,($output) + vmovdqu %xmm2,0x10($output) + vmovdqu %xmm3,0x20($output) + vmovdqu %xmm4,0x30($output) + add \$0x50,$output + vmovdqa %xmm5,%xmm8 + jmp .L_done_${rndsuffix} +___ + } + + $code .= "\n.L_num_blocks_is_4_${rndsuffix}:\n"; + + initialize("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", + "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", + "%xmm13", "%xmm14", "%xmm15", 4); + + { + $code .= <<___; + add \$0x40,$input + and \$0xf,$length + je .L_done_4_${rndsuffix} + + .L_steal_cipher_4_${rndsuffix}: + xor $gf_poly_8b_temp, $gf_poly_8b_temp + shl \$1, $TEMPLOW + adc $TEMPHIGH, $TEMPHIGH + cmovc $gf_poly_8b, $gf_poly_8b_temp + xor $gf_poly_8b_temp, $TEMPLOW + mov $TEMPLOW,0x10($TW) + mov $TEMPHIGH,0x18($TW) + vmovdqa64 %xmm12,%xmm13 + vmovdqa 0x10($TW),%xmm12 +___ + } + + decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", + "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", + "%xmm13", "%xmm14", "%xmm15", "%xmm0", 4, 1, $is_128); + + { + $code .= <<___; + vmovdqu %xmm1,($output) + vmovdqu %xmm2,0x10($output) + vmovdqu %xmm3,0x20($output) + add \$0x40,$output + vmovdqa %xmm13,%xmm0 + vmovdqa %xmm4,%xmm8 + jmp .L_steal_cipher_${rndsuffix} +___ + } + + $code .= "\n.L_done_4_${rndsuffix}:\n"; + decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", + "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", + "%xmm13", "%xmm14", "%xmm15", "%xmm0", 4, 1, $is_128); + + { + $code .= <<___; + vmovdqu %xmm1,($output) + vmovdqu %xmm2,0x10($output) + vmovdqu %xmm3,0x20($output) + add \$0x40,$output + vmovdqa %xmm4,%xmm8 + jmp .L_done_${rndsuffix} +___ + } + + $code .= "\n.L_num_blocks_is_3_${rndsuffix}:\n"; + + initialize("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", + "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", + "%xmm13", "%xmm14", "%xmm15", 3); + + { + $code .= <<___; + add \$0x30,$input + and \$0xf,$length + je .L_done_3_${rndsuffix} + + .L_steal_cipher_3_${rndsuffix}: + xor $gf_poly_8b_temp, $gf_poly_8b_temp + shl \$1, $TEMPLOW + adc $TEMPHIGH, $TEMPHIGH + cmovc $gf_poly_8b, $gf_poly_8b_temp + xor $gf_poly_8b_temp, $TEMPLOW + mov $TEMPLOW,0x10($TW) + mov $TEMPHIGH,0x18($TW) + vmovdqa64 %xmm11,%xmm12 + vmovdqa 0x10($TW),%xmm11 +___ + } + + decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", + "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", + "%xmm13", "%xmm14", "%xmm15", "%xmm0", 3, 1, $is_128); + + { + $code .= <<___; + vmovdqu %xmm1,($output) + vmovdqu %xmm2,0x10($output) + add \$0x30,$output + vmovdqa %xmm12,%xmm0 + vmovdqa %xmm3,%xmm8 + jmp .L_steal_cipher_${rndsuffix} +___ + } + $code .= "\n.L_done_3_${rndsuffix}:\n"; + decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", + "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", + "%xmm13", "%xmm14", "%xmm15", "%xmm0", 3, 1, $is_128); + + { + $code .= <<___; + vmovdqu %xmm1,($output) + vmovdqu %xmm2,0x10($output) + add \$0x30,$output + vmovdqa %xmm3,%xmm8 + jmp .L_done_${rndsuffix} +___ + } + + $code .= "\n.L_num_blocks_is_2_${rndsuffix}:\n"; + + initialize("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", + "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", + "%xmm13", "%xmm14", "%xmm15", 2); + + { + $code .= <<___; + add \$0x20,$input + and \$0xf,$length + je .L_done_2_${rndsuffix} + + .L_steal_cipher_2_${rndsuffix}: + xor $gf_poly_8b_temp, $gf_poly_8b_temp + shl \$1, $TEMPLOW + adc $TEMPHIGH, $TEMPHIGH + cmovc $gf_poly_8b, $gf_poly_8b_temp + xor $gf_poly_8b_temp, $TEMPLOW + mov $TEMPLOW,0x10($TW) + mov $TEMPHIGH,0x18($TW) + vmovdqa64 %xmm10,%xmm11 + vmovdqa 0x10($TW),%xmm10 +___ + } + + decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", + "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", + "%xmm13", "%xmm14", "%xmm15", "%xmm0", 2, 1, $is_128); + + { + $code .= <<___; + vmovdqu %xmm1,($output) + add \$0x20,$output + vmovdqa %xmm11,%xmm0 + vmovdqa %xmm2,%xmm8 + jmp .L_steal_cipher_${rndsuffix} +___ + } + + $code .= "\n.L_done_2_${rndsuffix}:\n"; + decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", + "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", + "%xmm13", "%xmm14", "%xmm15", "%xmm0", 2, 1, $is_128); + + { + $code .= <<___; + vmovdqu %xmm1,($output) + add \$0x20,$output + vmovdqa %xmm2,%xmm8 + jmp .L_done_${rndsuffix} +___ + } + + $code .= "\n.L_num_blocks_is_1_${rndsuffix}:\n"; + + initialize("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", + "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", + "%xmm13", "%xmm14", "%xmm15", 1); + + { + $code .= <<___; + add \$0x10,$input + and \$0xf,$length + je .L_done_1_${rndsuffix} + + .L_steal_cipher_1_${rndsuffix}: + xor $gf_poly_8b_temp, $gf_poly_8b_temp + shl \$1, $TEMPLOW + adc $TEMPHIGH, $TEMPHIGH + cmovc $gf_poly_8b, $gf_poly_8b_temp + xor $gf_poly_8b_temp, $TEMPLOW + mov $TEMPLOW,0x10($TW) + mov $TEMPHIGH,0x18($TW) + vmovdqa64 %xmm9,%xmm10 + vmovdqa 0x10($TW),%xmm9 +___ + } + decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", + "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", + "%xmm13", "%xmm14", "%xmm15", "%xmm0", 1, 1, $is_128); + + { + $code .= <<___; + add \$0x10,$output + vmovdqa %xmm10,%xmm0 + vmovdqa %xmm1,%xmm8 + jmp .L_steal_cipher_${rndsuffix} +___ + } + $code .= "\n.L_done_1_${rndsuffix}:\n"; + decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", + "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", + "%xmm13", "%xmm14", "%xmm15", "%xmm0", 1, 1, $is_128); + + { + $code .= <<___; + add \$0x10,$output + vmovdqa %xmm1,%xmm8 + jmp .L_done_${rndsuffix} + .cfi_endproc +___ + } + + } + + # The only difference between AES-XTS-128 and -256 is the number of rounds, + # so we generate from the same perlasm base, extending to 14 rounds when + # `$is_128' is 0. + + enc(1); + dec(1); + + enc(0); + dec(0); + + $code .= <<___; + .section .rodata + .align 16 + + vpshufb_shf_table: + .quad 0x8786858483828100, 0x8f8e8d8c8b8a8988 + .quad 0x0706050403020100, 0x000e0d0c0b0a0908 + + mask1: + .quad 0x8080808080808080, 0x8080808080808080 + + const_dq3210: + .quad 0, 0, 1, 1, 2, 2, 3, 3 + const_dq5678: + .quad 8, 8, 7, 7, 6, 6, 5, 5 + const_dq7654: + .quad 4, 4, 5, 5, 6, 6, 7, 7 + const_dq1234: + .quad 4, 4, 3, 3, 2, 2, 1, 1 + + shufb_15_7: + .byte 15, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 7, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff + +.text +___ + +} else { + $code .= <<___; + .text + .globl aesni_xts_128_encrypt_avx512 + .globl aesni_xts_128_decrypt_avx512 + + aesni_xts_128_encrypt_avx512: + aesni_xts_128_decrypt_avx512: + .byte 0x0f,0x0b # ud2 + ret + + .globl aesni_xts_256_encrypt_avx512 + .globl aesni_xts_256_decrypt_avx512 + + aesni_xts_256_encrypt_avx512: + aesni_xts_256_decrypt_avx512: + .byte 0x0f,0x0b # ud2 + ret + + .globl aesni_xts_avx512_eligible + .type aesni_xts_avx512_eligible,\@abi-omnipotent + aesni_xts_avx512_eligible: + xor %eax,%eax + ret + .size aesni_xts_avx512_eligible, .-aesni_xts_avx512_eligible + +___ +} + +print $code; + +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/aes/asm/aesp8-ppc.pl b/crypto/openssl/crypto/aes/asm/aesp8-ppc.pl index f7f78d04b0e1..38b9405a283b 100755 --- a/crypto/openssl/crypto/aes/asm/aesp8-ppc.pl +++ b/crypto/openssl/crypto/aes/asm/aesp8-ppc.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2014-2024 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2014-2020 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy diff --git a/crypto/openssl/crypto/aes/asm/aesv8-armx.pl b/crypto/openssl/crypto/aes/asm/aesv8-armx.pl index 0aa14da30ac0..0fe126a5ef2f 100755 --- a/crypto/openssl/crypto/aes/asm/aesv8-armx.pl +++ b/crypto/openssl/crypto/aes/asm/aesv8-armx.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2014-2023 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2014-2025 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -110,6 +110,11 @@ my ($zero,$rcon,$mask,$in0,$in1,$tmp,$key)= $flavour=~/64/? map("q$_",(0..6)) : map("q$_",(0..3,8..10)); +# +# This file generates .s file for 64-bit and 32-bit CPUs. +# We don't implement .rodata on 32-bit CPUs yet. +# +$code.=".rodata\n" if ($flavour =~ /64/); $code.=<<___; .align 5 .type .Lrcon,%object @@ -118,7 +123,10 @@ $code.=<<___; .long 0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d // rotate-n-splat .long 0x1b,0x1b,0x1b,0x1b .size .Lrcon, . - .Lrcon +___ +$code.=".previous\n" if ($flavour =~ /64/); +$code.=<<___; .globl ${prefix}_set_encrypt_key .type ${prefix}_set_encrypt_key,%function .align 5 @@ -145,7 +153,15 @@ $code.=<<___; tst $bits,#0x3f b.ne .Lenc_key_abort +___ +$code.=<<___ if ($flavour =~ /64/); + adrp $ptr,.Lrcon + add $ptr,$ptr,:lo12:.Lrcon +___ +$code.=<<___ if ($flavour !~ /64/); adr $ptr,.Lrcon +___ +$code.=<<___; cmp $bits,#192 veor $zero,$zero,$zero @@ -423,7 +439,7 @@ ___ # If lsize < 3*16 bytes, treat them as the tail, interleave the # two blocks AES instructions. # There is one special case, if the original input data size dsize -# = 16 bytes, we will treat it seperately to improve the +# = 16 bytes, we will treat it separately to improve the # performance: one independent code block without LR, FP load and # store, just looks like what the original ECB implementation does. @@ -1758,6 +1774,755 @@ $code.=<<___; .size ${prefix}_cbc_encrypt,.-${prefix}_cbc_encrypt ___ }}} + +{{{ +my ($inp,$out,$len,$key,$ivp)=map("x$_",(0..4)); +my ($rounds,$roundsx,$cnt,$key_)=("w5","x5","w6","x7"); +my ($ctr,$tctr0,$tctr1,$tctr2)=map("w$_",(8..10,12)); +my ($tctr3,$tctr4,$tctr5,$tctr6)=map("w$_",(11,13..15)); +my ($tctr7,$tctr8,$tctr9,$tctr10,$tctr11)=map("w$_",(19..23)); + +# q0-q7 => v0-v7; q8-q23 => v16-v31; q24-q31 => v8-v15 +my ($ivec,$rndlast,$rndping,$rndpang)=map("q$_",(0..3)); +my ($in0,$in1,$in2,$in3,$in4,$in5)=map("q$_",(4..9)); +my ($in6,$in7,$in8,$in9,$in10,$in11)=map("q$_",(10..15)); +my ($dat0,$dat1,$dat2,$dat3,$dat4,$dat5)=map("q$_",(16..21)); +my ($dat6,$dat7,$dat8,$dat9,$dat10,$dat11)=map("q$_",(22..27)); +my ($tmp0,$tmp1,$tmp2)=map("q$_",(25..27)); + +#q_X => qX, for ldp & stp +my ($in0q,$in1q,$in2q,$in3q)=map("q_$_",(4..7)); +my ($in4q,$in5q,$in6q,$in7q,$in8q,$in9q,$in10q,$in11q)=map("q_$_",(16..23)); + +my ($dat8d,$dat9d,$dat10d,$dat11d)=map("d$_",(8..11)); + +$code.=<<___ if ($flavour =~ /64/); +.globl ${prefix}_ctr32_encrypt_blocks_unroll12_eor3 +.type ${prefix}_ctr32_encrypt_blocks_unroll12_eor3,%function +.align 5 +${prefix}_ctr32_encrypt_blocks_unroll12_eor3: + AARCH64_VALID_CALL_TARGET + // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. + stp x29,x30,[sp,#-80]! + stp d8,d9,[sp, #16] + stp d10,d11,[sp, #32] + stp d12,d13,[sp, #48] + stp d14,d15,[sp, #64] + add x29,sp,#0 + + ldr $rounds,[$key,#240] + + ldr $ctr, [$ivp, #12] +#ifdef __AARCH64EB__ + vld1.8 {$dat0},[$ivp] +#else + vld1.32 {$dat0},[$ivp] +#endif + vld1.32 {$rndping-$rndpang},[$key] // load key schedule... + sub $rounds,$rounds,#4 + cmp $len,#2 + add $key_,$key,$roundsx,lsl#4 // pointer to last round key + sub $rounds,$rounds,#2 + add $key_, $key_, #64 + vld1.32 {$rndlast},[$key_] + add $key_,$key,#32 + mov $cnt,$rounds +#ifndef __AARCH64EB__ + rev $ctr, $ctr +#endif + + vorr $dat1,$dat0,$dat0 + add $tctr1, $ctr, #1 + vorr $dat2,$dat0,$dat0 + add $ctr, $ctr, #2 + vorr $ivec,$dat0,$dat0 + rev $tctr1, $tctr1 + vmov.32 ${dat1}[3],$tctr1 + b.ls .Lctr32_tail_unroll + cmp $len,#6 + rev $tctr2, $ctr + sub $len,$len,#3 // bias + vmov.32 ${dat2}[3],$tctr2 + b.lo .Loop3x_ctr32_unroll + cmp $len,#9 + vorr $dat3,$dat0,$dat0 + add $tctr3, $ctr, #1 + vorr $dat4,$dat0,$dat0 + add $tctr4, $ctr, #2 + rev $tctr3, $tctr3 + vorr $dat5,$dat0,$dat0 + add $ctr, $ctr, #3 + rev $tctr4, $tctr4 + vmov.32 ${dat3}[3],$tctr3 + rev $tctr5, $ctr + vmov.32 ${dat4}[3],$tctr4 + vmov.32 ${dat5}[3],$tctr5 + sub $len,$len,#3 + b.lo .Loop6x_ctr32_unroll + + // push regs to stack when 12 data chunks are interleaved + stp x19,x20,[sp,#-16]! + stp x21,x22,[sp,#-16]! + stp x23,x24,[sp,#-16]! + stp $dat8d,$dat9d,[sp,#-32]! + stp $dat10d,$dat11d,[sp,#-32]! + + add $tctr6,$ctr,#1 + add $tctr7,$ctr,#2 + add $tctr8,$ctr,#3 + add $tctr9,$ctr,#4 + add $tctr10,$ctr,#5 + add $ctr,$ctr,#6 + vorr $dat6,$dat0,$dat0 + rev $tctr6,$tctr6 + vorr $dat7,$dat0,$dat0 + rev $tctr7,$tctr7 + vorr $dat8,$dat0,$dat0 + rev $tctr8,$tctr8 + vorr $dat9,$dat0,$dat0 + rev $tctr9,$tctr9 + vorr $dat10,$dat0,$dat0 + rev $tctr10,$tctr10 + vorr $dat11,$dat0,$dat0 + rev $tctr11,$ctr + + sub $len,$len,#6 // bias + vmov.32 ${dat6}[3],$tctr6 + vmov.32 ${dat7}[3],$tctr7 + vmov.32 ${dat8}[3],$tctr8 + vmov.32 ${dat9}[3],$tctr9 + vmov.32 ${dat10}[3],$tctr10 + vmov.32 ${dat11}[3],$tctr11 + b .Loop12x_ctr32_unroll + +.align 4 +.Loop12x_ctr32_unroll: + aese $dat0,$rndping + aesmc $dat0,$dat0 + aese $dat1,$rndping + aesmc $dat1,$dat1 + aese $dat2,$rndping + aesmc $dat2,$dat2 + aese $dat3,$rndping + aesmc $dat3,$dat3 + aese $dat4,$rndping + aesmc $dat4,$dat4 + aese $dat5,$rndping + aesmc $dat5,$dat5 + aese $dat6,$rndping + aesmc $dat6,$dat6 + aese $dat7,$rndping + aesmc $dat7,$dat7 + aese $dat8,$rndping + aesmc $dat8,$dat8 + aese $dat9,$rndping + aesmc $dat9,$dat9 + aese $dat10,$rndping + aesmc $dat10,$dat10 + aese $dat11,$rndping + aesmc $dat11,$dat11 + vld1.32 {$rndping},[$key_],#16 + subs $cnt,$cnt,#2 + aese $dat0,$rndpang + aesmc $dat0,$dat0 + aese $dat1,$rndpang + aesmc $dat1,$dat1 + aese $dat2,$rndpang + aesmc $dat2,$dat2 + aese $dat3,$rndpang + aesmc $dat3,$dat3 + aese $dat4,$rndpang + aesmc $dat4,$dat4 + aese $dat5,$rndpang + aesmc $dat5,$dat5 + aese $dat6,$rndpang + aesmc $dat6,$dat6 + aese $dat7,$rndpang + aesmc $dat7,$dat7 + aese $dat8,$rndpang + aesmc $dat8,$dat8 + aese $dat9,$rndpang + aesmc $dat9,$dat9 + aese $dat10,$rndpang + aesmc $dat10,$dat10 + aese $dat11,$rndpang + aesmc $dat11,$dat11 + vld1.32 {$rndpang},[$key_],#16 + b.gt .Loop12x_ctr32_unroll + + aese $dat0,$rndping + aesmc $dat0,$dat0 + aese $dat1,$rndping + aesmc $dat1,$dat1 + aese $dat2,$rndping + aesmc $dat2,$dat2 + aese $dat3,$rndping + aesmc $dat3,$dat3 + aese $dat4,$rndping + aesmc $dat4,$dat4 + aese $dat5,$rndping + aesmc $dat5,$dat5 + aese $dat6,$rndping + aesmc $dat6,$dat6 + aese $dat7,$rndping + aesmc $dat7,$dat7 + aese $dat8,$rndping + aesmc $dat8,$dat8 + aese $dat9,$rndping + aesmc $dat9,$dat9 + aese $dat10,$rndping + aesmc $dat10,$dat10 + aese $dat11,$rndping + aesmc $dat11,$dat11 + vld1.32 {$rndping},[$key_],#16 + + aese $dat0,$rndpang + aesmc $dat0,$dat0 + aese $dat1,$rndpang + aesmc $dat1,$dat1 + aese $dat2,$rndpang + aesmc $dat2,$dat2 + aese $dat3,$rndpang + aesmc $dat3,$dat3 + aese $dat4,$rndpang + aesmc $dat4,$dat4 + aese $dat5,$rndpang + aesmc $dat5,$dat5 + aese $dat6,$rndpang + aesmc $dat6,$dat6 + aese $dat7,$rndpang + aesmc $dat7,$dat7 + aese $dat8,$rndpang + aesmc $dat8,$dat8 + aese $dat9,$rndpang + aesmc $dat9,$dat9 + aese $dat10,$rndpang + aesmc $dat10,$dat10 + aese $dat11,$rndpang + aesmc $dat11,$dat11 + vld1.32 {$rndpang},[$key_],#16 + + aese $dat0,$rndping + aesmc $dat0,$dat0 + add $tctr0,$ctr,#1 + add $tctr1,$ctr,#2 + aese $dat1,$rndping + aesmc $dat1,$dat1 + add $tctr2,$ctr,#3 + add $tctr3,$ctr,#4 + aese $dat2,$rndping + aesmc $dat2,$dat2 + add $tctr4,$ctr,#5 + add $tctr5,$ctr,#6 + rev $tctr0,$tctr0 + aese $dat3,$rndping + aesmc $dat3,$dat3 + add $tctr6,$ctr,#7 + add $tctr7,$ctr,#8 + rev $tctr1,$tctr1 + rev $tctr2,$tctr2 + aese $dat4,$rndping + aesmc $dat4,$dat4 + add $tctr8,$ctr,#9 + add $tctr9,$ctr,#10 + rev $tctr3,$tctr3 + rev $tctr4,$tctr4 + aese $dat5,$rndping + aesmc $dat5,$dat5 + add $tctr10,$ctr,#11 + add $tctr11,$ctr,#12 + rev $tctr5,$tctr5 + rev $tctr6,$tctr6 + aese $dat6,$rndping + aesmc $dat6,$dat6 + rev $tctr7,$tctr7 + rev $tctr8,$tctr8 + aese $dat7,$rndping + aesmc $dat7,$dat7 + rev $tctr9,$tctr9 + rev $tctr10,$tctr10 + aese $dat8,$rndping + aesmc $dat8,$dat8 + rev $tctr11,$tctr11 + aese $dat9,$rndping + aesmc $dat9,$dat9 + aese $dat10,$rndping + aesmc $dat10,$dat10 + aese $dat11,$rndping + aesmc $dat11,$dat11 + vld1.32 {$rndping},[$key_],#16 + + aese $dat0,$rndpang + aesmc $dat0,$dat0 + aese $dat1,$rndpang + aesmc $dat1,$dat1 + aese $dat2,$rndpang + aesmc $dat2,$dat2 + aese $dat3,$rndpang + aesmc $dat3,$dat3 + vld1.8 {$in0,$in1,$in2,$in3},[$inp],#64 + aese $dat4,$rndpang + aesmc $dat4,$dat4 + aese $dat5,$rndpang + aesmc $dat5,$dat5 + aese $dat6,$rndpang + aesmc $dat6,$dat6 + aese $dat7,$rndpang + aesmc $dat7,$dat7 + vld1.8 {$in4,$in5,$in6,$in7},[$inp],#64 + aese $dat8,$rndpang + aesmc $dat8,$dat8 + aese $dat9,$rndpang + aesmc $dat9,$dat9 + aese $dat10,$rndpang + aesmc $dat10,$dat10 + aese $dat11,$rndpang + aesmc $dat11,$dat11 + vld1.8 {$in8,$in9,$in10,$in11},[$inp],#64 + vld1.32 {$rndpang},[$key_],#16 + + mov $key_, $key + aese $dat0,$rndping + aesmc $dat0,$dat0 + aese $dat1,$rndping + aesmc $dat1,$dat1 + aese $dat2,$rndping + aesmc $dat2,$dat2 + aese $dat3,$rndping + aesmc $dat3,$dat3 + aese $dat4,$rndping + aesmc $dat4,$dat4 + aese $dat5,$rndping + aesmc $dat5,$dat5 + aese $dat6,$rndping + aesmc $dat6,$dat6 + aese $dat7,$rndping + aesmc $dat7,$dat7 + aese $dat8,$rndping + aesmc $dat8,$dat8 + aese $dat9,$rndping + aesmc $dat9,$dat9 + aese $dat10,$rndping + aesmc $dat10,$dat10 + aese $dat11,$rndping + aesmc $dat11,$dat11 + vld1.32 {$rndping},[$key_],#16 // re-pre-load rndkey[0] + + aese $dat0,$rndpang + eor3 $in0,$in0,$rndlast,$dat0 + vorr $dat0,$ivec,$ivec + aese $dat1,$rndpang + eor3 $in1,$in1,$rndlast,$dat1 + vorr $dat1,$ivec,$ivec + aese $dat2,$rndpang + eor3 $in2,$in2,$rndlast,$dat2 + vorr $dat2,$ivec,$ivec + aese $dat3,$rndpang + eor3 $in3,$in3,$rndlast,$dat3 + vorr $dat3,$ivec,$ivec + aese $dat4,$rndpang + eor3 $in4,$in4,$rndlast,$dat4 + vorr $dat4,$ivec,$ivec + aese $dat5,$rndpang + eor3 $in5,$in5,$rndlast,$dat5 + vorr $dat5,$ivec,$ivec + aese $dat6,$rndpang + eor3 $in6,$in6,$rndlast,$dat6 + vorr $dat6,$ivec,$ivec + aese $dat7,$rndpang + eor3 $in7,$in7,$rndlast,$dat7 + vorr $dat7,$ivec,$ivec + aese $dat8,$rndpang + eor3 $in8,$in8,$rndlast,$dat8 + vorr $dat8,$ivec,$ivec + aese $dat9,$rndpang + eor3 $in9,$in9,$rndlast,$dat9 + vorr $dat9,$ivec,$ivec + aese $dat10,$rndpang + eor3 $in10,$in10,$rndlast,$dat10 + vorr $dat10,$ivec,$ivec + aese $dat11,$rndpang + eor3 $in11,$in11,$rndlast,$dat11 + vorr $dat11,$ivec,$ivec + vld1.32 {$rndpang},[$key_],#16 // re-pre-load rndkey[1] + + vmov.32 ${dat0}[3],$tctr0 + vmov.32 ${dat1}[3],$tctr1 + vmov.32 ${dat2}[3],$tctr2 + vmov.32 ${dat3}[3],$tctr3 + vst1.8 {$in0,$in1,$in2,$in3},[$out],#64 + vmov.32 ${dat4}[3],$tctr4 + vmov.32 ${dat5}[3],$tctr5 + vmov.32 ${dat6}[3],$tctr6 + vmov.32 ${dat7}[3],$tctr7 + vst1.8 {$in4,$in5,$in6,$in7},[$out],#64 + vmov.32 ${dat8}[3],$tctr8 + vmov.32 ${dat9}[3],$tctr9 + vmov.32 ${dat10}[3],$tctr10 + vmov.32 ${dat11}[3],$tctr11 + vst1.8 {$in8,$in9,$in10,$in11},[$out],#64 + + mov $cnt,$rounds + + add $ctr,$ctr,#12 + subs $len,$len,#12 + b.hs .Loop12x_ctr32_unroll + + // pop regs from stack when 12 data chunks are interleaved + ldp $dat10d,$dat11d,[sp],#32 + ldp $dat8d,$dat9d,[sp],#32 + ldp x23,x24,[sp],#16 + ldp x21,x22,[sp],#16 + ldp x19,x20,[sp],#16 + + add $len,$len,#12 + cbz $len,.Lctr32_done_unroll + sub $ctr,$ctr,#12 + + cmp $len,#2 + b.ls .Lctr32_tail_unroll + + cmp $len,#6 + sub $len,$len,#3 // bias + add $ctr,$ctr,#3 + b.lo .Loop3x_ctr32_unroll + + sub $len,$len,#3 + add $ctr,$ctr,#3 + b.lo .Loop6x_ctr32_unroll + +.align 4 +.Loop6x_ctr32_unroll: + aese $dat0,$rndping + aesmc $dat0,$dat0 + aese $dat1,$rndping + aesmc $dat1,$dat1 + aese $dat2,$rndping + aesmc $dat2,$dat2 + aese $dat3,$rndping + aesmc $dat3,$dat3 + aese $dat4,$rndping + aesmc $dat4,$dat4 + aese $dat5,$rndping + aesmc $dat5,$dat5 + vld1.32 {$rndping},[$key_],#16 + subs $cnt,$cnt,#2 + aese $dat0,$rndpang + aesmc $dat0,$dat0 + aese $dat1,$rndpang + aesmc $dat1,$dat1 + aese $dat2,$rndpang + aesmc $dat2,$dat2 + aese $dat3,$rndpang + aesmc $dat3,$dat3 + aese $dat4,$rndpang + aesmc $dat4,$dat4 + aese $dat5,$rndpang + aesmc $dat5,$dat5 + vld1.32 {$rndpang},[$key_],#16 + b.gt .Loop6x_ctr32_unroll + + aese $dat0,$rndping + aesmc $dat0,$dat0 + aese $dat1,$rndping + aesmc $dat1,$dat1 + aese $dat2,$rndping + aesmc $dat2,$dat2 + aese $dat3,$rndping + aesmc $dat3,$dat3 + aese $dat4,$rndping + aesmc $dat4,$dat4 + aese $dat5,$rndping + aesmc $dat5,$dat5 + vld1.32 {$rndping},[$key_],#16 + + aese $dat0,$rndpang + aesmc $dat0,$dat0 + aese $dat1,$rndpang + aesmc $dat1,$dat1 + aese $dat2,$rndpang + aesmc $dat2,$dat2 + aese $dat3,$rndpang + aesmc $dat3,$dat3 + aese $dat4,$rndpang + aesmc $dat4,$dat4 + aese $dat5,$rndpang + aesmc $dat5,$dat5 + vld1.32 {$rndpang},[$key_],#16 + + aese $dat0,$rndping + aesmc $dat0,$dat0 + add $tctr0,$ctr,#1 + add $tctr1,$ctr,#2 + aese $dat1,$rndping + aesmc $dat1,$dat1 + add $tctr2,$ctr,#3 + add $tctr3,$ctr,#4 + aese $dat2,$rndping + aesmc $dat2,$dat2 + add $tctr4,$ctr,#5 + add $tctr5,$ctr,#6 + rev $tctr0,$tctr0 + aese $dat3,$rndping + aesmc $dat3,$dat3 + rev $tctr1,$tctr1 + rev $tctr2,$tctr2 + aese $dat4,$rndping + aesmc $dat4,$dat4 + rev $tctr3,$tctr3 + rev $tctr4,$tctr4 + aese $dat5,$rndping + aesmc $dat5,$dat5 + rev $tctr5,$tctr5 + vld1.32 {$rndping},[$key_],#16 + + aese $dat0,$rndpang + aesmc $dat0,$dat0 + aese $dat1,$rndpang + aesmc $dat1,$dat1 + vld1.8 {$in0,$in1,$in2,$in3},[$inp],#64 + aese $dat2,$rndpang + aesmc $dat2,$dat2 + aese $dat3,$rndpang + aesmc $dat3,$dat3 + vld1.8 {$in4,$in5},[$inp],#32 + aese $dat4,$rndpang + aesmc $dat4,$dat4 + aese $dat5,$rndpang + aesmc $dat5,$dat5 + vld1.32 {$rndpang},[$key_],#16 + + mov $key_, $key + aese $dat0,$rndping + aesmc $dat0,$dat0 + aese $dat1,$rndping + aesmc $dat1,$dat1 + aese $dat2,$rndping + aesmc $dat2,$dat2 + aese $dat3,$rndping + aesmc $dat3,$dat3 + aese $dat4,$rndping + aesmc $dat4,$dat4 + aese $dat5,$rndping + aesmc $dat5,$dat5 + vld1.32 {$rndping},[$key_],#16 // re-pre-load rndkey[0] + + aese $dat0,$rndpang + eor3 $in0,$in0,$rndlast,$dat0 + aese $dat1,$rndpang + eor3 $in1,$in1,$rndlast,$dat1 + aese $dat2,$rndpang + eor3 $in2,$in2,$rndlast,$dat2 + aese $dat3,$rndpang + eor3 $in3,$in3,$rndlast,$dat3 + aese $dat4,$rndpang + eor3 $in4,$in4,$rndlast,$dat4 + aese $dat5,$rndpang + eor3 $in5,$in5,$rndlast,$dat5 + vld1.32 {$rndpang},[$key_],#16 // re-pre-load rndkey[1] + + vorr $dat0,$ivec,$ivec + vorr $dat1,$ivec,$ivec + vorr $dat2,$ivec,$ivec + vorr $dat3,$ivec,$ivec + vorr $dat4,$ivec,$ivec + vorr $dat5,$ivec,$ivec + + vmov.32 ${dat0}[3],$tctr0 + vmov.32 ${dat1}[3],$tctr1 + vst1.8 {$in0,$in1,$in2,$in3},[$out],#64 + vmov.32 ${dat2}[3],$tctr2 + vmov.32 ${dat3}[3],$tctr3 + vst1.8 {$in4,$in5},[$out],#32 + vmov.32 ${dat4}[3],$tctr4 + vmov.32 ${dat5}[3],$tctr5 + + cbz $len,.Lctr32_done_unroll + mov $cnt,$rounds + + cmp $len,#2 + b.ls .Lctr32_tail_unroll + + sub $len,$len,#3 // bias + add $ctr,$ctr,#3 + b .Loop3x_ctr32_unroll + +.align 4 +.Loop3x_ctr32_unroll: + aese $dat0,$rndping + aesmc $dat0,$dat0 + aese $dat1,$rndping + aesmc $dat1,$dat1 + aese $dat2,$rndping + aesmc $dat2,$dat2 + vld1.32 {$rndping},[$key_],#16 + subs $cnt,$cnt,#2 + aese $dat0,$rndpang + aesmc $dat0,$dat0 + aese $dat1,$rndpang + aesmc $dat1,$dat1 + aese $dat2,$rndpang + aesmc $dat2,$dat2 + vld1.32 {$rndpang},[$key_],#16 + b.gt .Loop3x_ctr32_unroll + + aese $dat0,$rndping + aesmc $tmp0,$dat0 + aese $dat1,$rndping + aesmc $tmp1,$dat1 + vld1.8 {$in0,$in1,$in2},[$inp],#48 + vorr $dat0,$ivec,$ivec + aese $dat2,$rndping + aesmc $dat2,$dat2 + vld1.32 {$rndping},[$key_],#16 + vorr $dat1,$ivec,$ivec + aese $tmp0,$rndpang + aesmc $tmp0,$tmp0 + aese $tmp1,$rndpang + aesmc $tmp1,$tmp1 + aese $dat2,$rndpang + aesmc $tmp2,$dat2 + vld1.32 {$rndpang},[$key_],#16 + vorr $dat2,$ivec,$ivec + add $tctr0,$ctr,#1 + aese $tmp0,$rndping + aesmc $tmp0,$tmp0 + aese $tmp1,$rndping + aesmc $tmp1,$tmp1 + add $tctr1,$ctr,#2 + aese $tmp2,$rndping + aesmc $tmp2,$tmp2 + vld1.32 {$rndping},[$key_],#16 + add $ctr,$ctr,#3 + aese $tmp0,$rndpang + aesmc $tmp0,$tmp0 + aese $tmp1,$rndpang + aesmc $tmp1,$tmp1 + + rev $tctr0,$tctr0 + aese $tmp2,$rndpang + aesmc $tmp2,$tmp2 + vld1.32 {$rndpang},[$key_],#16 + vmov.32 ${dat0}[3], $tctr0 + mov $key_,$key + rev $tctr1,$tctr1 + aese $tmp0,$rndping + aesmc $tmp0,$tmp0 + + aese $tmp1,$rndping + aesmc $tmp1,$tmp1 + vmov.32 ${dat1}[3], $tctr1 + rev $tctr2,$ctr + aese $tmp2,$rndping + aesmc $tmp2,$tmp2 + vmov.32 ${dat2}[3], $tctr2 + + aese $tmp0,$rndpang + aese $tmp1,$rndpang + aese $tmp2,$rndpang + + eor3 $in0,$in0,$rndlast,$tmp0 + vld1.32 {$rndping},[$key_],#16 // re-pre-load rndkey[0] + eor3 $in1,$in1,$rndlast,$tmp1 + mov $cnt,$rounds + eor3 $in2,$in2,$rndlast,$tmp2 + vld1.32 {$rndpang},[$key_],#16 // re-pre-load rndkey[1] + vst1.8 {$in0,$in1,$in2},[$out],#48 + + cbz $len,.Lctr32_done_unroll + +.Lctr32_tail_unroll: + cmp $len,#1 + b.eq .Lctr32_tail_1_unroll + +.Lctr32_tail_2_unroll: + aese $dat0,$rndping + aesmc $dat0,$dat0 + aese $dat1,$rndping + aesmc $dat1,$dat1 + vld1.32 {$rndping},[$key_],#16 + subs $cnt,$cnt,#2 + aese $dat0,$rndpang + aesmc $dat0,$dat0 + aese $dat1,$rndpang + aesmc $dat1,$dat1 + vld1.32 {$rndpang},[$key_],#16 + b.gt .Lctr32_tail_2_unroll + + aese $dat0,$rndping + aesmc $dat0,$dat0 + aese $dat1,$rndping + aesmc $dat1,$dat1 + vld1.32 {$rndping},[$key_],#16 + aese $dat0,$rndpang + aesmc $dat0,$dat0 + aese $dat1,$rndpang + aesmc $dat1,$dat1 + vld1.32 {$rndpang},[$key_],#16 + vld1.8 {$in0,$in1},[$inp],#32 + aese $dat0,$rndping + aesmc $dat0,$dat0 + aese $dat1,$rndping + aesmc $dat1,$dat1 + vld1.32 {$rndping},[$key_],#16 + aese $dat0,$rndpang + aesmc $dat0,$dat0 + aese $dat1,$rndpang + aesmc $dat1,$dat1 + vld1.32 {$rndpang},[$key_],#16 + aese $dat0,$rndping + aesmc $dat0,$dat0 + aese $dat1,$rndping + aesmc $dat1,$dat1 + aese $dat0,$rndpang + aese $dat1,$rndpang + + eor3 $in0,$in0,$rndlast,$dat0 + eor3 $in1,$in1,$rndlast,$dat1 + vst1.8 {$in0,$in1},[$out],#32 + b .Lctr32_done_unroll + +.Lctr32_tail_1_unroll: + aese $dat0,$rndping + aesmc $dat0,$dat0 + vld1.32 {$rndping},[$key_],#16 + subs $cnt,$cnt,#2 + aese $dat0,$rndpang + aesmc $dat0,$dat0 + vld1.32 {$rndpang},[$key_],#16 + b.gt .Lctr32_tail_1_unroll + + aese $dat0,$rndping + aesmc $dat0,$dat0 + vld1.32 {$rndping},[$key_],#16 + aese $dat0,$rndpang + aesmc $dat0,$dat0 + vld1.32 {$rndpang},[$key_],#16 + vld1.8 {$in0},[$inp] + aese $dat0,$rndping + aesmc $dat0,$dat0 + vld1.32 {$rndping},[$key_],#16 + aese $dat0,$rndpang + aesmc $dat0,$dat0 + vld1.32 {$rndpang},[$key_],#16 + aese $dat0,$rndping + aesmc $dat0,$dat0 + aese $dat0,$rndpang + + eor3 $in0,$in0,$rndlast,$dat0 + vst1.8 {$in0},[$out],#16 + +.Lctr32_done_unroll: + ldp d8,d9,[sp, #16] + ldp d10,d11,[sp, #32] + ldp d12,d13,[sp, #48] + ldp d14,d15,[sp, #64] + ldr x29,[sp],#80 + ret +.size ${prefix}_ctr32_encrypt_blocks_unroll12_eor3,.-${prefix}_ctr32_encrypt_blocks_unroll12_eor3 +___ +}}} + {{{ my ($inp,$out,$len,$key,$ivp)=("PTR(0)","PTR(1)","x2","PTR(3)","PTR(4)"); my ($rounds,$cnt,$key_)=("w5","w6","PTR(7)"); @@ -2229,7 +2994,7 @@ ___ # will be processed specially, which be integrated into the 5*16 bytes # loop to improve the efficiency. # There is one special case, if the original input data size dsize -# = 16 bytes, we will treat it seperately to improve the +# = 16 bytes, we will treat it separately to improve the # performance: one independent code block without LR, FP load and # store. # Encryption will process the (length -tailcnt) bytes as mentioned @@ -3568,7 +4333,7 @@ $code.=<<___ if ($flavour =~ /64/); cbnz x2,.Lxts_dec_1st_done vld1.8 {$dat0},[$inp],#16 - // Decrypt the last secod block to get the last plain text block + // Decrypt the last second block to get the last plain text block .Lxts_dec_1st_done: eor $tmpin,$dat0,$iv1 ldr $rounds,[$key1,#240] @@ -3651,7 +4416,8 @@ ___ if ($flavour =~ /64/) { ######## 64-bit code my %opcode = ( "aesd" => 0x4e285800, "aese" => 0x4e284800, - "aesimc"=> 0x4e287800, "aesmc" => 0x4e286800 ); + "aesimc"=> 0x4e287800, "aesmc" => 0x4e286800, + "eor3" => 0xce000000, ); local *unaes = sub { my ($mnemonic,$arg)=@_; @@ -3662,10 +4428,21 @@ if ($flavour =~ /64/) { ######## 64-bit code $mnemonic,$arg; }; + sub unsha3 { + my ($mnemonic,$arg)=@_; + + $arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv#]([0-9\-]+))?)?/ + && + sprintf ".inst\t0x%08x\t//%s %s", + $opcode{$mnemonic}|$1|($2<<5)|($3<<16)|(eval($4)<<10), + $mnemonic,$arg; + } + foreach(split("\n",$code)) { s/\`([^\`]*)\`/eval($1)/geo; - s/\bq([0-9]+)\b/"v".($1<8?$1:$1+8).".16b"/geo; # old->new registers + s/\bq([0-9]+)\b/"v".($1<8?$1:($1<24?$1+8:$1-16)).".16b"/geo; # old->new registers + s/\bq_([0-9]+)\b/"q".$1/geo; # old->new registers s/@\s/\/\//o; # old->new style commentary #s/[v]?(aes\w+)\s+([qv].*)/unaes($1,$2)/geo or @@ -3678,6 +4455,7 @@ if ($flavour =~ /64/) { ######## 64-bit code s/vshr/ushr/o or s/^(\s+)v/$1/o or # strip off v prefix s/\bbx\s+lr\b/ret/o; + s/\b(eor3)\s+(v.*)/unsha3($1,$2)/ge; # fix up remaining legacy suffixes s/\.[ui]?8//o; diff --git a/crypto/openssl/crypto/aes/asm/bsaes-armv8.pl b/crypto/openssl/crypto/aes/asm/bsaes-armv8.pl new file mode 100644 index 000000000000..912538622b39 --- /dev/null +++ b/crypto/openssl/crypto/aes/asm/bsaes-armv8.pl @@ -0,0 +1,2389 @@ +#!/usr/bin/env perl +# Copyright 2020-2025 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +use strict; + +my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; +my $xlate; + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; my $dir=$1; +( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate ) or +die "can't locate arm-xlate.pl"; + +open OUT,"| \"$^X\" $xlate $flavour $output"; +*STDOUT=*OUT; + +my $code = data(); +print $code; + +close STDOUT or die "error closing STDOUT: $!"; # enforce flush + +sub data +{ + local $/; + return ; +} + +__END__ +// Copyright 2021-2025 The OpenSSL Project Authors. All Rights Reserved. +// +// Licensed under the OpenSSL license (the "License"). You may not use +// this file except in compliance with the License. You can obtain a copy +// in the file LICENSE in the source distribution or at +// https://www.openssl.org/source/license.html +// +// ==================================================================== +// Written by Ben Avison for the OpenSSL +// project. Rights for redistribution and usage in source and binary +// forms are granted according to the OpenSSL license. +// ==================================================================== +// +// This implementation is a translation of bsaes-armv7 for AArch64. +// No attempt has been made to carry across the build switches for +// kernel targets, since the Linux kernel crypto support has moved on +// from when it was based on OpenSSL. + +// A lot of hand-scheduling has been performed. Consequently, this code +// doesn't factor out neatly into macros in the same way that the +// AArch32 version did, and there is little to be gained by wrapping it +// up in Perl, and it is presented as pure assembly. + + +#include "crypto/arm_arch.h" + +.text + +.extern AES_cbc_encrypt +.extern AES_encrypt +.extern AES_decrypt + +.type _bsaes_decrypt8,%function +.align 4 +// On entry: +// x9 -> key (previously expanded using _bsaes_key_convert) +// x10 = number of rounds +// v0-v7 input data +// On exit: +// x9-x11 corrupted +// other general-purpose registers preserved +// v0-v7 output data +// v11-v15 preserved +// other SIMD registers corrupted +_bsaes_decrypt8: + ldr q8, [x9], #16 + adrp x11, .LM0ISR + add x11, x11, #:lo12:.LM0ISR + movi v9.16b, #0x55 + ldr q10, [x11], #16 + movi v16.16b, #0x33 + movi v17.16b, #0x0f + sub x10, x10, #1 + eor v0.16b, v0.16b, v8.16b + eor v1.16b, v1.16b, v8.16b + eor v2.16b, v2.16b, v8.16b + eor v4.16b, v4.16b, v8.16b + eor v3.16b, v3.16b, v8.16b + eor v5.16b, v5.16b, v8.16b + tbl v0.16b, {v0.16b}, v10.16b + tbl v1.16b, {v1.16b}, v10.16b + tbl v2.16b, {v2.16b}, v10.16b + tbl v4.16b, {v4.16b}, v10.16b + eor v6.16b, v6.16b, v8.16b + eor v7.16b, v7.16b, v8.16b + tbl v3.16b, {v3.16b}, v10.16b + tbl v5.16b, {v5.16b}, v10.16b + tbl v6.16b, {v6.16b}, v10.16b + ushr v8.2d, v0.2d, #1 + tbl v7.16b, {v7.16b}, v10.16b + ushr v10.2d, v4.2d, #1 + ushr v18.2d, v2.2d, #1 + eor v8.16b, v8.16b, v1.16b + ushr v19.2d, v6.2d, #1 + eor v10.16b, v10.16b, v5.16b + eor v18.16b, v18.16b, v3.16b + and v8.16b, v8.16b, v9.16b + eor v19.16b, v19.16b, v7.16b + and v10.16b, v10.16b, v9.16b + and v18.16b, v18.16b, v9.16b + eor v1.16b, v1.16b, v8.16b + shl v8.2d, v8.2d, #1 + and v9.16b, v19.16b, v9.16b + eor v5.16b, v5.16b, v10.16b + shl v10.2d, v10.2d, #1 + eor v3.16b, v3.16b, v18.16b + shl v18.2d, v18.2d, #1 + eor v0.16b, v0.16b, v8.16b + shl v8.2d, v9.2d, #1 + eor v7.16b, v7.16b, v9.16b + eor v4.16b, v4.16b, v10.16b + eor v2.16b, v2.16b, v18.16b + ushr v9.2d, v1.2d, #2 + eor v6.16b, v6.16b, v8.16b + ushr v8.2d, v0.2d, #2 + ushr v10.2d, v5.2d, #2 + ushr v18.2d, v4.2d, #2 + eor v9.16b, v9.16b, v3.16b + eor v8.16b, v8.16b, v2.16b + eor v10.16b, v10.16b, v7.16b + eor v18.16b, v18.16b, v6.16b + and v9.16b, v9.16b, v16.16b + and v8.16b, v8.16b, v16.16b + and v10.16b, v10.16b, v16.16b + and v16.16b, v18.16b, v16.16b + eor v3.16b, v3.16b, v9.16b + shl v9.2d, v9.2d, #2 + eor v2.16b, v2.16b, v8.16b + shl v8.2d, v8.2d, #2 + eor v7.16b, v7.16b, v10.16b + shl v10.2d, v10.2d, #2 + eor v6.16b, v6.16b, v16.16b + shl v16.2d, v16.2d, #2 + eor v1.16b, v1.16b, v9.16b + eor v0.16b, v0.16b, v8.16b + eor v5.16b, v5.16b, v10.16b + eor v4.16b, v4.16b, v16.16b + ushr v8.2d, v3.2d, #4 + ushr v9.2d, v2.2d, #4 + ushr v10.2d, v1.2d, #4 + ushr v16.2d, v0.2d, #4 + eor v8.16b, v8.16b, v7.16b + eor v9.16b, v9.16b, v6.16b + eor v10.16b, v10.16b, v5.16b + eor v16.16b, v16.16b, v4.16b + and v8.16b, v8.16b, v17.16b + and v9.16b, v9.16b, v17.16b + and v10.16b, v10.16b, v17.16b + and v16.16b, v16.16b, v17.16b + eor v7.16b, v7.16b, v8.16b + shl v8.2d, v8.2d, #4 + eor v6.16b, v6.16b, v9.16b + shl v9.2d, v9.2d, #4 + eor v5.16b, v5.16b, v10.16b + shl v10.2d, v10.2d, #4 + eor v4.16b, v4.16b, v16.16b + shl v16.2d, v16.2d, #4 + eor v3.16b, v3.16b, v8.16b + eor v2.16b, v2.16b, v9.16b + eor v1.16b, v1.16b, v10.16b + eor v0.16b, v0.16b, v16.16b + b .Ldec_sbox +.align 4 +.Ldec_loop: + ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x9], #64 + ldp q8, q9, [x9], #32 + eor v0.16b, v16.16b, v0.16b + ldr q10, [x9], #16 + eor v1.16b, v17.16b, v1.16b + ldr q16, [x9], #16 + eor v2.16b, v18.16b, v2.16b + eor v3.16b, v19.16b, v3.16b + eor v4.16b, v8.16b, v4.16b + eor v5.16b, v9.16b, v5.16b + eor v6.16b, v10.16b, v6.16b + eor v7.16b, v16.16b, v7.16b + tbl v0.16b, {v0.16b}, v28.16b + tbl v1.16b, {v1.16b}, v28.16b + tbl v2.16b, {v2.16b}, v28.16b + tbl v3.16b, {v3.16b}, v28.16b + tbl v4.16b, {v4.16b}, v28.16b + tbl v5.16b, {v5.16b}, v28.16b + tbl v6.16b, {v6.16b}, v28.16b + tbl v7.16b, {v7.16b}, v28.16b +.Ldec_sbox: + eor v1.16b, v1.16b, v4.16b + eor v3.16b, v3.16b, v4.16b + subs x10, x10, #1 + eor v4.16b, v4.16b, v7.16b + eor v2.16b, v2.16b, v7.16b + eor v1.16b, v1.16b, v6.16b + eor v6.16b, v6.16b, v4.16b + eor v2.16b, v2.16b, v5.16b + eor v0.16b, v0.16b, v1.16b + eor v7.16b, v7.16b, v6.16b + eor v8.16b, v6.16b, v2.16b + and v9.16b, v4.16b, v6.16b + eor v10.16b, v2.16b, v6.16b + eor v3.16b, v3.16b, v0.16b + eor v5.16b, v5.16b, v0.16b + eor v16.16b, v7.16b, v4.16b + eor v17.16b, v4.16b, v0.16b + and v18.16b, v0.16b, v2.16b + eor v19.16b, v7.16b, v4.16b + eor v1.16b, v1.16b, v3.16b + eor v20.16b, v3.16b, v0.16b + eor v21.16b, v5.16b, v2.16b + eor v22.16b, v3.16b, v7.16b + and v8.16b, v17.16b, v8.16b + orr v17.16b, v3.16b, v5.16b + eor v23.16b, v1.16b, v6.16b + eor v24.16b, v20.16b, v16.16b + eor v25.16b, v1.16b, v5.16b + orr v26.16b, v20.16b, v21.16b + and v20.16b, v20.16b, v21.16b + and v27.16b, v7.16b, v1.16b + eor v21.16b, v21.16b, v23.16b + orr v28.16b, v16.16b, v23.16b + orr v29.16b, v22.16b, v25.16b + eor v26.16b, v26.16b, v8.16b + and v16.16b, v16.16b, v23.16b + and v22.16b, v22.16b, v25.16b + and v21.16b, v24.16b, v21.16b + eor v8.16b, v28.16b, v8.16b + eor v23.16b, v5.16b, v2.16b + eor v24.16b, v1.16b, v6.16b + eor v16.16b, v16.16b, v22.16b + eor v22.16b, v3.16b, v0.16b + eor v25.16b, v29.16b, v21.16b + eor v21.16b, v26.16b, v21.16b + eor v8.16b, v8.16b, v20.16b + eor v26.16b, v23.16b, v24.16b + eor v16.16b, v16.16b, v20.16b + eor v28.16b, v22.16b, v19.16b + eor v20.16b, v25.16b, v20.16b + eor v9.16b, v21.16b, v9.16b + eor v8.16b, v8.16b, v18.16b + eor v18.16b, v5.16b, v1.16b + eor v21.16b, v16.16b, v17.16b + eor v16.16b, v16.16b, v17.16b + eor v17.16b, v20.16b, v27.16b + eor v20.16b, v3.16b, v7.16b + eor v25.16b, v9.16b, v8.16b + eor v27.16b, v0.16b, v4.16b + and v29.16b, v9.16b, v17.16b + eor v30.16b, v8.16b, v29.16b + eor v31.16b, v21.16b, v29.16b + eor v29.16b, v21.16b, v29.16b + bsl v30.16b, v17.16b, v21.16b + bsl v31.16b, v9.16b, v8.16b + bsl v16.16b, v30.16b, v29.16b + bsl v21.16b, v29.16b, v30.16b + eor v8.16b, v31.16b, v30.16b + and v1.16b, v1.16b, v31.16b + and v9.16b, v16.16b, v31.16b + and v6.16b, v6.16b, v30.16b + eor v16.16b, v17.16b, v21.16b + and v4.16b, v4.16b, v30.16b + eor v17.16b, v8.16b, v30.16b + and v21.16b, v24.16b, v8.16b + eor v9.16b, v9.16b, v25.16b + and v19.16b, v19.16b, v8.16b + eor v24.16b, v30.16b, v16.16b + eor v25.16b, v30.16b, v16.16b + and v7.16b, v7.16b, v17.16b + and v10.16b, v10.16b, v16.16b + eor v29.16b, v9.16b, v16.16b + eor v30.16b, v31.16b, v9.16b + and v0.16b, v24.16b, v0.16b + and v9.16b, v18.16b, v9.16b + and v2.16b, v25.16b, v2.16b + eor v10.16b, v10.16b, v6.16b + eor v18.16b, v29.16b, v16.16b + and v5.16b, v30.16b, v5.16b + eor v24.16b, v8.16b, v29.16b + and v25.16b, v26.16b, v29.16b + and v26.16b, v28.16b, v29.16b + eor v8.16b, v8.16b, v29.16b + eor v17.16b, v17.16b, v18.16b + eor v5.16b, v1.16b, v5.16b + and v23.16b, v24.16b, v23.16b + eor v21.16b, v21.16b, v25.16b + eor v19.16b, v19.16b, v26.16b + eor v0.16b, v4.16b, v0.16b + and v3.16b, v17.16b, v3.16b + eor v1.16b, v9.16b, v1.16b + eor v9.16b, v25.16b, v23.16b + eor v5.16b, v5.16b, v21.16b + eor v2.16b, v6.16b, v2.16b + and v6.16b, v8.16b, v22.16b + eor v3.16b, v7.16b, v3.16b + and v8.16b, v20.16b, v18.16b + eor v10.16b, v10.16b, v9.16b + eor v0.16b, v0.16b, v19.16b + eor v9.16b, v1.16b, v9.16b + eor v1.16b, v2.16b, v21.16b + eor v3.16b, v3.16b, v19.16b + and v16.16b, v27.16b, v16.16b + eor v17.16b, v26.16b, v6.16b + eor v6.16b, v8.16b, v7.16b + eor v7.16b, v1.16b, v9.16b + eor v1.16b, v5.16b, v3.16b + eor v2.16b, v10.16b, v3.16b + eor v4.16b, v16.16b, v4.16b + eor v8.16b, v6.16b, v17.16b + eor v5.16b, v9.16b, v3.16b + eor v9.16b, v0.16b, v1.16b + eor v6.16b, v7.16b, v1.16b + eor v0.16b, v4.16b, v17.16b + eor v4.16b, v8.16b, v7.16b + eor v7.16b, v9.16b, v2.16b + eor v8.16b, v3.16b, v0.16b + eor v7.16b, v7.16b, v5.16b + eor v3.16b, v4.16b, v7.16b + eor v4.16b, v7.16b, v0.16b + eor v7.16b, v8.16b, v3.16b + bcc .Ldec_done + ext v8.16b, v0.16b, v0.16b, #8 + ext v9.16b, v1.16b, v1.16b, #8 + ldr q28, [x11] // load from .LISR in common case (x10 > 0) + ext v10.16b, v6.16b, v6.16b, #8 + ext v16.16b, v3.16b, v3.16b, #8 + ext v17.16b, v5.16b, v5.16b, #8 + ext v18.16b, v4.16b, v4.16b, #8 + eor v8.16b, v8.16b, v0.16b + eor v9.16b, v9.16b, v1.16b + eor v10.16b, v10.16b, v6.16b + eor v16.16b, v16.16b, v3.16b + eor v17.16b, v17.16b, v5.16b + ext v19.16b, v2.16b, v2.16b, #8 + ext v20.16b, v7.16b, v7.16b, #8 + eor v18.16b, v18.16b, v4.16b + eor v6.16b, v6.16b, v8.16b + eor v8.16b, v2.16b, v10.16b + eor v4.16b, v4.16b, v9.16b + eor v2.16b, v19.16b, v2.16b + eor v9.16b, v20.16b, v7.16b + eor v0.16b, v0.16b, v16.16b + eor v1.16b, v1.16b, v16.16b + eor v6.16b, v6.16b, v17.16b + eor v8.16b, v8.16b, v16.16b + eor v7.16b, v7.16b, v18.16b + eor v4.16b, v4.16b, v16.16b + eor v2.16b, v3.16b, v2.16b + eor v1.16b, v1.16b, v17.16b + eor v3.16b, v5.16b, v9.16b + eor v5.16b, v8.16b, v17.16b + eor v7.16b, v7.16b, v17.16b + ext v8.16b, v0.16b, v0.16b, #12 + ext v9.16b, v6.16b, v6.16b, #12 + ext v10.16b, v4.16b, v4.16b, #12 + ext v16.16b, v1.16b, v1.16b, #12 + ext v17.16b, v5.16b, v5.16b, #12 + ext v18.16b, v7.16b, v7.16b, #12 + eor v0.16b, v0.16b, v8.16b + eor v6.16b, v6.16b, v9.16b + eor v4.16b, v4.16b, v10.16b + ext v19.16b, v2.16b, v2.16b, #12 + ext v20.16b, v3.16b, v3.16b, #12 + eor v1.16b, v1.16b, v16.16b + eor v5.16b, v5.16b, v17.16b + eor v7.16b, v7.16b, v18.16b + eor v2.16b, v2.16b, v19.16b + eor v16.16b, v16.16b, v0.16b + eor v3.16b, v3.16b, v20.16b + eor v17.16b, v17.16b, v4.16b + eor v10.16b, v10.16b, v6.16b + ext v0.16b, v0.16b, v0.16b, #8 + eor v9.16b, v9.16b, v1.16b + ext v1.16b, v1.16b, v1.16b, #8 + eor v8.16b, v8.16b, v3.16b + eor v16.16b, v16.16b, v3.16b + eor v18.16b, v18.16b, v5.16b + eor v19.16b, v19.16b, v7.16b + ext v21.16b, v5.16b, v5.16b, #8 + ext v5.16b, v7.16b, v7.16b, #8 + eor v7.16b, v20.16b, v2.16b + ext v4.16b, v4.16b, v4.16b, #8 + ext v20.16b, v3.16b, v3.16b, #8 + eor v17.16b, v17.16b, v3.16b + ext v2.16b, v2.16b, v2.16b, #8 + eor v3.16b, v10.16b, v3.16b + ext v10.16b, v6.16b, v6.16b, #8 + eor v0.16b, v0.16b, v8.16b + eor v1.16b, v1.16b, v16.16b + eor v5.16b, v5.16b, v18.16b + eor v3.16b, v3.16b, v4.16b + eor v7.16b, v20.16b, v7.16b + eor v6.16b, v2.16b, v19.16b + eor v4.16b, v21.16b, v17.16b + eor v2.16b, v10.16b, v9.16b + bne .Ldec_loop + ldr q28, [x11, #16]! // load from .LISRM0 on last round (x10 == 0) + b .Ldec_loop +.align 4 +.Ldec_done: + ushr v8.2d, v0.2d, #1 + movi v9.16b, #0x55 + ldr q10, [x9] + ushr v16.2d, v2.2d, #1 + movi v17.16b, #0x33 + ushr v18.2d, v6.2d, #1 + movi v19.16b, #0x0f + eor v8.16b, v8.16b, v1.16b + ushr v20.2d, v3.2d, #1 + eor v16.16b, v16.16b, v7.16b + eor v18.16b, v18.16b, v4.16b + and v8.16b, v8.16b, v9.16b + eor v20.16b, v20.16b, v5.16b + and v16.16b, v16.16b, v9.16b + and v18.16b, v18.16b, v9.16b + shl v21.2d, v8.2d, #1 + eor v1.16b, v1.16b, v8.16b + and v8.16b, v20.16b, v9.16b + eor v7.16b, v7.16b, v16.16b + shl v9.2d, v16.2d, #1 + eor v4.16b, v4.16b, v18.16b + shl v16.2d, v18.2d, #1 + eor v0.16b, v0.16b, v21.16b + shl v18.2d, v8.2d, #1 + eor v5.16b, v5.16b, v8.16b + eor v2.16b, v2.16b, v9.16b + eor v6.16b, v6.16b, v16.16b + ushr v8.2d, v1.2d, #2 + eor v3.16b, v3.16b, v18.16b + ushr v9.2d, v0.2d, #2 + ushr v16.2d, v7.2d, #2 + ushr v18.2d, v2.2d, #2 + eor v8.16b, v8.16b, v4.16b + eor v9.16b, v9.16b, v6.16b + eor v16.16b, v16.16b, v5.16b + eor v18.16b, v18.16b, v3.16b + and v8.16b, v8.16b, v17.16b + and v9.16b, v9.16b, v17.16b + and v16.16b, v16.16b, v17.16b + and v17.16b, v18.16b, v17.16b + eor v4.16b, v4.16b, v8.16b + shl v8.2d, v8.2d, #2 + eor v6.16b, v6.16b, v9.16b + shl v9.2d, v9.2d, #2 + eor v5.16b, v5.16b, v16.16b + shl v16.2d, v16.2d, #2 + eor v3.16b, v3.16b, v17.16b + shl v17.2d, v17.2d, #2 + eor v1.16b, v1.16b, v8.16b + eor v0.16b, v0.16b, v9.16b + eor v7.16b, v7.16b, v16.16b + eor v2.16b, v2.16b, v17.16b + ushr v8.2d, v4.2d, #4 + ushr v9.2d, v6.2d, #4 + ushr v16.2d, v1.2d, #4 + ushr v17.2d, v0.2d, #4 + eor v8.16b, v8.16b, v5.16b + eor v9.16b, v9.16b, v3.16b + eor v16.16b, v16.16b, v7.16b + eor v17.16b, v17.16b, v2.16b + and v8.16b, v8.16b, v19.16b + and v9.16b, v9.16b, v19.16b + and v16.16b, v16.16b, v19.16b + and v17.16b, v17.16b, v19.16b + eor v5.16b, v5.16b, v8.16b + shl v8.2d, v8.2d, #4 + eor v3.16b, v3.16b, v9.16b + shl v9.2d, v9.2d, #4 + eor v7.16b, v7.16b, v16.16b + shl v16.2d, v16.2d, #4 + eor v2.16b, v2.16b, v17.16b + shl v17.2d, v17.2d, #4 + eor v4.16b, v4.16b, v8.16b + eor v6.16b, v6.16b, v9.16b + eor v7.16b, v7.16b, v10.16b + eor v1.16b, v1.16b, v16.16b + eor v2.16b, v2.16b, v10.16b + eor v0.16b, v0.16b, v17.16b + eor v4.16b, v4.16b, v10.16b + eor v6.16b, v6.16b, v10.16b + eor v3.16b, v3.16b, v10.16b + eor v5.16b, v5.16b, v10.16b + eor v1.16b, v1.16b, v10.16b + eor v0.16b, v0.16b, v10.16b + ret +.size _bsaes_decrypt8,.-_bsaes_decrypt8 + +.rodata +.type _bsaes_consts,%object +.align 6 +_bsaes_consts: +// InvShiftRows constants +// Used in _bsaes_decrypt8, which assumes contiguity +// .LM0ISR used with round 0 key +// .LISR used with middle round keys +// .LISRM0 used with final round key +.LM0ISR: +.quad 0x0a0e0206070b0f03, 0x0004080c0d010509 +.LISR: +.quad 0x0504070602010003, 0x0f0e0d0c080b0a09 +.LISRM0: +.quad 0x01040b0e0205080f, 0x0306090c00070a0d + +// ShiftRows constants +// Used in _bsaes_encrypt8, which assumes contiguity +// .LM0SR used with round 0 key +// .LSR used with middle round keys +// .LSRM0 used with final round key +.LM0SR: +.quad 0x0a0e02060f03070b, 0x0004080c05090d01 +.LSR: +.quad 0x0504070600030201, 0x0f0e0d0c0a09080b +.LSRM0: +.quad 0x0304090e00050a0f, 0x01060b0c0207080d + +.LM0_bigendian: +.quad 0x02060a0e03070b0f, 0x0004080c0105090d +.LM0_littleendian: +.quad 0x0105090d0004080c, 0x03070b0f02060a0e + +// Used in ossl_bsaes_ctr32_encrypt_blocks, prior to dropping into +// _bsaes_encrypt8_alt, for round 0 key in place of .LM0SR +.LREVM0SR: +.quad 0x090d01050c000408, 0x03070b0f060a0e02 + +.align 6 +.size _bsaes_consts,.-_bsaes_consts + +.previous + +.type _bsaes_encrypt8,%function +.align 4 +// On entry: +// x9 -> key (previously expanded using _bsaes_key_convert) +// x10 = number of rounds +// v0-v7 input data +// On exit: +// x9-x11 corrupted +// other general-purpose registers preserved +// v0-v7 output data +// v11-v15 preserved +// other SIMD registers corrupted +_bsaes_encrypt8: + ldr q8, [x9], #16 + adrp x11, .LM0SR + add x11, x11, #:lo12:.LM0SR + ldr q9, [x11], #16 +_bsaes_encrypt8_alt: + eor v0.16b, v0.16b, v8.16b + eor v1.16b, v1.16b, v8.16b + sub x10, x10, #1 + eor v2.16b, v2.16b, v8.16b + eor v4.16b, v4.16b, v8.16b + eor v3.16b, v3.16b, v8.16b + eor v5.16b, v5.16b, v8.16b + tbl v0.16b, {v0.16b}, v9.16b + tbl v1.16b, {v1.16b}, v9.16b + tbl v2.16b, {v2.16b}, v9.16b + tbl v4.16b, {v4.16b}, v9.16b + eor v6.16b, v6.16b, v8.16b + eor v7.16b, v7.16b, v8.16b + tbl v3.16b, {v3.16b}, v9.16b + tbl v5.16b, {v5.16b}, v9.16b + tbl v6.16b, {v6.16b}, v9.16b + ushr v8.2d, v0.2d, #1 + movi v10.16b, #0x55 + tbl v7.16b, {v7.16b}, v9.16b + ushr v9.2d, v4.2d, #1 + movi v16.16b, #0x33 + ushr v17.2d, v2.2d, #1 + eor v8.16b, v8.16b, v1.16b + movi v18.16b, #0x0f + ushr v19.2d, v6.2d, #1 + eor v9.16b, v9.16b, v5.16b + eor v17.16b, v17.16b, v3.16b + and v8.16b, v8.16b, v10.16b + eor v19.16b, v19.16b, v7.16b + and v9.16b, v9.16b, v10.16b + and v17.16b, v17.16b, v10.16b + eor v1.16b, v1.16b, v8.16b + shl v8.2d, v8.2d, #1 + and v10.16b, v19.16b, v10.16b + eor v5.16b, v5.16b, v9.16b + shl v9.2d, v9.2d, #1 + eor v3.16b, v3.16b, v17.16b + shl v17.2d, v17.2d, #1 + eor v0.16b, v0.16b, v8.16b + shl v8.2d, v10.2d, #1 + eor v7.16b, v7.16b, v10.16b + eor v4.16b, v4.16b, v9.16b + eor v2.16b, v2.16b, v17.16b + ushr v9.2d, v1.2d, #2 + eor v6.16b, v6.16b, v8.16b + ushr v8.2d, v0.2d, #2 + ushr v10.2d, v5.2d, #2 + ushr v17.2d, v4.2d, #2 + eor v9.16b, v9.16b, v3.16b + eor v8.16b, v8.16b, v2.16b + eor v10.16b, v10.16b, v7.16b + eor v17.16b, v17.16b, v6.16b + and v9.16b, v9.16b, v16.16b + and v8.16b, v8.16b, v16.16b + and v10.16b, v10.16b, v16.16b + and v16.16b, v17.16b, v16.16b + eor v3.16b, v3.16b, v9.16b + shl v9.2d, v9.2d, #2 + eor v2.16b, v2.16b, v8.16b + shl v8.2d, v8.2d, #2 + eor v7.16b, v7.16b, v10.16b + shl v10.2d, v10.2d, #2 + eor v6.16b, v6.16b, v16.16b + shl v16.2d, v16.2d, #2 + eor v1.16b, v1.16b, v9.16b + eor v0.16b, v0.16b, v8.16b + eor v5.16b, v5.16b, v10.16b + eor v4.16b, v4.16b, v16.16b + ushr v8.2d, v3.2d, #4 + ushr v9.2d, v2.2d, #4 + ushr v10.2d, v1.2d, #4 + ushr v16.2d, v0.2d, #4 + eor v8.16b, v8.16b, v7.16b + eor v9.16b, v9.16b, v6.16b + eor v10.16b, v10.16b, v5.16b + eor v16.16b, v16.16b, v4.16b + and v8.16b, v8.16b, v18.16b + and v9.16b, v9.16b, v18.16b + and v10.16b, v10.16b, v18.16b + and v16.16b, v16.16b, v18.16b + eor v7.16b, v7.16b, v8.16b + shl v8.2d, v8.2d, #4 + eor v6.16b, v6.16b, v9.16b + shl v9.2d, v9.2d, #4 + eor v5.16b, v5.16b, v10.16b + shl v10.2d, v10.2d, #4 + eor v4.16b, v4.16b, v16.16b + shl v16.2d, v16.2d, #4 + eor v3.16b, v3.16b, v8.16b + eor v2.16b, v2.16b, v9.16b + eor v1.16b, v1.16b, v10.16b + eor v0.16b, v0.16b, v16.16b + b .Lenc_sbox +.align 4 +.Lenc_loop: + ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x9], #64 + ldp q8, q9, [x9], #32 + eor v0.16b, v16.16b, v0.16b + ldr q10, [x9], #16 + eor v1.16b, v17.16b, v1.16b + ldr q16, [x9], #16 + eor v2.16b, v18.16b, v2.16b + eor v3.16b, v19.16b, v3.16b + eor v4.16b, v8.16b, v4.16b + eor v5.16b, v9.16b, v5.16b + eor v6.16b, v10.16b, v6.16b + eor v7.16b, v16.16b, v7.16b + tbl v0.16b, {v0.16b}, v28.16b + tbl v1.16b, {v1.16b}, v28.16b + tbl v2.16b, {v2.16b}, v28.16b + tbl v3.16b, {v3.16b}, v28.16b + tbl v4.16b, {v4.16b}, v28.16b + tbl v5.16b, {v5.16b}, v28.16b + tbl v6.16b, {v6.16b}, v28.16b + tbl v7.16b, {v7.16b}, v28.16b +.Lenc_sbox: + eor v5.16b, v5.16b, v6.16b + eor v3.16b, v3.16b, v0.16b + subs x10, x10, #1 + eor v2.16b, v2.16b, v1.16b + eor v5.16b, v5.16b, v0.16b + eor v8.16b, v3.16b, v7.16b + eor v6.16b, v6.16b, v2.16b + eor v7.16b, v7.16b, v5.16b + eor v8.16b, v8.16b, v4.16b + eor v3.16b, v6.16b, v3.16b + eor v4.16b, v4.16b, v5.16b + eor v6.16b, v1.16b, v5.16b + eor v2.16b, v2.16b, v7.16b + eor v1.16b, v8.16b, v1.16b + eor v8.16b, v7.16b, v4.16b + eor v9.16b, v3.16b, v0.16b + eor v10.16b, v7.16b, v6.16b + eor v16.16b, v5.16b, v3.16b + eor v17.16b, v6.16b, v2.16b + eor v18.16b, v5.16b, v1.16b + eor v19.16b, v2.16b, v4.16b + eor v20.16b, v1.16b, v0.16b + orr v21.16b, v8.16b, v9.16b + orr v22.16b, v10.16b, v16.16b + eor v23.16b, v8.16b, v17.16b + eor v24.16b, v9.16b, v18.16b + and v19.16b, v19.16b, v20.16b + orr v20.16b, v17.16b, v18.16b + and v8.16b, v8.16b, v9.16b + and v9.16b, v17.16b, v18.16b + and v17.16b, v23.16b, v24.16b + and v10.16b, v10.16b, v16.16b + eor v16.16b, v21.16b, v19.16b + eor v18.16b, v20.16b, v19.16b + and v19.16b, v2.16b, v1.16b + and v20.16b, v6.16b, v5.16b + eor v21.16b, v22.16b, v17.16b + eor v9.16b, v9.16b, v10.16b + eor v10.16b, v16.16b, v17.16b + eor v16.16b, v18.16b, v8.16b + and v17.16b, v4.16b, v0.16b + orr v18.16b, v7.16b, v3.16b + eor v21.16b, v21.16b, v8.16b + eor v8.16b, v9.16b, v8.16b + eor v9.16b, v10.16b, v19.16b + eor v10.16b, v3.16b, v0.16b + eor v16.16b, v16.16b, v17.16b + eor v17.16b, v5.16b, v1.16b + eor v19.16b, v21.16b, v20.16b + eor v20.16b, v8.16b, v18.16b + eor v8.16b, v8.16b, v18.16b + eor v18.16b, v7.16b, v4.16b + eor v21.16b, v9.16b, v16.16b + eor v22.16b, v6.16b, v2.16b + and v23.16b, v9.16b, v19.16b + eor v24.16b, v10.16b, v17.16b + eor v25.16b, v0.16b, v1.16b + eor v26.16b, v7.16b, v6.16b + eor v27.16b, v18.16b, v22.16b + eor v28.16b, v3.16b, v5.16b + eor v29.16b, v16.16b, v23.16b + eor v30.16b, v20.16b, v23.16b + eor v23.16b, v20.16b, v23.16b + eor v31.16b, v4.16b, v2.16b + bsl v29.16b, v19.16b, v20.16b + bsl v30.16b, v9.16b, v16.16b + bsl v8.16b, v29.16b, v23.16b + bsl v20.16b, v23.16b, v29.16b + eor v9.16b, v30.16b, v29.16b + and v5.16b, v5.16b, v30.16b + and v8.16b, v8.16b, v30.16b + and v1.16b, v1.16b, v29.16b + eor v16.16b, v19.16b, v20.16b + and v2.16b, v2.16b, v29.16b + eor v19.16b, v9.16b, v29.16b + and v17.16b, v17.16b, v9.16b + eor v8.16b, v8.16b, v21.16b + and v20.16b, v22.16b, v9.16b + eor v21.16b, v29.16b, v16.16b + eor v22.16b, v29.16b, v16.16b + and v23.16b, v25.16b, v16.16b + and v6.16b, v6.16b, v19.16b + eor v25.16b, v8.16b, v16.16b + eor v29.16b, v30.16b, v8.16b + and v4.16b, v21.16b, v4.16b + and v8.16b, v28.16b, v8.16b + and v0.16b, v22.16b, v0.16b + eor v21.16b, v23.16b, v1.16b + eor v22.16b, v9.16b, v25.16b + eor v9.16b, v9.16b, v25.16b + eor v23.16b, v25.16b, v16.16b + and v3.16b, v29.16b, v3.16b + and v24.16b, v24.16b, v25.16b + and v25.16b, v27.16b, v25.16b + and v10.16b, v22.16b, v10.16b + and v9.16b, v9.16b, v18.16b + eor v18.16b, v19.16b, v23.16b + and v19.16b, v26.16b, v23.16b + eor v3.16b, v5.16b, v3.16b + eor v17.16b, v17.16b, v24.16b + eor v10.16b, v24.16b, v10.16b + and v16.16b, v31.16b, v16.16b + eor v20.16b, v20.16b, v25.16b + eor v9.16b, v25.16b, v9.16b + eor v4.16b, v2.16b, v4.16b + and v7.16b, v18.16b, v7.16b + eor v18.16b, v19.16b, v6.16b + eor v5.16b, v8.16b, v5.16b + eor v0.16b, v1.16b, v0.16b + eor v1.16b, v21.16b, v10.16b + eor v8.16b, v3.16b, v17.16b + eor v2.16b, v16.16b, v2.16b + eor v3.16b, v6.16b, v7.16b + eor v6.16b, v18.16b, v9.16b + eor v4.16b, v4.16b, v20.16b + eor v10.16b, v5.16b, v10.16b + eor v0.16b, v0.16b, v17.16b + eor v9.16b, v2.16b, v9.16b + eor v3.16b, v3.16b, v20.16b + eor v7.16b, v6.16b, v1.16b + eor v5.16b, v8.16b, v4.16b + eor v6.16b, v10.16b, v1.16b + eor v2.16b, v4.16b, v0.16b + eor v4.16b, v3.16b, v10.16b + eor v9.16b, v9.16b, v7.16b + eor v3.16b, v0.16b, v5.16b + eor v0.16b, v1.16b, v4.16b + eor v1.16b, v4.16b, v8.16b + eor v4.16b, v9.16b, v5.16b + eor v6.16b, v6.16b, v3.16b + bcc .Lenc_done + ext v8.16b, v0.16b, v0.16b, #12 + ext v9.16b, v4.16b, v4.16b, #12 + ldr q28, [x11] + ext v10.16b, v6.16b, v6.16b, #12 + ext v16.16b, v1.16b, v1.16b, #12 + ext v17.16b, v3.16b, v3.16b, #12 + ext v18.16b, v7.16b, v7.16b, #12 + eor v0.16b, v0.16b, v8.16b + eor v4.16b, v4.16b, v9.16b + eor v6.16b, v6.16b, v10.16b + ext v19.16b, v2.16b, v2.16b, #12 + ext v20.16b, v5.16b, v5.16b, #12 + eor v1.16b, v1.16b, v16.16b + eor v3.16b, v3.16b, v17.16b + eor v7.16b, v7.16b, v18.16b + eor v2.16b, v2.16b, v19.16b + eor v16.16b, v16.16b, v0.16b + eor v5.16b, v5.16b, v20.16b + eor v17.16b, v17.16b, v6.16b + eor v10.16b, v10.16b, v4.16b + ext v0.16b, v0.16b, v0.16b, #8 + eor v9.16b, v9.16b, v1.16b + ext v1.16b, v1.16b, v1.16b, #8 + eor v8.16b, v8.16b, v5.16b + eor v16.16b, v16.16b, v5.16b + eor v18.16b, v18.16b, v3.16b + eor v19.16b, v19.16b, v7.16b + ext v3.16b, v3.16b, v3.16b, #8 + ext v7.16b, v7.16b, v7.16b, #8 + eor v20.16b, v20.16b, v2.16b + ext v6.16b, v6.16b, v6.16b, #8 + ext v21.16b, v5.16b, v5.16b, #8 + eor v17.16b, v17.16b, v5.16b + ext v2.16b, v2.16b, v2.16b, #8 + eor v10.16b, v10.16b, v5.16b + ext v22.16b, v4.16b, v4.16b, #8 + eor v0.16b, v0.16b, v8.16b + eor v1.16b, v1.16b, v16.16b + eor v5.16b, v7.16b, v18.16b + eor v4.16b, v3.16b, v17.16b + eor v3.16b, v6.16b, v10.16b + eor v7.16b, v21.16b, v20.16b + eor v6.16b, v2.16b, v19.16b + eor v2.16b, v22.16b, v9.16b + bne .Lenc_loop + ldr q28, [x11, #16]! // load from .LSRM0 on last round (x10 == 0) + b .Lenc_loop +.align 4 +.Lenc_done: + ushr v8.2d, v0.2d, #1 + movi v9.16b, #0x55 + ldr q10, [x9] + ushr v16.2d, v3.2d, #1 + movi v17.16b, #0x33 + ushr v18.2d, v4.2d, #1 + movi v19.16b, #0x0f + eor v8.16b, v8.16b, v1.16b + ushr v20.2d, v2.2d, #1 + eor v16.16b, v16.16b, v7.16b + eor v18.16b, v18.16b, v6.16b + and v8.16b, v8.16b, v9.16b + eor v20.16b, v20.16b, v5.16b + and v16.16b, v16.16b, v9.16b + and v18.16b, v18.16b, v9.16b + shl v21.2d, v8.2d, #1 + eor v1.16b, v1.16b, v8.16b + and v8.16b, v20.16b, v9.16b + eor v7.16b, v7.16b, v16.16b + shl v9.2d, v16.2d, #1 + eor v6.16b, v6.16b, v18.16b + shl v16.2d, v18.2d, #1 + eor v0.16b, v0.16b, v21.16b + shl v18.2d, v8.2d, #1 + eor v5.16b, v5.16b, v8.16b + eor v3.16b, v3.16b, v9.16b + eor v4.16b, v4.16b, v16.16b + ushr v8.2d, v1.2d, #2 + eor v2.16b, v2.16b, v18.16b + ushr v9.2d, v0.2d, #2 + ushr v16.2d, v7.2d, #2 + ushr v18.2d, v3.2d, #2 + eor v8.16b, v8.16b, v6.16b + eor v9.16b, v9.16b, v4.16b + eor v16.16b, v16.16b, v5.16b + eor v18.16b, v18.16b, v2.16b + and v8.16b, v8.16b, v17.16b + and v9.16b, v9.16b, v17.16b + and v16.16b, v16.16b, v17.16b + and v17.16b, v18.16b, v17.16b + eor v6.16b, v6.16b, v8.16b + shl v8.2d, v8.2d, #2 + eor v4.16b, v4.16b, v9.16b + shl v9.2d, v9.2d, #2 + eor v5.16b, v5.16b, v16.16b + shl v16.2d, v16.2d, #2 + eor v2.16b, v2.16b, v17.16b + shl v17.2d, v17.2d, #2 + eor v1.16b, v1.16b, v8.16b + eor v0.16b, v0.16b, v9.16b + eor v7.16b, v7.16b, v16.16b + eor v3.16b, v3.16b, v17.16b + ushr v8.2d, v6.2d, #4 + ushr v9.2d, v4.2d, #4 + ushr v16.2d, v1.2d, #4 + ushr v17.2d, v0.2d, #4 + eor v8.16b, v8.16b, v5.16b + eor v9.16b, v9.16b, v2.16b + eor v16.16b, v16.16b, v7.16b + eor v17.16b, v17.16b, v3.16b + and v8.16b, v8.16b, v19.16b + and v9.16b, v9.16b, v19.16b + and v16.16b, v16.16b, v19.16b + and v17.16b, v17.16b, v19.16b + eor v5.16b, v5.16b, v8.16b + shl v8.2d, v8.2d, #4 + eor v2.16b, v2.16b, v9.16b + shl v9.2d, v9.2d, #4 + eor v7.16b, v7.16b, v16.16b + shl v16.2d, v16.2d, #4 + eor v3.16b, v3.16b, v17.16b + shl v17.2d, v17.2d, #4 + eor v6.16b, v6.16b, v8.16b + eor v4.16b, v4.16b, v9.16b + eor v7.16b, v7.16b, v10.16b + eor v1.16b, v1.16b, v16.16b + eor v3.16b, v3.16b, v10.16b + eor v0.16b, v0.16b, v17.16b + eor v6.16b, v6.16b, v10.16b + eor v4.16b, v4.16b, v10.16b + eor v2.16b, v2.16b, v10.16b + eor v5.16b, v5.16b, v10.16b + eor v1.16b, v1.16b, v10.16b + eor v0.16b, v0.16b, v10.16b + ret +.size _bsaes_encrypt8,.-_bsaes_encrypt8 + +.type _bsaes_key_convert,%function +.align 4 +// On entry: +// x9 -> input key (big-endian) +// x10 = number of rounds +// x17 -> output key (native endianness) +// On exit: +// x9, x10 corrupted +// x11 -> .LM0_bigendian +// x17 -> last quadword of output key +// other general-purpose registers preserved +// v2-v6 preserved +// v7.16b[] = 0x63 +// v8-v14 preserved +// v15 = last round key (converted to native endianness) +// other SIMD registers corrupted +_bsaes_key_convert: +#ifdef __AARCH64EL__ + adrp x11, .LM0_littleendian + add x11, x11, #:lo12:.LM0_littleendian +#else + adrp x11, .LM0_bigendian + add x11, x11, #:lo12:.LM0_bigendian +#endif + ldr q0, [x9], #16 // load round 0 key + ldr q1, [x11] // .LM0 + ldr q15, [x9], #16 // load round 1 key + + movi v7.16b, #0x63 // compose .L63 + movi v16.16b, #0x01 // bit masks + movi v17.16b, #0x02 + movi v18.16b, #0x04 + movi v19.16b, #0x08 + movi v20.16b, #0x10 + movi v21.16b, #0x20 + movi v22.16b, #0x40 + movi v23.16b, #0x80 + +#ifdef __AARCH64EL__ + rev32 v0.16b, v0.16b +#endif + sub x10, x10, #1 + str q0, [x17], #16 // save round 0 key + +.align 4 +.Lkey_loop: + tbl v0.16b, {v15.16b}, v1.16b + ldr q15, [x9], #16 // load next round key + + eor v0.16b, v0.16b, v7.16b + cmtst v24.16b, v0.16b, v16.16b + cmtst v25.16b, v0.16b, v17.16b + cmtst v26.16b, v0.16b, v18.16b + cmtst v27.16b, v0.16b, v19.16b + cmtst v28.16b, v0.16b, v20.16b + cmtst v29.16b, v0.16b, v21.16b + cmtst v30.16b, v0.16b, v22.16b + cmtst v31.16b, v0.16b, v23.16b + sub x10, x10, #1 + st1 {v24.16b-v27.16b}, [x17], #64 // write bit-sliced round key + st1 {v28.16b-v31.16b}, [x17], #64 + cbnz x10, .Lkey_loop + + // don't save last round key +#ifdef __AARCH64EL__ + rev32 v15.16b, v15.16b + adrp x11, .LM0_bigendian + add x11, x11, #:lo12:.LM0_bigendian +#endif + ret +.size _bsaes_key_convert,.-_bsaes_key_convert + +.globl ossl_bsaes_cbc_encrypt +.type ossl_bsaes_cbc_encrypt,%function +.align 4 +// On entry: +// x0 -> input ciphertext +// x1 -> output plaintext +// x2 = size of ciphertext and plaintext in bytes (assumed a multiple of 16) +// x3 -> key +// x4 -> 128-bit initialisation vector (or preceding 128-bit block of ciphertext if continuing after an earlier call) +// w5 must be == 0 +// On exit: +// Output plaintext filled in +// Initialisation vector overwritten with last quadword of ciphertext +// No output registers, usual AAPCS64 register preservation +ossl_bsaes_cbc_encrypt: + AARCH64_VALID_CALL_TARGET + cmp x2, #128 + bhs .Lcbc_do_bsaes + b AES_cbc_encrypt +.Lcbc_do_bsaes: + + // it is up to the caller to make sure we are called with enc == 0 + + stp x29, x30, [sp, #-48]! + stp d8, d9, [sp, #16] + stp d10, d15, [sp, #32] + lsr x2, x2, #4 // len in 16 byte blocks + + ldr w15, [x3, #240] // get # of rounds + mov x14, sp + + // allocate the key schedule on the stack + add x17, sp, #96 + sub x17, x17, x15, lsl #7 // 128 bytes per inner round key, less 96 bytes + + // populate the key schedule + mov x9, x3 // pass key + mov x10, x15 // pass # of rounds + mov sp, x17 // sp is sp + bl _bsaes_key_convert + ldr q6, [sp] + str q15, [x17] // save last round key + eor v6.16b, v6.16b, v7.16b // fix up round 0 key (by XORing with 0x63) + str q6, [sp] + + ldr q15, [x4] // load IV + b .Lcbc_dec_loop + +.align 4 +.Lcbc_dec_loop: + subs x2, x2, #0x8 + bmi .Lcbc_dec_loop_finish + + ldr q0, [x0], #16 // load input + mov x9, sp // pass the key + ldr q1, [x0], #16 + mov x10, x15 + ldr q2, [x0], #16 + ldr q3, [x0], #16 + ldr q4, [x0], #16 + ldr q5, [x0], #16 + ldr q6, [x0], #16 + ldr q7, [x0], #-7*16 + + bl _bsaes_decrypt8 + + ldr q16, [x0], #16 // reload input + eor v0.16b, v0.16b, v15.16b // ^= IV + eor v1.16b, v1.16b, v16.16b + str q0, [x1], #16 // write output + ldr q0, [x0], #16 + str q1, [x1], #16 + ldr q1, [x0], #16 + eor v1.16b, v4.16b, v1.16b + ldr q4, [x0], #16 + eor v2.16b, v2.16b, v4.16b + eor v0.16b, v6.16b, v0.16b + ldr q4, [x0], #16 + str q0, [x1], #16 + str q1, [x1], #16 + eor v0.16b, v7.16b, v4.16b + ldr q1, [x0], #16 + str q2, [x1], #16 + ldr q2, [x0], #16 + ldr q15, [x0], #16 + str q0, [x1], #16 + eor v0.16b, v5.16b, v2.16b + eor v1.16b, v3.16b, v1.16b + str q1, [x1], #16 + str q0, [x1], #16 + + b .Lcbc_dec_loop + +.Lcbc_dec_loop_finish: + adds x2, x2, #8 + beq .Lcbc_dec_done + + ldr q0, [x0], #16 // load input + cmp x2, #2 + blo .Lcbc_dec_one + ldr q1, [x0], #16 + mov x9, sp // pass the key + mov x10, x15 + beq .Lcbc_dec_two + ldr q2, [x0], #16 + cmp x2, #4 + blo .Lcbc_dec_three + ldr q3, [x0], #16 + beq .Lcbc_dec_four + ldr q4, [x0], #16 + cmp x2, #6 + blo .Lcbc_dec_five + ldr q5, [x0], #16 + beq .Lcbc_dec_six + ldr q6, [x0], #-6*16 + + bl _bsaes_decrypt8 + + ldr q5, [x0], #16 // reload input + eor v0.16b, v0.16b, v15.16b // ^= IV + ldr q8, [x0], #16 + ldr q9, [x0], #16 + ldr q10, [x0], #16 + str q0, [x1], #16 // write output + ldr q0, [x0], #16 + eor v1.16b, v1.16b, v5.16b + ldr q5, [x0], #16 + eor v6.16b, v6.16b, v8.16b + ldr q15, [x0] + eor v4.16b, v4.16b, v9.16b + eor v2.16b, v2.16b, v10.16b + str q1, [x1], #16 + eor v0.16b, v7.16b, v0.16b + str q6, [x1], #16 + eor v1.16b, v3.16b, v5.16b + str q4, [x1], #16 + str q2, [x1], #16 + str q0, [x1], #16 + str q1, [x1] + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_six: + sub x0, x0, #0x60 + bl _bsaes_decrypt8 + ldr q3, [x0], #16 // reload input + eor v0.16b, v0.16b, v15.16b // ^= IV + ldr q5, [x0], #16 + ldr q8, [x0], #16 + ldr q9, [x0], #16 + str q0, [x1], #16 // write output + ldr q0, [x0], #16 + eor v1.16b, v1.16b, v3.16b + ldr q15, [x0] + eor v3.16b, v6.16b, v5.16b + eor v4.16b, v4.16b, v8.16b + eor v2.16b, v2.16b, v9.16b + str q1, [x1], #16 + eor v0.16b, v7.16b, v0.16b + str q3, [x1], #16 + str q4, [x1], #16 + str q2, [x1], #16 + str q0, [x1] + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_five: + sub x0, x0, #0x50 + bl _bsaes_decrypt8 + ldr q3, [x0], #16 // reload input + eor v0.16b, v0.16b, v15.16b // ^= IV + ldr q5, [x0], #16 + ldr q7, [x0], #16 + ldr q8, [x0], #16 + str q0, [x1], #16 // write output + ldr q15, [x0] + eor v0.16b, v1.16b, v3.16b + eor v1.16b, v6.16b, v5.16b + eor v3.16b, v4.16b, v7.16b + str q0, [x1], #16 + eor v0.16b, v2.16b, v8.16b + str q1, [x1], #16 + str q3, [x1], #16 + str q0, [x1] + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_four: + sub x0, x0, #0x40 + bl _bsaes_decrypt8 + ldr q2, [x0], #16 // reload input + eor v0.16b, v0.16b, v15.16b // ^= IV + ldr q3, [x0], #16 + ldr q5, [x0], #16 + str q0, [x1], #16 // write output + ldr q15, [x0] + eor v0.16b, v1.16b, v2.16b + eor v1.16b, v6.16b, v3.16b + eor v2.16b, v4.16b, v5.16b + str q0, [x1], #16 + str q1, [x1], #16 + str q2, [x1] + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_three: + sub x0, x0, #0x30 + bl _bsaes_decrypt8 + ldr q2, [x0], #16 // reload input + eor v0.16b, v0.16b, v15.16b // ^= IV + ldr q3, [x0], #16 + ldr q15, [x0] + str q0, [x1], #16 // write output + eor v0.16b, v1.16b, v2.16b + eor v1.16b, v6.16b, v3.16b + str q0, [x1], #16 + str q1, [x1] + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_two: + sub x0, x0, #0x20 + bl _bsaes_decrypt8 + ldr q2, [x0], #16 // reload input + eor v0.16b, v0.16b, v15.16b // ^= IV + ldr q15, [x0] + str q0, [x1], #16 // write output + eor v0.16b, v1.16b, v2.16b + str q0, [x1] + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_one: + sub x0, x0, #0x10 + stp x1, x4, [sp, #-32]! + str x14, [sp, #16] + mov v8.16b, v15.16b + mov v15.16b, v0.16b + mov x2, x3 + bl AES_decrypt + ldr x14, [sp, #16] + ldp x1, x4, [sp], #32 + ldr q0, [x1] // load result + eor v0.16b, v0.16b, v8.16b // ^= IV + str q0, [x1] // write output + +.align 4 +.Lcbc_dec_done: + movi v0.16b, #0 + movi v1.16b, #0 +.Lcbc_dec_bzero:// wipe key schedule [if any] + stp q0, q1, [sp], #32 + cmp sp, x14 + bne .Lcbc_dec_bzero + str q15, [x4] // return IV + ldp d8, d9, [sp, #16] + ldp d10, d15, [sp, #32] + ldp x29, x30, [sp], #48 + ret +.size ossl_bsaes_cbc_encrypt,.-ossl_bsaes_cbc_encrypt + +.globl ossl_bsaes_ctr32_encrypt_blocks +.type ossl_bsaes_ctr32_encrypt_blocks,%function +.align 4 +// On entry: +// x0 -> input text (whole 16-byte blocks) +// x1 -> output text (whole 16-byte blocks) +// x2 = number of 16-byte blocks to encrypt/decrypt (> 0) +// x3 -> key +// x4 -> initial value of 128-bit counter (stored big-endian) which increments, modulo 2^32, for each block +// On exit: +// Output text filled in +// No output registers, usual AAPCS64 register preservation +ossl_bsaes_ctr32_encrypt_blocks: + AARCH64_VALID_CALL_TARGET + cmp x2, #8 // use plain AES for + blo .Lctr_enc_short // small sizes + + stp x29, x30, [sp, #-80]! + stp d8, d9, [sp, #16] + stp d10, d11, [sp, #32] + stp d12, d13, [sp, #48] + stp d14, d15, [sp, #64] + + ldr w15, [x3, #240] // get # of rounds + mov x14, sp + + // allocate the key schedule on the stack + add x17, sp, #96 + sub x17, x17, x15, lsl #7 // 128 bytes per inner round key, less 96 bytes + + // populate the key schedule + mov x9, x3 // pass key + mov x10, x15 // pass # of rounds + mov sp, x17 // sp is sp + bl _bsaes_key_convert + eor v7.16b, v7.16b, v15.16b // fix up last round key + str q7, [x17] // save last round key + + ldr q0, [x4] // load counter + add x13, x11, #.LREVM0SR-.LM0_bigendian + ldr q4, [sp] // load round0 key + + movi v8.4s, #1 // compose 1<<96 + movi v9.16b, #0 + rev32 v15.16b, v0.16b + rev32 v0.16b, v0.16b + ext v11.16b, v9.16b, v8.16b, #4 + rev32 v4.16b, v4.16b + add v12.4s, v11.4s, v11.4s // compose 2<<96 + str q4, [sp] // save adjusted round0 key + add v13.4s, v11.4s, v12.4s // compose 3<<96 + add v14.4s, v12.4s, v12.4s // compose 4<<96 + b .Lctr_enc_loop + +.align 4 +.Lctr_enc_loop: + // Intermix prologue from _bsaes_encrypt8 to use the opportunity + // to flip byte order in 32-bit counter + + add v1.4s, v15.4s, v11.4s // +1 + add x9, sp, #0x10 // pass next round key + add v2.4s, v15.4s, v12.4s // +2 + ldr q9, [x13] // .LREVM0SR + ldr q8, [sp] // load round0 key + add v3.4s, v15.4s, v13.4s // +3 + mov x10, x15 // pass rounds + sub x11, x13, #.LREVM0SR-.LSR // pass constants + add v6.4s, v2.4s, v14.4s + add v4.4s, v15.4s, v14.4s // +4 + add v7.4s, v3.4s, v14.4s + add v15.4s, v4.4s, v14.4s // next counter + add v5.4s, v1.4s, v14.4s + + bl _bsaes_encrypt8_alt + + subs x2, x2, #8 + blo .Lctr_enc_loop_done + + ldr q16, [x0], #16 + ldr q17, [x0], #16 + eor v1.16b, v1.16b, v17.16b + ldr q17, [x0], #16 + eor v0.16b, v0.16b, v16.16b + eor v4.16b, v4.16b, v17.16b + str q0, [x1], #16 + ldr q16, [x0], #16 + str q1, [x1], #16 + mov v0.16b, v15.16b + str q4, [x1], #16 + ldr q1, [x0], #16 + eor v4.16b, v6.16b, v16.16b + eor v1.16b, v3.16b, v1.16b + ldr q3, [x0], #16 + eor v3.16b, v7.16b, v3.16b + ldr q6, [x0], #16 + eor v2.16b, v2.16b, v6.16b + ldr q6, [x0], #16 + eor v5.16b, v5.16b, v6.16b + str q4, [x1], #16 + str q1, [x1], #16 + str q3, [x1], #16 + str q2, [x1], #16 + str q5, [x1], #16 + + bne .Lctr_enc_loop + b .Lctr_enc_done + +.align 4 +.Lctr_enc_loop_done: + add x2, x2, #8 + ldr q16, [x0], #16 // load input + eor v0.16b, v0.16b, v16.16b + str q0, [x1], #16 // write output + cmp x2, #2 + blo .Lctr_enc_done + ldr q17, [x0], #16 + eor v1.16b, v1.16b, v17.16b + str q1, [x1], #16 + beq .Lctr_enc_done + ldr q18, [x0], #16 + eor v4.16b, v4.16b, v18.16b + str q4, [x1], #16 + cmp x2, #4 + blo .Lctr_enc_done + ldr q19, [x0], #16 + eor v6.16b, v6.16b, v19.16b + str q6, [x1], #16 + beq .Lctr_enc_done + ldr q20, [x0], #16 + eor v3.16b, v3.16b, v20.16b + str q3, [x1], #16 + cmp x2, #6 + blo .Lctr_enc_done + ldr q21, [x0], #16 + eor v7.16b, v7.16b, v21.16b + str q7, [x1], #16 + beq .Lctr_enc_done + ldr q22, [x0] + eor v2.16b, v2.16b, v22.16b + str q2, [x1], #16 + +.Lctr_enc_done: + movi v0.16b, #0 + movi v1.16b, #0 +.Lctr_enc_bzero: // wipe key schedule [if any] + stp q0, q1, [sp], #32 + cmp sp, x14 + bne .Lctr_enc_bzero + + ldp d8, d9, [sp, #16] + ldp d10, d11, [sp, #32] + ldp d12, d13, [sp, #48] + ldp d14, d15, [sp, #64] + ldp x29, x30, [sp], #80 + ret + +.Lctr_enc_short: + stp x29, x30, [sp, #-96]! + stp x19, x20, [sp, #16] + stp x21, x22, [sp, #32] + str x23, [sp, #48] + + mov x19, x0 // copy arguments + mov x20, x1 + mov x21, x2 + mov x22, x3 + ldr w23, [x4, #12] // load counter .LSW + ldr q1, [x4] // load whole counter value +#ifdef __AARCH64EL__ + rev w23, w23 +#endif + str q1, [sp, #80] // copy counter value + +.Lctr_enc_short_loop: + add x0, sp, #80 // input counter value + add x1, sp, #64 // output on the stack + mov x2, x22 // key + + bl AES_encrypt + + ldr q0, [x19], #16 // load input + ldr q1, [sp, #64] // load encrypted counter + add x23, x23, #1 +#ifdef __AARCH64EL__ + rev w0, w23 + str w0, [sp, #80+12] // next counter value +#else + str w23, [sp, #80+12] // next counter value +#endif + eor v0.16b, v0.16b, v1.16b + str q0, [x20], #16 // store output + subs x21, x21, #1 + bne .Lctr_enc_short_loop + + movi v0.16b, #0 + movi v1.16b, #0 + stp q0, q1, [sp, #64] + + ldr x23, [sp, #48] + ldp x21, x22, [sp, #32] + ldp x19, x20, [sp, #16] + ldp x29, x30, [sp], #96 + ret +.size ossl_bsaes_ctr32_encrypt_blocks,.-ossl_bsaes_ctr32_encrypt_blocks + +.globl ossl_bsaes_xts_encrypt +.type ossl_bsaes_xts_encrypt,%function +.align 4 +// On entry: +// x0 -> input plaintext +// x1 -> output ciphertext +// x2 -> length of text in bytes (must be at least 16) +// x3 -> key1 (used to encrypt the XORed plaintext blocks) +// x4 -> key2 (used to encrypt the initial vector to yield the initial tweak) +// x5 -> 16-byte initial vector (typically, sector number) +// On exit: +// Output ciphertext filled in +// No output registers, usual AAPCS64 register preservation +ossl_bsaes_xts_encrypt: + AARCH64_VALID_CALL_TARGET + // Stack layout: + // sp -> + // nrounds*128-96 bytes: key schedule + // x19 -> + // 16 bytes: frame record + // 4*16 bytes: tweak storage across _bsaes_encrypt8 + // 6*8 bytes: storage for 5 callee-saved general-purpose registers + // 8*8 bytes: storage for 8 callee-saved SIMD registers + stp x29, x30, [sp, #-192]! + stp x19, x20, [sp, #80] + stp x21, x22, [sp, #96] + str x23, [sp, #112] + stp d8, d9, [sp, #128] + stp d10, d11, [sp, #144] + stp d12, d13, [sp, #160] + stp d14, d15, [sp, #176] + + mov x19, sp + mov x20, x0 + mov x21, x1 + mov x22, x2 + mov x23, x3 + + // generate initial tweak + sub sp, sp, #16 + mov x0, x5 // iv[] + mov x1, sp + mov x2, x4 // key2 + bl AES_encrypt + ldr q11, [sp], #16 + + ldr w1, [x23, #240] // get # of rounds + // allocate the key schedule on the stack + add x17, sp, #96 + sub x17, x17, x1, lsl #7 // 128 bytes per inner round key, less 96 bytes + + // populate the key schedule + mov x9, x23 // pass key + mov x10, x1 // pass # of rounds + mov sp, x17 + bl _bsaes_key_convert + eor v15.16b, v15.16b, v7.16b // fix up last round key + str q15, [x17] // save last round key + + subs x22, x22, #0x80 + blo .Lxts_enc_short + b .Lxts_enc_loop + +.align 4 +.Lxts_enc_loop: + ldr q8, .Lxts_magic + mov x10, x1 // pass rounds + add x2, x19, #16 + ldr q0, [x20], #16 + sshr v1.2d, v11.2d, #63 + mov x9, sp // pass key schedule + ldr q6, .Lxts_magic+16 + add v2.2d, v11.2d, v11.2d + cmtst v3.2d, v11.2d, v6.2d + and v1.16b, v1.16b, v8.16b + ext v1.16b, v1.16b, v1.16b, #8 + and v3.16b, v3.16b, v8.16b + ldr q4, [x20], #16 + eor v12.16b, v2.16b, v1.16b + eor v1.16b, v4.16b, v12.16b + eor v0.16b, v0.16b, v11.16b + cmtst v2.2d, v12.2d, v6.2d + add v4.2d, v12.2d, v12.2d + add x0, x19, #16 + ext v3.16b, v3.16b, v3.16b, #8 + and v2.16b, v2.16b, v8.16b + eor v13.16b, v4.16b, v3.16b + ldr q3, [x20], #16 + ext v4.16b, v2.16b, v2.16b, #8 + eor v2.16b, v3.16b, v13.16b + ldr q3, [x20], #16 + add v5.2d, v13.2d, v13.2d + cmtst v7.2d, v13.2d, v6.2d + and v7.16b, v7.16b, v8.16b + ldr q9, [x20], #16 + ext v7.16b, v7.16b, v7.16b, #8 + ldr q10, [x20], #16 + eor v14.16b, v5.16b, v4.16b + ldr q16, [x20], #16 + add v4.2d, v14.2d, v14.2d + eor v3.16b, v3.16b, v14.16b + eor v15.16b, v4.16b, v7.16b + add v5.2d, v15.2d, v15.2d + ldr q7, [x20], #16 + cmtst v4.2d, v14.2d, v6.2d + and v17.16b, v4.16b, v8.16b + cmtst v18.2d, v15.2d, v6.2d + eor v4.16b, v9.16b, v15.16b + ext v9.16b, v17.16b, v17.16b, #8 + eor v9.16b, v5.16b, v9.16b + add v17.2d, v9.2d, v9.2d + and v18.16b, v18.16b, v8.16b + eor v5.16b, v10.16b, v9.16b + str q9, [x2], #16 + ext v10.16b, v18.16b, v18.16b, #8 + cmtst v9.2d, v9.2d, v6.2d + and v9.16b, v9.16b, v8.16b + eor v10.16b, v17.16b, v10.16b + cmtst v17.2d, v10.2d, v6.2d + eor v6.16b, v16.16b, v10.16b + str q10, [x2], #16 + ext v9.16b, v9.16b, v9.16b, #8 + add v10.2d, v10.2d, v10.2d + eor v9.16b, v10.16b, v9.16b + str q9, [x2], #16 + eor v7.16b, v7.16b, v9.16b + add v9.2d, v9.2d, v9.2d + and v8.16b, v17.16b, v8.16b + ext v8.16b, v8.16b, v8.16b, #8 + eor v8.16b, v9.16b, v8.16b + str q8, [x2] // next round tweak + + bl _bsaes_encrypt8 + + ldr q8, [x0], #16 + eor v0.16b, v0.16b, v11.16b + eor v1.16b, v1.16b, v12.16b + ldr q9, [x0], #16 + eor v4.16b, v4.16b, v13.16b + eor v6.16b, v6.16b, v14.16b + ldr q10, [x0], #16 + eor v3.16b, v3.16b, v15.16b + subs x22, x22, #0x80 + str q0, [x21], #16 + ldr q11, [x0] // next round tweak + str q1, [x21], #16 + eor v0.16b, v7.16b, v8.16b + eor v1.16b, v2.16b, v9.16b + str q4, [x21], #16 + eor v2.16b, v5.16b, v10.16b + str q6, [x21], #16 + str q3, [x21], #16 + str q0, [x21], #16 + str q1, [x21], #16 + str q2, [x21], #16 + bpl .Lxts_enc_loop + +.Lxts_enc_short: + adds x22, x22, #0x70 + bmi .Lxts_enc_done + + ldr q8, .Lxts_magic + sshr v1.2d, v11.2d, #63 + add v2.2d, v11.2d, v11.2d + ldr q9, .Lxts_magic+16 + subs x22, x22, #0x10 + ldr q0, [x20], #16 + and v1.16b, v1.16b, v8.16b + cmtst v3.2d, v11.2d, v9.2d + ext v1.16b, v1.16b, v1.16b, #8 + and v3.16b, v3.16b, v8.16b + eor v12.16b, v2.16b, v1.16b + ext v1.16b, v3.16b, v3.16b, #8 + add v2.2d, v12.2d, v12.2d + cmtst v3.2d, v12.2d, v9.2d + eor v13.16b, v2.16b, v1.16b + and v22.16b, v3.16b, v8.16b + bmi .Lxts_enc_1 + + ext v2.16b, v22.16b, v22.16b, #8 + add v3.2d, v13.2d, v13.2d + ldr q1, [x20], #16 + cmtst v4.2d, v13.2d, v9.2d + subs x22, x22, #0x10 + eor v14.16b, v3.16b, v2.16b + and v23.16b, v4.16b, v8.16b + bmi .Lxts_enc_2 + + ext v3.16b, v23.16b, v23.16b, #8 + add v4.2d, v14.2d, v14.2d + ldr q2, [x20], #16 + cmtst v5.2d, v14.2d, v9.2d + eor v0.16b, v0.16b, v11.16b + subs x22, x22, #0x10 + eor v15.16b, v4.16b, v3.16b + and v24.16b, v5.16b, v8.16b + bmi .Lxts_enc_3 + + ext v4.16b, v24.16b, v24.16b, #8 + add v5.2d, v15.2d, v15.2d + ldr q3, [x20], #16 + cmtst v6.2d, v15.2d, v9.2d + eor v1.16b, v1.16b, v12.16b + subs x22, x22, #0x10 + eor v16.16b, v5.16b, v4.16b + and v25.16b, v6.16b, v8.16b + bmi .Lxts_enc_4 + + ext v5.16b, v25.16b, v25.16b, #8 + add v6.2d, v16.2d, v16.2d + add x0, x19, #16 + cmtst v7.2d, v16.2d, v9.2d + ldr q4, [x20], #16 + eor v2.16b, v2.16b, v13.16b + str q16, [x0], #16 + subs x22, x22, #0x10 + eor v17.16b, v6.16b, v5.16b + and v26.16b, v7.16b, v8.16b + bmi .Lxts_enc_5 + + ext v7.16b, v26.16b, v26.16b, #8 + add v18.2d, v17.2d, v17.2d + ldr q5, [x20], #16 + eor v3.16b, v3.16b, v14.16b + str q17, [x0], #16 + subs x22, x22, #0x10 + eor v18.16b, v18.16b, v7.16b + bmi .Lxts_enc_6 + + ldr q6, [x20], #16 + eor v4.16b, v4.16b, v15.16b + eor v5.16b, v5.16b, v16.16b + str q18, [x0] // next round tweak + mov x9, sp // pass key schedule + mov x10, x1 + add x0, x19, #16 + sub x22, x22, #0x10 + eor v6.16b, v6.16b, v17.16b + + bl _bsaes_encrypt8 + + ldr q16, [x0], #16 + eor v0.16b, v0.16b, v11.16b + eor v1.16b, v1.16b, v12.16b + ldr q17, [x0], #16 + eor v4.16b, v4.16b, v13.16b + eor v6.16b, v6.16b, v14.16b + eor v3.16b, v3.16b, v15.16b + ldr q11, [x0] // next round tweak + str q0, [x21], #16 + str q1, [x21], #16 + eor v0.16b, v7.16b, v16.16b + eor v1.16b, v2.16b, v17.16b + str q4, [x21], #16 + str q6, [x21], #16 + str q3, [x21], #16 + str q0, [x21], #16 + str q1, [x21], #16 + b .Lxts_enc_done + +.align 4 +.Lxts_enc_6: + eor v4.16b, v4.16b, v15.16b + eor v5.16b, v5.16b, v16.16b + mov x9, sp // pass key schedule + mov x10, x1 // pass rounds + add x0, x19, #16 + + bl _bsaes_encrypt8 + + ldr q16, [x0], #16 + eor v0.16b, v0.16b, v11.16b + eor v1.16b, v1.16b, v12.16b + eor v4.16b, v4.16b, v13.16b + eor v6.16b, v6.16b, v14.16b + ldr q11, [x0] // next round tweak + eor v3.16b, v3.16b, v15.16b + str q0, [x21], #16 + str q1, [x21], #16 + eor v0.16b, v7.16b, v16.16b + str q4, [x21], #16 + str q6, [x21], #16 + str q3, [x21], #16 + str q0, [x21], #16 + b .Lxts_enc_done + +.align 4 +.Lxts_enc_5: + eor v3.16b, v3.16b, v14.16b + eor v4.16b, v4.16b, v15.16b + mov x9, sp // pass key schedule + mov x10, x1 // pass rounds + add x0, x19, #16 + + bl _bsaes_encrypt8 + + eor v0.16b, v0.16b, v11.16b + eor v1.16b, v1.16b, v12.16b + ldr q11, [x0] // next round tweak + eor v4.16b, v4.16b, v13.16b + eor v6.16b, v6.16b, v14.16b + eor v3.16b, v3.16b, v15.16b + str q0, [x21], #16 + str q1, [x21], #16 + str q4, [x21], #16 + str q6, [x21], #16 + str q3, [x21], #16 + b .Lxts_enc_done + +.align 4 +.Lxts_enc_4: + eor v2.16b, v2.16b, v13.16b + eor v3.16b, v3.16b, v14.16b + mov x9, sp // pass key schedule + mov x10, x1 // pass rounds + add x0, x19, #16 + + bl _bsaes_encrypt8 + + eor v0.16b, v0.16b, v11.16b + eor v1.16b, v1.16b, v12.16b + eor v4.16b, v4.16b, v13.16b + eor v6.16b, v6.16b, v14.16b + mov v11.16b, v15.16b // next round tweak + str q0, [x21], #16 + str q1, [x21], #16 + str q4, [x21], #16 + str q6, [x21], #16 + b .Lxts_enc_done + +.align 4 +.Lxts_enc_3: + eor v1.16b, v1.16b, v12.16b + eor v2.16b, v2.16b, v13.16b + mov x9, sp // pass key schedule + mov x10, x1 // pass rounds + add x0, x19, #16 + + bl _bsaes_encrypt8 + + eor v0.16b, v0.16b, v11.16b + eor v1.16b, v1.16b, v12.16b + eor v4.16b, v4.16b, v13.16b + mov v11.16b, v14.16b // next round tweak + str q0, [x21], #16 + str q1, [x21], #16 + str q4, [x21], #16 + b .Lxts_enc_done + +.align 4 +.Lxts_enc_2: + eor v0.16b, v0.16b, v11.16b + eor v1.16b, v1.16b, v12.16b + mov x9, sp // pass key schedule + mov x10, x1 // pass rounds + add x0, x19, #16 + + bl _bsaes_encrypt8 + + eor v0.16b, v0.16b, v11.16b + eor v1.16b, v1.16b, v12.16b + mov v11.16b, v13.16b // next round tweak + str q0, [x21], #16 + str q1, [x21], #16 + b .Lxts_enc_done + +.align 4 +.Lxts_enc_1: + eor v0.16b, v0.16b, v11.16b + sub x0, sp, #16 + sub x1, sp, #16 + mov x2, x23 + mov v13.d[0], v11.d[1] // just in case AES_encrypt corrupts top half of callee-saved SIMD registers + mov v14.d[0], v12.d[1] + str q0, [sp, #-16]! + + bl AES_encrypt + + ldr q0, [sp], #16 + trn1 v13.2d, v11.2d, v13.2d + trn1 v11.2d, v12.2d, v14.2d // next round tweak + eor v0.16b, v0.16b, v13.16b + str q0, [x21], #16 + +.Lxts_enc_done: + adds x22, x22, #0x10 + beq .Lxts_enc_ret + + sub x6, x21, #0x10 + // Penultimate plaintext block produces final ciphertext part-block + // plus remaining part of final plaintext block. Move ciphertext part + // to final position and reuse penultimate ciphertext block buffer to + // construct final plaintext block +.Lxts_enc_steal: + ldrb w0, [x20], #1 + ldrb w1, [x21, #-0x10] + strb w0, [x21, #-0x10] + strb w1, [x21], #1 + + subs x22, x22, #1 + bhi .Lxts_enc_steal + + // Finally encrypt the penultimate ciphertext block using the + // last tweak + ldr q0, [x6] + eor v0.16b, v0.16b, v11.16b + str q0, [sp, #-16]! + mov x0, sp + mov x1, sp + mov x2, x23 + mov x21, x6 + mov v13.d[0], v11.d[1] // just in case AES_encrypt corrupts top half of callee-saved SIMD registers + + bl AES_encrypt + + trn1 v11.2d, v11.2d, v13.2d + ldr q0, [sp], #16 + eor v0.16b, v0.16b, v11.16b + str q0, [x21] + +.Lxts_enc_ret: + + movi v0.16b, #0 + movi v1.16b, #0 +.Lxts_enc_bzero: // wipe key schedule + stp q0, q1, [sp], #32 + cmp sp, x19 + bne .Lxts_enc_bzero + + ldp x19, x20, [sp, #80] + ldp x21, x22, [sp, #96] + ldr x23, [sp, #112] + ldp d8, d9, [sp, #128] + ldp d10, d11, [sp, #144] + ldp d12, d13, [sp, #160] + ldp d14, d15, [sp, #176] + ldp x29, x30, [sp], #192 + ret +.size ossl_bsaes_xts_encrypt,.-ossl_bsaes_xts_encrypt + +// The assembler doesn't seem capable of de-duplicating these when expressed +// using `ldr qd,=` syntax, so assign a symbolic address +.align 5 +.Lxts_magic: +.quad 1, 0x87, 0x4000000000000000, 0x4000000000000000 + +.globl ossl_bsaes_xts_decrypt +.type ossl_bsaes_xts_decrypt,%function +.align 4 +// On entry: +// x0 -> input ciphertext +// x1 -> output plaintext +// x2 -> length of text in bytes (must be at least 16) +// x3 -> key1 (used to decrypt the XORed ciphertext blocks) +// x4 -> key2 (used to encrypt the initial vector to yield the initial tweak) +// x5 -> 16-byte initial vector (typically, sector number) +// On exit: +// Output plaintext filled in +// No output registers, usual AAPCS64 register preservation +ossl_bsaes_xts_decrypt: + AARCH64_VALID_CALL_TARGET + // Stack layout: + // sp -> + // nrounds*128-96 bytes: key schedule + // x19 -> + // 16 bytes: frame record + // 4*16 bytes: tweak storage across _bsaes_decrypt8 + // 6*8 bytes: storage for 5 callee-saved general-purpose registers + // 8*8 bytes: storage for 8 callee-saved SIMD registers + stp x29, x30, [sp, #-192]! + stp x19, x20, [sp, #80] + stp x21, x22, [sp, #96] + str x23, [sp, #112] + stp d8, d9, [sp, #128] + stp d10, d11, [sp, #144] + stp d12, d13, [sp, #160] + stp d14, d15, [sp, #176] + + mov x19, sp + mov x20, x0 + mov x21, x1 + mov x22, x2 + mov x23, x3 + + // generate initial tweak + sub sp, sp, #16 + mov x0, x5 // iv[] + mov x1, sp + mov x2, x4 // key2 + bl AES_encrypt + ldr q11, [sp], #16 + + ldr w1, [x23, #240] // get # of rounds + // allocate the key schedule on the stack + add x17, sp, #96 + sub x17, x17, x1, lsl #7 // 128 bytes per inner round key, less 96 bytes + + // populate the key schedule + mov x9, x23 // pass key + mov x10, x1 // pass # of rounds + mov sp, x17 + bl _bsaes_key_convert + ldr q6, [sp] + str q15, [x17] // save last round key + eor v6.16b, v6.16b, v7.16b // fix up round 0 key (by XORing with 0x63) + str q6, [sp] + + sub x30, x22, #0x10 + tst x22, #0xf // if not multiple of 16 + csel x22, x30, x22, ne // subtract another 16 bytes + subs x22, x22, #0x80 + + blo .Lxts_dec_short + b .Lxts_dec_loop + +.align 4 +.Lxts_dec_loop: + ldr q8, .Lxts_magic + mov x10, x1 // pass rounds + add x2, x19, #16 + ldr q0, [x20], #16 + sshr v1.2d, v11.2d, #63 + mov x9, sp // pass key schedule + ldr q6, .Lxts_magic+16 + add v2.2d, v11.2d, v11.2d + cmtst v3.2d, v11.2d, v6.2d + and v1.16b, v1.16b, v8.16b + ext v1.16b, v1.16b, v1.16b, #8 + and v3.16b, v3.16b, v8.16b + ldr q4, [x20], #16 + eor v12.16b, v2.16b, v1.16b + eor v1.16b, v4.16b, v12.16b + eor v0.16b, v0.16b, v11.16b + cmtst v2.2d, v12.2d, v6.2d + add v4.2d, v12.2d, v12.2d + add x0, x19, #16 + ext v3.16b, v3.16b, v3.16b, #8 + and v2.16b, v2.16b, v8.16b + eor v13.16b, v4.16b, v3.16b + ldr q3, [x20], #16 + ext v4.16b, v2.16b, v2.16b, #8 + eor v2.16b, v3.16b, v13.16b + ldr q3, [x20], #16 + add v5.2d, v13.2d, v13.2d + cmtst v7.2d, v13.2d, v6.2d + and v7.16b, v7.16b, v8.16b + ldr q9, [x20], #16 + ext v7.16b, v7.16b, v7.16b, #8 + ldr q10, [x20], #16 + eor v14.16b, v5.16b, v4.16b + ldr q16, [x20], #16 + add v4.2d, v14.2d, v14.2d + eor v3.16b, v3.16b, v14.16b + eor v15.16b, v4.16b, v7.16b + add v5.2d, v15.2d, v15.2d + ldr q7, [x20], #16 + cmtst v4.2d, v14.2d, v6.2d + and v17.16b, v4.16b, v8.16b + cmtst v18.2d, v15.2d, v6.2d + eor v4.16b, v9.16b, v15.16b + ext v9.16b, v17.16b, v17.16b, #8 + eor v9.16b, v5.16b, v9.16b + add v17.2d, v9.2d, v9.2d + and v18.16b, v18.16b, v8.16b + eor v5.16b, v10.16b, v9.16b + str q9, [x2], #16 + ext v10.16b, v18.16b, v18.16b, #8 + cmtst v9.2d, v9.2d, v6.2d + and v9.16b, v9.16b, v8.16b + eor v10.16b, v17.16b, v10.16b + cmtst v17.2d, v10.2d, v6.2d + eor v6.16b, v16.16b, v10.16b + str q10, [x2], #16 + ext v9.16b, v9.16b, v9.16b, #8 + add v10.2d, v10.2d, v10.2d + eor v9.16b, v10.16b, v9.16b + str q9, [x2], #16 + eor v7.16b, v7.16b, v9.16b + add v9.2d, v9.2d, v9.2d + and v8.16b, v17.16b, v8.16b + ext v8.16b, v8.16b, v8.16b, #8 + eor v8.16b, v9.16b, v8.16b + str q8, [x2] // next round tweak + + bl _bsaes_decrypt8 + + eor v6.16b, v6.16b, v13.16b + eor v0.16b, v0.16b, v11.16b + ldr q8, [x0], #16 + eor v7.16b, v7.16b, v8.16b + str q0, [x21], #16 + eor v0.16b, v1.16b, v12.16b + ldr q1, [x0], #16 + eor v1.16b, v3.16b, v1.16b + subs x22, x22, #0x80 + eor v2.16b, v2.16b, v15.16b + eor v3.16b, v4.16b, v14.16b + ldr q4, [x0], #16 + str q0, [x21], #16 + ldr q11, [x0] // next round tweak + eor v0.16b, v5.16b, v4.16b + str q6, [x21], #16 + str q3, [x21], #16 + str q2, [x21], #16 + str q7, [x21], #16 + str q1, [x21], #16 + str q0, [x21], #16 + bpl .Lxts_dec_loop + +.Lxts_dec_short: + adds x22, x22, #0x70 + bmi .Lxts_dec_done + + ldr q8, .Lxts_magic + sshr v1.2d, v11.2d, #63 + add v2.2d, v11.2d, v11.2d + ldr q9, .Lxts_magic+16 + subs x22, x22, #0x10 + ldr q0, [x20], #16 + and v1.16b, v1.16b, v8.16b + cmtst v3.2d, v11.2d, v9.2d + ext v1.16b, v1.16b, v1.16b, #8 + and v3.16b, v3.16b, v8.16b + eor v12.16b, v2.16b, v1.16b + ext v1.16b, v3.16b, v3.16b, #8 + add v2.2d, v12.2d, v12.2d + cmtst v3.2d, v12.2d, v9.2d + eor v13.16b, v2.16b, v1.16b + and v22.16b, v3.16b, v8.16b + bmi .Lxts_dec_1 + + ext v2.16b, v22.16b, v22.16b, #8 + add v3.2d, v13.2d, v13.2d + ldr q1, [x20], #16 + cmtst v4.2d, v13.2d, v9.2d + subs x22, x22, #0x10 + eor v14.16b, v3.16b, v2.16b + and v23.16b, v4.16b, v8.16b + bmi .Lxts_dec_2 + + ext v3.16b, v23.16b, v23.16b, #8 + add v4.2d, v14.2d, v14.2d + ldr q2, [x20], #16 + cmtst v5.2d, v14.2d, v9.2d + eor v0.16b, v0.16b, v11.16b + subs x22, x22, #0x10 + eor v15.16b, v4.16b, v3.16b + and v24.16b, v5.16b, v8.16b + bmi .Lxts_dec_3 + + ext v4.16b, v24.16b, v24.16b, #8 + add v5.2d, v15.2d, v15.2d + ldr q3, [x20], #16 + cmtst v6.2d, v15.2d, v9.2d + eor v1.16b, v1.16b, v12.16b + subs x22, x22, #0x10 + eor v16.16b, v5.16b, v4.16b + and v25.16b, v6.16b, v8.16b + bmi .Lxts_dec_4 + + ext v5.16b, v25.16b, v25.16b, #8 + add v6.2d, v16.2d, v16.2d + add x0, x19, #16 + cmtst v7.2d, v16.2d, v9.2d + ldr q4, [x20], #16 + eor v2.16b, v2.16b, v13.16b + str q16, [x0], #16 + subs x22, x22, #0x10 + eor v17.16b, v6.16b, v5.16b + and v26.16b, v7.16b, v8.16b + bmi .Lxts_dec_5 + + ext v7.16b, v26.16b, v26.16b, #8 + add v18.2d, v17.2d, v17.2d + ldr q5, [x20], #16 + eor v3.16b, v3.16b, v14.16b + str q17, [x0], #16 + subs x22, x22, #0x10 + eor v18.16b, v18.16b, v7.16b + bmi .Lxts_dec_6 + + ldr q6, [x20], #16 + eor v4.16b, v4.16b, v15.16b + eor v5.16b, v5.16b, v16.16b + str q18, [x0] // next round tweak + mov x9, sp // pass key schedule + mov x10, x1 + add x0, x19, #16 + sub x22, x22, #0x10 + eor v6.16b, v6.16b, v17.16b + + bl _bsaes_decrypt8 + + ldr q16, [x0], #16 + eor v0.16b, v0.16b, v11.16b + eor v1.16b, v1.16b, v12.16b + ldr q17, [x0], #16 + eor v6.16b, v6.16b, v13.16b + eor v4.16b, v4.16b, v14.16b + eor v2.16b, v2.16b, v15.16b + ldr q11, [x0] // next round tweak + str q0, [x21], #16 + str q1, [x21], #16 + eor v0.16b, v7.16b, v16.16b + eor v1.16b, v3.16b, v17.16b + str q6, [x21], #16 + str q4, [x21], #16 + str q2, [x21], #16 + str q0, [x21], #16 + str q1, [x21], #16 + b .Lxts_dec_done + +.align 4 +.Lxts_dec_6: + eor v4.16b, v4.16b, v15.16b + eor v5.16b, v5.16b, v16.16b + mov x9, sp // pass key schedule + mov x10, x1 // pass rounds + add x0, x19, #16 + + bl _bsaes_decrypt8 + + ldr q16, [x0], #16 + eor v0.16b, v0.16b, v11.16b + eor v1.16b, v1.16b, v12.16b + eor v6.16b, v6.16b, v13.16b + eor v4.16b, v4.16b, v14.16b + ldr q11, [x0] // next round tweak + eor v2.16b, v2.16b, v15.16b + str q0, [x21], #16 + str q1, [x21], #16 + eor v0.16b, v7.16b, v16.16b + str q6, [x21], #16 + str q4, [x21], #16 + str q2, [x21], #16 + str q0, [x21], #16 + b .Lxts_dec_done + +.align 4 +.Lxts_dec_5: + eor v3.16b, v3.16b, v14.16b + eor v4.16b, v4.16b, v15.16b + mov x9, sp // pass key schedule + mov x10, x1 // pass rounds + add x0, x19, #16 + + bl _bsaes_decrypt8 + + eor v0.16b, v0.16b, v11.16b + eor v1.16b, v1.16b, v12.16b + ldr q11, [x0] // next round tweak + eor v6.16b, v6.16b, v13.16b + eor v4.16b, v4.16b, v14.16b + eor v2.16b, v2.16b, v15.16b + str q0, [x21], #16 + str q1, [x21], #16 + str q6, [x21], #16 + str q4, [x21], #16 + str q2, [x21], #16 + b .Lxts_dec_done + +.align 4 +.Lxts_dec_4: + eor v2.16b, v2.16b, v13.16b + eor v3.16b, v3.16b, v14.16b + mov x9, sp // pass key schedule + mov x10, x1 // pass rounds + add x0, x19, #16 + + bl _bsaes_decrypt8 + + eor v0.16b, v0.16b, v11.16b + eor v1.16b, v1.16b, v12.16b + eor v6.16b, v6.16b, v13.16b + eor v4.16b, v4.16b, v14.16b + mov v11.16b, v15.16b // next round tweak + str q0, [x21], #16 + str q1, [x21], #16 + str q6, [x21], #16 + str q4, [x21], #16 + b .Lxts_dec_done + +.align 4 +.Lxts_dec_3: + eor v1.16b, v1.16b, v12.16b + eor v2.16b, v2.16b, v13.16b + mov x9, sp // pass key schedule + mov x10, x1 // pass rounds + add x0, x19, #16 + + bl _bsaes_decrypt8 + + eor v0.16b, v0.16b, v11.16b + eor v1.16b, v1.16b, v12.16b + eor v6.16b, v6.16b, v13.16b + mov v11.16b, v14.16b // next round tweak + str q0, [x21], #16 + str q1, [x21], #16 + str q6, [x21], #16 + b .Lxts_dec_done + +.align 4 +.Lxts_dec_2: + eor v0.16b, v0.16b, v11.16b + eor v1.16b, v1.16b, v12.16b + mov x9, sp // pass key schedule + mov x10, x1 // pass rounds + add x0, x19, #16 + + bl _bsaes_decrypt8 + + eor v0.16b, v0.16b, v11.16b + eor v1.16b, v1.16b, v12.16b + mov v11.16b, v13.16b // next round tweak + str q0, [x21], #16 + str q1, [x21], #16 + b .Lxts_dec_done + +.align 4 +.Lxts_dec_1: + eor v0.16b, v0.16b, v11.16b + sub x0, sp, #16 + sub x1, sp, #16 + mov x2, x23 + mov v13.d[0], v11.d[1] // just in case AES_decrypt corrupts top half of callee-saved SIMD registers + mov v14.d[0], v12.d[1] + str q0, [sp, #-16]! + + bl AES_decrypt + + ldr q0, [sp], #16 + trn1 v13.2d, v11.2d, v13.2d + trn1 v11.2d, v12.2d, v14.2d // next round tweak + eor v0.16b, v0.16b, v13.16b + str q0, [x21], #16 + +.Lxts_dec_done: + adds x22, x22, #0x10 + beq .Lxts_dec_ret + + // calculate one round of extra tweak for the stolen ciphertext + ldr q8, .Lxts_magic + sshr v6.2d, v11.2d, #63 + and v6.16b, v6.16b, v8.16b + add v12.2d, v11.2d, v11.2d + ext v6.16b, v6.16b, v6.16b, #8 + eor v12.16b, v12.16b, v6.16b + + // perform the final decryption with the last tweak value + ldr q0, [x20], #16 + eor v0.16b, v0.16b, v12.16b + str q0, [sp, #-16]! + mov x0, sp + mov x1, sp + mov x2, x23 + mov v13.d[0], v11.d[1] // just in case AES_decrypt corrupts top half of callee-saved SIMD registers + mov v14.d[0], v12.d[1] + + bl AES_decrypt + + trn1 v12.2d, v12.2d, v14.2d + trn1 v11.2d, v11.2d, v13.2d + ldr q0, [sp], #16 + eor v0.16b, v0.16b, v12.16b + str q0, [x21] + + mov x6, x21 + // Penultimate ciphertext block produces final plaintext part-block + // plus remaining part of final ciphertext block. Move plaintext part + // to final position and reuse penultimate plaintext block buffer to + // construct final ciphertext block +.Lxts_dec_steal: + ldrb w1, [x21] + ldrb w0, [x20], #1 + strb w1, [x21, #0x10] + strb w0, [x21], #1 + + subs x22, x22, #1 + bhi .Lxts_dec_steal + + // Finally decrypt the penultimate plaintext block using the + // penultimate tweak + ldr q0, [x6] + eor v0.16b, v0.16b, v11.16b + str q0, [sp, #-16]! + mov x0, sp + mov x1, sp + mov x2, x23 + mov x21, x6 + + bl AES_decrypt + + trn1 v11.2d, v11.2d, v13.2d + ldr q0, [sp], #16 + eor v0.16b, v0.16b, v11.16b + str q0, [x21] + +.Lxts_dec_ret: + + movi v0.16b, #0 + movi v1.16b, #0 +.Lxts_dec_bzero: // wipe key schedule + stp q0, q1, [sp], #32 + cmp sp, x19 + bne .Lxts_dec_bzero + + ldp x19, x20, [sp, #80] + ldp x21, x22, [sp, #96] + ldr x23, [sp, #112] + ldp d8, d9, [sp, #128] + ldp d10, d11, [sp, #144] + ldp d12, d13, [sp, #160] + ldp d14, d15, [sp, #176] + ldp x29, x30, [sp], #192 + ret +.size ossl_bsaes_xts_decrypt,.-ossl_bsaes_xts_decrypt diff --git a/crypto/openssl/crypto/aes/asm/bsaes-x86_64.pl b/crypto/openssl/crypto/aes/asm/bsaes-x86_64.pl index 6498cfe908c8..7230f760856a 100644 --- a/crypto/openssl/crypto/aes/asm/bsaes-x86_64.pl +++ b/crypto/openssl/crypto/aes/asm/bsaes-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2011-2021 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2011-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -2182,6 +2182,7 @@ $code.=<<___; .align 16 ossl_bsaes_xts_encrypt: .cfi_startproc + endbranch mov %rsp, %rax .Lxts_enc_prologue: push %rbp @@ -2581,6 +2582,7 @@ $code.=<<___; .align 16 ossl_bsaes_xts_decrypt: .cfi_startproc + endbranch mov %rsp, %rax .Lxts_dec_prologue: push %rbp @@ -3003,6 +3005,7 @@ ___ } $code.=<<___; .type _bsaes_const,\@object +.section .rodata align=64 .align 64 _bsaes_const: .LM0ISR: # InvShiftRows constants @@ -3054,9 +3057,9 @@ _bsaes_const: .quad 0x02060a0e03070b0f, 0x0004080c0105090d .L63: .quad 0x6363636363636363, 0x6363636363636363 -.asciz "Bit-sliced AES for x86_64/SSSE3, Emilia Käsper, Peter Schwabe, Andy Polyakov" .align 64 .size _bsaes_const,.-_bsaes_const +.asciz "Bit-sliced AES for x86_64/SSSE3, Emilia Käsper, Peter Schwabe, Andy Polyakov" ___ # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, diff --git a/crypto/openssl/crypto/aes/asm/vpaes-armv8.pl b/crypto/openssl/crypto/aes/asm/vpaes-armv8.pl index e00f824c465f..189bc01ca464 100755 --- a/crypto/openssl/crypto/aes/asm/vpaes-armv8.pl +++ b/crypto/openssl/crypto/aes/asm/vpaes-armv8.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2015-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2015-2025 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -55,7 +55,7 @@ open OUT,"| \"$^X\" $xlate $flavour \"$output\"" $code.=<<___; #include "arm_arch.h" -.text +.rodata .type _vpaes_consts,%object .align 7 // totally strategic alignment @@ -146,6 +146,9 @@ _vpaes_consts: .asciz "Vector Permutation AES for ARMv8, Mike Hamburg (Stanford University)" .size _vpaes_consts,.-_vpaes_consts .align 6 + +.text + ___ { @@ -165,7 +168,8 @@ $code.=<<___; .type _vpaes_encrypt_preheat,%function .align 4 _vpaes_encrypt_preheat: - adr PTR(10), .Lk_inv + adrp PTR(10), .Lk_inv + add PTR(10), PTR(10), :lo12:.Lk_inv movi v17.16b, #0x0f ld1 {v18.2d-v19.2d}, [PTR(10)],#32 // .Lk_inv ld1 {v20.2d-v23.2d}, [PTR(10)],#64 // .Lk_ipt, .Lk_sbo @@ -193,7 +197,8 @@ _vpaes_encrypt_preheat: _vpaes_encrypt_core: mov PTR(9), $key ldr w8, [$key,#240] // pull rounds - adr PTR(11), .Lk_mc_forward+16 + adrp PTR(11), .Lk_mc_forward+16 + add PTR(11), PTR(11), :lo12:.Lk_mc_forward+16 // vmovdqa .Lk_ipt(%rip), %xmm2 # iptlo ld1 {v16.2d}, [PTR(9)], #16 // vmovdqu (%r9), %xmm5 # round0 key and v1.16b, v7.16b, v17.16b // vpand %xmm9, %xmm0, %xmm1 @@ -285,7 +290,8 @@ vpaes_encrypt: _vpaes_encrypt_2x: mov PTR(9), $key ldr w8, [$key,#240] // pull rounds - adr PTR(11), .Lk_mc_forward+16 + adrp PTR(11), .Lk_mc_forward+16 + add PTR(11), PTR(11), :lo12:.Lk_mc_forward+16 // vmovdqa .Lk_ipt(%rip), %xmm2 # iptlo ld1 {v16.2d}, [PTR(9)], #16 // vmovdqu (%r9), %xmm5 # round0 key and v1.16b, v14.16b, v17.16b // vpand %xmm9, %xmm0, %xmm1 @@ -393,9 +399,11 @@ _vpaes_encrypt_2x: .type _vpaes_decrypt_preheat,%function .align 4 _vpaes_decrypt_preheat: - adr PTR(10), .Lk_inv + adrp PTR(10), .Lk_inv + add PTR(10), PTR(10), :lo12:.Lk_inv movi v17.16b, #0x0f - adr PTR(11), .Lk_dipt + adrp PTR(11), .Lk_dipt + add PTR(11), PTR(11), :lo12:.Lk_dipt ld1 {v18.2d-v19.2d}, [PTR(10)],#32 // .Lk_inv ld1 {v20.2d-v23.2d}, [PTR(11)],#64 // .Lk_dipt, .Lk_dsbo ld1 {v24.2d-v27.2d}, [PTR(11)],#64 // .Lk_dsb9, .Lk_dsbd @@ -417,10 +425,12 @@ _vpaes_decrypt_core: // vmovdqa .Lk_dipt(%rip), %xmm2 # iptlo lsl x11, x8, #4 // mov %rax, %r11; shl \$4, %r11 eor x11, x11, #0x30 // xor \$0x30, %r11 - adr PTR(10), .Lk_sr + adrp PTR(10), .Lk_sr + add PTR(10), PTR(10), :lo12:.Lk_sr and x11, x11, #0x30 // and \$0x30, %r11 add PTR(11), PTR(10), x11 - adr PTR(10), .Lk_mc_forward+48 + adrp PTR(10), .Lk_mc_forward+48 + add PTR(10), PTR(10), :lo12:.Lk_mc_forward+48 ld1 {v16.2d}, [PTR(9)],#16 // vmovdqu (%r9), %xmm4 # round0 key and v1.16b, v7.16b, v17.16b // vpand %xmm9, %xmm0, %xmm1 @@ -528,10 +538,12 @@ _vpaes_decrypt_2x: // vmovdqa .Lk_dipt(%rip), %xmm2 # iptlo lsl x11, x8, #4 // mov %rax, %r11; shl \$4, %r11 eor x11, x11, #0x30 // xor \$0x30, %r11 - adr PTR(10), .Lk_sr + adrp PTR(10), .Lk_sr + add PTR(10), PTR(10), :lo12:.Lk_sr and x11, x11, #0x30 // and \$0x30, %r11 add PTR(11), PTR(10), x11 - adr PTR(10), .Lk_mc_forward+48 + adrp PTR(10), .Lk_mc_forward+48 + add PTR(10), PTR(10), :lo12:.Lk_mc_forward+48 ld1 {v16.2d}, [PTR(9)],#16 // vmovdqu (%r9), %xmm4 # round0 key and v1.16b, v14.16b, v17.16b // vpand %xmm9, %xmm0, %xmm1 @@ -668,14 +680,18 @@ $code.=<<___; .type _vpaes_key_preheat,%function .align 4 _vpaes_key_preheat: - adr PTR(10), .Lk_inv + adrp PTR(10), .Lk_inv + add PTR(10), PTR(10), :lo12:.Lk_inv movi v16.16b, #0x5b // .Lk_s63 - adr PTR(11), .Lk_sb1 + adrp PTR(11), .Lk_sb1 + add PTR(11), PTR(11), :lo12:.Lk_sb1 movi v17.16b, #0x0f // .Lk_s0F ld1 {v18.2d-v21.2d}, [PTR(10)] // .Lk_inv, .Lk_ipt - adr PTR(10), .Lk_dksd + adrp PTR(10), .Lk_dksd + add PTR(10), PTR(10), :lo12:.Lk_dksd ld1 {v22.2d-v23.2d}, [PTR(11)] // .Lk_sb1 - adr PTR(11), .Lk_mc_forward + adrp PTR(11), .Lk_mc_forward + add PTR(11), PTR(11), :lo12:.Lk_mc_forward ld1 {v24.2d-v27.2d}, [PTR(10)],#64 // .Lk_dksd, .Lk_dksb ld1 {v28.2d-v31.2d}, [PTR(10)],#64 // .Lk_dkse, .Lk_dks9 ld1 {v8.2d}, [PTR(10)] // .Lk_rcon @@ -699,7 +715,8 @@ _vpaes_schedule_core: bl _vpaes_schedule_transform mov v7.16b, v0.16b // vmovdqa %xmm0, %xmm7 - adr PTR(10), .Lk_sr // lea .Lk_sr(%rip),%r10 + adrp PTR(10), .Lk_sr // lea .Lk_sr(%rip),%r10 + add PTR(10), PTR(10), :lo12:.Lk_sr add PTR(8), PTR(10), x8 cbnz $dir, .Lschedule_am_decrypting @@ -830,12 +847,14 @@ _vpaes_schedule_core: .align 4 .Lschedule_mangle_last: // schedule last round key from xmm0 - adr PTR(11), .Lk_deskew // lea .Lk_deskew(%rip),%r11 # prepare to deskew + adrp PTR(11), .Lk_deskew // lea .Lk_deskew(%rip),%r11 # prepare to deskew + add PTR(11), PTR(11), :lo12:.Lk_deskew cbnz $dir, .Lschedule_mangle_last_dec // encrypting ld1 {v1.2d}, [PTR(8)] // vmovdqa (%r8,%r10),%xmm1 - adr PTR(11), .Lk_opt // lea .Lk_opt(%rip), %r11 # prepare to output transform + adrp PTR(11), .Lk_opt // lea .Lk_opt(%rip), %r11 # prepare to output transform + add PTR(11), PTR(11), :lo12:.Lk_opt add $out, $out, #32 // add \$32, %rdx tbl v0.16b, {v0.16b}, v1.16b // vpshufb %xmm1, %xmm0, %xmm0 # output permute diff --git a/crypto/openssl/crypto/aes/asm/vpaes-loongarch64.pl b/crypto/openssl/crypto/aes/asm/vpaes-loongarch64.pl new file mode 100644 index 000000000000..42967f526605 --- /dev/null +++ b/crypto/openssl/crypto/aes/asm/vpaes-loongarch64.pl @@ -0,0 +1,1004 @@ +#! /usr/bin/env perl +# Copyright 2015-2025 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +###################################################################### +## Constant-time SSSE3 AES core implementation. +## version 0.1 +## +## By Mike Hamburg (Stanford University), 2009 +## Public domain. +## +## For details see http://shiftleft.org/papers/vector_aes/ and +## http://crypto.stanford.edu/vpaes/. +## +###################################################################### + +# Loongarch64 LSX adaptation by , +# and +# + +($zero,$ra,$tp,$sp)=map("\$r$_",(0..3)); +($a0,$a1,$a2,$a3,$a4,$a5,$a6,$a7)=map("\$r$_",(4..11)); +($t0,$t1,$t2,$t3,$t4,$t5,$t6,$t7,$t8,$t9)=map("\$r$_",(12..21)); +($s0,$s1,$s2,$s3,$s4,$s5,$s6,$s7)=map("\$r$_",(23..30)); +($vr0,$vr1,$vr2,$vr3,$vr4,$vr5,$vr6,$vr7,$vr8,$vr9,$vr10,$vr11,$vr12,$vr13,$vr14,$vr15,$vr16,$vr17,$vr18,$vr19)=map("\$vr$_",(0..19)); +($fp)=map("\$r$_",(22)); + +# $output is the last argument if it looks like a file (it has an extension) +my $output; +$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +open STDOUT,">$output"; + +$PREFIX="vpaes"; + +$code.=<<___; + +## +## _aes_encrypt_core +## +## AES-encrypt %vr0. +## +## Inputs: +## %vr0 = input +## %vr9-%vr15 as in _vpaes_preheat +## (%a2) = scheduled keys +## +## Output in %vr0 +## Clobbers %vr1-%vr5, %r9, %r10, %r11, %t5 +## Preserves %vr6 - %vr8 so you get some local vectors +## +## +##.type _vpaes_encrypt_core +.align 4 +_vpaes_encrypt_core: +.cfi_startproc + move $a5,$a2 + li.d $a7,0x10 + ld.w $t5,$a2,240 + vori.b $vr1,$vr9,0 + la.local $t0,Lk_ipt + vld $vr2,$t0,0 # iptlo + vandn.v $vr1,$vr1,$vr0 + vld $vr5,$a5,0 # round0 key + vsrli.w $vr1,$vr1,4 + vand.v $vr0,$vr0,$vr9 + vshuf.b $vr2,$vr18,$vr2,$vr0 + vld $vr0,$t0,16 # ipthi + vshuf.b $vr0,$vr18,$vr0,$vr1 + vxor.v $vr2,$vr2,$vr5 + addi.d $a5,$a5,16 + vxor.v $vr0,$vr0,$vr2 + la.local $a6,Lk_mc_backward + b .Lenc_entry + +.align 4 +.Lenc_loop: + # middle of middle round + vori.b $vr4,$vr13,0 # 4 : sb1u + vori.b $vr0,$vr12,0 # 0 : sb1t + vshuf.b $vr4,$vr18,$vr4,$vr2 # 4 = sb1u + vshuf.b $vr0,$vr18,$vr0,$vr3 # 0 = sb1t + vxor.v $vr4,$vr4,$vr5 # 4 = sb1u + k + vori.b $vr5,$vr15,0 # 4 : sb2u + vxor.v $vr0,$vr0,$vr4 # 0 = A + add.d $t0,$a7,$a6 # Lk_mc_forward[] + vld $vr1,$t0,-0x40 + vshuf.b $vr5,$vr18,$vr5,$vr2 # 4 = sb2u + vld $vr4,$t0,0 # Lk_mc_backward[] + vori.b $vr2,$vr14,0 # 2 : sb2t + vshuf.b $vr2,$vr18,$vr2,$vr3 # 2 = sb2t + vori.b $vr3,$vr0,0 # 3 = A + vxor.v $vr2,$vr5,$vr2 # 2 = 2A + vshuf.b $vr0,$vr18,$vr0,$vr1 # 0 = B + addi.d $a5,$a5,16 # next key + vxor.v $vr0,$vr0,$vr2 # 0 = 2A+B + vshuf.b $vr3,$vr18,$vr3,$vr4 # 3 = D + addi.d $a7,$a7,16 # next mc + vxor.v $vr3,$vr3,$vr0 # 3 = 2A+B+D + vshuf.b $vr0,$vr18,$vr0,$vr1 # 0 = 2B+C + andi $a7,$a7,0x30 # ... mod 4 + addi.d $t5,$t5,-1 # nr-- + vxor.v $vr0,$vr0,$vr3 # 0 = 2A+3B+C+D + +.Lenc_entry: + # top of round + vori.b $vr1,$vr9,0 # 1 : i + vori.b $vr5,$vr11,0 # 2 : a/k + vandn.v $vr1,$vr1,$vr0 # 1 = i<<4 + vsrli.w $vr1,$vr1,4 # 1 = i + vand.v $vr0,$vr0,$vr9 # 0 = k + vshuf.b $vr5,$vr18,$vr5,$vr0 # 2 = a/k + vori.b $vr3,$vr10,0 # 3 : 1/i + vxor.v $vr0,$vr0,$vr1 # 0 = j + vshuf.b $vr3,$vr18,$vr3,$vr1 # 3 = 1/i + vori.b $vr4,$vr10,0 # 4 : 1/j + vxor.v $vr3,$vr3,$vr5 # 3 = iak = 1/i + a/k + vshuf.b $vr4,$vr18,$vr4,$vr0 # 4 = 1/j + vori.b $vr2,$vr10,0 # 2 : 1/iak + vxor.v $vr4,$vr4,$vr5 # 4 = jak = 1/j + a/k + vshuf.b $vr2,$vr18,$vr2,$vr3 # 2 = 1/iak + vori.b $vr3,$vr10,0 # 3 : 1/jak + vxor.v $vr2,$vr2,$vr0 # 2 = io + vshuf.b $vr3,$vr18,$vr3,$vr4 # 3 = 1/jak + vld $vr5,$a5,0 + vxor.v $vr3,$vr3,$vr1 # 3 = jo + bnez $t5,.Lenc_loop + + # middle of last round + vld $vr4,$a6, -0x60 # 3 : sbou Lk_sbo + vld $vr0,$a6, -0x50 # 0 : sbot Lk_sbo+16 + vshuf.b $vr4,$vr18,$vr4,$vr2 # 4 = sbou + vxor.v $vr4,$vr4,$vr5 # 4 = sb1u + k + vshuf.b $vr0,$vr18,$vr0,$vr3 # 0 = sb1t + add.d $t0,$a7,$a6 # Lk_sr[] + vld $vr1,$t0,0x40 + vxor.v $vr0,$vr0,$vr4 # 0 = A + vshuf.b $vr0,$vr18,$vr0,$vr1 + jr $ra +.cfi_endproc +.size _vpaes_encrypt_core,.-_vpaes_encrypt_core + +## +## Decryption core +## +## Same API as encryption core. +## +#.type _vpaes_decrypt_core,\@abi-omnipotent +.align 4 +_vpaes_decrypt_core: +.cfi_startproc + move $a5,$a2 # load key + ld.w $t5,$a2,240 + vori.b $vr1,$vr9,0 + la.local $t0,Lk_dipt + vld $vr2,$t0,0 # iptlo + vandn.v $vr1,$vr1,$vr0 + move $a7,$t5 + vsrli.w $vr1,$vr1,4 + vld $vr5,$a5,0 # round0 key + slli.d $a7,$a7,4 + vand.v $vr0,$vr9,$vr0 + vshuf.b $vr2,$vr18,$vr2,$vr0 + vld $vr0,$t0,16 # ipthi + xori $a7,$a7,0x30 + la.local $a6,Lk_dsbd + vshuf.b $vr0,$vr18,$vr0,$vr1 + andi $a7,$a7,0x30 + vxor.v $vr2,$vr2,$vr5 + la.local $t0,Lk_mc_forward + vld $vr5,$t0,48 + vxor.v $vr0,$vr0,$vr2 + addi.d $a5,$a5,16 + add.d $a7,$a7,$a6 + b .Ldec_entry + +.align 4 +.Ldec_loop: +## +## Inverse mix columns +## + vld $vr4,$a6,-0x20 # 4 : sb9u + vld $vr1,$a6,-0x10 # 0 : sb9t + vshuf.b $vr4,$vr18,$vr4,$vr2 # 4 = sb9u + vshuf.b $vr1,$vr18,$vr1,$vr3 # 0 = sb9t + vxor.v $vr0,$vr0,$vr4 + vld $vr4,$a6,0x0 # 4 : sbdu + vxor.v $vr0,$vr0,$vr1 # 0 = ch + vld $vr1,$a6,0x10 # 0 : sbdt + vshuf.b $vr4,$vr18,$vr4,$vr2 # 4 = sbdu + vshuf.b $vr0,$vr18,$vr0,$vr5 # MC ch + vshuf.b $vr1,$vr18,$vr1,$vr3 # 0 = sbdt + vxor.v $vr0,$vr0,$vr4 # 4 = ch + vld $vr4,$a6,0x20 # 4 : sbbu + vxor.v $vr0,$vr0,$vr1 # 0 = ch + vld $vr1,$a6,0x30 # 0 : sbbt + vshuf.b $vr4,$vr18,$vr4,$vr2 # 4 = sbbu + vshuf.b $vr0,$vr18,$vr0,$vr5 # MC ch + vshuf.b $vr1,$vr18,$vr1,$vr3 # 0 = sbbt + vxor.v $vr0,$vr0,$vr4 # 4 = ch + vld $vr4,$a6,0x40 # 4 : sbeu + vxor.v $vr0,$vr0,$vr1 # 0 = ch + vld $vr1,$a6,0x50 # 0 : sbet + vshuf.b $vr4,$vr18,$vr4,$vr2 # 4 = sbeu + vshuf.b $vr0,$vr18,$vr0,$vr5 # MC ch + vshuf.b $vr1,$vr18,$vr1,$vr3 # 0 = sbet + vxor.v $vr0,$vr0,$vr4 # 4 = ch + addi.d $a5,$a5, 16 # next round key + vbsrl.v $vr16,$vr5,0xc + vbsll.v $vr5,$vr5,0x4 + vor.v $vr5,$vr5,$vr16 + vxor.v $vr0,$vr0,$vr1 # 0 = ch + addi.d $t5,$t5,-1 # nr-- + +.Ldec_entry: + # top of round + vori.b $vr1,$vr9,0 # 1 : i + vandn.v $vr1,$vr1,$vr0 # 1 = i<<4 + vori.b $vr2,$vr11,0 # 2 : a/k + vsrli.w $vr1,$vr1,4 # 1 = i + vand.v $vr0,$vr0,$vr9 # 0 = k + vshuf.b $vr2,$vr18,$vr2,$vr0 # 2 = a/k + vori.b $vr3,$vr10,0 # 3 : 1/i + vxor.v $vr0,$vr0,$vr1 # 0 = j + vshuf.b $vr3,$vr18,$vr3,$vr1 # 3 = 1/i + vori.b $vr4,$vr10,0 # 4 : 1/j + vxor.v $vr3,$vr3,$vr2 # 3 = iak = 1/i + a/k + vshuf.b $vr4,$vr18,$vr4,$vr0 # 4 = 1/j + vxor.v $vr4,$vr4,$vr2 # 4 = jak = 1/j + a/k + vori.b $vr2,$vr10,0 # 2 : 1/iak + vshuf.b $vr2,$vr18,$vr2,$vr3 # 2 = 1/iak + vori.b $vr3,$vr10,0 # 3 : 1/jak + vxor.v $vr2,$vr2,$vr0 # 2 = io + vshuf.b $vr3,$vr18,$vr3,$vr4 # 3 = 1/jak + vld $vr0,$a5,0 + vxor.v $vr3,$vr3,$vr1 # 3 = jo + bnez $t5,.Ldec_loop + + # middle of last round + vld $vr4,$a6,0x60 # 3 : sbou + vshuf.b $vr4,$vr18,$vr4,$vr2 # 4 = sbou + vxor.v $vr4,$vr4,$vr0 # 4 = sb1u + k + vld $vr0,$a6,0x70 # 0 : sbot + vld $vr2,$a7,-0x160 # Lk_sr-.Lk_dsbd=-0x160 + vshuf.b $vr0,$vr18,$vr0,$vr3 # 0 = sb1t + vxor.v $vr0,$vr0,$vr4 # 0 = A + vshuf.b $vr0,$vr18,$vr0,$vr2 + jr $ra +.cfi_endproc +.size _vpaes_decrypt_core,.-_vpaes_decrypt_core + +######################################################## +## ## +## AES key schedule ## +## ## +######################################################## +#.type _vpaes_schedule_core,\@abi-omnipotent +.align 4 +_vpaes_schedule_core: +.cfi_startproc + # a0 = key + # a1 = size in bits + # a2 = buffer + # a3 = direction. 0=encrypt, 1=decrypt + + addi.d $sp,$sp,-48 + st.d $ra,$sp,40 + st.d $fp,$sp,32 + + bl _vpaes_preheat # load the tables + la.local $t0,Lk_rcon + vld $vr8,$t0,0 # load rcon + vld $vr0,$a0,0 # load key (unaligned) + + # input transform + vori.b $vr3,$vr0,0 + la.local $a7,Lk_ipt + bl _vpaes_schedule_transform + vori.b $vr7,$vr0,0 + + la.local $a6,Lk_sr + bnez $a3,.Lschedule_am_decrypting + + # encrypting, output zeroth round key after transform + vst $vr0,$a2,0 + b .Lschedule_go + +.Lschedule_am_decrypting: + # decrypting, output zeroth round key after shiftrows + add.d $t2,$a4,$a6 + vld $vr1,$t2,0 + vshuf.b $vr3,$vr18,$vr3,$vr1 + vst $vr3,$a2,0 + xori $a4,$a4,0x30 + +.Lschedule_go: + li.d $t6,192 + bltu $t6,$a1,.Lschedule_256 + beq $t6,$a1,.Lschedule_192 + # 128: fall though + +## +## .schedule_128 +## +## 128-bit specific part of key schedule. +## +## This schedule is really simple, because all its parts +## are accomplished by the subroutines. +## +.Lschedule_128: + li.w $a1,10 + +.Loop_schedule_128: + bl _vpaes_schedule_round + addi.w $a1,$a1,-1 + beqz $a1,.Lschedule_mangle_last + bl _vpaes_schedule_mangle + b .Loop_schedule_128 + +## +## .aes_schedule_192 +## +## 192-bit specific part of key schedule. +## +## The main body of this schedule is the same as the 128-bit +## schedule, but with more smearing. The long, high side is +## stored in %vr7 as before, and the short, low side is in +## the high bits of %vr6. +## +## This schedule is somewhat nastier, however, because each +## round produces 192 bits of key material, or 1.5 round keys. +## Therefore, on each cycle we do 2 rounds and produce 3 round +## keys. +## +.align 4 +.Lschedule_192: + vld $vr0,$a0,8 #load key part 2 + bl _vpaes_schedule_transform #input transform + vaddi.du $vr6,$vr0,0x0 #save short part + vxor.v $vr4,$vr4,$vr4 #clear 4 + vpackod.d $vr6,$vr6,$vr4 #clobber low side with zeros + li.w $a1,4 + +.Loop_schedule_192: + bl _vpaes_schedule_round + vbsrl.v $vr16,$vr6,0x8 + vbsll.v $vr0,$vr0,0x8 + vor.v $vr0,$vr0,$vr16 + + bl _vpaes_schedule_mangle # save key n + bl _vpaes_schedule_192_smear + bl _vpaes_schedule_mangle # save key n+1 + bl _vpaes_schedule_round + addi.w $a1,$a1,-1 + beqz $a1,.Lschedule_mangle_last + bl _vpaes_schedule_mangle # save key n+2 + bl _vpaes_schedule_192_smear + b .Loop_schedule_192 + +## +## .aes_schedule_256 +## +## 256-bit specific part of key schedule. +## +## The structure here is very similar to the 128-bit +## schedule, but with an additional "low side" in +## %vr6. The low side's rounds are the same as the +## high side's, except no rcon and no rotation. +## +.align 4 +.Lschedule_256: + vld $vr0,$a0,16 # load key part 2 (unaligned) + bl _vpaes_schedule_transform # input transform + addi.w $a1,$zero,7 + +.Loop_schedule_256: + bl _vpaes_schedule_mangle # output low result + vori.b $vr6,$vr0,0 # save cur_lo in vr6 + + # high round + bl _vpaes_schedule_round + addi.d $a1,$a1,-1 + beqz $a1,.Lschedule_mangle_last + bl _vpaes_schedule_mangle + + # low round. swap vr7 and vr6 + vshuf4i.w $vr0,$vr0,0xFF + vori.b $vr5,$vr7,0 + vori.b $vr7,$vr6,0 + bl _vpaes_schedule_low_round + vori.b $vr7,$vr5,0 + + b .Loop_schedule_256 + + +## +## .aes_schedule_mangle_last +## +## Mangler for last round of key schedule +## Mangles %vr0 +## when encrypting, outputs out(%vr0) ^ 63 +## when decrypting, outputs unskew(%vr0) +## +## Always called right before return... jumps to cleanup and exits +## +.align 4 +.Lschedule_mangle_last: + # schedule last round key from vr0 + la.local $a7,Lk_deskew # prepare to deskew + bnez $a3,.Lschedule_mangle_last_dec + + # encrypting + add.d $t0,$a4,$a6 + vld $vr1,$t0,0 + vshuf.b $vr0,$vr18,$vr0,$vr1 # output permute + la.local $a7,Lk_opt # prepare to output transform + addi.d $a2,$a2,32 + +.Lschedule_mangle_last_dec: + addi.d $a2,$a2,-16 + la.local $t0,Lk_s63 + vld $vr16,$t0,0 + vxor.v $vr0,$vr0,$vr16 + bl _vpaes_schedule_transform # output transform + vst $vr0,$a2,0 # save last key + + # cleanup + vxor.v $vr0,$vr0,$vr0 + vxor.v $vr1,$vr1,$vr1 + vxor.v $vr2,$vr2,$vr2 + vxor.v $vr3,$vr3,$vr3 + vxor.v $vr4,$vr4,$vr4 + vxor.v $vr5,$vr5,$vr5 + vxor.v $vr6,$vr6,$vr6 + vxor.v $vr7,$vr7,$vr7 + ld.d $ra,$sp,40 + ld.d $fp,$sp,32 + addi.d $sp,$sp,48 + jr $ra +.cfi_endproc +.size _vpaes_schedule_core,.-_vpaes_schedule_core + +## +## .aes_schedule_192_smear +## +## Smear the short, low side in the 192-bit key schedule. +## +## Inputs: +## %vr7: high side, b a x y +## %vr6: low side, d c 0 0 +## %vr13: 0 +## +## Outputs: +## %vr6: b+c+d b+c 0 0 +## %vr0: b+c+d b+c b a +## +#.type _vpaes_schedule_192_smear,\@abi-omnipotent +.align 4 +_vpaes_schedule_192_smear: +.cfi_startproc + vshuf4i.w $vr1,$vr6,0x80 # d c 0 0 -> c 0 0 0 + vshuf4i.w $vr0,$vr7,0xFE # b a _ _ -> b b b a + vxor.v $vr6,$vr6,$vr1 # -> c+d c 0 0 + vxor.v $vr1,$vr1,$vr1 + vxor.v $vr6,$vr6,$vr0 # -> b+c+d b+c b a + vori.b $vr0,$vr6,0 + vilvh.d $vr6,$vr6,$vr1 # clobber low side with zeros + jr $ra +.cfi_endproc +.size _vpaes_schedule_192_smear,.-_vpaes_schedule_192_smear + +## +## .aes_schedule_round +## +## Runs one main round of the key schedule on %vr0, %vr7 +## +## Specifically, runs subbytes on the high dword of %vr0 +## then rotates it by one byte and xors into the low dword of +## %vr7. +## +## Adds rcon from low byte of %vr8, then rotates %vr8 for +## next rcon. +## +## Smears the dwords of %vr7 by xoring the low into the +## second low, result into third, result into highest. +## +## Returns results in %vr7 = %vr0. +## Clobbers %vr1-%vr4, %a7. +## +#.type _vpaes_schedule_round,\@abi-omnipotent +.align 4 +_vpaes_schedule_round: +.cfi_startproc + # extract rcon from vr8 + vxor.v $vr1,$vr1,$vr1 + vbsrl.v $vr16,$vr8,0xf + vbsll.v $vr1,$vr1,0x1 + vor.v $vr1,$vr1,$vr16 + vbsrl.v $vr16,$vr8,0xf + vbsll.v $vr8,$vr8,0x1 + vor.v $vr8,$vr8,$vr16 + + vxor.v $vr7,$vr7,$vr1 + + # rotate + vshuf4i.w $vr0,$vr0,0xff #put $vr0 lowest 32 bit to each words + vbsrl.v $vr16,$vr0,0x1 + vbsll.v $vr0,$vr0,0xf + vor.v $vr0,$vr0,$vr16 + + # fall through... + + # low round: same as high round, but no rotation and no rcon. +_vpaes_schedule_low_round: + # smear vr7 + vaddi.du $vr1,$vr7,0x0 + vbsll.v $vr7,$vr7,0x4 + vxor.v $vr7,$vr7,$vr1 + vaddi.du $vr1,$vr7,0x0 + vbsll.v $vr7,$vr7,0x8 + vxor.v $vr7,$vr7,$vr1 + vxori.b $vr7,$vr7,0x5B + + # subbytes + vaddi.du $vr1,$vr9,0x0 + vandn.v $vr1,$vr1,$vr0 + vsrli.w $vr1,$vr1,0x4 # 1 = i + vand.v $vr0,$vr0,$vr9 # 0 = k + vaddi.du $vr2,$vr11,0x0 # 2 : a/k + vshuf.b $vr2,$vr18,$vr2,$vr0 # 2 = a/k + vxor.v $vr0,$vr0,$vr1 # 0 = j + vaddi.du $vr3,$vr10,0x0 # 3 : 1/i + vshuf.b $vr3,$vr18,$vr3,$vr1 # 3 = 1/i + vxor.v $vr3,$vr3,$vr2 # 3 = iak = 1/i + a/k + vaddi.du $vr4,$vr10,0x0 # 4 : 1/j + vshuf.b $vr4,$vr18,$vr4,$vr0 # 4 = 1/j + vxor.v $vr4,$vr4,$vr2 # 4 = jak = 1/j + a/k + vaddi.du $vr2,$vr10,0x0 # 2 : 1/iak + vshuf.b $vr2,$vr18,$vr2,$vr3 # 2 = 1/iak + vxor.v $vr2,$vr2,$vr0 # 2 = io + vaddi.du $vr3,$vr10,0x0 # 3 : 1/jak + vshuf.b $vr3,$vr18,$vr3,$vr4 # 3 = 1/jak + vxor.v $vr3,$vr3,$vr1 # 3 = jo + vaddi.du $vr4,$vr13,0x0 # 4 : sbou + vshuf.b $vr4,$vr18,$vr4,$vr2 # 4 = sbou + vaddi.du $vr0,$vr12,0x0 # 0 : sbot + vshuf.b $vr0,$vr18,$vr0,$vr3 # 0 = sb1t + vxor.v $vr0,$vr0,$vr4 # 0 = sbox output + + # add in smeared stuff + vxor.v $vr0,$vr0,$vr7 + vaddi.du $vr7,$vr0,0x0 + jr $ra +.cfi_endproc +.size _vpaes_schedule_round,.-_vpaes_schedule_round + +## +## .aes_schedule_transform +## +## Linear-transform %vr0 according to tables at (%r11) +## +## Requires that %vr9 = 0x0F0F... as in preheat +## Output in %vr0 +## Clobbers %vr1, %vr2 +## +#.type _vpaes_schedule_transform,\@abi-omnipotent +.align 4 +_vpaes_schedule_transform: +.cfi_startproc + vori.b $vr1,$vr9,0 + vandn.v $vr1,$vr1,$vr0 + vsrli.w $vr1,$vr1,4 + vand.v $vr0,$vr0,$vr9 + vld $vr2,$a7,0 # lo + vshuf.b $vr2,$vr18,$vr2,$vr0 + vld $vr0,$a7,16 # hi + vshuf.b $vr0,$vr18,$vr0,$vr1 + vxor.v $vr0,$vr0,$vr2 + jr $ra +.cfi_endproc +.size _vpaes_schedule_transform,.-_vpaes_schedule_transform + +## +## .aes_schedule_mangle +## +## Mangle vr0 from (basis-transformed) standard version +## to our version. +## +## On encrypt, +## xor with 0x63 +## multiply by circulant 0,1,1,1 +## apply shiftrows transform +## +## On decrypt, +## xor with 0x63 +## multiply by "inverse mixcolumns" circulant E,B,D,9 +## deskew +## apply shiftrows transform +## +## +## Writes out to (%a2), and increments or decrements it +## Keeps track of round number mod 4 in %a4 +## Preserves vr0 +## Clobbers vr1-vr5 +## +#.type _vpaes_schedule_mangle,\@abi-omnipotent +.align 4 +_vpaes_schedule_mangle: +.cfi_startproc + vori.b $vr4,$vr0,0 # save vr0 for later + la.local $t0,Lk_mc_forward + vld $vr5,$t0,0 + bnez $a3,.Lschedule_mangle_dec + + # encrypting + addi.d $a2,$a2,16 + la.local $t0,Lk_s63 + vld $vr16,$t0,0 + vxor.v $vr4,$vr4,$vr16 + vshuf.b $vr4,$vr18,$vr4,$vr5 + vori.b $vr3,$vr4,0 + vshuf.b $vr4,$vr18,$vr4,$vr5 + vxor.v $vr3,$vr3,$vr4 + vshuf.b $vr4,$vr18,$vr4,$vr5 + vxor.v $vr3,$vr3,$vr4 + + b .Lschedule_mangle_both +.align 4 +.Lschedule_mangle_dec: + # inverse mix columns + la.local $a7,Lk_dksd + vori.b $vr1,$vr9,0 + vandn.v $vr1,$vr1,$vr4 + vsrli.w $vr1,$vr1,4 # 1 = hi + vand.v $vr4,$vr4,$vr9 # 4 = lo + + vld $vr2,$a7,0 + vshuf.b $vr2,$vr18,$vr2,$vr4 + vld $vr3,$a7,0x10 + vshuf.b $vr3,$vr18,$vr3,$vr1 + vxor.v $vr3,$vr3,$vr2 + vshuf.b $vr3,$vr18,$vr3,$vr5 + + vld $vr2,$a7,0x20 + vshuf.b $vr2,$vr18,$vr2,$vr4 + vxor.v $vr2,$vr2,$vr3 + vld $vr3,$a7,0x30 + vshuf.b $vr3,$vr18,$vr3,$vr1 + vxor.v $vr3,$vr3,$vr2 + vshuf.b $vr3,$vr18,$vr3,$vr5 + + vld $vr2,$a7,0x40 + vshuf.b $vr2,$vr18,$vr2,$vr4 + vxor.v $vr2,$vr2,$vr3 + vld $vr3,$a7,0x50 + vshuf.b $vr3,$vr18,$vr3,$vr1 + vxor.v $vr3,$vr3,$vr2 + vshuf.b $vr3,$vr18,$vr3,$vr5 + + vld $vr2,$a7,0x60 + vshuf.b $vr2,$vr18,$vr2,$vr4 + vxor.v $vr2,$vr2,$vr3 + vld $vr3,$a7,0x70 + vshuf.b $vr3,$vr18,$vr3,$vr1 + vxor.v $vr3,$vr3,$vr2 + + addi.d $a2,$a2,-16 + +.Lschedule_mangle_both: + add.d $t2,$a4,$a6 + vld $vr1,$t2,0 + vshuf.b $vr3,$vr18,$vr3,$vr1 + addi.d $a4,$a4,-16 + andi $a4,$a4,0x30 + vst $vr3,$a2,0 + jirl $zero,$ra,0 +.cfi_endproc +.size _vpaes_schedule_mangle,.-_vpaes_schedule_mangle + +# +# Interface to OpenSSL +# +.globl ${PREFIX}_set_encrypt_key +#.type ${PREFIX}_set_encrypt_key,\@function,3 +.align 4 +${PREFIX}_set_encrypt_key: +.cfi_startproc +___ +$code.=<<___; + addi.d $sp,$sp,-48 + st.d $ra,$sp,40 + st.d $fp,$sp,32 + move $t5,$a1 + srli.w $t5,$t5,0x5 + addi.w $t5,$t5,0x5 + st.w $t5,$a2,240 # AES_KEY->rounds = nbits/32+5; + + move $a3,$zero + li.d $a4,0x30 + bl _vpaes_schedule_core +___ +$code.=<<___; + xor $a0,$a0,$a0 + ld.d $ra,$sp,40 + ld.d $fp,$sp,32 + addi.d $sp,$sp,48 + jirl $zero,$ra,0 +.cfi_endproc +.size ${PREFIX}_set_encrypt_key,.-${PREFIX}_set_encrypt_key + +.globl ${PREFIX}_set_decrypt_key +#.type ${PREFIX}_set_decrypt_key,\@function,3 +.align 4 +${PREFIX}_set_decrypt_key: +.cfi_startproc + +.Ldec_key_body: +___ +$code.=<<___; + addi.d $sp,$sp,-48 + st.d $ra,$sp,40 + st.d $fp,$sp,32 + + move $t5,$a1 + srli.w $t5,$t5,5 + addi.w $t5,$t5,5 + st.w $t5,$a2,240 # AES_KEY->rounds = nbits/32+5; + slli.w $t5,$t5,4 + add.d $t0,$a2,$t5 + addi.d $a2,$t0,16 + + li.d $a3,0x1 + move $a4,$a1 + srli.w $a4,$a4,1 + andi $a4,$a4,32 + xori $a4,$a4,32 # nbits==192?0:32 + bl _vpaes_schedule_core + +.Ldec_key_epilogue: +___ +$code.=<<___; + xor $a0,$a0,$a0 + ld.d $ra,$sp,40 + ld.d $fp,$sp,32 + addi.d $sp,$sp,48 + jirl $zero,$ra,0 +.cfi_endproc +.size ${PREFIX}_set_decrypt_key,.-${PREFIX}_set_decrypt_key + +.globl ${PREFIX}_encrypt +#.type ${PREFIX}_encrypt,\@function,3 +.align 4 +${PREFIX}_encrypt: +.cfi_startproc +.Lenc_body: +___ +$code.=<<___; + addi.d $sp,$sp,-48 + st.d $ra,$sp,40 + st.d $fp,$sp,32 + vld $vr0,$a0,0x0 + bl _vpaes_preheat + bl _vpaes_encrypt_core + vst $vr0,$a1,0x0 +.Lenc_epilogue: +___ +$code.=<<___; + ld.d $ra,$sp,40 + ld.d $fp,$sp,32 + addi.d $sp,$sp,48 + jirl $zero,$ra,0 +.cfi_endproc +.size ${PREFIX}_encrypt,.-${PREFIX}_encrypt + +.globl ${PREFIX}_decrypt +#.type ${PREFIX}_decrypt,\@function,3 +.align 4 +${PREFIX}_decrypt: +.cfi_startproc +___ +$code.=<<___; + addi.d $sp,$sp,-48 + st.d $ra,$sp,40 + st.d $fp,$sp,32 + vld $vr0,$a0,0x0 + bl _vpaes_preheat + bl _vpaes_decrypt_core + vst $vr0,$a1,0x0 +___ +$code.=<<___; + ld.d $ra,$sp,40 + ld.d $fp,$sp,32 + addi.d $sp,$sp,48 + jirl $zero,$ra,0 +.cfi_endproc +.size ${PREFIX}_decrypt,.-${PREFIX}_decrypt +___ +{ +my ($inp,$out,$len,$key,$ivp,$enc)=("$a0","$a1","$a2","$a3","$a4","$a5"); +# void AES_cbc_encrypt (const void char *inp, unsigned char *out, +# size_t length, const AES_KEY *key, +# unsigned char *ivp,const int enc); +$code.=<<___; +.globl ${PREFIX}_cbc_encrypt +#.type ${PREFIX}_cbc_encrypt,\@function,6 +.align 4 +${PREFIX}_cbc_encrypt: +.cfi_startproc + addi.d $sp,$sp,-48 + st.d $ra,$sp,40 + st.d $fp,$sp,32 + + ori $t0,$len,0 + ori $len,$key,0 + ori $key,$t0,0 +___ +($len,$key)=($key,$len); +$code.=<<___; + addi.d $len,$len,-16 + blt $len,$zero,.Lcbc_abort +___ +$code.=<<___; + vld $vr6,$ivp,0 # load IV + sub.d $out,$out,$inp + bl _vpaes_preheat + beqz $a5,.Lcbc_dec_loop + b .Lcbc_enc_loop +.align 4 +.Lcbc_enc_loop: + vld $vr0,$inp,0 + vxor.v $vr0,$vr0,$vr6 + bl _vpaes_encrypt_core + vori.b $vr6,$vr0,0 + add.d $t0,$out,$inp + vst $vr0,$t0,0 + addi.d $inp,$inp,16 + addi.d $len,$len,-16 + bge $len,$zero,.Lcbc_enc_loop + b .Lcbc_done +.align 4 +.Lcbc_dec_loop: + vld $vr0,$inp,0 + vori.b $vr7,$vr0,0 + bl _vpaes_decrypt_core + vxor.v $vr0,$vr0,$vr6 + vori.b $vr6,$vr7,0 + add.d $t0,$out,$inp + vst $vr0,$t0,0 + addi.d $inp,$inp,16 + addi.d $len,$len,-16 + bge $len,$zero,.Lcbc_dec_loop +.Lcbc_done: + vst $vr6,$ivp,0 # save IV +___ +$code.=<<___; +.Lcbc_abort: + ld.d $ra,$sp,40 + ld.d $fp,$sp,32 + addi.d $sp,$sp,48 + jirl $zero,$ra,0 +.cfi_endproc +.size ${PREFIX}_cbc_encrypt,.-${PREFIX}_cbc_encrypt +___ +} +{ +$code.=<<___; +## +## _aes_preheat +## +## Fills register %a6 -> .aes_consts (so you can -fPIC) +## and %vr9-%vr15 as specified below. +## +#.type _vpaes_preheat,\@abi-omnipotent +.align 4 +_vpaes_preheat: +.cfi_startproc + la.local $a6,Lk_s0F + vld $vr10,$a6,-0x20 # Lk_inv + vld $vr11,$a6,-0x10 # Lk_inv+16 + vld $vr9,$a6,0 # Lk_s0F + vld $vr13,$a6,0x30 # Lk_sb1 + vld $vr12,$a6,0x40 # Lk_sb1+16 + vld $vr15,$a6,0x50 # Lk_sb2 + vld $vr14,$a6,0x60 # Lk_sb2+16 + vldi $vr18,0 # $vr18 in this program is equal to 0 + jirl $zero,$ra,0 +.cfi_endproc +.size _vpaes_preheat,.-_vpaes_preheat +___ +} +######################################################## +## ## +## Constants ## +## ## +######################################################## +$code.=<<___; +.section .rodata +.align 6 +Lk_inv: # inv, inva + .quad 0x0E05060F0D080110, 0x040703090A0B0C02 + .quad 0x01040A060F0B0710, 0x030D0E0C02050809 + +Lk_s0F: # s0F + .quad 0x0F0F0F0F0F0F0F0F, 0x0F0F0F0F0F0F0F0F + +Lk_ipt: # input transform (lo, hi) + .quad 0xC2B2E8985A2A7000, 0xCABAE09052227808 + .quad 0x4C01307D317C4D00, 0xCD80B1FCB0FDCC81 + +Lk_sb1: # sb1u, sb1t + .quad 0xB19BE18FCB503E00, 0xA5DF7A6E142AF544 + .quad 0x3618D415FAE22300, 0x3BF7CCC10D2ED9EF +Lk_sb2: # sb2u, sb2t + .quad 0xE27A93C60B712400, 0x5EB7E955BC982FCD + .quad 0x69EB88400AE12900, 0xC2A163C8AB82234A +Lk_sbo: # sbou, sbot + .quad 0xD0D26D176FBDC700, 0x15AABF7AC502A878 + .quad 0xCFE474A55FBB6A00, 0x8E1E90D1412B35FA + +Lk_mc_forward: # mc_forward + .quad 0x0407060500030201, 0x0C0F0E0D080B0A09 + .quad 0x080B0A0904070605, 0x000302010C0F0E0D + .quad 0x0C0F0E0D080B0A09, 0x0407060500030201 + .quad 0x000302010C0F0E0D, 0x080B0A0904070605 + +Lk_mc_backward:# mc_backward + .quad 0x0605040702010003, 0x0E0D0C0F0A09080B + .quad 0x020100030E0D0C0F, 0x0A09080B06050407 + .quad 0x0E0D0C0F0A09080B, 0x0605040702010003 + .quad 0x0A09080B06050407, 0x020100030E0D0C0F + +Lk_sr: # sr + .quad 0x0706050403020100, 0x0F0E0D0C0B0A0908 + .quad 0x030E09040F0A0500, 0x0B06010C07020D08 + .quad 0x0F060D040B020900, 0x070E050C030A0108 + .quad 0x0B0E0104070A0D00, 0x0306090C0F020508 + +Lk_rcon: # rcon + .quad 0x1F8391B9AF9DEEB6, 0x702A98084D7C7D81 + +Lk_s63: # s63: all equal to 0x63 transformed + .quad 0x5B5B5B5B5B5B5B5B, 0x5B5B5B5B5B5B5B5B + +Lk_opt: # output transform + .quad 0xFF9F4929D6B66000, 0xF7974121DEBE6808 + .quad 0x01EDBD5150BCEC00, 0xE10D5DB1B05C0CE0 + +Lk_deskew: # deskew tables: inverts the sbox's "skew" + .quad 0x07E4A34047A4E300, 0x1DFEB95A5DBEF91A + .quad 0x5F36B5DC83EA6900, 0x2841C2ABF49D1E77 + +## +## Decryption stuff +## Key schedule constants +## +Lk_dksd: # decryption key schedule: invskew x*D + .quad 0xFEB91A5DA3E44700, 0x0740E3A45A1DBEF9 + .quad 0x41C277F4B5368300, 0x5FDC69EAAB289D1E +Lk_dksb: # decryption key schedule: invskew x*B + .quad 0x9A4FCA1F8550D500, 0x03D653861CC94C99 + .quad 0x115BEDA7B6FC4A00, 0xD993256F7E3482C8 +Lk_dkse: # decryption key schedule: invskew x*E + 0x63 + .quad 0xD5031CCA1FC9D600, 0x53859A4C994F5086 + .quad 0xA23196054FDC7BE8, 0xCD5EF96A20B31487 +Lk_dks9: # decryption key schedule: invskew x*9 + .quad 0xB6116FC87ED9A700, 0x4AED933482255BFC + .quad 0x4576516227143300, 0x8BB89FACE9DAFDCE + +## +## Decryption stuff +## Round function constants +## +Lk_dipt: # decryption input transform + .quad 0x0F505B040B545F00, 0x154A411E114E451A + .quad 0x86E383E660056500, 0x12771772F491F194 + +Lk_dsb9: # decryption sbox output *9*u, *9*t + .quad 0x851C03539A86D600, 0xCAD51F504F994CC9 + .quad 0xC03B1789ECD74900, 0x725E2C9EB2FBA565 +Lk_dsbd: # decryption sbox output *D*u, *D*t + .quad 0x7D57CCDFE6B1A200, 0xF56E9B13882A4439 + .quad 0x3CE2FAF724C6CB00, 0x2931180D15DEEFD3 +Lk_dsbb: # decryption sbox output *B*u, *B*t + .quad 0xD022649296B44200, 0x602646F6B0F2D404 + .quad 0xC19498A6CD596700, 0xF3FF0C3E3255AA6B +Lk_dsbe: # decryption sbox output *E*u, *E*t + .quad 0x46F2929626D4D000, 0x2242600464B4F6B0 + .quad 0x0C55A6CDFFAAC100, 0x9467F36B98593E32 +Lk_dsbo: # decryption sbox final output + .quad 0x1387EA537EF94000, 0xC7AA6DB9D4943E2D + .quad 0x12D7560F93441D00, 0xCA4B8159D8C58E9C +.asciz "Vector Permutation AES for loongarch64/lsx, Mike Hamburg (Stanford University)" +.align 6 +___ + + +$code =~ s/\`([^\`]*)\`/eval($1)/gem; + +print $code; + +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/aes/asm/vpaes-x86_64.pl b/crypto/openssl/crypto/aes/asm/vpaes-x86_64.pl index 845528f41ac2..f62818841526 100755 --- a/crypto/openssl/crypto/aes/asm/vpaes-x86_64.pl +++ b/crypto/openssl/crypto/aes/asm/vpaes-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2011-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2011-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -1006,6 +1006,7 @@ _vpaes_preheat: ## ## ######################################################## .type _vpaes_consts,\@object +.section .rodata align=64 .align 64 _vpaes_consts: .Lk_inv: # inv, inva @@ -1101,9 +1102,9 @@ _vpaes_consts: .Lk_dsbo: # decryption sbox final output .quad 0x1387EA537EF94000, 0xC7AA6DB9D4943E2D .quad 0x12D7560F93441D00, 0xCA4B8159D8C58E9C -.asciz "Vector Permutation AES for x86_64/SSSE3, Mike Hamburg (Stanford University)" .align 64 .size _vpaes_consts,.-_vpaes_consts +.asciz "Vector Permutation AES for x86_64/SSSE3, Mike Hamburg (Stanford University)" ___ if ($win64) { diff --git a/crypto/openssl/crypto/bn/asm/armv4-gf2m.pl b/crypto/openssl/crypto/bn/asm/armv4-gf2m.pl index d380c89f1f2b..5733a0174660 100755 --- a/crypto/openssl/crypto/bn/asm/armv4-gf2m.pl +++ b/crypto/openssl/crypto/bn/asm/armv4-gf2m.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2011-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2011-2023 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -325,7 +325,8 @@ $code.=<<___; .align 5 #if __ARM_MAX_ARCH__>=7 -.comm OPENSSL_armcap_P,4,4 +.extern OPENSSL_armcap_P +.hidden OPENSSL_armcap_P #endif ___ diff --git a/crypto/openssl/crypto/bn/asm/armv4-mont.pl b/crypto/openssl/crypto/bn/asm/armv4-mont.pl index c26df751a5eb..6e839d845ca8 100755 --- a/crypto/openssl/crypto/bn/asm/armv4-mont.pl +++ b/crypto/openssl/crypto/bn/asm/armv4-mont.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2007-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2007-2023 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -749,7 +749,8 @@ $code.=<<___; .asciz "Montgomery multiplication for ARMv4/NEON, CRYPTOGAMS by " .align 2 #if __ARM_MAX_ARCH__>=7 -.comm OPENSSL_armcap_P,4,4 +.extern OPENSSL_armcap_P +.hidden OPENSSL_armcap_P #endif ___ diff --git a/crypto/openssl/crypto/bn/asm/armv8-mont.pl b/crypto/openssl/crypto/bn/asm/armv8-mont.pl index 9fe1c46805f5..6acba75c9136 100755 --- a/crypto/openssl/crypto/bn/asm/armv8-mont.pl +++ b/crypto/openssl/crypto/bn/asm/armv8-mont.pl @@ -2051,6 +2051,7 @@ __bn_mul4x_mont: ___ } $code.=<<___; +.rodata .asciz "Montgomery Multiplication for ARMv8, CRYPTOGAMS by " .align 4 ___ diff --git a/crypto/openssl/crypto/bn/asm/mips.pl b/crypto/openssl/crypto/bn/asm/mips.pl index bc18826d08f8..44a6f10d4dbd 100755 --- a/crypto/openssl/crypto/bn/asm/mips.pl +++ b/crypto/openssl/crypto/bn/asm/mips.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2010-2021 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2010-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -802,7 +802,7 @@ $code.=<<___; #if 0 /* - * The bn_div_3_words entry point is re-used for constant-time interface. + * The bn_div_3_words entry point is reused for constant-time interface. * Implementation is retained as historical reference. */ .align 5 diff --git a/crypto/openssl/crypto/bn/asm/ppc64-mont-fixed.pl b/crypto/openssl/crypto/bn/asm/ppc64-mont-fixed.pl index e69de29bb2d1..e27d0ad93d85 100755 --- a/crypto/openssl/crypto/bn/asm/ppc64-mont-fixed.pl +++ b/crypto/openssl/crypto/bn/asm/ppc64-mont-fixed.pl @@ -0,0 +1,583 @@ +#! /usr/bin/env perl +# Copyright 2021-2022 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +# ==================================================================== +# Written by Amitay Isaacs , Martin Schwenke +# & Alastair D'Silva for +# the OpenSSL project. +# ==================================================================== + +# +# Fixed length (n=6), unrolled PPC Montgomery Multiplication +# + +# 2021 +# +# Although this is a generic implementation for unrolling Montgomery +# Multiplication for arbitrary values of n, this is currently only +# used for n = 6 to improve the performance of ECC p384. +# +# Unrolling allows intermediate results to be stored in registers, +# rather than on the stack, improving performance by ~7% compared to +# the existing PPC assembly code. +# +# The ISA 3.0 implementation uses combination multiply/add +# instructions (maddld, maddhdu) to improve performance by an +# additional ~10% on Power 9. +# +# Finally, saving non-volatile registers into volatile vector +# registers instead of onto the stack saves a little more. +# +# On a Power 9 machine we see an overall improvement of ~18%. +# + +use strict; +use warnings; + +my ($flavour, $output, $dir, $xlate); + +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or +die "can't locate ppc-xlate.pl"; + +open STDOUT,"| $^X $xlate $flavour \"$output\"" + or die "can't call $xlate: $!"; + +if ($flavour !~ /64/) { + die "bad flavour ($flavour) - only ppc64 permitted"; +} + +my $SIZE_T= 8; + +# Registers are global so the code is remotely readable + +# Parameters for Montgomery multiplication +my $ze = "r0"; +my $sp = "r1"; +my $toc = "r2"; +my $rp = "r3"; +my $ap = "r4"; +my $bp = "r5"; +my $np = "r6"; +my $n0 = "r7"; +my $num = "r8"; + +my $i = "r9"; +my $c0 = "r10"; +my $bp0 = "r11"; +my $bpi = "r11"; +my $bpj = "r11"; +my $tj = "r12"; +my $apj = "r12"; +my $npj = "r12"; +my $lo = "r14"; +my $c1 = "r14"; + +# Non-volatile registers used for tp[i] +# +# 12 registers are available but the limit on unrolling is 10, +# since registers from $tp[0] to $tp[$n+1] are used. +my @tp = ("r20" .. "r31"); + +# volatile VSRs for saving non-volatile GPRs - faster than stack +my @vsrs = ("v32" .. "v46"); + +package Mont; + +sub new($$) +{ + my ($class, $n) = @_; + + if ($n > 10) { + die "Can't unroll for BN length ${n} (maximum 10)" + } + + my $self = { + code => "", + n => $n, + }; + bless $self, $class; + + return $self; +} + +sub add_code($$) +{ + my ($self, $c) = @_; + + $self->{code} .= $c; +} + +sub get_code($) +{ + my ($self) = @_; + + return $self->{code}; +} + +sub get_function_name($) +{ + my ($self) = @_; + + return "bn_mul_mont_fixed_n" . $self->{n}; +} + +sub get_label($$) +{ + my ($self, $l) = @_; + + return "L" . $l . "_" . $self->{n}; +} + +sub get_labels($@) +{ + my ($self, @labels) = @_; + + my %out = (); + + foreach my $l (@labels) { + $out{"$l"} = $self->get_label("$l"); + } + + return \%out; +} + +sub nl($) +{ + my ($self) = @_; + + $self->add_code("\n"); +} + +sub copy_result($) +{ + my ($self) = @_; + + my ($n) = $self->{n}; + + for (my $j = 0; $j < $n; $j++) { + $self->add_code(<<___); + std $tp[$j],`$j*$SIZE_T`($rp) +___ + } + +} + +sub mul_mont_fixed($) +{ + my ($self) = @_; + + my ($n) = $self->{n}; + my $fname = $self->get_function_name(); + my $label = $self->get_labels("outer", "enter", "sub", "copy", "end"); + + $self->add_code(<<___); + +.globl .${fname} +.align 5 +.${fname}: + +___ + + $self->save_registers(); + + $self->add_code(<<___); + li $ze,0 + ld $n0,0($n0) + + ld $bp0,0($bp) + + ld $apj,0($ap) +___ + + $self->mul_c_0($tp[0], $apj, $bp0, $c0); + + for (my $j = 1; $j < $n - 1; $j++) { + $self->add_code(<<___); + ld $apj,`$j*$SIZE_T`($ap) +___ + $self->mul($tp[$j], $apj, $bp0, $c0); + } + + $self->add_code(<<___); + ld $apj,`($n-1)*$SIZE_T`($ap) +___ + + $self->mul_last($tp[$n-1], $tp[$n], $apj, $bp0, $c0); + + $self->add_code(<<___); + li $tp[$n+1],0 + +___ + + $self->add_code(<<___); + li $i,0 + mtctr $num + b $label->{"enter"} + +.align 4 +$label->{"outer"}: + ldx $bpi,$bp,$i + + ld $apj,0($ap) +___ + + $self->mul_add_c_0($tp[0], $tp[0], $apj, $bpi, $c0); + + for (my $j = 1; $j < $n; $j++) { + $self->add_code(<<___); + ld $apj,`$j*$SIZE_T`($ap) +___ + $self->mul_add($tp[$j], $tp[$j], $apj, $bpi, $c0); + } + + $self->add_code(<<___); + addc $tp[$n],$tp[$n],$c0 + addze $tp[$n+1],$ze +___ + + $self->add_code(<<___); +.align 4 +$label->{"enter"}: + mulld $bpi,$tp[0],$n0 + + ld $npj,0($np) +___ + + $self->mul_add_c_0($lo, $tp[0], $bpi, $npj, $c0); + + for (my $j = 1; $j < $n; $j++) { + $self->add_code(<<___); + ld $npj,`$j*$SIZE_T`($np) +___ + $self->mul_add($tp[$j-1], $tp[$j], $npj, $bpi, $c0); + } + + $self->add_code(<<___); + addc $tp[$n-1],$tp[$n],$c0 + addze $tp[$n],$tp[$n+1] + + addi $i,$i,$SIZE_T + bdnz $label->{"outer"} + + and. $tp[$n],$tp[$n],$tp[$n] + bne $label->{"sub"} + + cmpld $tp[$n-1],$npj + blt $label->{"copy"} + +$label->{"sub"}: +___ + + # + # Reduction + # + + $self->add_code(<<___); + ld $bpj,`0*$SIZE_T`($np) + subfc $c1,$bpj,$tp[0] + std $c1,`0*$SIZE_T`($rp) + +___ + for (my $j = 1; $j < $n - 1; $j++) { + $self->add_code(<<___); + ld $bpj,`$j*$SIZE_T`($np) + subfe $c1,$bpj,$tp[$j] + std $c1,`$j*$SIZE_T`($rp) + +___ + } + + $self->add_code(<<___); + subfe $c1,$npj,$tp[$n-1] + std $c1,`($n-1)*$SIZE_T`($rp) + +___ + + $self->add_code(<<___); + addme. $tp[$n],$tp[$n] + beq $label->{"end"} + +$label->{"copy"}: +___ + + $self->copy_result(); + + $self->add_code(<<___); + +$label->{"end"}: +___ + + $self->restore_registers(); + + $self->add_code(<<___); + li r3,1 + blr +.size .${fname},.-.${fname} +___ + +} + +package Mont::GPR; + +our @ISA = ('Mont'); + +sub new($$) +{ + my ($class, $n) = @_; + + return $class->SUPER::new($n); +} + +sub save_registers($) +{ + my ($self) = @_; + + my $n = $self->{n}; + + $self->add_code(<<___); + std $lo,-8($sp) +___ + + for (my $j = 0; $j <= $n+1; $j++) { + $self->{code}.=<<___; + std $tp[$j],-`($j+2)*8`($sp) +___ + } + + $self->add_code(<<___); + +___ +} + +sub restore_registers($) +{ + my ($self) = @_; + + my $n = $self->{n}; + + $self->add_code(<<___); + ld $lo,-8($sp) +___ + + for (my $j = 0; $j <= $n+1; $j++) { + $self->{code}.=<<___; + ld $tp[$j],-`($j+2)*8`($sp) +___ + } + + $self->{code} .=<<___; + +___ +} + +# Direct translation of C mul() +sub mul($$$$$) +{ + my ($self, $r, $a, $w, $c) = @_; + + $self->add_code(<<___); + mulld $lo,$a,$w + addc $r,$lo,$c + mulhdu $c,$a,$w + addze $c,$c + +___ +} + +# Like mul() but $c is ignored as an input - an optimisation to save a +# preliminary instruction that would set input $c to 0 +sub mul_c_0($$$$$) +{ + my ($self, $r, $a, $w, $c) = @_; + + $self->add_code(<<___); + mulld $r,$a,$w + mulhdu $c,$a,$w + +___ +} + +# Like mul() but does not to the final addition of CA into $c - an +# optimisation to save an instruction +sub mul_last($$$$$$) +{ + my ($self, $r1, $r2, $a, $w, $c) = @_; + + $self->add_code(<<___); + mulld $lo,$a,$w + addc $r1,$lo,$c + mulhdu $c,$a,$w + + addze $r2,$c +___ +} + +# Like C mul_add() but allow $r_out and $r_in to be different +sub mul_add($$$$$$) +{ + my ($self, $r_out, $r_in, $a, $w, $c) = @_; + + $self->add_code(<<___); + mulld $lo,$a,$w + addc $lo,$lo,$c + mulhdu $c,$a,$w + addze $c,$c + addc $r_out,$r_in,$lo + addze $c,$c + +___ +} + +# Like mul_add() but $c is ignored as an input - an optimisation to save a +# preliminary instruction that would set input $c to 0 +sub mul_add_c_0($$$$$$) +{ + my ($self, $r_out, $r_in, $a, $w, $c) = @_; + + $self->add_code(<<___); + mulld $lo,$a,$w + addc $r_out,$r_in,$lo + mulhdu $c,$a,$w + addze $c,$c + +___ +} + +package Mont::GPR_300; + +our @ISA = ('Mont::GPR'); + +sub new($$) +{ + my ($class, $n) = @_; + + my $mont = $class->SUPER::new($n); + + return $mont; +} + +sub get_function_name($) +{ + my ($self) = @_; + + return "bn_mul_mont_300_fixed_n" . $self->{n}; +} + +sub get_label($$) +{ + my ($self, $l) = @_; + + return "L" . $l . "_300_" . $self->{n}; +} + +# Direct translation of C mul() +sub mul($$$$$) +{ + my ($self, $r, $a, $w, $c, $last) = @_; + + $self->add_code(<<___); + maddld $r,$a,$w,$c + maddhdu $c,$a,$w,$c + +___ +} + +# Save the last carry as the final entry +sub mul_last($$$$$) +{ + my ($self, $r1, $r2, $a, $w, $c) = @_; + + $self->add_code(<<___); + maddld $r1,$a,$w,$c + maddhdu $r2,$a,$w,$c + +___ +} + +# Like mul() but $c is ignored as an input - an optimisation to save a +# preliminary instruction that would set input $c to 0 +sub mul_c_0($$$$$) +{ + my ($self, $r, $a, $w, $c) = @_; + + $self->add_code(<<___); + mulld $r,$a,$w + mulhdu $c,$a,$w + +___ +} + +# Like C mul_add() but allow $r_out and $r_in to be different +sub mul_add($$$$$$) +{ + my ($self, $r_out, $r_in, $a, $w, $c) = @_; + + $self->add_code(<<___); + maddld $lo,$a,$w,$c + maddhdu $c,$a,$w,$c + addc $r_out,$r_in,$lo + addze $c,$c + +___ +} + +# Like mul_add() but $c is ignored as an input - an optimisation to save a +# preliminary instruction that would set input $c to 0 +sub mul_add_c_0($$$$$$) +{ + my ($self, $r_out, $r_in, $a, $w, $c) = @_; + + $self->add_code(<<___); + maddld $lo,$a,$w,$r_in + maddhdu $c,$a,$w,$r_in +___ + + if ($r_out ne $lo) { + $self->add_code(<<___); + mr $r_out,$lo +___ + } + + $self->nl(); +} + + +package main; + +my $code; + +$code.=<<___; +.machine "any" +.text +___ + +my $mont; + +$mont = new Mont::GPR(6); +$mont->mul_mont_fixed(); +$code .= $mont->get_code(); + +$mont = new Mont::GPR_300(6); +$mont->mul_mont_fixed(); +$code .= $mont->get_code(); + +$code =~ s/\`([^\`]*)\`/eval $1/gem; + +$code.=<<___; +.asciz "Montgomery Multiplication for PPC by , " +___ + +print $code; +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/bn/asm/rsaz-avx512.pl b/crypto/openssl/crypto/bn/asm/rsaz-2k-avx512.pl similarity index 70% rename from crypto/openssl/crypto/bn/asm/rsaz-avx512.pl rename to crypto/openssl/crypto/bn/asm/rsaz-2k-avx512.pl index 8d1d19f6c728..131e315324c1 100644 --- a/crypto/openssl/crypto/bn/asm/rsaz-avx512.pl +++ b/crypto/openssl/crypto/bn/asm/rsaz-2k-avx512.pl @@ -1,4 +1,4 @@ -# Copyright 2020-2022 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2020-2024 The OpenSSL Project Authors. All Rights Reserved. # Copyright (c) 2020, Intel Corporation. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use @@ -7,7 +7,8 @@ # https://www.openssl.org/source/license.html # # -# Originally written by Ilya Albrekht, Sergey Kirillov and Andrey Matyukov +# Originally written by Sergey Kirillov and Andrey Matyukov. +# Special thanks to Ilya Albrekht for his valuable hints. # Intel Corporation # # December 2020 @@ -43,12 +44,12 @@ if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` $avx512ifma = ($1>=2.26); } -if (!$avx512 && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && +if (!$avx512ifma && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?/) { $avx512ifma = ($1==2.11 && $2>=8) + ($1>=2.12); } -if (!$avx512 && `$ENV{CC} -v 2>&1` +if (!$avx512ifma && `$ENV{CC} -v 2>&1` =~ /(Apple)?\s*((?:clang|LLVM) version|.*based on LLVM) ([0-9]+)\.([0-9]+)\.([0-9]+)?/) { my $ver = $3 + $4/100.0 + $5/10000.0; # 3.1.0->3.01, 3.10.1->3.1001 if ($1) { @@ -57,7 +58,7 @@ if (!$avx512 && `$ENV{CC} -v 2>&1` # clang 7.0.0 is Apple clang 10.0.1 $avx512ifma = ($ver>=10.0001) } else { - $avx512ifma = ($3>=7.0); + $avx512ifma = ($ver>=7.0); } } @@ -86,26 +87,29 @@ ___ ############################################################################### # Almost Montgomery Multiplication (AMM) for 20-digit number in radix 2^52. # -# AMM is defined as presented in the paper -# "Efficient Software Implementations of Modular Exponentiation" by Shay Gueron. +# AMM is defined as presented in the paper [1]. # # The input and output are presented in 2^52 radix domain, i.e. # |res|, |a|, |b|, |m| are arrays of 20 64-bit qwords with 12 high bits zeroed. # |k0| is a Montgomery coefficient, which is here k0 = -1/m mod 2^64 -# (note, the implementation counts only 52 bits from it). # -# NB: the AMM implementation does not perform "conditional" subtraction step as -# specified in the original algorithm as according to the paper "Enhanced Montgomery -# Multiplication" by Shay Gueron (see Lemma 1), the result will be always < 2*2^1024 -# and can be used as a direct input to the next AMM iteration. -# This post-condition is true, provided the correct parameter |s| is choosen, i.e. -# s >= n + 2 * k, which matches our case: 1040 > 1024 + 2 * 1. +# NB: the AMM implementation does not perform "conditional" subtraction step +# specified in the original algorithm as according to the Lemma 1 from the paper +# [2], the result will be always < 2*m and can be used as a direct input to +# the next AMM iteration. This post-condition is true, provided the correct +# parameter |s| (notion of the Lemma 1 from [2]) is chosen, i.e. s >= n + 2 * k, +# which matches our case: 1040 > 1024 + 2 * 1. # -# void ossl_rsaz_amm52x20_x1_256(BN_ULONG *res, -# const BN_ULONG *a, -# const BN_ULONG *b, -# const BN_ULONG *m, -# BN_ULONG k0); +# [1] Gueron, S. Efficient software implementations of modular exponentiation. +# DOI: 10.1007/s13389-012-0031-5 +# [2] Gueron, S. Enhanced Montgomery Multiplication. +# DOI: 10.1007/3-540-36400-5_5 +# +# void ossl_rsaz_amm52x20_x1_ifma256(BN_ULONG *res, +# const BN_ULONG *a, +# const BN_ULONG *b, +# const BN_ULONG *m, +# BN_ULONG k0); ############################################################################### { # input parameters ("%rdi","%rsi","%rdx","%rcx","%r8") @@ -121,16 +125,13 @@ my $b_ptr = "%r11"; my $iter = "%ebx"; my $zero = "%ymm0"; -my ($R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0) = ("%ymm1", map("%ymm$_",(16..19))); -my ($R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1) = ("%ymm2", map("%ymm$_",(20..23))); -my $Bi = "%ymm3"; -my $Yi = "%ymm4"; +my $Bi = "%ymm1"; +my $Yi = "%ymm2"; +my ($R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0) = ("%ymm3",map("%ymm$_",(16..19))); +my ($R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1) = ("%ymm4",map("%ymm$_",(20..23))); # Registers mapping for normalization. -# We can reuse Bi, Yi registers here. -my $TMP = $Bi; -my $mask52x4 = $Yi; -my ($T0,$T0h,$T1,$T1h,$T2) = map("%ymm$_", (24..28)); +my ($T0,$T0h,$T1,$T1h,$T2) = ("$zero", "$Bi", "$Yi", map("%ymm$_", (25..26))); sub amm52x20_x1() { # _data_offset - offset in the |a| or |m| arrays pointing to the beginning @@ -199,16 +200,16 @@ $code.=<<___; ___ } -# Normalization routine: handles carry bits in R0..R2 QWs and -# gets R0..R2 back to normalized 2^52 representation. +# Normalization routine: handles carry bits and gets bignum qwords to normalized +# 2^52 representation. # # Uses %r8-14,%e[bcd]x sub amm52x20_x1_norm { my ($_acc,$_R0,$_R0h,$_R1,$_R1h,$_R2) = @_; $code.=<<___; # Put accumulator to low qword in R0 - vpbroadcastq $_acc, $TMP - vpblendd \$3, $TMP, $_R0, $_R0 + vpbroadcastq $_acc, $T0 + vpblendd \$3, $T0, $_R0, $_R0 # Extract "carries" (12 high bits) from each QW of R0..R2 # Save them to LSB of QWs in T0..T2 @@ -223,14 +224,14 @@ $code.=<<___; valignq \$3, $T1, $T1h, $T1h valignq \$3, $T0h, $T1, $T1 valignq \$3, $T0, $T0h, $T0h - valignq \$3, $zero, $T0, $T0 + valignq \$3, .Lzeros(%rip), $T0, $T0 # Drop "carries" from R0..R2 QWs - vpandq $mask52x4, $_R0, $_R0 - vpandq $mask52x4, $_R0h, $_R0h - vpandq $mask52x4, $_R1, $_R1 - vpandq $mask52x4, $_R1h, $_R1h - vpandq $mask52x4, $_R2, $_R2 + vpandq .Lmask52x4(%rip), $_R0, $_R0 + vpandq .Lmask52x4(%rip), $_R0h, $_R0h + vpandq .Lmask52x4(%rip), $_R1, $_R1 + vpandq .Lmask52x4(%rip), $_R1h, $_R1h + vpandq .Lmask52x4(%rip), $_R2, $_R2 # Sum R0..R2 with corresponding adjusted carries vpaddq $T0, $_R0, $_R0 @@ -241,11 +242,11 @@ $code.=<<___; # Now handle carry bits from this addition # Get mask of QWs which 52-bit parts overflow... - vpcmpuq \$1, $_R0, $mask52x4, %k1 # OP=lt - vpcmpuq \$1, $_R0h, $mask52x4, %k2 - vpcmpuq \$1, $_R1, $mask52x4, %k3 - vpcmpuq \$1, $_R1h, $mask52x4, %k4 - vpcmpuq \$1, $_R2, $mask52x4, %k5 + vpcmpuq \$6, .Lmask52x4(%rip), $_R0, %k1 # OP=nle (i.e. gt) + vpcmpuq \$6, .Lmask52x4(%rip), $_R0h, %k2 + vpcmpuq \$6, .Lmask52x4(%rip), $_R1, %k3 + vpcmpuq \$6, .Lmask52x4(%rip), $_R1h, %k4 + vpcmpuq \$6, .Lmask52x4(%rip), $_R2, %k5 kmovb %k1, %r14d # k1 kmovb %k2, %r13d # k1h kmovb %k3, %r12d # k2 @@ -253,11 +254,11 @@ $code.=<<___; kmovb %k5, %r10d # k3 # ...or saturated - vpcmpuq \$0, $_R0, $mask52x4, %k1 # OP=eq - vpcmpuq \$0, $_R0h, $mask52x4, %k2 - vpcmpuq \$0, $_R1, $mask52x4, %k3 - vpcmpuq \$0, $_R1h, $mask52x4, %k4 - vpcmpuq \$0, $_R2, $mask52x4, %k5 + vpcmpuq \$0, .Lmask52x4(%rip), $_R0, %k1 # OP=eq + vpcmpuq \$0, .Lmask52x4(%rip), $_R0h, %k2 + vpcmpuq \$0, .Lmask52x4(%rip), $_R1, %k3 + vpcmpuq \$0, .Lmask52x4(%rip), $_R1h, %k4 + vpcmpuq \$0, .Lmask52x4(%rip), $_R2, %k5 kmovb %k1, %r9d # k4 kmovb %k2, %r8d # k4h kmovb %k3, %ebx # k5 @@ -297,27 +298,27 @@ $code.=<<___; kmovb %r10d, %k5 # Add carries according to the obtained mask - vpsubq $mask52x4, $_R0, ${_R0}{%k1} - vpsubq $mask52x4, $_R0h, ${_R0h}{%k2} - vpsubq $mask52x4, $_R1, ${_R1}{%k3} - vpsubq $mask52x4, $_R1h, ${_R1h}{%k4} - vpsubq $mask52x4, $_R2, ${_R2}{%k5} + vpsubq .Lmask52x4(%rip), $_R0, ${_R0}{%k1} + vpsubq .Lmask52x4(%rip), $_R0h, ${_R0h}{%k2} + vpsubq .Lmask52x4(%rip), $_R1, ${_R1}{%k3} + vpsubq .Lmask52x4(%rip), $_R1h, ${_R1h}{%k4} + vpsubq .Lmask52x4(%rip), $_R2, ${_R2}{%k5} - vpandq $mask52x4, $_R0, $_R0 - vpandq $mask52x4, $_R0h, $_R0h - vpandq $mask52x4, $_R1, $_R1 - vpandq $mask52x4, $_R1h, $_R1h - vpandq $mask52x4, $_R2, $_R2 + vpandq .Lmask52x4(%rip), $_R0, $_R0 + vpandq .Lmask52x4(%rip), $_R0h, $_R0h + vpandq .Lmask52x4(%rip), $_R1, $_R1 + vpandq .Lmask52x4(%rip), $_R1h, $_R1h + vpandq .Lmask52x4(%rip), $_R2, $_R2 ___ } $code.=<<___; .text -.globl ossl_rsaz_amm52x20_x1_256 -.type ossl_rsaz_amm52x20_x1_256,\@function,5 +.globl ossl_rsaz_amm52x20_x1_ifma256 +.type ossl_rsaz_amm52x20_x1_ifma256,\@function,5 .align 32 -ossl_rsaz_amm52x20_x1_256: +ossl_rsaz_amm52x20_x1_ifma256: .cfi_startproc endbranch push %rbx @@ -332,7 +333,7 @@ ossl_rsaz_amm52x20_x1_256: .cfi_push %r14 push %r15 .cfi_push %r15 -.Lrsaz_amm52x20_x1_256_body: +.Lossl_rsaz_amm52x20_x1_ifma256_body: # Zeroing accumulators vpxord $zero, $zero, $zero @@ -360,17 +361,15 @@ $code.=<<___; lea `4*8`($b_ptr), $b_ptr dec $iter jne .Lloop5 - - vmovdqa64 .Lmask52x4(%rip), $mask52x4 ___ &amm52x20_x1_norm($acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0); $code.=<<___; - vmovdqu64 $R0_0, ($res) - vmovdqu64 $R0_0h, 32($res) - vmovdqu64 $R1_0, 64($res) - vmovdqu64 $R1_0h, 96($res) - vmovdqu64 $R2_0, 128($res) + vmovdqu64 $R0_0, `0*32`($res) + vmovdqu64 $R0_0h, `1*32`($res) + vmovdqu64 $R1_0, `2*32`($res) + vmovdqu64 $R1_0h, `3*32`($res) + vmovdqu64 $R2_0, `4*32`($res) vzeroupper mov 0(%rsp),%r15 @@ -387,14 +386,14 @@ $code.=<<___; .cfi_restore %rbx lea 48(%rsp),%rsp .cfi_adjust_cfa_offset -48 -.Lrsaz_amm52x20_x1_256_epilogue: +.Lossl_rsaz_amm52x20_x1_ifma256_epilogue: ret .cfi_endproc -.size ossl_rsaz_amm52x20_x1_256, .-ossl_rsaz_amm52x20_x1_256 +.size ossl_rsaz_amm52x20_x1_ifma256, .-ossl_rsaz_amm52x20_x1_ifma256 ___ $code.=<<___; -.data +.section .rodata align=32 .align 32 .Lmask52x4: .quad 0xfffffffffffff @@ -406,25 +405,25 @@ ___ ############################################################################### # Dual Almost Montgomery Multiplication for 20-digit number in radix 2^52 # -# See description of ossl_rsaz_amm52x20_x1_256() above for details about Almost +# See description of ossl_rsaz_amm52x20_x1_ifma256() above for details about Almost # Montgomery Multiplication algorithm and function input parameters description. # # This function does two AMMs for two independent inputs, hence dual. # -# void ossl_rsaz_amm52x20_x2_256(BN_ULONG out[2][20], -# const BN_ULONG a[2][20], -# const BN_ULONG b[2][20], -# const BN_ULONG m[2][20], -# const BN_ULONG k0[2]); +# void ossl_rsaz_amm52x20_x2_ifma256(BN_ULONG out[2][20], +# const BN_ULONG a[2][20], +# const BN_ULONG b[2][20], +# const BN_ULONG m[2][20], +# const BN_ULONG k0[2]); ############################################################################### $code.=<<___; .text -.globl ossl_rsaz_amm52x20_x2_256 -.type ossl_rsaz_amm52x20_x2_256,\@function,5 +.globl ossl_rsaz_amm52x20_x2_ifma256 +.type ossl_rsaz_amm52x20_x2_ifma256,\@function,5 .align 32 -ossl_rsaz_amm52x20_x2_256: +ossl_rsaz_amm52x20_x2_ifma256: .cfi_startproc endbranch push %rbx @@ -439,7 +438,7 @@ ossl_rsaz_amm52x20_x2_256: .cfi_push %r14 push %r15 .cfi_push %r15 -.Lrsaz_amm52x20_x2_256_body: +.Lossl_rsaz_amm52x20_x2_ifma256_body: # Zeroing accumulators vpxord $zero, $zero, $zero @@ -472,24 +471,22 @@ $code.=<<___; lea 8($b_ptr), $b_ptr dec $iter jne .Lloop20 - - vmovdqa64 .Lmask52x4(%rip), $mask52x4 ___ &amm52x20_x1_norm($acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0); &amm52x20_x1_norm($acc0_1,$R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1); $code.=<<___; - vmovdqu64 $R0_0, ($res) - vmovdqu64 $R0_0h, 32($res) - vmovdqu64 $R1_0, 64($res) - vmovdqu64 $R1_0h, 96($res) - vmovdqu64 $R2_0, 128($res) + vmovdqu64 $R0_0, `0*32`($res) + vmovdqu64 $R0_0h, `1*32`($res) + vmovdqu64 $R1_0, `2*32`($res) + vmovdqu64 $R1_0h, `3*32`($res) + vmovdqu64 $R2_0, `4*32`($res) - vmovdqu64 $R0_1, 160($res) - vmovdqu64 $R0_1h, 192($res) - vmovdqu64 $R1_1, 224($res) - vmovdqu64 $R1_1h, 256($res) - vmovdqu64 $R2_1, 288($res) + vmovdqu64 $R0_1, `5*32`($res) + vmovdqu64 $R0_1h, `6*32`($res) + vmovdqu64 $R1_1, `7*32`($res) + vmovdqu64 $R1_1h, `8*32`($res) + vmovdqu64 $R2_1, `9*32`($res) vzeroupper mov 0(%rsp),%r15 @@ -506,10 +503,10 @@ $code.=<<___; .cfi_restore %rbx lea 48(%rsp),%rsp .cfi_adjust_cfa_offset -48 -.Lrsaz_amm52x20_x2_256_epilogue: +.Lossl_rsaz_amm52x20_x2_ifma256_epilogue: ret .cfi_endproc -.size ossl_rsaz_amm52x20_x2_256, .-ossl_rsaz_amm52x20_x2_256 +.size ossl_rsaz_amm52x20_x2_ifma256, .-ossl_rsaz_amm52x20_x2_ifma256 ___ } @@ -517,86 +514,87 @@ ___ # Constant time extraction from the precomputed table of powers base^i, where # i = 0..2^EXP_WIN_SIZE-1 # -# The input |red_table| contains precomputations for two independent base values, -# so the |tbl_idx| indicates for which base shall we extract the value. -# |red_table_idx| is a power index. +# The input |red_table| contains precomputations for two independent base values. +# |red_table_idx1| and |red_table_idx2| are corresponding power indexes. # -# Extracted value (output) is 20 digit number in 2^52 radix. +# Extracted value (output) is 2 20 digit numbers in 2^52 radix. # # void ossl_extract_multiplier_2x20_win5(BN_ULONG *red_Y, # const BN_ULONG red_table[1 << EXP_WIN_SIZE][2][20], -# int red_table_idx, -# int tbl_idx); # 0 or 1 +# int red_table_idx1, int red_table_idx2); # # EXP_WIN_SIZE = 5 ############################################################################### { # input parameters -my ($out,$red_tbl,$red_tbl_idx,$tbl_idx) = @_6_args_universal_ABI; +my ($out,$red_tbl,$red_tbl_idx1,$red_tbl_idx2)=$win64 ? ("%rcx","%rdx","%r8", "%r9") : # Win64 order + ("%rdi","%rsi","%rdx","%rcx"); # Unix order -my ($t0,$t1,$t2,$t3,$t4) = map("%ymm$_", (0..4)); -my $t4xmm = $t4; -$t4xmm =~ s/%y/%x/; -my ($tmp0,$tmp1,$tmp2,$tmp3,$tmp4) = map("%ymm$_", (16..20)); -my ($cur_idx,$idx,$ones) = map("%ymm$_", (21..23)); +my ($t0,$t1,$t2,$t3,$t4,$t5) = map("%ymm$_", (0..5)); +my ($t6,$t7,$t8,$t9) = map("%ymm$_", (16..19)); +my ($tmp,$cur_idx,$idx1,$idx2,$ones) = map("%ymm$_", (20..24)); + +my @t = ($t0,$t1,$t2,$t3,$t4,$t5,$t6,$t7,$t8,$t9); +my $t0xmm = $t0; +$t0xmm =~ s/%y/%x/; $code.=<<___; .text .align 32 .globl ossl_extract_multiplier_2x20_win5 -.type ossl_extract_multiplier_2x20_win5,\@function,4 +.type ossl_extract_multiplier_2x20_win5,\@abi-omnipotent ossl_extract_multiplier_2x20_win5: .cfi_startproc endbranch - leaq ($tbl_idx,$tbl_idx,4), %rax - salq \$5, %rax - addq %rax, $red_tbl - vmovdqa64 .Lones(%rip), $ones # broadcast ones - vpbroadcastq $red_tbl_idx, $idx + vpbroadcastq $red_tbl_idx1, $idx1 + vpbroadcastq $red_tbl_idx2, $idx2 leaq `(1<<5)*2*20*8`($red_tbl), %rax # holds end of the tbl - vpxor $t4xmm, $t4xmm, $t4xmm - vmovdqa64 $t4, $t3 # zeroing t0..4, cur_idx - vmovdqa64 $t4, $t2 - vmovdqa64 $t4, $t1 - vmovdqa64 $t4, $t0 - vmovdqa64 $t4, $cur_idx + # zeroing t0..n, cur_idx + vpxor $t0xmm, $t0xmm, $t0xmm + vmovdqa64 $t0, $cur_idx +___ +foreach (1..9) { + $code.="vmovdqa64 $t0, $t[$_] \n"; +} +$code.=<<___; .align 32 .Lloop: - vpcmpq \$0, $cur_idx, $idx, %k1 # mask of (idx == cur_idx) - addq \$320, $red_tbl # 320 = 2 * 20 digits * 8 bytes - vpaddq $ones, $cur_idx, $cur_idx # increment cur_idx - vmovdqu64 -320($red_tbl), $tmp0 # load data from red_tbl - vmovdqu64 -288($red_tbl), $tmp1 - vmovdqu64 -256($red_tbl), $tmp2 - vmovdqu64 -224($red_tbl), $tmp3 - vmovdqu64 -192($red_tbl), $tmp4 - vpblendmq $tmp0, $t0, ${t0}{%k1} # extract data when mask is not zero - vpblendmq $tmp1, $t1, ${t1}{%k1} - vpblendmq $tmp2, $t2, ${t2}{%k1} - vpblendmq $tmp3, $t3, ${t3}{%k1} - vpblendmq $tmp4, $t4, ${t4}{%k1} + vpcmpq \$0, $cur_idx, $idx1, %k1 # mask of (idx1 == cur_idx) + vpcmpq \$0, $cur_idx, $idx2, %k2 # mask of (idx2 == cur_idx) +___ +foreach (0..9) { + my $mask = $_<5?"%k1":"%k2"; +$code.=<<___; + vmovdqu64 `${_}*32`($red_tbl), $tmp # load data from red_tbl + vpblendmq $tmp, $t[$_], ${t[$_]}{$mask} # extract data when mask is not zero +___ +} +$code.=<<___; + vpaddq $ones, $cur_idx, $cur_idx # increment cur_idx + addq \$`2*20*8`, $red_tbl cmpq $red_tbl, %rax jne .Lloop - - vmovdqu64 $t0, ($out) # store t0..4 - vmovdqu64 $t1, 32($out) - vmovdqu64 $t2, 64($out) - vmovdqu64 $t3, 96($out) - vmovdqu64 $t4, 128($out) - +___ +# store t0..n +foreach (0..9) { + $code.="vmovdqu64 $t[$_], `${_}*32`($out) \n"; +} +$code.=<<___; ret .cfi_endproc .size ossl_extract_multiplier_2x20_win5, .-ossl_extract_multiplier_2x20_win5 ___ $code.=<<___; -.data +.section .rodata align=32 .align 32 .Lones: .quad 1,1,1,1 +.Lzeros: + .quad 0,0,0,0 ___ } @@ -606,7 +604,7 @@ $frame="%rdx"; $context="%r8"; $disp="%r9"; -$code.=<<___ +$code.=<<___; .extern __imp_RtlVirtualUnwind .type rsaz_def_handler,\@abi-omnipotent .align 16 @@ -697,32 +695,24 @@ rsaz_def_handler: .section .pdata .align 4 - .rva .LSEH_begin_ossl_rsaz_amm52x20_x1_256 - .rva .LSEH_end_ossl_rsaz_amm52x20_x1_256 - .rva .LSEH_info_ossl_rsaz_amm52x20_x1_256 + .rva .LSEH_begin_ossl_rsaz_amm52x20_x1_ifma256 + .rva .LSEH_end_ossl_rsaz_amm52x20_x1_ifma256 + .rva .LSEH_info_ossl_rsaz_amm52x20_x1_ifma256 - .rva .LSEH_begin_ossl_rsaz_amm52x20_x2_256 - .rva .LSEH_end_ossl_rsaz_amm52x20_x2_256 - .rva .LSEH_info_ossl_rsaz_amm52x20_x2_256 - - .rva .LSEH_begin_ossl_extract_multiplier_2x20_win5 - .rva .LSEH_end_ossl_extract_multiplier_2x20_win5 - .rva .LSEH_info_ossl_extract_multiplier_2x20_win5 + .rva .LSEH_begin_ossl_rsaz_amm52x20_x2_ifma256 + .rva .LSEH_end_ossl_rsaz_amm52x20_x2_ifma256 + .rva .LSEH_info_ossl_rsaz_amm52x20_x2_ifma256 .section .xdata .align 8 -.LSEH_info_ossl_rsaz_amm52x20_x1_256: +.LSEH_info_ossl_rsaz_amm52x20_x1_ifma256: .byte 9,0,0,0 .rva rsaz_def_handler - .rva .Lrsaz_amm52x20_x1_256_body,.Lrsaz_amm52x20_x1_256_epilogue -.LSEH_info_ossl_rsaz_amm52x20_x2_256: + .rva .Lossl_rsaz_amm52x20_x1_ifma256_body,.Lossl_rsaz_amm52x20_x1_ifma256_epilogue +.LSEH_info_ossl_rsaz_amm52x20_x2_ifma256: .byte 9,0,0,0 .rva rsaz_def_handler - .rva .Lrsaz_amm52x20_x2_256_body,.Lrsaz_amm52x20_x2_256_epilogue -.LSEH_info_ossl_extract_multiplier_2x20_win5: - .byte 9,0,0,0 - .rva rsaz_def_handler - .rva .LSEH_begin_ossl_extract_multiplier_2x20_win5,.LSEH_begin_ossl_extract_multiplier_2x20_win5 + .rva .Lossl_rsaz_amm52x20_x2_ifma256_body,.Lossl_rsaz_amm52x20_x2_ifma256_epilogue ___ } }}} else {{{ # fallback for old assembler @@ -736,16 +726,16 @@ ossl_rsaz_avx512ifma_eligible: ret .size ossl_rsaz_avx512ifma_eligible, .-ossl_rsaz_avx512ifma_eligible -.globl ossl_rsaz_amm52x20_x1_256 -.globl ossl_rsaz_amm52x20_x2_256 +.globl ossl_rsaz_amm52x20_x1_ifma256 +.globl ossl_rsaz_amm52x20_x2_ifma256 .globl ossl_extract_multiplier_2x20_win5 -.type ossl_rsaz_amm52x20_x1_256,\@abi-omnipotent -ossl_rsaz_amm52x20_x1_256: -ossl_rsaz_amm52x20_x2_256: +.type ossl_rsaz_amm52x20_x1_ifma256,\@abi-omnipotent +ossl_rsaz_amm52x20_x1_ifma256: +ossl_rsaz_amm52x20_x2_ifma256: ossl_extract_multiplier_2x20_win5: .byte 0x0f,0x0b # ud2 ret -.size ossl_rsaz_amm52x20_x1_256, .-ossl_rsaz_amm52x20_x1_256 +.size ossl_rsaz_amm52x20_x1_ifma256, .-ossl_rsaz_amm52x20_x1_ifma256 ___ }}} diff --git a/crypto/openssl/crypto/bn/asm/rsaz-2k-avxifma.pl b/crypto/openssl/crypto/bn/asm/rsaz-2k-avxifma.pl new file mode 100644 index 000000000000..0c962f248dd9 --- /dev/null +++ b/crypto/openssl/crypto/bn/asm/rsaz-2k-avxifma.pl @@ -0,0 +1,867 @@ +# Copyright 2024-2025 The OpenSSL Project Authors. All Rights Reserved. +# Copyright (c) 2024, Intel Corporation. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html +# +# Written by Zhiguo Zhou and Wangyang Guo . +# Special thanks to Tomasz Kantecki for his valuable suggestions. +# +# October 2024 +# +# Initial release. +# + +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); +$avxifma=0; + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or +die "can't locate x86_64-xlate.pl"; + +if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` + =~ /GNU assembler version ([2-9]\.[0-9]+)/) { + $avxifma = ($1>=2.40); +} + +if (!$avxifma && `$ENV{CC} -v 2>&1` + =~ /\s*((?:clang|LLVM) version|.*based on LLVM) ([0-9]+)\.([0-9]+)\.([0-9]+)?/) { + my $ver = $2 + $3/100.0 + $4/10000.0; # 3.1.0->3.01, 3.10.1->3.1001 + $avxifma = ($ver>=16.0); +} + +if ($win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && + `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?(rc[0-9]+)?/) { + $avxifma = ($1>2.16) + ($1==2.16 && ((!defined($2) && !defined($3)) || (defined($2)))); +} + +open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"" + or die "can't call $xlate: $!"; +*STDOUT=*OUT; + +if ($avxifma>0) {{{ +@_6_args_universal_ABI = ("%rdi","%rsi","%rdx","%rcx","%r8","%r9"); + +$code.=<<___; +.text +.extern OPENSSL_ia32cap_P +.globl ossl_rsaz_avxifma_eligible +.type ossl_rsaz_avxifma_eligible,\@abi-omnipotent +.align 32 +ossl_rsaz_avxifma_eligible: + mov OPENSSL_ia32cap_P+20(%rip), %ecx + xor %eax,%eax + and \$`1<<23`, %ecx # avxifma + cmp \$`1<<23`, %ecx + cmove %ecx,%eax + ret +.size ossl_rsaz_avxifma_eligible, .-ossl_rsaz_avxifma_eligible +___ + +############################################################################### +# Almost Montgomery Multiplication (AMM) for 20-digit number in radix 2^52. +# +# AMM is defined as presented in the paper [1]. +# +# The input and output are presented in 2^52 radix domain, i.e. +# |res|, |a|, |b|, |m| are arrays of 20 64-bit qwords with 12 high bits zeroed. +# |k0| is a Montgomery coefficient, which is here k0 = -1/m mod 2^64 +# +# NB: the AMM implementation does not perform "conditional" subtraction step +# specified in the original algorithm as according to the Lemma 1 from the paper +# [2], the result will be always < 2*m and can be used as a direct input to +# the next AMM iteration. This post-condition is true, provided the correct +# parameter |s| (notion of the Lemma 1 from [2]) is chosen, i.e. s >= n + 2 * k, +# which matches our case: 1040 > 1024 + 2 * 1. +# +# [1] Gueron, S. Efficient software implementations of modular exponentiation. +# DOI: 10.1007/s13389-012-0031-5 +# [2] Gueron, S. Enhanced Montgomery Multiplication. +# DOI: 10.1007/3-540-36400-5_5 +# +# void ossl_rsaz_amm52x20_x1_avxifma256(BN_ULONG *res, +# const BN_ULONG *a, +# const BN_ULONG *b, +# const BN_ULONG *m, +# BN_ULONG k0); +############################################################################### +{ +# input parameters ("%rdi","%rsi","%rdx","%rcx","%r8") +my ($res,$a,$b,$m,$k0) = @_6_args_universal_ABI; + +my $mask52 = "%rax"; +my $acc0_0 = "%r9"; +my $acc0_0_low = "%r9d"; +my $acc0_1 = "%r15"; +my $acc0_1_low = "%r15d"; +my $b_ptr = "%r11"; + +my $iter = "%ebx"; + +my $zero = "%ymm0"; +my $Bi = "%ymm1"; +my $Yi = "%ymm2"; +my $Yi_xmm = "%xmm2"; +my ($R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0) = ("%ymm3",map("%ymm$_",(5..8))); +my ($R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1) = ("%ymm4",map("%ymm$_",(9..12))); + +# Registers mapping for normalization. +my ($T0,$T0h,$T1,$T1h,$T2,$tmp) = ("$zero", "$Bi", "$Yi", map("%ymm$_", (13..15))); +my $T0_xmm = "%xmm0"; + +sub amm52x20_x1() { +# _data_offset - offset in the |a| or |m| arrays pointing to the beginning +# of data for corresponding AMM operation; +# _b_offset - offset in the |b| array pointing to the next qword digit; +my ($_data_offset,$_b_offset,$_acc,$_R0,$_R0h,$_R1,$_R1h,$_R2,$_k0) = @_; +$code.=<<___; + movq $_b_offset($b_ptr), %r13 # b[i] + + vpbroadcastq $_b_offset($b_ptr), $Bi # broadcast b[i] + movq $_data_offset($a), %rdx + mulx %r13, %r13, %r12 # a[0]*b[i] = (t0,t2) + addq %r13, $_acc # acc += t0 + movq %r12, %r10 + adcq \$0, %r10 # t2 += CF + + movq $_k0, %r13 + imulq $_acc, %r13 # acc * k0 + andq $mask52, %r13 # yi = (acc * k0) & mask52 + + vmovq %r13, $Yi_xmm + vpbroadcastq $Yi_xmm, $Yi # broadcast y[i] + movq $_data_offset($m), %rdx + mulx %r13, %r13, %r12 # yi * m[0] = (t0,t1) + addq %r13, $_acc # acc += t0 + adcq %r12, %r10 # t2 += (t1 + CF) + + shrq \$52, $_acc + salq \$12, %r10 + or %r10, $_acc # acc = ((acc >> 52) | (t2 << 12)) + + lea -168(%rsp), %rsp + {vex} vpmadd52luq `$_data_offset`($a), $Bi, $_R0 + {vex} vpmadd52luq `$_data_offset+32`($a), $Bi, $_R0h + {vex} vpmadd52luq `$_data_offset+64*1`($a), $Bi, $_R1 + {vex} vpmadd52luq `$_data_offset+64*1+32`($a), $Bi, $_R1h + {vex} vpmadd52luq `$_data_offset+64*2`($a), $Bi, $_R2 + + {vex} vpmadd52luq `$_data_offset`($m), $Yi, $_R0 + {vex} vpmadd52luq `$_data_offset+32`($m), $Yi, $_R0h + {vex} vpmadd52luq `$_data_offset+64*1`($m), $Yi, $_R1 + {vex} vpmadd52luq `$_data_offset+64*1+32`($m), $Yi, $_R1h + {vex} vpmadd52luq `$_data_offset+64*2`($m), $Yi, $_R2 + + # Shift accumulators right by 1 qword, zero extending the highest one + vmovdqu $_R0, `32*0`(%rsp) + vmovdqu $_R0h, `32*1`(%rsp) + vmovdqu $_R1, `32*2`(%rsp) + vmovdqu $_R1h, `32*3`(%rsp) + vmovdqu $_R2, `32*4`(%rsp) + movq \$0, `32*5`(%rsp) + + vmovdqu `32*0 + 8`(%rsp), $_R0 + vmovdqu `32*1 + 8`(%rsp), $_R0h + vmovdqu `32*2 + 8`(%rsp), $_R1 + vmovdqu `32*3 + 8`(%rsp), $_R1h + vmovdqu `32*4 + 8`(%rsp), $_R2 + + addq 8(%rsp), $_acc # acc += R0[0] + + {vex} vpmadd52huq `$_data_offset`($a), $Bi, $_R0 + {vex} vpmadd52huq `$_data_offset+32`($a), $Bi, $_R0h + {vex} vpmadd52huq `$_data_offset+64*1`($a), $Bi, $_R1 + {vex} vpmadd52huq `$_data_offset+64*1+32`($a), $Bi, $_R1h + {vex} vpmadd52huq `$_data_offset+64*2`($a), $Bi, $_R2 + + {vex} vpmadd52huq `$_data_offset`($m), $Yi, $_R0 + {vex} vpmadd52huq `$_data_offset+32`($m), $Yi, $_R0h + {vex} vpmadd52huq `$_data_offset+64*1`($m), $Yi, $_R1 + {vex} vpmadd52huq `$_data_offset+64*1+32`($m), $Yi, $_R1h + {vex} vpmadd52huq `$_data_offset+64*2`($m), $Yi, $_R2 + lea 168(%rsp),%rsp +___ +} + +# Normalization routine: handles carry bits and gets bignum qwords to normalized +# 2^52 representation. +# +# Uses %r8-14,%e[bcd]x +sub amm52x20_x1_norm { +my ($_acc,$_R0,$_R0h,$_R1,$_R1h,$_R2) = @_; +$code.=<<___; + # Put accumulator to low qword in R0 + vmovq $_acc, $T0_xmm + vpbroadcastq $T0_xmm, $T0 + vpblendd \$3, $T0, $_R0, $_R0 + + # Extract "carries" (12 high bits) from each QW of R0..R2 + # Save them to LSB of QWs in T0..T2 + vpsrlq \$52, $_R0, $T0 + vpsrlq \$52, $_R0h, $T0h + vpsrlq \$52, $_R1, $T1 + vpsrlq \$52, $_R1h, $T1h + vpsrlq \$52, $_R2, $T2 + + # "Shift left" T0..T2 by 1 QW + vpermq \$144, $T2, $T2 + vpermq \$3, $T1h, $tmp + vblendpd \$1, $tmp, $T2, $T2 + + vpermq \$144, $T1h, $T1h + vpermq \$3, $T1, $tmp + vblendpd \$1, $tmp, $T1h, $T1h + + vpermq \$144, $T1, $T1 + vpermq \$3, $T0h, $tmp + vblendpd \$1, $tmp, $T1, $T1 + + vpermq \$144, $T0h, $T0h + vpermq \$3, $T0, $tmp + vblendpd \$1, $tmp, $T0h, $T0h + + vpermq \$144, $T0, $T0 + vpand .Lhigh64x3(%rip), $T0, $T0 + + # Drop "carries" from R0..R2 QWs + vpand .Lmask52x4(%rip), $_R0, $_R0 + vpand .Lmask52x4(%rip), $_R0h, $_R0h + vpand .Lmask52x4(%rip), $_R1, $_R1 + vpand .Lmask52x4(%rip), $_R1h, $_R1h + vpand .Lmask52x4(%rip), $_R2, $_R2 + + # Sum R0..R2 with corresponding adjusted carries + vpaddq $T0, $_R0, $_R0 + vpaddq $T0h, $_R0h, $_R0h + vpaddq $T1, $_R1, $_R1 + vpaddq $T1h, $_R1h, $_R1h + vpaddq $T2, $_R2, $_R2 + + # Now handle carry bits from this addition + # Get mask of QWs which 52-bit parts overflow... + vpcmpgtq .Lmask52x4(%rip), $_R0, $T0 + vpcmpgtq .Lmask52x4(%rip), $_R0h, $T0h + vpcmpgtq .Lmask52x4(%rip), $_R1, $T1 + vpcmpgtq .Lmask52x4(%rip), $_R1h, $T1h + vpcmpgtq .Lmask52x4(%rip), $_R2, $T2 + vmovmskpd $T0, %r14d + vmovmskpd $T0h, %r13d + vmovmskpd $T1, %r12d + vmovmskpd $T1h, %r11d + vmovmskpd $T2, %r10d + + # ...or saturated + vpcmpeqq .Lmask52x4(%rip), $_R0, $T0 + vpcmpeqq .Lmask52x4(%rip), $_R0h, $T0h + vpcmpeqq .Lmask52x4(%rip), $_R1, $T1 + vpcmpeqq .Lmask52x4(%rip), $_R1h, $T1h + vpcmpeqq .Lmask52x4(%rip), $_R2, $T2 + vmovmskpd $T0, %r9d + vmovmskpd $T0h, %r8d + vmovmskpd $T1, %ebx + vmovmskpd $T1h, %ecx + vmovmskpd $T2, %edx + + # Get mask of QWs where carries shall be propagated to. + # Merge 4-bit masks to 8-bit values to use add with carry. + shl \$4, %r13b + or %r13b, %r14b + shl \$4, %r11b + or %r11b, %r12b + + add %r14b, %r14b + adc %r12b, %r12b + adc %r10b, %r10b + + shl \$4, %r8b + or %r8b,%r9b + shl \$4, %cl + or %cl, %bl + + add %r9b, %r14b + adc %bl, %r12b + adc %dl, %r10b + + xor %r9b, %r14b + xor %bl, %r12b + xor %dl, %r10b + + lea .Lkmasklut(%rip), %rdx + + mov %r14b, %r13b + and \$0xf, %r14 + vpsubq .Lmask52x4(%rip), $_R0, $T0 + shl \$5, %r14 + vmovapd (%rdx,%r14), $T1 + vblendvpd $T1, $T0, $_R0, $_R0 + + shr \$4, %r13b + and \$0xf, %r13 + vpsubq .Lmask52x4(%rip), $_R0h, $T0 + shl \$5, %r13 + vmovapd (%rdx,%r13), $T1 + vblendvpd $T1, $T0, $_R0h, $_R0h + + mov %r12b, %r11b + and \$0xf, %r12 + vpsubq .Lmask52x4(%rip), $_R1, $T0 + shl \$5, %r12 + vmovapd (%rdx,%r12), $T1 + vblendvpd $T1, $T0, $_R1, $_R1 + + shr \$4, %r11b + and \$0xf, %r11 + vpsubq .Lmask52x4(%rip), $_R1h, $T0 + shl \$5, %r11 + vmovapd (%rdx,%r11), $T1 + vblendvpd $T1, $T0, $_R1h, $_R1h + + and \$0xf, %r10 + vpsubq .Lmask52x4(%rip), $_R2, $T0 + shl \$5, %r10 + vmovapd (%rdx,%r10), $T1 + vblendvpd $T1, $T0, $_R2, $_R2 + + # Add carries according to the obtained mask + vpand .Lmask52x4(%rip), $_R0, $_R0 + vpand .Lmask52x4(%rip), $_R0h, $_R0h + vpand .Lmask52x4(%rip), $_R1, $_R1 + vpand .Lmask52x4(%rip), $_R1h, $_R1h + vpand .Lmask52x4(%rip), $_R2, $_R2 +___ +} + +$code.=<<___; +.text + +.globl ossl_rsaz_amm52x20_x1_avxifma256 +.type ossl_rsaz_amm52x20_x1_avxifma256,\@function,5 +.align 32 +ossl_rsaz_amm52x20_x1_avxifma256: +.cfi_startproc + endbranch + push %rbx +.cfi_push %rbx + push %rbp +.cfi_push %rbp + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +.Lossl_rsaz_amm52x20_x1_avxifma256_body: + + # Zeroing accumulators + vpxor $zero, $zero, $zero + vmovapd $zero, $R0_0 + vmovapd $zero, $R0_0h + vmovapd $zero, $R1_0 + vmovapd $zero, $R1_0h + vmovapd $zero, $R2_0 + + xorl $acc0_0_low, $acc0_0_low + + movq $b, $b_ptr # backup address of b + movq \$0xfffffffffffff, $mask52 # 52-bit mask + + # Loop over 20 digits unrolled by 4 + mov \$5, $iter + +.align 32 +.Lloop5: +___ + foreach my $idx (0..3) { + &amm52x20_x1(0,8*$idx,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$k0); + } +$code.=<<___; + lea `4*8`($b_ptr), $b_ptr + dec $iter + jne .Lloop5 +___ + &amm52x20_x1_norm($acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0); +$code.=<<___; + + vmovdqu $R0_0, `0*32`($res) + vmovdqu $R0_0h, `1*32`($res) + vmovdqu $R1_0, `2*32`($res) + vmovdqu $R1_0h, `3*32`($res) + vmovdqu $R2_0, `4*32`($res) + + vzeroupper + mov 0(%rsp),%r15 +.cfi_restore %r15 + mov 8(%rsp),%r14 +.cfi_restore %r14 + mov 16(%rsp),%r13 +.cfi_restore %r13 + mov 24(%rsp),%r12 +.cfi_restore %r12 + mov 32(%rsp),%rbp +.cfi_restore %rbp + mov 40(%rsp),%rbx +.cfi_restore %rbx + lea 48(%rsp),%rsp +.cfi_adjust_cfa_offset -48 +.Lossl_rsaz_amm52x20_x1_avxifma256_epilogue: + ret +.cfi_endproc +.size ossl_rsaz_amm52x20_x1_avxifma256, .-ossl_rsaz_amm52x20_x1_avxifma256 +___ + +$code.=<<___; +.section .rodata align=32 +.align 32 +.Lmask52x4: + .quad 0xfffffffffffff + .quad 0xfffffffffffff + .quad 0xfffffffffffff + .quad 0xfffffffffffff +.Lhigh64x3: + .quad 0x0 + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff +.Lkmasklut: + #0000 + .quad 0x0 + .quad 0x0 + .quad 0x0 + .quad 0x0 + #0001 + .quad 0xffffffffffffffff + .quad 0x0 + .quad 0x0 + .quad 0x0 + #0010 + .quad 0x0 + .quad 0xffffffffffffffff + .quad 0x0 + .quad 0x0 + #0011 + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + .quad 0x0 + .quad 0x0 + #0100 + .quad 0x0 + .quad 0x0 + .quad 0xffffffffffffffff + .quad 0x0 + #0101 + .quad 0xffffffffffffffff + .quad 0x0 + .quad 0xffffffffffffffff + .quad 0x0 + #0110 + .quad 0x0 + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + .quad 0x0 + #0111 + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + .quad 0x0 + #1000 + .quad 0x0 + .quad 0x0 + .quad 0x0 + .quad 0xffffffffffffffff + #1001 + .quad 0xffffffffffffffff + .quad 0x0 + .quad 0x0 + .quad 0xffffffffffffffff + #1010 + .quad 0x0 + .quad 0xffffffffffffffff + .quad 0x0 + .quad 0xffffffffffffffff + #1011 + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + .quad 0x0 + .quad 0xffffffffffffffff + #1100 + .quad 0x0 + .quad 0x0 + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + #1101 + .quad 0xffffffffffffffff + .quad 0x0 + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + #1110 + .quad 0x0 + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + #1111 + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff +___ + +############################################################################### +# Dual Almost Montgomery Multiplication for 20-digit number in radix 2^52 +# +# See description of ossl_rsaz_amm52x20_x1_ifma256() above for details about Almost +# Montgomery Multiplication algorithm and function input parameters description. +# +# This function does two AMMs for two independent inputs, hence dual. +# +# void ossl_rsaz_amm52x20_x2_avxifma256(BN_ULONG out[2][20], +# const BN_ULONG a[2][20], +# const BN_ULONG b[2][20], +# const BN_ULONG m[2][20], +# const BN_ULONG k0[2]); +############################################################################### + +$code.=<<___; +.text + +.globl ossl_rsaz_amm52x20_x2_avxifma256 +.type ossl_rsaz_amm52x20_x2_avxifma256,\@function,5 +.align 32 +ossl_rsaz_amm52x20_x2_avxifma256: +.cfi_startproc + endbranch + push %rbx +.cfi_push %rbx + push %rbp +.cfi_push %rbp + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +.Lossl_rsaz_amm52x20_x2_avxifma256_body: + + # Zeroing accumulators + vpxor $zero, $zero, $zero + vmovapd $zero, $R0_0 + vmovapd $zero, $R0_0h + vmovapd $zero, $R1_0 + vmovapd $zero, $R1_0h + vmovapd $zero, $R2_0 + vmovapd $zero, $R0_1 + vmovapd $zero, $R0_1h + vmovapd $zero, $R1_1 + vmovapd $zero, $R1_1h + vmovapd $zero, $R2_1 + + xorl $acc0_0_low, $acc0_0_low + xorl $acc0_1_low, $acc0_1_low + + movq $b, $b_ptr # backup address of b + movq \$0xfffffffffffff, $mask52 # 52-bit mask + + mov \$20, $iter + +.align 32 +.Lloop20: +___ + &amm52x20_x1( 0, 0,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,"($k0)"); + # 20*8 = offset of the next dimension in two-dimension array + &amm52x20_x1(20*8,20*8,$acc0_1,$R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1,"8($k0)"); +$code.=<<___; + lea 8($b_ptr), $b_ptr + dec $iter + jne .Lloop20 +___ + &amm52x20_x1_norm($acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0); + &amm52x20_x1_norm($acc0_1,$R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1); +$code.=<<___; + + vmovdqu $R0_0, `0*32`($res) + vmovdqu $R0_0h, `1*32`($res) + vmovdqu $R1_0, `2*32`($res) + vmovdqu $R1_0h, `3*32`($res) + vmovdqu $R2_0, `4*32`($res) + + vmovdqu $R0_1, `5*32`($res) + vmovdqu $R0_1h, `6*32`($res) + vmovdqu $R1_1, `7*32`($res) + vmovdqu $R1_1h, `8*32`($res) + vmovdqu $R2_1, `9*32`($res) + + vzeroupper + mov 0(%rsp),%r15 +.cfi_restore %r15 + mov 8(%rsp),%r14 +.cfi_restore %r14 + mov 16(%rsp),%r13 +.cfi_restore %r13 + mov 24(%rsp),%r12 +.cfi_restore %r12 + mov 32(%rsp),%rbp +.cfi_restore %rbp + mov 40(%rsp),%rbx +.cfi_restore %rbx + lea 48(%rsp),%rsp +.cfi_adjust_cfa_offset -48 +.Lossl_rsaz_amm52x20_x2_avxifma256_epilogue: + ret +.cfi_endproc +.size ossl_rsaz_amm52x20_x2_avxifma256, .-ossl_rsaz_amm52x20_x2_avxifma256 +___ +} + +############################################################################### +# Constant time extraction from the precomputed table of powers base^i, where +# i = 0..2^EXP_WIN_SIZE-1 +# +# The input |red_table| contains precomputations for two independent base values. +# |red_table_idx1| and |red_table_idx2| are corresponding power indexes. +# +# Extracted value (output) is 2 20 digit numbers in 2^52 radix. +# +# void ossl_extract_multiplier_2x20_win5_avx(BN_ULONG *red_Y, +# const BN_ULONG red_table[1 << EXP_WIN_SIZE][2][20], +# int red_table_idx1, int red_table_idx2); +# +# EXP_WIN_SIZE = 5 +############################################################################### +{ +# input parameters +my ($out,$red_tbl,$red_tbl_idx1,$red_tbl_idx2)=$win64 ? ("%rcx","%rdx","%r8", "%r9") : # Win64 order + ("%rdi","%rsi","%rdx","%rcx"); # Unix order + +my ($t0,$t1,$t2,$t3,$t4,$t5) = map("%ymm$_", (0..5)); +my ($t6,$t7,$t8,$t9) = map("%ymm$_", (6..9)); +my ($tmp,$cur_idx,$idx1,$idx2,$ones,$mask) = map("%ymm$_", (10..15)); + +my @t = ($t0,$t1,$t2,$t3,$t4,$t5,$t6,$t7,$t8,$t9); +my $t0xmm = $t0; +my $tmp_xmm = "%xmm10"; +$t0xmm =~ s/%y/%x/; + +$code.=<<___; +.text + +.align 32 +.globl ossl_extract_multiplier_2x20_win5_avx +.type ossl_extract_multiplier_2x20_win5_avx,\@abi-omnipotent +ossl_extract_multiplier_2x20_win5_avx: +.cfi_startproc + endbranch + vmovapd .Lones(%rip), $ones # broadcast ones + vmovq $red_tbl_idx1, $tmp_xmm + vpbroadcastq $tmp_xmm, $idx1 + vmovq $red_tbl_idx2, $tmp_xmm + vpbroadcastq $tmp_xmm, $idx2 + leaq `(1<<5)*2*20*8`($red_tbl), %rax # holds end of the tbl + + # zeroing t0..n, cur_idx + vpxor $t0xmm, $t0xmm, $t0xmm + vmovapd $t0, $cur_idx +___ +foreach (1..9) { + $code.="vmovapd $t0, $t[$_] \n"; +} +$code.=<<___; + +.align 32 +.Lloop: + vpcmpeqq $cur_idx, $idx1, $mask # mask of (idx1 == cur_idx) +___ +foreach (0..4) { +$code.=<<___; + vmovdqu `${_}*32`($red_tbl), $tmp # load data from red_tbl + vblendvpd $mask, $tmp, $t[$_], ${t[$_]} # extract data when mask is not zero +___ +} +$code.=<<___; + vpcmpeqq $cur_idx, $idx2, $mask # mask of (idx2 == cur_idx) +___ +foreach (5..9) { +$code.=<<___; + vmovdqu `${_}*32`($red_tbl), $tmp # load data from red_tbl + vblendvpd $mask, $tmp, $t[$_], ${t[$_]} # extract data when mask is not zero +___ +} +$code.=<<___; + vpaddq $ones, $cur_idx, $cur_idx # increment cur_idx + addq \$`2*20*8`, $red_tbl + cmpq $red_tbl, %rax + jne .Lloop +___ +# store t0..n +foreach (0..9) { + $code.="vmovdqu $t[$_], `${_}*32`($out) \n"; +} +$code.=<<___; + ret +.cfi_endproc +.size ossl_extract_multiplier_2x20_win5_avx, .-ossl_extract_multiplier_2x20_win5_avx +___ +$code.=<<___; +.section .rodata align=32 +.align 32 +.Lones: + .quad 1,1,1,1 +.Lzeros: + .quad 0,0,0,0 +___ +} + +if ($win64) { +$rec="%rcx"; +$frame="%rdx"; +$context="%r8"; +$disp="%r9"; + +$code.=<<___; +.extern __imp_RtlVirtualUnwind +.type rsaz_def_handler,\@abi-omnipotent +.align 16 +rsaz_def_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + mov 8($disp),%rsi # disp->ImageBase + mov 56($disp),%r11 # disp->HandlerData + + mov 0(%r11),%r10d # HandlerData[0] + lea (%rsi,%r10),%r10 # prologue label + cmp %r10,%rbx # context->Rip<.Lprologue + jb .Lcommon_seh_tail + + mov 152($context),%rax # pull context->Rsp + + mov 4(%r11),%r10d # HandlerData[1] + lea (%rsi,%r10),%r10 # epilogue label + cmp %r10,%rbx # context->Rip>=.Lepilogue + jae .Lcommon_seh_tail + + lea 48(%rax),%rax + + mov -8(%rax),%rbx + mov -16(%rax),%rbp + mov -24(%rax),%r12 + mov -32(%rax),%r13 + mov -40(%rax),%r14 + mov -48(%rax),%r15 + mov %rbx,144($context) # restore context->Rbx + mov %rbp,160($context) # restore context->Rbp + mov %r12,216($context) # restore context->R12 + mov %r13,224($context) # restore context->R13 + mov %r14,232($context) # restore context->R14 + mov %r15,240($context) # restore context->R14 + +.Lcommon_seh_tail: + mov 8(%rax),%rdi + mov 16(%rax),%rsi + mov %rax,152($context) # restore context->Rsp + mov %rsi,168($context) # restore context->Rsi + mov %rdi,176($context) # restore context->Rdi + + mov 40($disp),%rdi # disp->ContextRecord + mov $context,%rsi # context + mov \$154,%ecx # sizeof(CONTEXT) + .long 0xa548f3fc # cld; rep movsq + + mov $disp,%rsi + xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER + mov 8(%rsi),%rdx # arg2, disp->ImageBase + mov 0(%rsi),%r8 # arg3, disp->ControlPc + mov 16(%rsi),%r9 # arg4, disp->FunctionEntry + mov 40(%rsi),%r10 # disp->ContextRecord + lea 56(%rsi),%r11 # &disp->HandlerData + lea 24(%rsi),%r12 # &disp->EstablisherFrame + mov %r10,32(%rsp) # arg5 + mov %r11,40(%rsp) # arg6 + mov %r12,48(%rsp) # arg7 + mov %rcx,56(%rsp) # arg8, (NULL) + call *__imp_RtlVirtualUnwind(%rip) + + mov \$1,%eax # ExceptionContinueSearch + add \$64,%rsp + popfq + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbp + pop %rbx + pop %rdi + pop %rsi + ret +.size rsaz_def_handler,.-rsaz_def_handler + +.section .pdata +.align 4 + .rva .LSEH_begin_ossl_rsaz_amm52x20_x1_avxifma256 + .rva .LSEH_end_ossl_rsaz_amm52x20_x1_avxifma256 + .rva .LSEH_info_ossl_rsaz_amm52x20_x1_avxifma256 + + .rva .LSEH_begin_ossl_rsaz_amm52x20_x2_avxifma256 + .rva .LSEH_end_ossl_rsaz_amm52x20_x2_avxifma256 + .rva .LSEH_info_ossl_rsaz_amm52x20_x2_avxifma256 + +.section .xdata +.align 8 +.LSEH_info_ossl_rsaz_amm52x20_x1_avxifma256: + .byte 9,0,0,0 + .rva rsaz_def_handler + .rva .Lossl_rsaz_amm52x20_x1_avxifma256_body,.Lossl_rsaz_amm52x20_x1_avxifma256_epilogue +.LSEH_info_ossl_rsaz_amm52x20_x2_avxifma256: + .byte 9,0,0,0 + .rva rsaz_def_handler + .rva .Lossl_rsaz_amm52x20_x2_avxifma256_body,.Lossl_rsaz_amm52x20_x2_avxifma256_epilogue +___ +} +}}} else {{{ # fallback for old assembler +$code.=<<___; +.text + +.globl ossl_rsaz_avxifma_eligible +.type ossl_rsaz_avxifma_eligible,\@abi-omnipotent +ossl_rsaz_avxifma_eligible: + xor %eax,%eax + ret +.size ossl_rsaz_avxifma_eligible, .-ossl_rsaz_avxifma_eligible + +.globl ossl_rsaz_amm52x20_x1_avxifma256 +.globl ossl_rsaz_amm52x20_x2_avxifma256 +.globl ossl_extract_multiplier_2x20_win5_avx +.type ossl_rsaz_amm52x20_x1_avxifma256,\@abi-omnipotent +ossl_rsaz_amm52x20_x1_avxifma256: +ossl_rsaz_amm52x20_x2_avxifma256: +ossl_extract_multiplier_2x20_win5_avx: + .byte 0x0f,0x0b # ud2 + ret +.size ossl_rsaz_amm52x20_x1_avxifma256, .-ossl_rsaz_amm52x20_x1_avxifma256 +___ +}}} + +$code =~ s/\`([^\`]*)\`/eval $1/gem; +print $code; +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/bn/asm/rsaz-3k-avx512.pl b/crypto/openssl/crypto/bn/asm/rsaz-3k-avx512.pl new file mode 100644 index 000000000000..235268a68ad7 --- /dev/null +++ b/crypto/openssl/crypto/bn/asm/rsaz-3k-avx512.pl @@ -0,0 +1,883 @@ +# Copyright 2021-2024 The OpenSSL Project Authors. All Rights Reserved. +# Copyright (c) 2021, Intel Corporation. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html +# +# +# Originally written by Sergey Kirillov and Andrey Matyukov +# Intel Corporation +# +# March 2021 +# +# Initial release. +# +# Implementation utilizes 256-bit (ymm) registers to avoid frequency scaling issues. +# +# IceLake-Client @ 1.3GHz +# |---------+-----------------------+---------------+-------------| +# | | OpenSSL 3.0.0-alpha15 | this | Unit | +# |---------+-----------------------+---------------+-------------| +# | rsa3072 | 6 397 637 | 2 866 593 | cycles/sign | +# | | 203.2 | 453.5 / +123% | sign/s | +# |---------+-----------------------+---------------+-------------| +# + +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); +$avx512ifma=0; + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or +die "can't locate x86_64-xlate.pl"; + +if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` + =~ /GNU assembler version ([2-9]\.[0-9]+)/) { + $avx512ifma = ($1>=2.26); +} + +if (!$avx512ifma && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && + `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?/) { + $avx512ifma = ($1==2.11 && $2>=8) + ($1>=2.12); +} + +if (!$avx512ifma && `$ENV{CC} -v 2>&1` + =~ /(Apple)?\s*((?:clang|LLVM) version|.*based on LLVM) ([0-9]+)\.([0-9]+)\.([0-9]+)?/) { + my $ver = $3 + $4/100.0 + $5/10000.0; # 3.1.0->3.01, 3.10.1->3.1001 + if ($1) { + # Apple conditions, they use a different version series, see + # https://en.wikipedia.org/wiki/Xcode#Xcode_7.0_-_10.x_(since_Free_On-Device_Development)_2 + # clang 7.0.0 is Apple clang 10.0.1 + $avx512ifma = ($ver>=10.0001) + } else { + $avx512ifma = ($ver>=7.0); + } +} + +open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"" + or die "can't call $xlate: $!"; +*STDOUT=*OUT; + +if ($avx512ifma>0) {{{ +@_6_args_universal_ABI = ("%rdi","%rsi","%rdx","%rcx","%r8","%r9"); + +############################################################################### +# Almost Montgomery Multiplication (AMM) for 30-digit number in radix 2^52. +# +# AMM is defined as presented in the paper [1]. +# +# The input and output are presented in 2^52 radix domain, i.e. +# |res|, |a|, |b|, |m| are arrays of 32 64-bit qwords with 12 high bits zeroed +# +# NOTE: the function uses zero-padded data - 2 high QWs is a padding. +# +# |k0| is a Montgomery coefficient, which is here k0 = -1/m mod 2^64 +# +# NB: the AMM implementation does not perform "conditional" subtraction step +# specified in the original algorithm as according to the Lemma 1 from the paper +# [2], the result will be always < 2*m and can be used as a direct input to +# the next AMM iteration. This post-condition is true, provided the correct +# parameter |s| (notion of the Lemma 1 from [2]) is chosen, i.e. s >= n + 2 * k, +# which matches our case: 1560 > 1536 + 2 * 1. +# +# [1] Gueron, S. Efficient software implementations of modular exponentiation. +# DOI: 10.1007/s13389-012-0031-5 +# [2] Gueron, S. Enhanced Montgomery Multiplication. +# DOI: 10.1007/3-540-36400-5_5 +# +# void ossl_rsaz_amm52x30_x1_ifma256(BN_ULONG *res, +# const BN_ULONG *a, +# const BN_ULONG *b, +# const BN_ULONG *m, +# BN_ULONG k0); +############################################################################### +{ +# input parameters ("%rdi","%rsi","%rdx","%rcx","%r8") +my ($res,$a,$b,$m,$k0) = @_6_args_universal_ABI; + +my $mask52 = "%rax"; +my $acc0_0 = "%r9"; +my $acc0_0_low = "%r9d"; +my $acc0_1 = "%r15"; +my $acc0_1_low = "%r15d"; +my $b_ptr = "%r11"; + +my $iter = "%ebx"; + +my $zero = "%ymm0"; +my $Bi = "%ymm1"; +my $Yi = "%ymm2"; +my ($R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h) = map("%ymm$_",(3..10)); +my ($R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1,$R2_1h,$R3_1,$R3_1h) = map("%ymm$_",(11..18)); + +# Registers mapping for normalization +my ($T0,$T0h,$T1,$T1h,$T2,$T2h,$T3,$T3h) = ("$zero", "$Bi", "$Yi", map("%ymm$_", (19..23))); + +sub amm52x30_x1() { +# _data_offset - offset in the |a| or |m| arrays pointing to the beginning +# of data for corresponding AMM operation; +# _b_offset - offset in the |b| array pointing to the next qword digit; +my ($_data_offset,$_b_offset,$_acc,$_R0,$_R0h,$_R1,$_R1h,$_R2,$_R2h,$_R3,$_R3h,$_k0) = @_; +my $_R0_xmm = $_R0; +$_R0_xmm =~ s/%y/%x/; +$code.=<<___; + movq $_b_offset($b_ptr), %r13 # b[i] + + vpbroadcastq %r13, $Bi # broadcast b[i] + movq $_data_offset($a), %rdx + mulx %r13, %r13, %r12 # a[0]*b[i] = (t0,t2) + addq %r13, $_acc # acc += t0 + movq %r12, %r10 + adcq \$0, %r10 # t2 += CF + + movq $_k0, %r13 + imulq $_acc, %r13 # acc * k0 + andq $mask52, %r13 # yi = (acc * k0) & mask52 + + vpbroadcastq %r13, $Yi # broadcast y[i] + movq $_data_offset($m), %rdx + mulx %r13, %r13, %r12 # yi * m[0] = (t0,t1) + addq %r13, $_acc # acc += t0 + adcq %r12, %r10 # t2 += (t1 + CF) + + shrq \$52, $_acc + salq \$12, %r10 + or %r10, $_acc # acc = ((acc >> 52) | (t2 << 12)) + + vpmadd52luq `$_data_offset+64*0`($a), $Bi, $_R0 + vpmadd52luq `$_data_offset+64*0+32`($a), $Bi, $_R0h + vpmadd52luq `$_data_offset+64*1`($a), $Bi, $_R1 + vpmadd52luq `$_data_offset+64*1+32`($a), $Bi, $_R1h + vpmadd52luq `$_data_offset+64*2`($a), $Bi, $_R2 + vpmadd52luq `$_data_offset+64*2+32`($a), $Bi, $_R2h + vpmadd52luq `$_data_offset+64*3`($a), $Bi, $_R3 + vpmadd52luq `$_data_offset+64*3+32`($a), $Bi, $_R3h + + vpmadd52luq `$_data_offset+64*0`($m), $Yi, $_R0 + vpmadd52luq `$_data_offset+64*0+32`($m), $Yi, $_R0h + vpmadd52luq `$_data_offset+64*1`($m), $Yi, $_R1 + vpmadd52luq `$_data_offset+64*1+32`($m), $Yi, $_R1h + vpmadd52luq `$_data_offset+64*2`($m), $Yi, $_R2 + vpmadd52luq `$_data_offset+64*2+32`($m), $Yi, $_R2h + vpmadd52luq `$_data_offset+64*3`($m), $Yi, $_R3 + vpmadd52luq `$_data_offset+64*3+32`($m), $Yi, $_R3h + + # Shift accumulators right by 1 qword, zero extending the highest one + valignq \$1, $_R0, $_R0h, $_R0 + valignq \$1, $_R0h, $_R1, $_R0h + valignq \$1, $_R1, $_R1h, $_R1 + valignq \$1, $_R1h, $_R2, $_R1h + valignq \$1, $_R2, $_R2h, $_R2 + valignq \$1, $_R2h, $_R3, $_R2h + valignq \$1, $_R3, $_R3h, $_R3 + valignq \$1, $_R3h, $zero, $_R3h + + vmovq $_R0_xmm, %r13 + addq %r13, $_acc # acc += R0[0] + + vpmadd52huq `$_data_offset+64*0`($a), $Bi, $_R0 + vpmadd52huq `$_data_offset+64*0+32`($a), $Bi, $_R0h + vpmadd52huq `$_data_offset+64*1`($a), $Bi, $_R1 + vpmadd52huq `$_data_offset+64*1+32`($a), $Bi, $_R1h + vpmadd52huq `$_data_offset+64*2`($a), $Bi, $_R2 + vpmadd52huq `$_data_offset+64*2+32`($a), $Bi, $_R2h + vpmadd52huq `$_data_offset+64*3`($a), $Bi, $_R3 + vpmadd52huq `$_data_offset+64*3+32`($a), $Bi, $_R3h + + vpmadd52huq `$_data_offset+64*0`($m), $Yi, $_R0 + vpmadd52huq `$_data_offset+64*0+32`($m), $Yi, $_R0h + vpmadd52huq `$_data_offset+64*1`($m), $Yi, $_R1 + vpmadd52huq `$_data_offset+64*1+32`($m), $Yi, $_R1h + vpmadd52huq `$_data_offset+64*2`($m), $Yi, $_R2 + vpmadd52huq `$_data_offset+64*2+32`($m), $Yi, $_R2h + vpmadd52huq `$_data_offset+64*3`($m), $Yi, $_R3 + vpmadd52huq `$_data_offset+64*3+32`($m), $Yi, $_R3h +___ +} + +# Normalization routine: handles carry bits and gets bignum qwords to normalized +# 2^52 representation. +# +# Uses %r8-14,%e[abcd]x +sub amm52x30_x1_norm { +my ($_acc,$_R0,$_R0h,$_R1,$_R1h,$_R2,$_R2h,$_R3,$_R3h) = @_; +$code.=<<___; + # Put accumulator to low qword in R0 + vpbroadcastq $_acc, $T0 + vpblendd \$3, $T0, $_R0, $_R0 + + # Extract "carries" (12 high bits) from each QW of the bignum + # Save them to LSB of QWs in T0..Tn + vpsrlq \$52, $_R0, $T0 + vpsrlq \$52, $_R0h, $T0h + vpsrlq \$52, $_R1, $T1 + vpsrlq \$52, $_R1h, $T1h + vpsrlq \$52, $_R2, $T2 + vpsrlq \$52, $_R2h, $T2h + vpsrlq \$52, $_R3, $T3 + vpsrlq \$52, $_R3h, $T3h + + # "Shift left" T0..Tn by 1 QW + valignq \$3, $T3, $T3h, $T3h + valignq \$3, $T2h, $T3, $T3 + valignq \$3, $T2, $T2h, $T2h + valignq \$3, $T1h, $T2, $T2 + valignq \$3, $T1, $T1h, $T1h + valignq \$3, $T0h, $T1, $T1 + valignq \$3, $T0, $T0h, $T0h + valignq \$3, .Lzeros(%rip), $T0, $T0 + + # Drop "carries" from R0..Rn QWs + vpandq .Lmask52x4(%rip), $_R0, $_R0 + vpandq .Lmask52x4(%rip), $_R0h, $_R0h + vpandq .Lmask52x4(%rip), $_R1, $_R1 + vpandq .Lmask52x4(%rip), $_R1h, $_R1h + vpandq .Lmask52x4(%rip), $_R2, $_R2 + vpandq .Lmask52x4(%rip), $_R2h, $_R2h + vpandq .Lmask52x4(%rip), $_R3, $_R3 + vpandq .Lmask52x4(%rip), $_R3h, $_R3h + + # Sum R0..Rn with corresponding adjusted carries + vpaddq $T0, $_R0, $_R0 + vpaddq $T0h, $_R0h, $_R0h + vpaddq $T1, $_R1, $_R1 + vpaddq $T1h, $_R1h, $_R1h + vpaddq $T2, $_R2, $_R2 + vpaddq $T2h, $_R2h, $_R2h + vpaddq $T3, $_R3, $_R3 + vpaddq $T3h, $_R3h, $_R3h + + # Now handle carry bits from this addition + # Get mask of QWs whose 52-bit parts overflow + vpcmpuq \$6,.Lmask52x4(%rip),${_R0},%k1 # OP=nle (i.e. gt) + vpcmpuq \$6,.Lmask52x4(%rip),${_R0h},%k2 + kmovb %k1,%r14d + kmovb %k2,%r13d + shl \$4,%r13b + or %r13b,%r14b + + vpcmpuq \$6,.Lmask52x4(%rip),${_R1},%k1 + vpcmpuq \$6,.Lmask52x4(%rip),${_R1h},%k2 + kmovb %k1,%r13d + kmovb %k2,%r12d + shl \$4,%r12b + or %r12b,%r13b + + vpcmpuq \$6,.Lmask52x4(%rip),${_R2},%k1 + vpcmpuq \$6,.Lmask52x4(%rip),${_R2h},%k2 + kmovb %k1,%r12d + kmovb %k2,%r11d + shl \$4,%r11b + or %r11b,%r12b + + vpcmpuq \$6,.Lmask52x4(%rip),${_R3},%k1 + vpcmpuq \$6,.Lmask52x4(%rip),${_R3h},%k2 + kmovb %k1,%r11d + kmovb %k2,%r10d + shl \$4,%r10b + or %r10b,%r11b + + addb %r14b,%r14b + adcb %r13b,%r13b + adcb %r12b,%r12b + adcb %r11b,%r11b + + # Get mask of QWs whose 52-bit parts saturated + vpcmpuq \$0,.Lmask52x4(%rip),${_R0},%k1 # OP=eq + vpcmpuq \$0,.Lmask52x4(%rip),${_R0h},%k2 + kmovb %k1,%r9d + kmovb %k2,%r8d + shl \$4,%r8b + or %r8b,%r9b + + vpcmpuq \$0,.Lmask52x4(%rip),${_R1},%k1 + vpcmpuq \$0,.Lmask52x4(%rip),${_R1h},%k2 + kmovb %k1,%r8d + kmovb %k2,%edx + shl \$4,%dl + or %dl,%r8b + + vpcmpuq \$0,.Lmask52x4(%rip),${_R2},%k1 + vpcmpuq \$0,.Lmask52x4(%rip),${_R2h},%k2 + kmovb %k1,%edx + kmovb %k2,%ecx + shl \$4,%cl + or %cl,%dl + + vpcmpuq \$0,.Lmask52x4(%rip),${_R3},%k1 + vpcmpuq \$0,.Lmask52x4(%rip),${_R3h},%k2 + kmovb %k1,%ecx + kmovb %k2,%ebx + shl \$4,%bl + or %bl,%cl + + addb %r9b,%r14b + adcb %r8b,%r13b + adcb %dl,%r12b + adcb %cl,%r11b + + xor %r9b,%r14b + xor %r8b,%r13b + xor %dl,%r12b + xor %cl,%r11b + + kmovb %r14d,%k1 + shr \$4,%r14b + kmovb %r14d,%k2 + kmovb %r13d,%k3 + shr \$4,%r13b + kmovb %r13d,%k4 + kmovb %r12d,%k5 + shr \$4,%r12b + kmovb %r12d,%k6 + kmovb %r11d,%k7 + + vpsubq .Lmask52x4(%rip), $_R0, ${_R0}{%k1} + vpsubq .Lmask52x4(%rip), $_R0h, ${_R0h}{%k2} + vpsubq .Lmask52x4(%rip), $_R1, ${_R1}{%k3} + vpsubq .Lmask52x4(%rip), $_R1h, ${_R1h}{%k4} + vpsubq .Lmask52x4(%rip), $_R2, ${_R2}{%k5} + vpsubq .Lmask52x4(%rip), $_R2h, ${_R2h}{%k6} + vpsubq .Lmask52x4(%rip), $_R3, ${_R3}{%k7} + + vpandq .Lmask52x4(%rip), $_R0, $_R0 + vpandq .Lmask52x4(%rip), $_R0h, $_R0h + vpandq .Lmask52x4(%rip), $_R1, $_R1 + vpandq .Lmask52x4(%rip), $_R1h, $_R1h + vpandq .Lmask52x4(%rip), $_R2, $_R2 + vpandq .Lmask52x4(%rip), $_R2h, $_R2h + vpandq .Lmask52x4(%rip), $_R3, $_R3 + + shr \$4,%r11b + kmovb %r11d,%k1 + + vpsubq .Lmask52x4(%rip), $_R3h, ${_R3h}{%k1} + + vpandq .Lmask52x4(%rip), $_R3h, $_R3h +___ +} + +$code.=<<___; +.text + +.globl ossl_rsaz_amm52x30_x1_ifma256 +.type ossl_rsaz_amm52x30_x1_ifma256,\@function,5 +.align 32 +ossl_rsaz_amm52x30_x1_ifma256: +.cfi_startproc + endbranch + push %rbx +.cfi_push %rbx + push %rbp +.cfi_push %rbp + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +___ +$code.=<<___ if ($win64); + lea -168(%rsp),%rsp # 16*10 + (8 bytes to get correct 16-byte SIMD alignment) + vmovdqa64 %xmm6, `0*16`(%rsp) # save non-volatile registers + vmovdqa64 %xmm7, `1*16`(%rsp) + vmovdqa64 %xmm8, `2*16`(%rsp) + vmovdqa64 %xmm9, `3*16`(%rsp) + vmovdqa64 %xmm10,`4*16`(%rsp) + vmovdqa64 %xmm11,`5*16`(%rsp) + vmovdqa64 %xmm12,`6*16`(%rsp) + vmovdqa64 %xmm13,`7*16`(%rsp) + vmovdqa64 %xmm14,`8*16`(%rsp) + vmovdqa64 %xmm15,`9*16`(%rsp) +.Lossl_rsaz_amm52x30_x1_ifma256_body: +___ +$code.=<<___; + # Zeroing accumulators + vpxord $zero, $zero, $zero + vmovdqa64 $zero, $R0_0 + vmovdqa64 $zero, $R0_0h + vmovdqa64 $zero, $R1_0 + vmovdqa64 $zero, $R1_0h + vmovdqa64 $zero, $R2_0 + vmovdqa64 $zero, $R2_0h + vmovdqa64 $zero, $R3_0 + vmovdqa64 $zero, $R3_0h + + xorl $acc0_0_low, $acc0_0_low + + movq $b, $b_ptr # backup address of b + movq \$0xfffffffffffff, $mask52 # 52-bit mask + + # Loop over 30 digits unrolled by 4 + mov \$7, $iter + +.align 32 +.Lloop7: +___ + foreach my $idx (0..3) { + &amm52x30_x1(0,8*$idx,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,$k0); + } +$code.=<<___; + lea `4*8`($b_ptr), $b_ptr + dec $iter + jne .Lloop7 +___ + &amm52x30_x1(0,8*0,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,$k0); + &amm52x30_x1(0,8*1,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,$k0); + + &amm52x30_x1_norm($acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h); +$code.=<<___; + + vmovdqu64 $R0_0, `0*32`($res) + vmovdqu64 $R0_0h, `1*32`($res) + vmovdqu64 $R1_0, `2*32`($res) + vmovdqu64 $R1_0h, `3*32`($res) + vmovdqu64 $R2_0, `4*32`($res) + vmovdqu64 $R2_0h, `5*32`($res) + vmovdqu64 $R3_0, `6*32`($res) + vmovdqu64 $R3_0h, `7*32`($res) + + vzeroupper + lea (%rsp),%rax +.cfi_def_cfa_register %rax +___ +$code.=<<___ if ($win64); + vmovdqa64 `0*16`(%rax),%xmm6 + vmovdqa64 `1*16`(%rax),%xmm7 + vmovdqa64 `2*16`(%rax),%xmm8 + vmovdqa64 `3*16`(%rax),%xmm9 + vmovdqa64 `4*16`(%rax),%xmm10 + vmovdqa64 `5*16`(%rax),%xmm11 + vmovdqa64 `6*16`(%rax),%xmm12 + vmovdqa64 `7*16`(%rax),%xmm13 + vmovdqa64 `8*16`(%rax),%xmm14 + vmovdqa64 `9*16`(%rax),%xmm15 + lea 168(%rsp),%rax +___ +$code.=<<___; + mov 0(%rax),%r15 +.cfi_restore %r15 + mov 8(%rax),%r14 +.cfi_restore %r14 + mov 16(%rax),%r13 +.cfi_restore %r13 + mov 24(%rax),%r12 +.cfi_restore %r12 + mov 32(%rax),%rbp +.cfi_restore %rbp + mov 40(%rax),%rbx +.cfi_restore %rbx + lea 48(%rax),%rsp # restore rsp +.cfi_def_cfa %rsp,8 +.Lossl_rsaz_amm52x30_x1_ifma256_epilogue: + ret +.cfi_endproc +.size ossl_rsaz_amm52x30_x1_ifma256, .-ossl_rsaz_amm52x30_x1_ifma256 +___ + +$code.=<<___; +.section .rodata align=32 +.align 32 +.Lmask52x4: + .quad 0xfffffffffffff + .quad 0xfffffffffffff + .quad 0xfffffffffffff + .quad 0xfffffffffffff +___ + +############################################################################### +# Dual Almost Montgomery Multiplication for 30-digit number in radix 2^52 +# +# See description of ossl_rsaz_amm52x30_x1_ifma256() above for details about Almost +# Montgomery Multiplication algorithm and function input parameters description. +# +# This function does two AMMs for two independent inputs, hence dual. +# +# NOTE: the function uses zero-padded data - 2 high QWs is a padding. +# +# void ossl_rsaz_amm52x30_x2_ifma256(BN_ULONG out[2][32], +# const BN_ULONG a[2][32], +# const BN_ULONG b[2][32], +# const BN_ULONG m[2][32], +# const BN_ULONG k0[2]); +############################################################################### + +$code.=<<___; +.text + +.globl ossl_rsaz_amm52x30_x2_ifma256 +.type ossl_rsaz_amm52x30_x2_ifma256,\@function,5 +.align 32 +ossl_rsaz_amm52x30_x2_ifma256: +.cfi_startproc + endbranch + push %rbx +.cfi_push %rbx + push %rbp +.cfi_push %rbp + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +___ +$code.=<<___ if ($win64); + lea -168(%rsp),%rsp + vmovdqa64 %xmm6, `0*16`(%rsp) # save non-volatile registers + vmovdqa64 %xmm7, `1*16`(%rsp) + vmovdqa64 %xmm8, `2*16`(%rsp) + vmovdqa64 %xmm9, `3*16`(%rsp) + vmovdqa64 %xmm10,`4*16`(%rsp) + vmovdqa64 %xmm11,`5*16`(%rsp) + vmovdqa64 %xmm12,`6*16`(%rsp) + vmovdqa64 %xmm13,`7*16`(%rsp) + vmovdqa64 %xmm14,`8*16`(%rsp) + vmovdqa64 %xmm15,`9*16`(%rsp) +.Lossl_rsaz_amm52x30_x2_ifma256_body: +___ +$code.=<<___; + # Zeroing accumulators + vpxord $zero, $zero, $zero + vmovdqa64 $zero, $R0_0 + vmovdqa64 $zero, $R0_0h + vmovdqa64 $zero, $R1_0 + vmovdqa64 $zero, $R1_0h + vmovdqa64 $zero, $R2_0 + vmovdqa64 $zero, $R2_0h + vmovdqa64 $zero, $R3_0 + vmovdqa64 $zero, $R3_0h + + vmovdqa64 $zero, $R0_1 + vmovdqa64 $zero, $R0_1h + vmovdqa64 $zero, $R1_1 + vmovdqa64 $zero, $R1_1h + vmovdqa64 $zero, $R2_1 + vmovdqa64 $zero, $R2_1h + vmovdqa64 $zero, $R3_1 + vmovdqa64 $zero, $R3_1h + + + xorl $acc0_0_low, $acc0_0_low + xorl $acc0_1_low, $acc0_1_low + + movq $b, $b_ptr # backup address of b + movq \$0xfffffffffffff, $mask52 # 52-bit mask + + mov \$30, $iter + +.align 32 +.Lloop30: +___ + &amm52x30_x1( 0, 0,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,"($k0)"); + # 32*8 = offset of the next dimension in two-dimension array + &amm52x30_x1(32*8,32*8,$acc0_1,$R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1,$R2_1h,$R3_1,$R3_1h,"8($k0)"); +$code.=<<___; + lea 8($b_ptr), $b_ptr + dec $iter + jne .Lloop30 +___ + &amm52x30_x1_norm($acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h); + &amm52x30_x1_norm($acc0_1,$R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1,$R2_1h,$R3_1,$R3_1h); +$code.=<<___; + + vmovdqu64 $R0_0, `0*32`($res) + vmovdqu64 $R0_0h, `1*32`($res) + vmovdqu64 $R1_0, `2*32`($res) + vmovdqu64 $R1_0h, `3*32`($res) + vmovdqu64 $R2_0, `4*32`($res) + vmovdqu64 $R2_0h, `5*32`($res) + vmovdqu64 $R3_0, `6*32`($res) + vmovdqu64 $R3_0h, `7*32`($res) + + vmovdqu64 $R0_1, `8*32`($res) + vmovdqu64 $R0_1h, `9*32`($res) + vmovdqu64 $R1_1, `10*32`($res) + vmovdqu64 $R1_1h, `11*32`($res) + vmovdqu64 $R2_1, `12*32`($res) + vmovdqu64 $R2_1h, `13*32`($res) + vmovdqu64 $R3_1, `14*32`($res) + vmovdqu64 $R3_1h, `15*32`($res) + + vzeroupper + lea (%rsp),%rax +.cfi_def_cfa_register %rax +___ +$code.=<<___ if ($win64); + vmovdqa64 `0*16`(%rax),%xmm6 + vmovdqa64 `1*16`(%rax),%xmm7 + vmovdqa64 `2*16`(%rax),%xmm8 + vmovdqa64 `3*16`(%rax),%xmm9 + vmovdqa64 `4*16`(%rax),%xmm10 + vmovdqa64 `5*16`(%rax),%xmm11 + vmovdqa64 `6*16`(%rax),%xmm12 + vmovdqa64 `7*16`(%rax),%xmm13 + vmovdqa64 `8*16`(%rax),%xmm14 + vmovdqa64 `9*16`(%rax),%xmm15 + lea 168(%rsp),%rax +___ +$code.=<<___; + mov 0(%rax),%r15 +.cfi_restore %r15 + mov 8(%rax),%r14 +.cfi_restore %r14 + mov 16(%rax),%r13 +.cfi_restore %r13 + mov 24(%rax),%r12 +.cfi_restore %r12 + mov 32(%rax),%rbp +.cfi_restore %rbp + mov 40(%rax),%rbx +.cfi_restore %rbx + lea 48(%rax),%rsp +.cfi_def_cfa %rsp,8 +.Lossl_rsaz_amm52x30_x2_ifma256_epilogue: + ret +.cfi_endproc +.size ossl_rsaz_amm52x30_x2_ifma256, .-ossl_rsaz_amm52x30_x2_ifma256 +___ +} + +############################################################################### +# Constant time extraction from the precomputed table of powers base^i, where +# i = 0..2^EXP_WIN_SIZE-1 +# +# The input |red_table| contains precomputations for two independent base values. +# |red_table_idx1| and |red_table_idx2| are corresponding power indexes. +# +# Extracted value (output) is 2 (30 + 2) digits numbers in 2^52 radix. +# (2 high QW is zero padding) +# +# void ossl_extract_multiplier_2x30_win5(BN_ULONG *red_Y, +# const BN_ULONG red_table[1 << EXP_WIN_SIZE][2][32], +# int red_table_idx1, int red_table_idx2); +# +# EXP_WIN_SIZE = 5 +############################################################################### +{ +# input parameters +my ($out,$red_tbl,$red_tbl_idx1,$red_tbl_idx2)=$win64 ? ("%rcx","%rdx","%r8", "%r9") : # Win64 order + ("%rdi","%rsi","%rdx","%rcx"); # Unix order + +my ($t0,$t1,$t2,$t3,$t4,$t5) = map("%ymm$_", (0..5)); +my ($t6,$t7,$t8,$t9,$t10,$t11,$t12,$t13,$t14,$t15) = map("%ymm$_", (16..25)); +my ($tmp,$cur_idx,$idx1,$idx2,$ones) = map("%ymm$_", (26..30)); + +my @t = ($t0,$t1,$t2,$t3,$t4,$t5,$t6,$t7,$t8,$t9,$t10,$t11,$t12,$t13,$t14,$t15); +my $t0xmm = $t0; +$t0xmm =~ s/%y/%x/; + +$code.=<<___; +.text + +.align 32 +.globl ossl_extract_multiplier_2x30_win5 +.type ossl_extract_multiplier_2x30_win5,\@abi-omnipotent +ossl_extract_multiplier_2x30_win5: +.cfi_startproc + endbranch + vmovdqa64 .Lones(%rip), $ones # broadcast ones + vpbroadcastq $red_tbl_idx1, $idx1 + vpbroadcastq $red_tbl_idx2, $idx2 + leaq `(1<<5)*2*32*8`($red_tbl), %rax # holds end of the tbl + + # zeroing t0..n, cur_idx + vpxor $t0xmm, $t0xmm, $t0xmm + vmovdqa64 $t0, $cur_idx +___ +foreach (1..15) { + $code.="vmovdqa64 $t0, $t[$_] \n"; +} +$code.=<<___; + +.align 32 +.Lloop: + vpcmpq \$0, $cur_idx, $idx1, %k1 # mask of (idx1 == cur_idx) + vpcmpq \$0, $cur_idx, $idx2, %k2 # mask of (idx2 == cur_idx) +___ +foreach (0..15) { + my $mask = $_<8?"%k1":"%k2"; +$code.=<<___; + vmovdqu64 `${_}*32`($red_tbl), $tmp # load data from red_tbl + vpblendmq $tmp, $t[$_], ${t[$_]}{$mask} # extract data when mask is not zero +___ +} +$code.=<<___; + vpaddq $ones, $cur_idx, $cur_idx # increment cur_idx + addq \$`2*32*8`, $red_tbl + cmpq $red_tbl, %rax + jne .Lloop +___ +# store t0..n +foreach (0..15) { + $code.="vmovdqu64 $t[$_], `${_}*32`($out) \n"; +} +$code.=<<___; + + ret +.cfi_endproc +.size ossl_extract_multiplier_2x30_win5, .-ossl_extract_multiplier_2x30_win5 +___ +$code.=<<___; +.section .rodata align=32 +.align 32 +.Lones: + .quad 1,1,1,1 +.Lzeros: + .quad 0,0,0,0 +___ +} + +if ($win64) { +$rec="%rcx"; +$frame="%rdx"; +$context="%r8"; +$disp="%r9"; + +$code.=<<___; +.extern __imp_RtlVirtualUnwind +.type rsaz_avx_handler,\@abi-omnipotent +.align 16 +rsaz_avx_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + mov 8($disp),%rsi # disp->ImageBase + mov 56($disp),%r11 # disp->HandlerData + + mov 0(%r11),%r10d # HandlerData[0] + lea (%rsi,%r10),%r10 # prologue label + cmp %r10,%rbx # context->Rip<.Lprologue + jb .Lcommon_seh_tail + + mov 4(%r11),%r10d # HandlerData[1] + lea (%rsi,%r10),%r10 # epilogue label + cmp %r10,%rbx # context->Rip>=.Lepilogue + jae .Lcommon_seh_tail + + mov 152($context),%rax # pull context->Rsp + + lea (%rax),%rsi # %xmm save area + lea 512($context),%rdi # & context.Xmm6 + mov \$20,%ecx # 10*sizeof(%xmm0)/sizeof(%rax) + .long 0xa548f3fc # cld; rep movsq + + lea `48+168`(%rax),%rax + + mov -8(%rax),%rbx + mov -16(%rax),%rbp + mov -24(%rax),%r12 + mov -32(%rax),%r13 + mov -40(%rax),%r14 + mov -48(%rax),%r15 + mov %rbx,144($context) # restore context->Rbx + mov %rbp,160($context) # restore context->Rbp + mov %r12,216($context) # restore context->R12 + mov %r13,224($context) # restore context->R13 + mov %r14,232($context) # restore context->R14 + mov %r15,240($context) # restore context->R14 + +.Lcommon_seh_tail: + mov 8(%rax),%rdi + mov 16(%rax),%rsi + mov %rax,152($context) # restore context->Rsp + mov %rsi,168($context) # restore context->Rsi + mov %rdi,176($context) # restore context->Rdi + + mov 40($disp),%rdi # disp->ContextRecord + mov $context,%rsi # context + mov \$154,%ecx # sizeof(CONTEXT) + .long 0xa548f3fc # cld; rep movsq + + mov $disp,%rsi + xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER + mov 8(%rsi),%rdx # arg2, disp->ImageBase + mov 0(%rsi),%r8 # arg3, disp->ControlPc + mov 16(%rsi),%r9 # arg4, disp->FunctionEntry + mov 40(%rsi),%r10 # disp->ContextRecord + lea 56(%rsi),%r11 # &disp->HandlerData + lea 24(%rsi),%r12 # &disp->EstablisherFrame + mov %r10,32(%rsp) # arg5 + mov %r11,40(%rsp) # arg6 + mov %r12,48(%rsp) # arg7 + mov %rcx,56(%rsp) # arg8, (NULL) + call *__imp_RtlVirtualUnwind(%rip) + + mov \$1,%eax # ExceptionContinueSearch + add \$64,%rsp + popfq + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbp + pop %rbx + pop %rdi + pop %rsi + ret +.size rsaz_avx_handler,.-rsaz_avx_handler + +.section .pdata +.align 4 + .rva .LSEH_begin_ossl_rsaz_amm52x30_x1_ifma256 + .rva .LSEH_end_ossl_rsaz_amm52x30_x1_ifma256 + .rva .LSEH_info_ossl_rsaz_amm52x30_x1_ifma256 + + .rva .LSEH_begin_ossl_rsaz_amm52x30_x2_ifma256 + .rva .LSEH_end_ossl_rsaz_amm52x30_x2_ifma256 + .rva .LSEH_info_ossl_rsaz_amm52x30_x2_ifma256 + +.section .xdata +.align 8 +.LSEH_info_ossl_rsaz_amm52x30_x1_ifma256: + .byte 9,0,0,0 + .rva rsaz_avx_handler + .rva .Lossl_rsaz_amm52x30_x1_ifma256_body,.Lossl_rsaz_amm52x30_x1_ifma256_epilogue +.LSEH_info_ossl_rsaz_amm52x30_x2_ifma256: + .byte 9,0,0,0 + .rva rsaz_avx_handler + .rva .Lossl_rsaz_amm52x30_x2_ifma256_body,.Lossl_rsaz_amm52x30_x2_ifma256_epilogue +___ +} +}}} else {{{ # fallback for old assembler +$code.=<<___; +.text + +.globl ossl_rsaz_amm52x30_x1_ifma256 +.globl ossl_rsaz_amm52x30_x2_ifma256 +.globl ossl_extract_multiplier_2x30_win5 +.type ossl_rsaz_amm52x30_x1_ifma256,\@abi-omnipotent +ossl_rsaz_amm52x30_x1_ifma256: +ossl_rsaz_amm52x30_x2_ifma256: +ossl_extract_multiplier_2x30_win5: + .byte 0x0f,0x0b # ud2 + ret +.size ossl_rsaz_amm52x30_x1_ifma256, .-ossl_rsaz_amm52x30_x1_ifma256 +___ +}}} + +$code =~ s/\`([^\`]*)\`/eval $1/gem; +print $code; +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/bn/asm/rsaz-3k-avxifma.pl b/crypto/openssl/crypto/bn/asm/rsaz-3k-avxifma.pl new file mode 100644 index 000000000000..a19cb5aaa309 --- /dev/null +++ b/crypto/openssl/crypto/bn/asm/rsaz-3k-avxifma.pl @@ -0,0 +1,1091 @@ +# Copyright 2024-2025 The OpenSSL Project Authors. All Rights Reserved. +# Copyright (c) 2024, Intel Corporation. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html +# +# Written by Zhiguo Zhou and Wangyang Guo . +# Special thanks to Tomasz Kantecki for his valuable suggestions. +# +# October 2024 +# +# Initial release. +# + +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); +$avxifma=0; + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or +die "can't locate x86_64-xlate.pl"; + +if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` + =~ /GNU assembler version ([2-9]\.[0-9]+)/) { + $avxifma = ($1>=2.40); +} + +if (!$avxifma && `$ENV{CC} -v 2>&1` + =~ /\s*((?:clang|LLVM) version|.*based on LLVM) ([0-9]+)\.([0-9]+)\.([0-9]+)?/) { + my $ver = $2 + $3/100.0 + $4/10000.0; # 3.1.0->3.01, 3.10.1->3.1001 + $avxifma = ($ver>=16.0); +} + +if ($win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && + `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?(rc[0-9]+)?/) { + $avxifma = ($1>2.16) + ($1==2.16 && ((!defined($2) && !defined($3)) || (defined($2)))); +} + +open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"" + or die "can't call $xlate: $!"; +*STDOUT=*OUT; + +if ($avxifma>0) {{{ +@_6_args_universal_ABI = ("%rdi","%rsi","%rdx","%rcx","%r8","%r9"); + +############################################################################### +# Almost Montgomery Multiplication (AMM) for 30-digit number in radix 2^52. +# +# AMM is defined as presented in the paper [1]. +# +# The input and output are presented in 2^52 radix domain, i.e. +# |res|, |a|, |b|, |m| are arrays of 32 64-bit qwords with 12 high bits zeroed +# +# NOTE: the function uses zero-padded data - 2 high QWs is a padding. +# +# |k0| is a Montgomery coefficient, which is here k0 = -1/m mod 2^64 +# +# NB: the AMM implementation does not perform "conditional" subtraction step +# specified in the original algorithm as according to the Lemma 1 from the paper +# [2], the result will be always < 2*m and can be used as a direct input to +# the next AMM iteration. This post-condition is true, provided the correct +# parameter |s| (notion of the Lemma 1 from [2]) is chosen, i.e. s >= n + 2 * k, +# which matches our case: 1560 > 1536 + 2 * 1. +# +# [1] Gueron, S. Efficient software implementations of modular exponentiation. +# DOI: 10.1007/s13389-012-0031-5 +# [2] Gueron, S. Enhanced Montgomery Multiplication. +# DOI: 10.1007/3-540-36400-5_5 +# +# void ossl_rsaz_amm52x30_x1_avxifma256(BN_ULONG *res, +# const BN_ULONG *a, +# const BN_ULONG *b, +# const BN_ULONG *m, +# BN_ULONG k0); +############################################################################### +{ +# input parameters ("%rdi","%rsi","%rdx","%rcx","%r8") +my ($res,$a,$b,$m,$k0) = @_6_args_universal_ABI; + +my $mask52 = "%rax"; +my $acc0_0 = "%r9"; +my $acc0_0_low = "%r9d"; +my $acc0_1 = "%r15"; +my $acc0_1_low = "%r15d"; +my $b_ptr = "%r11"; + +my $iter = "%ebx"; + +my $zero = "%ymm0"; +my $Bi = "%ymm1"; +my $Bi_xmm = "%xmm1"; +my $Yi = "%ymm2"; +my $Yi_xmm = "%xmm2"; +my ($R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h) = map("%ymm$_",(3..10)); + +# Registers mapping for normalization +my ($T0,$T0h,$T1,$T1h,$T2,$T2h,$T3,$T3h) = ("$zero", "$Bi", "$Yi", map("%ymm$_", (11..15))); +my $T0_xmm = "%xmm0"; + +sub amm52x30_x1() { +# _data_offset - offset in the |a| or |m| arrays pointing to the beginning +# of data for corresponding AMM operation; +# _b_offset - offset in the |b| array pointing to the next qword digit; +my ($_data_offset,$_b_offset,$_acc,$_R0,$_R0h,$_R1,$_R1h,$_R2,$_R2h,$_R3,$_R3h,$_k0) = @_; +$code.=<<___; + movq $_b_offset($b_ptr), %r13 # b[i] + + vpbroadcastq $_b_offset($b_ptr), $Bi # broadcast b[i] + movq $_data_offset($a), %rdx + mulx %r13, %r13, %r12 # a[0]*b[i] = (t0,t2) + addq %r13, $_acc # acc += t0 + movq %r12, %r10 + adcq \$0, %r10 # t2 += CF + + movq $_k0, %r13 + imulq $_acc, %r13 # acc * k0 + andq $mask52, %r13 # yi = (acc * k0) & mask52 + + vmovq %r13, $Yi_xmm + vpbroadcastq $Yi_xmm, $Yi # broadcast y[i] + movq $_data_offset($m), %rdx + mulx %r13, %r13, %r12 # yi * m[0] = (t0,t1) + addq %r13, $_acc # acc += t0 + adcq %r12, %r10 # t2 += (t1 + CF) + + shrq \$52, $_acc + salq \$12, %r10 + or %r10, $_acc # acc = ((acc >> 52) | (t2 << 12)) + + lea -264(%rsp), %rsp + + {vex} vpmadd52luq `$_data_offset`($a), $Bi, $_R0 + {vex} vpmadd52luq `$_data_offset+32`($a), $Bi, $_R0h + {vex} vpmadd52luq `$_data_offset+64*1`($a), $Bi, $_R1 + {vex} vpmadd52luq `$_data_offset+64*1+32`($a), $Bi, $_R1h + {vex} vpmadd52luq `$_data_offset+64*2`($a), $Bi, $_R2 + {vex} vpmadd52luq `$_data_offset+64*2+32`($a), $Bi, $_R2h + {vex} vpmadd52luq `$_data_offset+64*3`($a), $Bi, $_R3 + {vex} vpmadd52luq `$_data_offset+64*3+32`($a), $Bi, $_R3h + + {vex} vpmadd52luq `$_data_offset`($m), $Yi, $_R0 + {vex} vpmadd52luq `$_data_offset+32`($m), $Yi, $_R0h + {vex} vpmadd52luq `$_data_offset+64*1`($m), $Yi, $_R1 + {vex} vpmadd52luq `$_data_offset+64*1+32`($m), $Yi, $_R1h + {vex} vpmadd52luq `$_data_offset+64*2`($m), $Yi, $_R2 + {vex} vpmadd52luq `$_data_offset+64*2+32`($m), $Yi, $_R2h + {vex} vpmadd52luq `$_data_offset+64*3`($m), $Yi, $_R3 + {vex} vpmadd52luq `$_data_offset+64*3+32`($m), $Yi, $_R3h + + # Shift accumulators right by 1 qword, zero extending the highest one + vmovdqu $_R0, `32*0`(%rsp) + vmovdqu $_R0h, `32*1`(%rsp) + vmovdqu $_R1, `32*2`(%rsp) + vmovdqu $_R1h, `32*3`(%rsp) + vmovdqu $_R2, `32*4`(%rsp) + vmovdqu $_R2h, `32*5`(%rsp) + vmovdqu $_R3, `32*6`(%rsp) + vmovdqu $_R3h, `32*7`(%rsp) + movq \$0, `32*8`(%rsp) + + vmovdqu `32*0 + 8`(%rsp), $_R0 + vmovdqu `32*1 + 8`(%rsp), $_R0h + vmovdqu `32*2 + 8`(%rsp), $_R1 + vmovdqu `32*3 + 8`(%rsp), $_R1h + vmovdqu `32*4 + 8`(%rsp), $_R2 + vmovdqu `32*5 + 8`(%rsp), $_R2h + vmovdqu `32*6 + 8`(%rsp), $_R3 + vmovdqu `32*7 + 8`(%rsp), $_R3h + + addq 8(%rsp), $_acc # acc += R0[0] + + {vex} vpmadd52huq `$_data_offset`($a), $Bi, $_R0 + {vex} vpmadd52huq `$_data_offset+32`($a), $Bi, $_R0h + {vex} vpmadd52huq `$_data_offset+64*1`($a), $Bi, $_R1 + {vex} vpmadd52huq `$_data_offset+64*1+32`($a), $Bi, $_R1h + {vex} vpmadd52huq `$_data_offset+64*2`($a), $Bi, $_R2 + {vex} vpmadd52huq `$_data_offset+64*2+32`($a), $Bi, $_R2h + {vex} vpmadd52huq `$_data_offset+64*3`($a), $Bi, $_R3 + {vex} vpmadd52huq `$_data_offset+64*3+32`($a), $Bi, $_R3h + + {vex} vpmadd52huq `$_data_offset`($m), $Yi, $_R0 + {vex} vpmadd52huq `$_data_offset+32`($m), $Yi, $_R0h + {vex} vpmadd52huq `$_data_offset+64*1`($m), $Yi, $_R1 + {vex} vpmadd52huq `$_data_offset+64*1+32`($m), $Yi, $_R1h + {vex} vpmadd52huq `$_data_offset+64*2`($m), $Yi, $_R2 + {vex} vpmadd52huq `$_data_offset+64*2+32`($m), $Yi, $_R2h + {vex} vpmadd52huq `$_data_offset+64*3`($m), $Yi, $_R3 + {vex} vpmadd52huq `$_data_offset+64*3+32`($m), $Yi, $_R3h + + lea 264(%rsp),%rsp +___ +} + +# Normalization routine: handles carry bits and gets bignum qwords to normalized +# 2^52 representation. +# +# Uses %r8-14,%e[abcd]x +sub amm52x30_x1_norm { +my ($_acc,$_R0,$_R0h,$_R1,$_R1h,$_R2,$_R2h,$_R3,$_R3h) = @_; +$code.=<<___; + # Put accumulator to low qword in R0 + vmovq $_acc, $T0_xmm + vpbroadcastq $T0_xmm, $T0 + vpblendd \$3, $T0, $_R0, $_R0 + + # Extract "carries" (12 high bits) from each QW of the bignum + # Save them to LSB of QWs in T0..Tn + vpsrlq \$52, $_R0, $T0 + vpsrlq \$52, $_R0h, $T0h + vpsrlq \$52, $_R1, $T1 + vpsrlq \$52, $_R1h, $T1h + vpsrlq \$52, $_R2, $T2 + vpsrlq \$52, $_R2h, $T2h + vpsrlq \$52, $_R3, $T3 + vpsrlq \$52, $_R3h, $T3h + + lea -32(%rsp),%rsp + vmovupd $_R0, (%rsp) + + # "Shift left" T0..Tn by 1 QW + vpermq \$144, $T3h, $T3h + vpermq \$3, $T3, $_R0 + vblendpd \$1, $_R0, $T3h, $T3h + + vpermq \$144, $T3, $T3 + vpermq \$3, $T2h, $_R0 + vblendpd \$1, $_R0, $T3, $T3 + + vpermq \$144, $T2h, $T2h + vpermq \$3, $T2, $_R0 + vblendpd \$1, $_R0, $T2h, $T2h + + vpermq \$144, $T2, $T2 + vpermq \$3, $T1h, $_R0 + vblendpd \$1, $_R0, $T2, $T2 + + vpermq \$144, $T1h, $T1h + vpermq \$3, $T1, $_R0 + vblendpd \$1, $_R0, $T1h, $T1h + + vpermq \$144, $T1, $T1 + vpermq \$3, $T0h, $_R0 + vblendpd \$1, $_R0, $T1, $T1 + + vpermq \$144, $T0h, $T0h + vpermq \$3, $T0, $_R0 + vblendpd \$1, $_R0, $T0h, $T0h + + vpermq \$144, $T0, $T0 + vpand .Lhigh64x3(%rip), $T0, $T0 + + vmovupd (%rsp), $_R0 + lea 32(%rsp),%rsp + + # Drop "carries" from R0..Rn QWs + vpand .Lmask52x4(%rip), $_R0, $_R0 + vpand .Lmask52x4(%rip), $_R0h, $_R0h + vpand .Lmask52x4(%rip), $_R1, $_R1 + vpand .Lmask52x4(%rip), $_R1h, $_R1h + vpand .Lmask52x4(%rip), $_R2, $_R2 + vpand .Lmask52x4(%rip), $_R2h, $_R2h + vpand .Lmask52x4(%rip), $_R3, $_R3 + vpand .Lmask52x4(%rip), $_R3h, $_R3h + + # Sum R0..Rn with corresponding adjusted carries + vpaddq $T0, $_R0, $_R0 + vpaddq $T0h, $_R0h, $_R0h + vpaddq $T1, $_R1, $_R1 + vpaddq $T1h, $_R1h, $_R1h + vpaddq $T2, $_R2, $_R2 + vpaddq $T2h, $_R2h, $_R2h + vpaddq $T3, $_R3, $_R3 + vpaddq $T3h, $_R3h, $_R3h + + # Now handle carry bits from this addition + # Get mask of QWs whose 52-bit parts overflow + vpcmpgtq .Lmask52x4(%rip),${_R0},$T0 # OP=nle (i.e. gt) + vpcmpgtq .Lmask52x4(%rip),${_R0h},$T0h + vmovmskpd $T0, %r14d + vmovmskpd $T0h,%r13d + shl \$4,%r13b + or %r13b,%r14b + + vpcmpgtq .Lmask52x4(%rip),${_R1},$T1 + vpcmpgtq .Lmask52x4(%rip),${_R1h},$T1h + vmovmskpd $T1, %r13d + vmovmskpd $T1h,%r12d + shl \$4,%r12b + or %r12b,%r13b + + vpcmpgtq .Lmask52x4(%rip),${_R2},$T2 + vpcmpgtq .Lmask52x4(%rip),${_R2h},$T2h + vmovmskpd $T2, %r12d + vmovmskpd $T2h,%r11d + shl \$4,%r11b + or %r11b,%r12b + + vpcmpgtq .Lmask52x4(%rip),${_R3},$T3 + vpcmpgtq .Lmask52x4(%rip),${_R3h},$T3h + vmovmskpd $T3, %r11d + vmovmskpd $T3h,%r10d + shl \$4,%r10b + or %r10b,%r11b + + addb %r14b,%r14b + adcb %r13b,%r13b + adcb %r12b,%r12b + adcb %r11b,%r11b + + # Get mask of QWs whose 52-bit parts saturated + vpcmpeqq .Lmask52x4(%rip),${_R0},$T0 # OP=eq + vpcmpeqq .Lmask52x4(%rip),${_R0h},$T0h + vmovmskpd $T0, %r9d + vmovmskpd $T0h,%r8d + shl \$4,%r8b + or %r8b,%r9b + + vpcmpeqq .Lmask52x4(%rip),${_R1},$T1 + vpcmpeqq .Lmask52x4(%rip),${_R1h},$T1h + vmovmskpd $T1, %r8d + vmovmskpd $T1h,%edx + shl \$4,%dl + or %dl,%r8b + + vpcmpeqq .Lmask52x4(%rip),${_R2},$T2 + vpcmpeqq .Lmask52x4(%rip),${_R2h},$T2h + vmovmskpd $T2, %edx + vmovmskpd $T2h,%ecx + shl \$4,%cl + or %cl,%dl + + vpcmpeqq .Lmask52x4(%rip),${_R3},$T3 + vpcmpeqq .Lmask52x4(%rip),${_R3h},$T3h + vmovmskpd $T3, %ecx + vmovmskpd $T3h,%ebx + shl \$4,%bl + or %bl,%cl + + addb %r9b,%r14b + adcb %r8b,%r13b + adcb %dl,%r12b + adcb %cl,%r11b + + xor %r9b,%r14b + xor %r8b,%r13b + xor %dl,%r12b + xor %cl,%r11b + + lea .Lkmasklut(%rip), %rdx + + mov %r14b, %r10b + and \$0xf, %r14 + vpsubq .Lmask52x4(%rip), $_R0, $T0 + shl \$5, %r14 + vmovapd (%rdx,%r14), $T1 + vblendvpd $T1, $T0, $_R0, $_R0 + + shr \$4, %r10b + and \$0xf, %r10 + vpsubq .Lmask52x4(%rip), $_R0h, $T0 + shl \$5, %r10 + vmovapd (%rdx,%r10), $T1 + vblendvpd $T1, $T0, $_R0h, $_R0h + + mov %r13b, %r10b + and \$0xf, %r13 + vpsubq .Lmask52x4(%rip), $_R1, $T0 + shl \$5, %r13 + vmovapd (%rdx,%r13), $T1 + vblendvpd $T1, $T0, $_R1, $_R1 + + shr \$4, %r10b + and \$0xf, %r10 + vpsubq .Lmask52x4(%rip), $_R1h, $T0 + shl \$5, %r10 + vmovapd (%rdx,%r10), $T1 + vblendvpd $T1, $T0, $_R1h, $_R1h + + mov %r12b, %r10b + and \$0xf, %r12 + vpsubq .Lmask52x4(%rip), $_R2, $T0 + shl \$5, %r12 + vmovapd (%rdx,%r12), $T1 + vblendvpd $T1, $T0, $_R2, $_R2 + + shr \$4, %r10b + and \$0xf, %r10 + vpsubq .Lmask52x4(%rip), $_R2h, $T0 + shl \$5, %r10 + vmovapd (%rdx,%r10), $T1 + vblendvpd $T1, $T0, $_R2h, $_R2h + + mov %r11b, %r10b + and \$0xf, %r11 + vpsubq .Lmask52x4(%rip), $_R3, $T0 + shl \$5, %r11 + vmovapd (%rdx,%r11), $T1 + vblendvpd $T1, $T0, $_R3, $_R3 + + shr \$4, %r10b + and \$0xf, %r10 + vpsubq .Lmask52x4(%rip), $_R3h, $T0 + shl \$5, %r10 + vmovapd (%rdx,%r10), $T1 + vblendvpd $T1, $T0, $_R3h, $_R3h + + vpand .Lmask52x4(%rip), $_R0, $_R0 + vpand .Lmask52x4(%rip), $_R0h, $_R0h + vpand .Lmask52x4(%rip), $_R1, $_R1 + vpand .Lmask52x4(%rip), $_R1h, $_R1h + vpand .Lmask52x4(%rip), $_R2, $_R2 + vpand .Lmask52x4(%rip), $_R2h, $_R2h + vpand .Lmask52x4(%rip), $_R3, $_R3 + + vpand .Lmask52x4(%rip), $_R3h, $_R3h +___ +} + +$code.=<<___; +.text + +.globl ossl_rsaz_amm52x30_x1_avxifma256 +.type ossl_rsaz_amm52x30_x1_avxifma256,\@function,5 +.align 32 +ossl_rsaz_amm52x30_x1_avxifma256: +.cfi_startproc + endbranch + push %rbx +.cfi_push %rbx + push %rbp +.cfi_push %rbp + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +___ +$code.=<<___ if ($win64); + lea -168(%rsp),%rsp # 16*10 + (8 bytes to get correct 16-byte SIMD alignment) + vmovapd %xmm6, `0*16`(%rsp) # save non-volatile registers + vmovapd %xmm7, `1*16`(%rsp) + vmovapd %xmm8, `2*16`(%rsp) + vmovapd %xmm9, `3*16`(%rsp) + vmovapd %xmm10,`4*16`(%rsp) + vmovapd %xmm11,`5*16`(%rsp) + vmovapd %xmm12,`6*16`(%rsp) + vmovapd %xmm13,`7*16`(%rsp) + vmovapd %xmm14,`8*16`(%rsp) + vmovapd %xmm15,`9*16`(%rsp) +.Lossl_rsaz_amm52x30_x1_avxifma256_body: +___ +$code.=<<___; + # Zeroing accumulators + vpxor $zero, $zero, $zero + vmovapd $zero, $R0_0 + vmovapd $zero, $R0_0h + vmovapd $zero, $R1_0 + vmovapd $zero, $R1_0h + vmovapd $zero, $R2_0 + vmovapd $zero, $R2_0h + vmovapd $zero, $R3_0 + vmovapd $zero, $R3_0h + + xorl $acc0_0_low, $acc0_0_low + + movq $b, $b_ptr # backup address of b + movq \$0xfffffffffffff, $mask52 # 52-bit mask + + # Loop over 30 digits unrolled by 4 + mov \$7, $iter + +.align 32 +.Lloop7: +___ + foreach my $idx (0..3) { + &amm52x30_x1(0,8*$idx,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,$k0); + } +$code.=<<___; + lea `4*8`($b_ptr), $b_ptr + dec $iter + jne .Lloop7 +___ + &amm52x30_x1(0,8*0,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,$k0); + &amm52x30_x1(0,8*1,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,$k0); + + &amm52x30_x1_norm($acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h); +$code.=<<___; + + vmovdqu $R0_0, `0*32`($res) + vmovdqu $R0_0h, `1*32`($res) + vmovdqu $R1_0, `2*32`($res) + vmovdqu $R1_0h, `3*32`($res) + vmovdqu $R2_0, `4*32`($res) + vmovdqu $R2_0h, `5*32`($res) + vmovdqu $R3_0, `6*32`($res) + vmovdqu $R3_0h, `7*32`($res) + + vzeroupper + lea (%rsp),%rax +.cfi_def_cfa_register %rax +___ +$code.=<<___ if ($win64); + vmovapd `0*16`(%rax),%xmm6 + vmovapd `1*16`(%rax),%xmm7 + vmovapd `2*16`(%rax),%xmm8 + vmovapd `3*16`(%rax),%xmm9 + vmovapd `4*16`(%rax),%xmm10 + vmovapd `5*16`(%rax),%xmm11 + vmovapd `6*16`(%rax),%xmm12 + vmovapd `7*16`(%rax),%xmm13 + vmovapd `8*16`(%rax),%xmm14 + vmovapd `9*16`(%rax),%xmm15 + lea 168(%rsp),%rax +___ +$code.=<<___; + mov 0(%rax),%r15 +.cfi_restore %r15 + mov 8(%rax),%r14 +.cfi_restore %r14 + mov 16(%rax),%r13 +.cfi_restore %r13 + mov 24(%rax),%r12 +.cfi_restore %r12 + mov 32(%rax),%rbp +.cfi_restore %rbp + mov 40(%rax),%rbx +.cfi_restore %rbx + lea 48(%rax),%rsp # restore rsp +.cfi_def_cfa %rsp,8 +.Lossl_rsaz_amm52x30_x1_avxifma256_epilogue: + ret +.cfi_endproc +.size ossl_rsaz_amm52x30_x1_avxifma256, .-ossl_rsaz_amm52x30_x1_avxifma256 +___ + +$code.=<<___; +.section .rodata align=32 +.align 32 +.Lmask52x4: + .quad 0xfffffffffffff + .quad 0xfffffffffffff + .quad 0xfffffffffffff + .quad 0xfffffffffffff +.Lhigh64x3: + .quad 0x0 + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff +.Lkmasklut: + #0000 + .quad 0x0 + .quad 0x0 + .quad 0x0 + .quad 0x0 + #0001 + .quad 0xffffffffffffffff + .quad 0x0 + .quad 0x0 + .quad 0x0 + #0010 + .quad 0x0 + .quad 0xffffffffffffffff + .quad 0x0 + .quad 0x0 + #0011 + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + .quad 0x0 + .quad 0x0 + #0100 + .quad 0x0 + .quad 0x0 + .quad 0xffffffffffffffff + .quad 0x0 + #0101 + .quad 0xffffffffffffffff + .quad 0x0 + .quad 0xffffffffffffffff + .quad 0x0 + #0110 + .quad 0x0 + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + .quad 0x0 + #0111 + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + .quad 0x0 + #1000 + .quad 0x0 + .quad 0x0 + .quad 0x0 + .quad 0xffffffffffffffff + #1001 + .quad 0xffffffffffffffff + .quad 0x0 + .quad 0x0 + .quad 0xffffffffffffffff + #1010 + .quad 0x0 + .quad 0xffffffffffffffff + .quad 0x0 + .quad 0xffffffffffffffff + #1011 + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + .quad 0x0 + .quad 0xffffffffffffffff + #1100 + .quad 0x0 + .quad 0x0 + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + #1101 + .quad 0xffffffffffffffff + .quad 0x0 + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + #1110 + .quad 0x0 + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + #1111 + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff +___ + +############################################################################### +# Dual Almost Montgomery Multiplication for 30-digit number in radix 2^52 +# +# See description of ossl_rsaz_amm52x30_x1_ifma256() above for details about Almost +# Montgomery Multiplication algorithm and function input parameters description. +# +# This function does two AMMs for two independent inputs, hence dual. +# +# NOTE: the function uses zero-padded data - 2 high QWs is a padding. +# +# void ossl_rsaz_amm52x30_x2_avxifma256(BN_ULONG out[2][32], +# const BN_ULONG a[2][32], +# const BN_ULONG b[2][32], +# const BN_ULONG m[2][32], +# const BN_ULONG k0[2]); +############################################################################### + +$code.=<<___; +.text + +.globl ossl_rsaz_amm52x30_x2_avxifma256 +.type ossl_rsaz_amm52x30_x2_avxifma256,\@function,5 +.align 32 +ossl_rsaz_amm52x30_x2_avxifma256: +.cfi_startproc + endbranch + push %rbx +.cfi_push %rbx + push %rbp +.cfi_push %rbp + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +___ +$code.=<<___ if ($win64); + lea -168(%rsp),%rsp + vmovapd %xmm6, `0*16`(%rsp) # save non-volatile registers + vmovapd %xmm7, `1*16`(%rsp) + vmovapd %xmm8, `2*16`(%rsp) + vmovapd %xmm9, `3*16`(%rsp) + vmovapd %xmm10,`4*16`(%rsp) + vmovapd %xmm11,`5*16`(%rsp) + vmovapd %xmm12,`6*16`(%rsp) + vmovapd %xmm13,`7*16`(%rsp) + vmovapd %xmm14,`8*16`(%rsp) + vmovapd %xmm15,`9*16`(%rsp) +.Lossl_rsaz_amm52x30_x2_avxifma256_body: +___ +$code.=<<___; + # Zeroing accumulators + vpxor $zero, $zero, $zero + vmovapd $zero, $R0_0 + vmovapd $zero, $R0_0h + vmovapd $zero, $R1_0 + vmovapd $zero, $R1_0h + vmovapd $zero, $R2_0 + vmovapd $zero, $R2_0h + vmovapd $zero, $R3_0 + vmovapd $zero, $R3_0h + + xorl $acc0_0_low, $acc0_0_low + + movq $b, $b_ptr # backup address of b + movq \$0xfffffffffffff, $mask52 # 52-bit mask + + mov \$30, $iter + +.align 32 +.Lloop30: +___ + &amm52x30_x1( 0, 0,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,"($k0)"); +$code.=<<___; + lea 8($b_ptr), $b_ptr + dec $iter + jne .Lloop30 + + push $b_ptr + push $a + push $m + push $k0 +___ + &amm52x30_x1_norm($acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h); +$code.=<<___; + pop $k0 + pop $m + pop $a + pop $b_ptr + + vmovdqu $R0_0, `0*32`($res) + vmovdqu $R0_0h, `1*32`($res) + vmovdqu $R1_0, `2*32`($res) + vmovdqu $R1_0h, `3*32`($res) + vmovdqu $R2_0, `4*32`($res) + vmovdqu $R2_0h, `5*32`($res) + vmovdqu $R3_0, `6*32`($res) + vmovdqu $R3_0h, `7*32`($res) + + xorl $acc0_1_low, $acc0_1_low + + lea 16($b_ptr), $b_ptr + movq \$0xfffffffffffff, $mask52 # 52-bit mask + + mov \$30, $iter + + vpxor $zero, $zero, $zero + vmovapd $zero, $R0_0 + vmovapd $zero, $R0_0h + vmovapd $zero, $R1_0 + vmovapd $zero, $R1_0h + vmovapd $zero, $R2_0 + vmovapd $zero, $R2_0h + vmovapd $zero, $R3_0 + vmovapd $zero, $R3_0h +.align 32 +.Lloop40: +___ + # 32*8 = offset of the next dimension in two-dimension array + &amm52x30_x1(32*8, 0,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,"8($k0)"); +$code.=<<___; + lea 8($b_ptr), $b_ptr + dec $iter + jne .Lloop40 +___ + &amm52x30_x1_norm($acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h); +$code.=<<___; + + vmovdqu $R0_0, `8*32`($res) + vmovdqu $R0_0h, `9*32`($res) + vmovdqu $R1_0, `10*32`($res) + vmovdqu $R1_0h, `11*32`($res) + vmovdqu $R2_0, `12*32`($res) + vmovdqu $R2_0h, `13*32`($res) + vmovdqu $R3_0, `14*32`($res) + vmovdqu $R3_0h, `15*32`($res) + + vzeroupper + lea (%rsp),%rax +.cfi_def_cfa_register %rax +___ +$code.=<<___ if ($win64); + vmovapd `0*16`(%rax),%xmm6 + vmovapd `1*16`(%rax),%xmm7 + vmovapd `2*16`(%rax),%xmm8 + vmovapd `3*16`(%rax),%xmm9 + vmovapd `4*16`(%rax),%xmm10 + vmovapd `5*16`(%rax),%xmm11 + vmovapd `6*16`(%rax),%xmm12 + vmovapd `7*16`(%rax),%xmm13 + vmovapd `8*16`(%rax),%xmm14 + vmovapd `9*16`(%rax),%xmm15 + lea 168(%rsp),%rax +___ +$code.=<<___; + mov 0(%rax),%r15 +.cfi_restore %r15 + mov 8(%rax),%r14 +.cfi_restore %r14 + mov 16(%rax),%r13 +.cfi_restore %r13 + mov 24(%rax),%r12 +.cfi_restore %r12 + mov 32(%rax),%rbp +.cfi_restore %rbp + mov 40(%rax),%rbx +.cfi_restore %rbx + lea 48(%rax),%rsp +.cfi_def_cfa %rsp,8 +.Lossl_rsaz_amm52x30_x2_avxifma256_epilogue: + ret +.cfi_endproc +.size ossl_rsaz_amm52x30_x2_avxifma256, .-ossl_rsaz_amm52x30_x2_avxifma256 +___ +} + +############################################################################### +# Constant time extraction from the precomputed table of powers base^i, where +# i = 0..2^EXP_WIN_SIZE-1 +# +# The input |red_table| contains precomputations for two independent base values. +# |red_table_idx1| and |red_table_idx2| are corresponding power indexes. +# +# Extracted value (output) is 2 (30 + 2) digits numbers in 2^52 radix. +# (2 high QW is zero padding) +# +# void ossl_extract_multiplier_2x30_win5_avx(BN_ULONG *red_Y, +# const BN_ULONG red_table[1 << EXP_WIN_SIZE][2][32], +# int red_table_idx1, int red_table_idx2); +# +# EXP_WIN_SIZE = 5 +############################################################################### +{ +# input parameters +my ($out,$red_tbl,$red_tbl_idx1,$red_tbl_idx2)=$win64 ? ("%rcx","%rdx","%r8", "%r9") : # Win64 order + ("%rdi","%rsi","%rdx","%rcx"); # Unix order + +my ($t0,$t1,$t2,$t3,$t4,$t5,$t6,$t7) = map("%ymm$_", (0..7)); +my ($t8,$t9,$t10,$t11,$t12,$t13,$t14,$t15) = map("%ymm$_", (0..7)); +my ($tmp,$cur_idx,$idx1,$idx2,$ones,$mask) = map("%ymm$_", (8..13)); + +my @t = ($t0,$t1,$t2,$t3,$t4,$t5,$t6,$t7,$t8,$t9,$t10,$t11,$t12,$t13,$t14,$t15); +my $t0xmm = $t0; +my $tmp_xmm = "%xmm8"; +$t0xmm =~ s/%y/%x/; + +$code.=<<___; +.text + +.align 32 +.globl ossl_extract_multiplier_2x30_win5_avx +.type ossl_extract_multiplier_2x30_win5_avx,\@abi-omnipotent +ossl_extract_multiplier_2x30_win5_avx: +.cfi_startproc + endbranch + vmovapd .Lones(%rip), $ones # broadcast ones + vmovq $red_tbl_idx1, $tmp_xmm + vpbroadcastq $tmp_xmm, $idx1 + vmovq $red_tbl_idx2, $tmp_xmm + vpbroadcastq $tmp_xmm, $idx2 + leaq `(1<<5)*2*32*8`($red_tbl), %rax # holds end of the tbl + + # zeroing t0..n, cur_idx + vpxor $t0xmm, $t0xmm, $t0xmm + vmovapd $t0, $cur_idx +___ +foreach (1..7) { + $code.="vmovapd $t0, $t[$_] \n"; +} +$code.=<<___; + +.align 32 +.Lloop: + vpcmpeqq $cur_idx, $idx1, $mask # mask of (idx1 == cur_idx) +___ +foreach (0..7) { +$code.=<<___; + vmovdqu `${_}*32`($red_tbl), $tmp # load data from red_tbl + #vpblendmq $tmp, $t[$_], ${t[$_]}{%k1} # extract data when mask is not zero + vblendvpd $mask, $tmp, $t[$_], ${t[$_]} # extract data when mask is not zero +___ +} +$code.=<<___; + vpaddq $ones, $cur_idx, $cur_idx # increment cur_idx + addq \$`2*32*8`, $red_tbl + cmpq $red_tbl, %rax + jne .Lloop +___ +# store t0..n +foreach (0..7) { + $code.="vmovdqu $t[$_], `${_}*32`($out) \n"; +} + +#second half +$code.=<<___; + leaq `-(1<<5)*2*32*8`(%rax), $red_tbl + + # zeroing t0..n, cur_idx + vpxor $t0xmm, $t0xmm, $t0xmm + vmovapd $t0, $cur_idx +___ +foreach (8..15) { + $code.="vmovapd $t0, $t[$_] \n"; +} +$code.=<<___; + +.align 32 +.Lloop_8_15: + vpcmpeqq $cur_idx, $idx2, $mask # mask of (idx2 == cur_idx) +___ +foreach (8..15) { +$code.=<<___; + vmovdqu `${_}*32`($red_tbl), $tmp # load data from red_tbl + #vpblendmq $tmp, $t[$_], ${t[$_]}{%k2} # extract data when mask is not zero + vblendvpd $mask, $tmp, $t[$_], ${t[$_]} # extract data when mask is not zero +___ +} +$code.=<<___; + vpaddq $ones, $cur_idx, $cur_idx # increment cur_idx + addq \$`2*32*8`, $red_tbl + cmpq $red_tbl, %rax + jne .Lloop_8_15 +___ +# store t0..n +foreach (8..15) { + $code.="vmovdqu $t[$_], `${_}*32`($out) \n"; +} + + +$code.=<<___; + + ret +.cfi_endproc +.size ossl_extract_multiplier_2x30_win5_avx, .-ossl_extract_multiplier_2x30_win5_avx +___ +$code.=<<___; +.section .rodata align=32 +.align 32 +.Lones: + .quad 1,1,1,1 +.Lzeros: + .quad 0,0,0,0 +___ +} + +if ($win64) { +$rec="%rcx"; +$frame="%rdx"; +$context="%r8"; +$disp="%r9"; + +$code.=<<___; +.extern __imp_RtlVirtualUnwind +.type rsaz_avx_handler,\@abi-omnipotent +.align 16 +rsaz_avx_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + mov 8($disp),%rsi # disp->ImageBase + mov 56($disp),%r11 # disp->HandlerData + + mov 0(%r11),%r10d # HandlerData[0] + lea (%rsi,%r10),%r10 # prologue label + cmp %r10,%rbx # context->Rip<.Lprologue + jb .Lcommon_seh_tail + + mov 4(%r11),%r10d # HandlerData[1] + lea (%rsi,%r10),%r10 # epilogue label + cmp %r10,%rbx # context->Rip>=.Lepilogue + jae .Lcommon_seh_tail + + mov 152($context),%rax # pull context->Rsp + + lea (%rax),%rsi # %xmm save area + lea 512($context),%rdi # & context.Xmm6 + mov \$20,%ecx # 10*sizeof(%xmm0)/sizeof(%rax) + .long 0xa548f3fc # cld; rep movsq + + lea `48+168`(%rax),%rax + + mov -8(%rax),%rbx + mov -16(%rax),%rbp + mov -24(%rax),%r12 + mov -32(%rax),%r13 + mov -40(%rax),%r14 + mov -48(%rax),%r15 + mov %rbx,144($context) # restore context->Rbx + mov %rbp,160($context) # restore context->Rbp + mov %r12,216($context) # restore context->R12 + mov %r13,224($context) # restore context->R13 + mov %r14,232($context) # restore context->R14 + mov %r15,240($context) # restore context->R14 + +.Lcommon_seh_tail: + mov 8(%rax),%rdi + mov 16(%rax),%rsi + mov %rax,152($context) # restore context->Rsp + mov %rsi,168($context) # restore context->Rsi + mov %rdi,176($context) # restore context->Rdi + + mov 40($disp),%rdi # disp->ContextRecord + mov $context,%rsi # context + mov \$154,%ecx # sizeof(CONTEXT) + .long 0xa548f3fc # cld; rep movsq + + mov $disp,%rsi + xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER + mov 8(%rsi),%rdx # arg2, disp->ImageBase + mov 0(%rsi),%r8 # arg3, disp->ControlPc + mov 16(%rsi),%r9 # arg4, disp->FunctionEntry + mov 40(%rsi),%r10 # disp->ContextRecord + lea 56(%rsi),%r11 # &disp->HandlerData + lea 24(%rsi),%r12 # &disp->EstablisherFrame + mov %r10,32(%rsp) # arg5 + mov %r11,40(%rsp) # arg6 + mov %r12,48(%rsp) # arg7 + mov %rcx,56(%rsp) # arg8, (NULL) + call *__imp_RtlVirtualUnwind(%rip) + + mov \$1,%eax # ExceptionContinueSearch + add \$64,%rsp + popfq + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbp + pop %rbx + pop %rdi + pop %rsi + ret +.size rsaz_avx_handler,.-rsaz_avx_handler + +.section .pdata +.align 4 + .rva .LSEH_begin_ossl_rsaz_amm52x30_x1_avxifma256 + .rva .LSEH_end_ossl_rsaz_amm52x30_x1_avxifma256 + .rva .LSEH_info_ossl_rsaz_amm52x30_x1_avxifma256 + + .rva .LSEH_begin_ossl_rsaz_amm52x30_x2_avxifma256 + .rva .LSEH_end_ossl_rsaz_amm52x30_x2_avxifma256 + .rva .LSEH_info_ossl_rsaz_amm52x30_x2_avxifma256 + +.section .xdata +.align 8 +.LSEH_info_ossl_rsaz_amm52x30_x1_avxifma256: + .byte 9,0,0,0 + .rva rsaz_avx_handler + .rva .Lossl_rsaz_amm52x30_x1_avxifma256_body,.Lossl_rsaz_amm52x30_x1_avxifma256_epilogue +.LSEH_info_ossl_rsaz_amm52x30_x2_avxifma256: + .byte 9,0,0,0 + .rva rsaz_avx_handler + .rva .Lossl_rsaz_amm52x30_x2_avxifma256_body,.Lossl_rsaz_amm52x30_x2_avxifma256_epilogue +___ +} +}}} else {{{ # fallback for old assembler +$code.=<<___; +.text + +.globl ossl_rsaz_amm52x30_x1_avxifma256 +.globl ossl_rsaz_amm52x30_x2_avxifma256 +.globl ossl_extract_multiplier_2x30_win5_avx +.type ossl_rsaz_amm52x30_x1_avxifma256,\@abi-omnipotent +ossl_rsaz_amm52x30_x1_avxifma256: +ossl_rsaz_amm52x30_x2_avxifma256: +ossl_extract_multiplier_2x30_win5_avx: + .byte 0x0f,0x0b # ud2 + ret +.size ossl_rsaz_amm52x30_x1_avxifma256, .-ossl_rsaz_amm52x30_x1_avxifma256 +___ +}}} + +$code =~ s/\`([^\`]*)\`/eval $1/gem; +print $code; +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/bn/asm/rsaz-4k-avx512.pl b/crypto/openssl/crypto/bn/asm/rsaz-4k-avx512.pl new file mode 100644 index 000000000000..e39374362b6e --- /dev/null +++ b/crypto/openssl/crypto/bn/asm/rsaz-4k-avx512.pl @@ -0,0 +1,939 @@ +# Copyright 2021-2024 The OpenSSL Project Authors. All Rights Reserved. +# Copyright (c) 2021, Intel Corporation. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html +# +# +# Originally written by Sergey Kirillov and Andrey Matyukov +# Intel Corporation +# +# March 2021 +# +# Initial release. +# +# Implementation utilizes 256-bit (ymm) registers to avoid frequency scaling issues. +# +# IceLake-Client @ 1.3GHz +# |---------+-----------------------+---------------+-------------| +# | | OpenSSL 3.0.0-alpha15 | this | Unit | +# |---------+-----------------------+---------------+-------------| +# | rsa4096 | 14 301 4300 | 5 813 953 | cycles/sign | +# | | 90.9 | 223.6 / +146% | sign/s | +# |---------+-----------------------+---------------+-------------| +# + +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); +$avx512ifma=0; + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or +die "can't locate x86_64-xlate.pl"; + +if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` + =~ /GNU assembler version ([2-9]\.[0-9]+)/) { + $avx512ifma = ($1>=2.26); +} + +if (!$avx512ifma && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && + `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?/) { + $avx512ifma = ($1==2.11 && $2>=8) + ($1>=2.12); +} + +if (!$avx512ifma && `$ENV{CC} -v 2>&1` + =~ /(Apple)?\s*((?:clang|LLVM) version|.*based on LLVM) ([0-9]+)\.([0-9]+)\.([0-9]+)?/) { + my $ver = $3 + $4/100.0 + $5/10000.0; # 3.1.0->3.01, 3.10.1->3.1001 + if ($1) { + # Apple conditions, they use a different version series, see + # https://en.wikipedia.org/wiki/Xcode#Xcode_7.0_-_10.x_(since_Free_On-Device_Development)_2 + # clang 7.0.0 is Apple clang 10.0.1 + $avx512ifma = ($ver>=10.0001) + } else { + $avx512ifma = ($ver>=7.0); + } +} + +open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"" + or die "can't call $xlate: $!"; +*STDOUT=*OUT; + +if ($avx512ifma>0) {{{ +@_6_args_universal_ABI = ("%rdi","%rsi","%rdx","%rcx","%r8","%r9"); + +############################################################################### +# Almost Montgomery Multiplication (AMM) for 40-digit number in radix 2^52. +# +# AMM is defined as presented in the paper [1]. +# +# The input and output are presented in 2^52 radix domain, i.e. +# |res|, |a|, |b|, |m| are arrays of 40 64-bit qwords with 12 high bits zeroed. +# |k0| is a Montgomery coefficient, which is here k0 = -1/m mod 2^64 +# +# NB: the AMM implementation does not perform "conditional" subtraction step +# specified in the original algorithm as according to the Lemma 1 from the paper +# [2], the result will be always < 2*m and can be used as a direct input to +# the next AMM iteration. This post-condition is true, provided the correct +# parameter |s| (notion of the Lemma 1 from [2]) is chosen, i.e. s >= n + 2 * k, +# which matches our case: 2080 > 2048 + 2 * 1. +# +# [1] Gueron, S. Efficient software implementations of modular exponentiation. +# DOI: 10.1007/s13389-012-0031-5 +# [2] Gueron, S. Enhanced Montgomery Multiplication. +# DOI: 10.1007/3-540-36400-5_5 +# +# void ossl_rsaz_amm52x40_x1_ifma256(BN_ULONG *res, +# const BN_ULONG *a, +# const BN_ULONG *b, +# const BN_ULONG *m, +# BN_ULONG k0); +############################################################################### +{ +# input parameters ("%rdi","%rsi","%rdx","%rcx","%r8") +my ($res,$a,$b,$m,$k0) = @_6_args_universal_ABI; + +my $mask52 = "%rax"; +my $acc0_0 = "%r9"; +my $acc0_0_low = "%r9d"; +my $acc0_1 = "%r15"; +my $acc0_1_low = "%r15d"; +my $b_ptr = "%r11"; + +my $iter = "%ebx"; + +my $zero = "%ymm0"; +my $Bi = "%ymm1"; +my $Yi = "%ymm2"; +my ($R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,$R4_0,$R4_0h) = map("%ymm$_",(3..12)); +my ($R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1,$R2_1h,$R3_1,$R3_1h,$R4_1,$R4_1h) = map("%ymm$_",(13..22)); + +# Registers mapping for normalization +my ($T0,$T0h,$T1,$T1h,$T2,$T2h,$T3,$T3h,$T4,$T4h) = ("$zero", "$Bi", "$Yi", map("%ymm$_", (23..29))); + +sub amm52x40_x1() { +# _data_offset - offset in the |a| or |m| arrays pointing to the beginning +# of data for corresponding AMM operation; +# _b_offset - offset in the |b| array pointing to the next qword digit; +my ($_data_offset,$_b_offset,$_acc,$_R0,$_R0h,$_R1,$_R1h,$_R2,$_R2h,$_R3,$_R3h,$_R4,$_R4h,$_k0) = @_; +my $_R0_xmm = $_R0; +$_R0_xmm =~ s/%y/%x/; +$code.=<<___; + movq $_b_offset($b_ptr), %r13 # b[i] + + vpbroadcastq %r13, $Bi # broadcast b[i] + movq $_data_offset($a), %rdx + mulx %r13, %r13, %r12 # a[0]*b[i] = (t0,t2) + addq %r13, $_acc # acc += t0 + movq %r12, %r10 + adcq \$0, %r10 # t2 += CF + + movq $_k0, %r13 + imulq $_acc, %r13 # acc * k0 + andq $mask52, %r13 # yi = (acc * k0) & mask52 + + vpbroadcastq %r13, $Yi # broadcast y[i] + movq $_data_offset($m), %rdx + mulx %r13, %r13, %r12 # yi * m[0] = (t0,t1) + addq %r13, $_acc # acc += t0 + adcq %r12, %r10 # t2 += (t1 + CF) + + shrq \$52, $_acc + salq \$12, %r10 + or %r10, $_acc # acc = ((acc >> 52) | (t2 << 12)) + + vpmadd52luq `$_data_offset+64*0`($a), $Bi, $_R0 + vpmadd52luq `$_data_offset+64*0+32`($a), $Bi, $_R0h + vpmadd52luq `$_data_offset+64*1`($a), $Bi, $_R1 + vpmadd52luq `$_data_offset+64*1+32`($a), $Bi, $_R1h + vpmadd52luq `$_data_offset+64*2`($a), $Bi, $_R2 + vpmadd52luq `$_data_offset+64*2+32`($a), $Bi, $_R2h + vpmadd52luq `$_data_offset+64*3`($a), $Bi, $_R3 + vpmadd52luq `$_data_offset+64*3+32`($a), $Bi, $_R3h + vpmadd52luq `$_data_offset+64*4`($a), $Bi, $_R4 + vpmadd52luq `$_data_offset+64*4+32`($a), $Bi, $_R4h + + vpmadd52luq `$_data_offset+64*0`($m), $Yi, $_R0 + vpmadd52luq `$_data_offset+64*0+32`($m), $Yi, $_R0h + vpmadd52luq `$_data_offset+64*1`($m), $Yi, $_R1 + vpmadd52luq `$_data_offset+64*1+32`($m), $Yi, $_R1h + vpmadd52luq `$_data_offset+64*2`($m), $Yi, $_R2 + vpmadd52luq `$_data_offset+64*2+32`($m), $Yi, $_R2h + vpmadd52luq `$_data_offset+64*3`($m), $Yi, $_R3 + vpmadd52luq `$_data_offset+64*3+32`($m), $Yi, $_R3h + vpmadd52luq `$_data_offset+64*4`($m), $Yi, $_R4 + vpmadd52luq `$_data_offset+64*4+32`($m), $Yi, $_R4h + + # Shift accumulators right by 1 qword, zero extending the highest one + valignq \$1, $_R0, $_R0h, $_R0 + valignq \$1, $_R0h, $_R1, $_R0h + valignq \$1, $_R1, $_R1h, $_R1 + valignq \$1, $_R1h, $_R2, $_R1h + valignq \$1, $_R2, $_R2h, $_R2 + valignq \$1, $_R2h, $_R3, $_R2h + valignq \$1, $_R3, $_R3h, $_R3 + valignq \$1, $_R3h, $_R4, $_R3h + valignq \$1, $_R4, $_R4h, $_R4 + valignq \$1, $_R4h, $zero, $_R4h + + vmovq $_R0_xmm, %r13 + addq %r13, $_acc # acc += R0[0] + + vpmadd52huq `$_data_offset+64*0`($a), $Bi, $_R0 + vpmadd52huq `$_data_offset+64*0+32`($a), $Bi, $_R0h + vpmadd52huq `$_data_offset+64*1`($a), $Bi, $_R1 + vpmadd52huq `$_data_offset+64*1+32`($a), $Bi, $_R1h + vpmadd52huq `$_data_offset+64*2`($a), $Bi, $_R2 + vpmadd52huq `$_data_offset+64*2+32`($a), $Bi, $_R2h + vpmadd52huq `$_data_offset+64*3`($a), $Bi, $_R3 + vpmadd52huq `$_data_offset+64*3+32`($a), $Bi, $_R3h + vpmadd52huq `$_data_offset+64*4`($a), $Bi, $_R4 + vpmadd52huq `$_data_offset+64*4+32`($a), $Bi, $_R4h + + vpmadd52huq `$_data_offset+64*0`($m), $Yi, $_R0 + vpmadd52huq `$_data_offset+64*0+32`($m), $Yi, $_R0h + vpmadd52huq `$_data_offset+64*1`($m), $Yi, $_R1 + vpmadd52huq `$_data_offset+64*1+32`($m), $Yi, $_R1h + vpmadd52huq `$_data_offset+64*2`($m), $Yi, $_R2 + vpmadd52huq `$_data_offset+64*2+32`($m), $Yi, $_R2h + vpmadd52huq `$_data_offset+64*3`($m), $Yi, $_R3 + vpmadd52huq `$_data_offset+64*3+32`($m), $Yi, $_R3h + vpmadd52huq `$_data_offset+64*4`($m), $Yi, $_R4 + vpmadd52huq `$_data_offset+64*4+32`($m), $Yi, $_R4h +___ +} + +# Normalization routine: handles carry bits and gets bignum qwords to normalized +# 2^52 representation. +# +# Uses %r8-14,%e[abcd]x +sub amm52x40_x1_norm { +my ($_acc,$_R0,$_R0h,$_R1,$_R1h,$_R2,$_R2h,$_R3,$_R3h,$_R4,$_R4h) = @_; +$code.=<<___; + # Put accumulator to low qword in R0 + vpbroadcastq $_acc, $T0 + vpblendd \$3, $T0, $_R0, $_R0 + + # Extract "carries" (12 high bits) from each QW of the bignum + # Save them to LSB of QWs in T0..Tn + vpsrlq \$52, $_R0, $T0 + vpsrlq \$52, $_R0h, $T0h + vpsrlq \$52, $_R1, $T1 + vpsrlq \$52, $_R1h, $T1h + vpsrlq \$52, $_R2, $T2 + vpsrlq \$52, $_R2h, $T2h + vpsrlq \$52, $_R3, $T3 + vpsrlq \$52, $_R3h, $T3h + vpsrlq \$52, $_R4, $T4 + vpsrlq \$52, $_R4h, $T4h + + # "Shift left" T0..Tn by 1 QW + valignq \$3, $T4, $T4h, $T4h + valignq \$3, $T3h, $T4, $T4 + valignq \$3, $T3, $T3h, $T3h + valignq \$3, $T2h, $T3, $T3 + valignq \$3, $T2, $T2h, $T2h + valignq \$3, $T1h, $T2, $T2 + valignq \$3, $T1, $T1h, $T1h + valignq \$3, $T0h, $T1, $T1 + valignq \$3, $T0, $T0h, $T0h + valignq \$3, .Lzeros(%rip), $T0, $T0 + + # Drop "carries" from R0..Rn QWs + vpandq .Lmask52x4(%rip), $_R0, $_R0 + vpandq .Lmask52x4(%rip), $_R0h, $_R0h + vpandq .Lmask52x4(%rip), $_R1, $_R1 + vpandq .Lmask52x4(%rip), $_R1h, $_R1h + vpandq .Lmask52x4(%rip), $_R2, $_R2 + vpandq .Lmask52x4(%rip), $_R2h, $_R2h + vpandq .Lmask52x4(%rip), $_R3, $_R3 + vpandq .Lmask52x4(%rip), $_R3h, $_R3h + vpandq .Lmask52x4(%rip), $_R4, $_R4 + vpandq .Lmask52x4(%rip), $_R4h, $_R4h + + # Sum R0..Rn with corresponding adjusted carries + vpaddq $T0, $_R0, $_R0 + vpaddq $T0h, $_R0h, $_R0h + vpaddq $T1, $_R1, $_R1 + vpaddq $T1h, $_R1h, $_R1h + vpaddq $T2, $_R2, $_R2 + vpaddq $T2h, $_R2h, $_R2h + vpaddq $T3, $_R3, $_R3 + vpaddq $T3h, $_R3h, $_R3h + vpaddq $T4, $_R4, $_R4 + vpaddq $T4h, $_R4h, $_R4h + + # Now handle carry bits from this addition + # Get mask of QWs whose 52-bit parts overflow + vpcmpuq \$6,.Lmask52x4(%rip),${_R0},%k1 # OP=nle (i.e. gt) + vpcmpuq \$6,.Lmask52x4(%rip),${_R0h},%k2 + kmovb %k1,%r14d + kmovb %k2,%r13d + shl \$4,%r13b + or %r13b,%r14b + + vpcmpuq \$6,.Lmask52x4(%rip),${_R1},%k1 + vpcmpuq \$6,.Lmask52x4(%rip),${_R1h},%k2 + kmovb %k1,%r13d + kmovb %k2,%r12d + shl \$4,%r12b + or %r12b,%r13b + + vpcmpuq \$6,.Lmask52x4(%rip),${_R2},%k1 + vpcmpuq \$6,.Lmask52x4(%rip),${_R2h},%k2 + kmovb %k1,%r12d + kmovb %k2,%r11d + shl \$4,%r11b + or %r11b,%r12b + + vpcmpuq \$6,.Lmask52x4(%rip),${_R3},%k1 + vpcmpuq \$6,.Lmask52x4(%rip),${_R3h},%k2 + kmovb %k1,%r11d + kmovb %k2,%r10d + shl \$4,%r10b + or %r10b,%r11b + + vpcmpuq \$6,.Lmask52x4(%rip),${_R4},%k1 + vpcmpuq \$6,.Lmask52x4(%rip),${_R4h},%k2 + kmovb %k1,%r10d + kmovb %k2,%r9d + shl \$4,%r9b + or %r9b,%r10b + + addb %r14b,%r14b + adcb %r13b,%r13b + adcb %r12b,%r12b + adcb %r11b,%r11b + adcb %r10b,%r10b + + # Get mask of QWs whose 52-bit parts saturated + vpcmpuq \$0,.Lmask52x4(%rip),${_R0},%k1 # OP=eq + vpcmpuq \$0,.Lmask52x4(%rip),${_R0h},%k2 + kmovb %k1,%r9d + kmovb %k2,%r8d + shl \$4,%r8b + or %r8b,%r9b + + vpcmpuq \$0,.Lmask52x4(%rip),${_R1},%k1 + vpcmpuq \$0,.Lmask52x4(%rip),${_R1h},%k2 + kmovb %k1,%r8d + kmovb %k2,%edx + shl \$4,%dl + or %dl,%r8b + + vpcmpuq \$0,.Lmask52x4(%rip),${_R2},%k1 + vpcmpuq \$0,.Lmask52x4(%rip),${_R2h},%k2 + kmovb %k1,%edx + kmovb %k2,%ecx + shl \$4,%cl + or %cl,%dl + + vpcmpuq \$0,.Lmask52x4(%rip),${_R3},%k1 + vpcmpuq \$0,.Lmask52x4(%rip),${_R3h},%k2 + kmovb %k1,%ecx + kmovb %k2,%ebx + shl \$4,%bl + or %bl,%cl + + vpcmpuq \$0,.Lmask52x4(%rip),${_R4},%k1 + vpcmpuq \$0,.Lmask52x4(%rip),${_R4h},%k2 + kmovb %k1,%ebx + kmovb %k2,%eax + shl \$4,%al + or %al,%bl + + addb %r9b,%r14b + adcb %r8b,%r13b + adcb %dl,%r12b + adcb %cl,%r11b + adcb %bl,%r10b + + xor %r9b,%r14b + xor %r8b,%r13b + xor %dl,%r12b + xor %cl,%r11b + xor %bl,%r10b + + kmovb %r14d,%k1 + shr \$4,%r14b + kmovb %r14d,%k2 + kmovb %r13d,%k3 + shr \$4,%r13b + kmovb %r13d,%k4 + kmovb %r12d,%k5 + shr \$4,%r12b + kmovb %r12d,%k6 + kmovb %r11d,%k7 + + vpsubq .Lmask52x4(%rip), $_R0, ${_R0}{%k1} + vpsubq .Lmask52x4(%rip), $_R0h, ${_R0h}{%k2} + vpsubq .Lmask52x4(%rip), $_R1, ${_R1}{%k3} + vpsubq .Lmask52x4(%rip), $_R1h, ${_R1h}{%k4} + vpsubq .Lmask52x4(%rip), $_R2, ${_R2}{%k5} + vpsubq .Lmask52x4(%rip), $_R2h, ${_R2h}{%k6} + vpsubq .Lmask52x4(%rip), $_R3, ${_R3}{%k7} + + vpandq .Lmask52x4(%rip), $_R0, $_R0 + vpandq .Lmask52x4(%rip), $_R0h, $_R0h + vpandq .Lmask52x4(%rip), $_R1, $_R1 + vpandq .Lmask52x4(%rip), $_R1h, $_R1h + vpandq .Lmask52x4(%rip), $_R2, $_R2 + vpandq .Lmask52x4(%rip), $_R2h, $_R2h + vpandq .Lmask52x4(%rip), $_R3, $_R3 + + shr \$4,%r11b + kmovb %r11d,%k1 + kmovb %r10d,%k2 + shr \$4,%r10b + kmovb %r10d,%k3 + + vpsubq .Lmask52x4(%rip), $_R3h, ${_R3h}{%k1} + vpsubq .Lmask52x4(%rip), $_R4, ${_R4}{%k2} + vpsubq .Lmask52x4(%rip), $_R4h, ${_R4h}{%k3} + + vpandq .Lmask52x4(%rip), $_R3h, $_R3h + vpandq .Lmask52x4(%rip), $_R4, $_R4 + vpandq .Lmask52x4(%rip), $_R4h, $_R4h +___ +} + +$code.=<<___; +.text + +.globl ossl_rsaz_amm52x40_x1_ifma256 +.type ossl_rsaz_amm52x40_x1_ifma256,\@function,5 +.align 32 +ossl_rsaz_amm52x40_x1_ifma256: +.cfi_startproc + endbranch + push %rbx +.cfi_push %rbx + push %rbp +.cfi_push %rbp + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +___ +$code.=<<___ if ($win64); + lea -168(%rsp),%rsp # 16*10 + (8 bytes to get correct 16-byte SIMD alignment) + vmovdqa64 %xmm6, `0*16`(%rsp) # save non-volatile registers + vmovdqa64 %xmm7, `1*16`(%rsp) + vmovdqa64 %xmm8, `2*16`(%rsp) + vmovdqa64 %xmm9, `3*16`(%rsp) + vmovdqa64 %xmm10,`4*16`(%rsp) + vmovdqa64 %xmm11,`5*16`(%rsp) + vmovdqa64 %xmm12,`6*16`(%rsp) + vmovdqa64 %xmm13,`7*16`(%rsp) + vmovdqa64 %xmm14,`8*16`(%rsp) + vmovdqa64 %xmm15,`9*16`(%rsp) +.Lossl_rsaz_amm52x40_x1_ifma256_body: +___ +$code.=<<___; + # Zeroing accumulators + vpxord $zero, $zero, $zero + vmovdqa64 $zero, $R0_0 + vmovdqa64 $zero, $R0_0h + vmovdqa64 $zero, $R1_0 + vmovdqa64 $zero, $R1_0h + vmovdqa64 $zero, $R2_0 + vmovdqa64 $zero, $R2_0h + vmovdqa64 $zero, $R3_0 + vmovdqa64 $zero, $R3_0h + vmovdqa64 $zero, $R4_0 + vmovdqa64 $zero, $R4_0h + + xorl $acc0_0_low, $acc0_0_low + + movq $b, $b_ptr # backup address of b + movq \$0xfffffffffffff, $mask52 # 52-bit mask + + # Loop over 40 digits unrolled by 4 + mov \$10, $iter + +.align 32 +.Lloop10: +___ + foreach my $idx (0..3) { + &amm52x40_x1(0,8*$idx,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,$R4_0,$R4_0h,$k0); + } +$code.=<<___; + lea `4*8`($b_ptr), $b_ptr + dec $iter + jne .Lloop10 +___ + &amm52x40_x1_norm($acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,$R4_0,$R4_0h); +$code.=<<___; + + vmovdqu64 $R0_0, `0*32`($res) + vmovdqu64 $R0_0h, `1*32`($res) + vmovdqu64 $R1_0, `2*32`($res) + vmovdqu64 $R1_0h, `3*32`($res) + vmovdqu64 $R2_0, `4*32`($res) + vmovdqu64 $R2_0h, `5*32`($res) + vmovdqu64 $R3_0, `6*32`($res) + vmovdqu64 $R3_0h, `7*32`($res) + vmovdqu64 $R4_0, `8*32`($res) + vmovdqu64 $R4_0h, `9*32`($res) + + vzeroupper + lea (%rsp),%rax +.cfi_def_cfa_register %rax +___ +$code.=<<___ if ($win64); + vmovdqa64 `0*16`(%rax),%xmm6 + vmovdqa64 `1*16`(%rax),%xmm7 + vmovdqa64 `2*16`(%rax),%xmm8 + vmovdqa64 `3*16`(%rax),%xmm9 + vmovdqa64 `4*16`(%rax),%xmm10 + vmovdqa64 `5*16`(%rax),%xmm11 + vmovdqa64 `6*16`(%rax),%xmm12 + vmovdqa64 `7*16`(%rax),%xmm13 + vmovdqa64 `8*16`(%rax),%xmm14 + vmovdqa64 `9*16`(%rax),%xmm15 + lea 168(%rsp),%rax +___ +$code.=<<___; + mov 0(%rax),%r15 +.cfi_restore %r15 + mov 8(%rax),%r14 +.cfi_restore %r14 + mov 16(%rax),%r13 +.cfi_restore %r13 + mov 24(%rax),%r12 +.cfi_restore %r12 + mov 32(%rax),%rbp +.cfi_restore %rbp + mov 40(%rax),%rbx +.cfi_restore %rbx + lea 48(%rax),%rsp # restore rsp +.cfi_def_cfa %rsp,8 +.Lossl_rsaz_amm52x40_x1_ifma256_epilogue: + + ret +.cfi_endproc +.size ossl_rsaz_amm52x40_x1_ifma256, .-ossl_rsaz_amm52x40_x1_ifma256 +___ + +$code.=<<___; +.section .rodata align=32 +.align 32 +.Lmask52x4: + .quad 0xfffffffffffff + .quad 0xfffffffffffff + .quad 0xfffffffffffff + .quad 0xfffffffffffff +___ + +############################################################################### +# Dual Almost Montgomery Multiplication for 40-digit number in radix 2^52 +# +# See description of ossl_rsaz_amm52x40_x1_ifma256() above for details about Almost +# Montgomery Multiplication algorithm and function input parameters description. +# +# This function does two AMMs for two independent inputs, hence dual. +# +# void ossl_rsaz_amm52x40_x2_ifma256(BN_ULONG out[2][40], +# const BN_ULONG a[2][40], +# const BN_ULONG b[2][40], +# const BN_ULONG m[2][40], +# const BN_ULONG k0[2]); +############################################################################### + +$code.=<<___; +.text + +.globl ossl_rsaz_amm52x40_x2_ifma256 +.type ossl_rsaz_amm52x40_x2_ifma256,\@function,5 +.align 32 +ossl_rsaz_amm52x40_x2_ifma256: +.cfi_startproc + endbranch + push %rbx +.cfi_push %rbx + push %rbp +.cfi_push %rbp + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +___ +$code.=<<___ if ($win64); + lea -168(%rsp),%rsp + vmovdqa64 %xmm6, `0*16`(%rsp) # save non-volatile registers + vmovdqa64 %xmm7, `1*16`(%rsp) + vmovdqa64 %xmm8, `2*16`(%rsp) + vmovdqa64 %xmm9, `3*16`(%rsp) + vmovdqa64 %xmm10,`4*16`(%rsp) + vmovdqa64 %xmm11,`5*16`(%rsp) + vmovdqa64 %xmm12,`6*16`(%rsp) + vmovdqa64 %xmm13,`7*16`(%rsp) + vmovdqa64 %xmm14,`8*16`(%rsp) + vmovdqa64 %xmm15,`9*16`(%rsp) +.Lossl_rsaz_amm52x40_x2_ifma256_body: +___ +$code.=<<___; + # Zeroing accumulators + vpxord $zero, $zero, $zero + vmovdqa64 $zero, $R0_0 + vmovdqa64 $zero, $R0_0h + vmovdqa64 $zero, $R1_0 + vmovdqa64 $zero, $R1_0h + vmovdqa64 $zero, $R2_0 + vmovdqa64 $zero, $R2_0h + vmovdqa64 $zero, $R3_0 + vmovdqa64 $zero, $R3_0h + vmovdqa64 $zero, $R4_0 + vmovdqa64 $zero, $R4_0h + + vmovdqa64 $zero, $R0_1 + vmovdqa64 $zero, $R0_1h + vmovdqa64 $zero, $R1_1 + vmovdqa64 $zero, $R1_1h + vmovdqa64 $zero, $R2_1 + vmovdqa64 $zero, $R2_1h + vmovdqa64 $zero, $R3_1 + vmovdqa64 $zero, $R3_1h + vmovdqa64 $zero, $R4_1 + vmovdqa64 $zero, $R4_1h + + + xorl $acc0_0_low, $acc0_0_low + xorl $acc0_1_low, $acc0_1_low + + movq $b, $b_ptr # backup address of b + movq \$0xfffffffffffff, $mask52 # 52-bit mask + + mov \$40, $iter + +.align 32 +.Lloop40: +___ + &amm52x40_x1( 0, 0,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,$R4_0,$R4_0h,"($k0)"); + # 40*8 = offset of the next dimension in two-dimension array + &amm52x40_x1(40*8,40*8,$acc0_1,$R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1,$R2_1h,$R3_1,$R3_1h,$R4_1,$R4_1h,"8($k0)"); +$code.=<<___; + lea 8($b_ptr), $b_ptr + dec $iter + jne .Lloop40 +___ + &amm52x40_x1_norm($acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,$R4_0,$R4_0h); + &amm52x40_x1_norm($acc0_1,$R0_1,$R0_1h,$R1_1,$R1_1h,$R2_1,$R2_1h,$R3_1,$R3_1h,$R4_1,$R4_1h); +$code.=<<___; + + vmovdqu64 $R0_0, `0*32`($res) + vmovdqu64 $R0_0h, `1*32`($res) + vmovdqu64 $R1_0, `2*32`($res) + vmovdqu64 $R1_0h, `3*32`($res) + vmovdqu64 $R2_0, `4*32`($res) + vmovdqu64 $R2_0h, `5*32`($res) + vmovdqu64 $R3_0, `6*32`($res) + vmovdqu64 $R3_0h, `7*32`($res) + vmovdqu64 $R4_0, `8*32`($res) + vmovdqu64 $R4_0h, `9*32`($res) + + vmovdqu64 $R0_1, `10*32`($res) + vmovdqu64 $R0_1h, `11*32`($res) + vmovdqu64 $R1_1, `12*32`($res) + vmovdqu64 $R1_1h, `13*32`($res) + vmovdqu64 $R2_1, `14*32`($res) + vmovdqu64 $R2_1h, `15*32`($res) + vmovdqu64 $R3_1, `16*32`($res) + vmovdqu64 $R3_1h, `17*32`($res) + vmovdqu64 $R4_1, `18*32`($res) + vmovdqu64 $R4_1h, `19*32`($res) + + vzeroupper + lea (%rsp),%rax +.cfi_def_cfa_register %rax +___ +$code.=<<___ if ($win64); + vmovdqa64 `0*16`(%rax),%xmm6 + vmovdqa64 `1*16`(%rax),%xmm7 + vmovdqa64 `2*16`(%rax),%xmm8 + vmovdqa64 `3*16`(%rax),%xmm9 + vmovdqa64 `4*16`(%rax),%xmm10 + vmovdqa64 `5*16`(%rax),%xmm11 + vmovdqa64 `6*16`(%rax),%xmm12 + vmovdqa64 `7*16`(%rax),%xmm13 + vmovdqa64 `8*16`(%rax),%xmm14 + vmovdqa64 `9*16`(%rax),%xmm15 + lea 168(%rsp),%rax +___ +$code.=<<___; + mov 0(%rax),%r15 +.cfi_restore %r15 + mov 8(%rax),%r14 +.cfi_restore %r14 + mov 16(%rax),%r13 +.cfi_restore %r13 + mov 24(%rax),%r12 +.cfi_restore %r12 + mov 32(%rax),%rbp +.cfi_restore %rbp + mov 40(%rax),%rbx +.cfi_restore %rbx + lea 48(%rax),%rsp +.cfi_def_cfa %rsp,8 +.Lossl_rsaz_amm52x40_x2_ifma256_epilogue: + ret +.cfi_endproc +.size ossl_rsaz_amm52x40_x2_ifma256, .-ossl_rsaz_amm52x40_x2_ifma256 +___ +} + +############################################################################### +# Constant time extraction from the precomputed table of powers base^i, where +# i = 0..2^EXP_WIN_SIZE-1 +# +# The input |red_table| contains precomputations for two independent base values. +# |red_table_idx1| and |red_table_idx2| are corresponding power indexes. +# +# Extracted value (output) is 2 40 digits numbers in 2^52 radix. +# +# void ossl_extract_multiplier_2x40_win5(BN_ULONG *red_Y, +# const BN_ULONG red_table[1 << EXP_WIN_SIZE][2][40], +# int red_table_idx1, int red_table_idx2); +# +# EXP_WIN_SIZE = 5 +############################################################################### +{ +# input parameters +my ($out,$red_tbl,$red_tbl_idx1,$red_tbl_idx2)=$win64 ? ("%rcx","%rdx","%r8", "%r9") : # Win64 order + ("%rdi","%rsi","%rdx","%rcx"); # Unix order + +my ($t0,$t1,$t2,$t3,$t4,$t5) = map("%ymm$_", (0..5)); +my ($t6,$t7,$t8,$t9) = map("%ymm$_", (16..19)); +my ($tmp,$cur_idx,$idx1,$idx2,$ones) = map("%ymm$_", (20..24)); + +my @t = ($t0,$t1,$t2,$t3,$t4,$t5,$t6,$t7,$t8,$t9); +my $t0xmm = $t0; +$t0xmm =~ s/%y/%x/; + +sub get_table_value_consttime() { +my ($_idx,$_offset) = @_; +$code.=<<___; + vpxorq $cur_idx, $cur_idx, $cur_idx +.align 32 +.Lloop_$_offset: + vpcmpq \$0, $cur_idx, $_idx, %k1 # mask of (idx == cur_idx) +___ +foreach (0..9) { +$code.=<<___; + vmovdqu64 `$_offset+${_}*32`($red_tbl), $tmp # load data from red_tbl + vpblendmq $tmp, $t[$_], ${t[$_]}{%k1} # extract data when mask is not zero +___ +} +$code.=<<___; + vpaddq $ones, $cur_idx, $cur_idx # increment cur_idx + addq \$`2*40*8`, $red_tbl + cmpq $red_tbl, %rax + jne .Lloop_$_offset +___ +} + +$code.=<<___; +.text + +.align 32 +.globl ossl_extract_multiplier_2x40_win5 +.type ossl_extract_multiplier_2x40_win5,\@abi-omnipotent +ossl_extract_multiplier_2x40_win5: +.cfi_startproc + endbranch + vmovdqa64 .Lones(%rip), $ones # broadcast ones + vpbroadcastq $red_tbl_idx1, $idx1 + vpbroadcastq $red_tbl_idx2, $idx2 + leaq `(1<<5)*2*40*8`($red_tbl), %rax # holds end of the tbl + + # backup red_tbl address + movq $red_tbl, %r10 + + # zeroing t0..n, cur_idx + vpxor $t0xmm, $t0xmm, $t0xmm +___ +foreach (1..9) { + $code.="vmovdqa64 $t0, $t[$_] \n"; +} + +&get_table_value_consttime($idx1, 0); +foreach (0..9) { + $code.="vmovdqu64 $t[$_], `(0+$_)*32`($out) \n"; +} +$code.="movq %r10, $red_tbl \n"; +&get_table_value_consttime($idx2, 40*8); +foreach (0..9) { + $code.="vmovdqu64 $t[$_], `(10+$_)*32`($out) \n"; +} +$code.=<<___; + + ret +.cfi_endproc +.size ossl_extract_multiplier_2x40_win5, .-ossl_extract_multiplier_2x40_win5 +___ +$code.=<<___; +.section .rodata align=32 +.align 32 +.Lones: + .quad 1,1,1,1 +.Lzeros: + .quad 0,0,0,0 +___ +} + +if ($win64) { +$rec="%rcx"; +$frame="%rdx"; +$context="%r8"; +$disp="%r9"; + +$code.=<<___; +.extern __imp_RtlVirtualUnwind +.type rsaz_avx_handler,\@abi-omnipotent +.align 16 +rsaz_avx_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + mov 8($disp),%rsi # disp->ImageBase + mov 56($disp),%r11 # disp->HandlerData + + mov 0(%r11),%r10d # HandlerData[0] + lea (%rsi,%r10),%r10 # prologue label + cmp %r10,%rbx # context->Rip<.Lprologue + jb .Lcommon_seh_tail + + mov 4(%r11),%r10d # HandlerData[1] + lea (%rsi,%r10),%r10 # epilogue label + cmp %r10,%rbx # context->Rip>=.Lepilogue + jae .Lcommon_seh_tail + + mov 152($context),%rax # pull context->Rsp + + lea (%rax),%rsi # %xmm save area + lea 512($context),%rdi # & context.Xmm6 + mov \$20,%ecx # 10*sizeof(%xmm0)/sizeof(%rax) + .long 0xa548f3fc # cld; rep movsq + + lea `48+168`(%rax),%rax + + mov -8(%rax),%rbx + mov -16(%rax),%rbp + mov -24(%rax),%r12 + mov -32(%rax),%r13 + mov -40(%rax),%r14 + mov -48(%rax),%r15 + mov %rbx,144($context) # restore context->Rbx + mov %rbp,160($context) # restore context->Rbp + mov %r12,216($context) # restore context->R12 + mov %r13,224($context) # restore context->R13 + mov %r14,232($context) # restore context->R14 + mov %r15,240($context) # restore context->R14 + +.Lcommon_seh_tail: + mov 8(%rax),%rdi + mov 16(%rax),%rsi + mov %rax,152($context) # restore context->Rsp + mov %rsi,168($context) # restore context->Rsi + mov %rdi,176($context) # restore context->Rdi + + mov 40($disp),%rdi # disp->ContextRecord + mov $context,%rsi # context + mov \$154,%ecx # sizeof(CONTEXT) + .long 0xa548f3fc # cld; rep movsq + + mov $disp,%rsi + xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER + mov 8(%rsi),%rdx # arg2, disp->ImageBase + mov 0(%rsi),%r8 # arg3, disp->ControlPc + mov 16(%rsi),%r9 # arg4, disp->FunctionEntry + mov 40(%rsi),%r10 # disp->ContextRecord + lea 56(%rsi),%r11 # &disp->HandlerData + lea 24(%rsi),%r12 # &disp->EstablisherFrame + mov %r10,32(%rsp) # arg5 + mov %r11,40(%rsp) # arg6 + mov %r12,48(%rsp) # arg7 + mov %rcx,56(%rsp) # arg8, (NULL) + call *__imp_RtlVirtualUnwind(%rip) + + mov \$1,%eax # ExceptionContinueSearch + add \$64,%rsp + popfq + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbp + pop %rbx + pop %rdi + pop %rsi + ret +.size rsaz_avx_handler,.-rsaz_avx_handler + +.section .pdata +.align 4 + .rva .LSEH_begin_ossl_rsaz_amm52x40_x1_ifma256 + .rva .LSEH_end_ossl_rsaz_amm52x40_x1_ifma256 + .rva .LSEH_info_ossl_rsaz_amm52x40_x1_ifma256 + + .rva .LSEH_begin_ossl_rsaz_amm52x40_x2_ifma256 + .rva .LSEH_end_ossl_rsaz_amm52x40_x2_ifma256 + .rva .LSEH_info_ossl_rsaz_amm52x40_x2_ifma256 + +.section .xdata +.align 8 +.LSEH_info_ossl_rsaz_amm52x40_x1_ifma256: + .byte 9,0,0,0 + .rva rsaz_avx_handler + .rva .Lossl_rsaz_amm52x40_x1_ifma256_body,.Lossl_rsaz_amm52x40_x1_ifma256_epilogue +.LSEH_info_ossl_rsaz_amm52x40_x2_ifma256: + .byte 9,0,0,0 + .rva rsaz_avx_handler + .rva .Lossl_rsaz_amm52x40_x2_ifma256_body,.Lossl_rsaz_amm52x40_x2_ifma256_epilogue +___ +} +}}} else {{{ # fallback for old assembler +$code.=<<___; +.text + +.globl ossl_rsaz_amm52x40_x1_ifma256 +.globl ossl_rsaz_amm52x40_x2_ifma256 +.globl ossl_extract_multiplier_2x40_win5 +.type ossl_rsaz_amm52x40_x1_ifma256,\@abi-omnipotent +ossl_rsaz_amm52x40_x1_ifma256: +ossl_rsaz_amm52x40_x2_ifma256: +ossl_extract_multiplier_2x40_win5: + .byte 0x0f,0x0b # ud2 + ret +.size ossl_rsaz_amm52x40_x1_ifma256, .-ossl_rsaz_amm52x40_x1_ifma256 +___ +}}} + +$code =~ s/\`([^\`]*)\`/eval $1/gem; +print $code; +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/bn/asm/rsaz-4k-avxifma.pl b/crypto/openssl/crypto/bn/asm/rsaz-4k-avxifma.pl new file mode 100644 index 000000000000..f15e2d74118c --- /dev/null +++ b/crypto/openssl/crypto/bn/asm/rsaz-4k-avxifma.pl @@ -0,0 +1,1161 @@ +# Copyright 2024-2025 The OpenSSL Project Authors. All Rights Reserved. +# Copyright (c) 2024, Intel Corporation. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html +# +# Written by Zhiguo Zhou and Wangyang Guo . +# Special thanks to Tomasz Kantecki for his valuable suggestions. +# +# October 2024 +# +# Initial release. +# + +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); +$avxifma=0; + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or +die "can't locate x86_64-xlate.pl"; + +if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` + =~ /GNU assembler version ([2-9]\.[0-9]+)/) { + $avxifma = ($1>=2.40); +} + +if (!$avxifma && `$ENV{CC} -v 2>&1` + =~ /\s*((?:clang|LLVM) version|.*based on LLVM) ([0-9]+)\.([0-9]+)\.([0-9]+)?/) { + my $ver = $2 + $3/100.0 + $4/10000.0; # 3.1.0->3.01, 3.10.1->3.1001 + $avxifma = ($ver>=16.0); +} + +if ($win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && + `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?(rc[0-9]+)?/) { + $avxifma = ($1>2.16) + ($1==2.16 && ((!defined($2) && !defined($3)) || (defined($2)))); +} + +open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"" + or die "can't call $xlate: $!"; +*STDOUT=*OUT; + +if ($avxifma>0) {{{ +@_6_args_universal_ABI = ("%rdi","%rsi","%rdx","%rcx","%r8","%r9"); + +############################################################################### +# Almost Montgomery Multiplication (AMM) for 40-digit number in radix 2^52. +# +# AMM is defined as presented in the paper [1]. +# +# The input and output are presented in 2^52 radix domain, i.e. +# |res|, |a|, |b|, |m| are arrays of 40 64-bit qwords with 12 high bits zeroed. +# |k0| is a Montgomery coefficient, which is here k0 = -1/m mod 2^64 +# +# NB: the AMM implementation does not perform "conditional" subtraction step +# specified in the original algorithm as according to the Lemma 1 from the paper +# [2], the result will be always < 2*m and can be used as a direct input to +# the next AMM iteration. This post-condition is true, provided the correct +# parameter |s| (notion of the Lemma 1 from [2]) is chosen, i.e. s >= n + 2 * k, +# which matches our case: 2080 > 2048 + 2 * 1. +# +# [1] Gueron, S. Efficient software implementations of modular exponentiation. +# DOI: 10.1007/s13389-012-0031-5 +# [2] Gueron, S. Enhanced Montgomery Multiplication. +# DOI: 10.1007/3-540-36400-5_5 +# +# void ossl_rsaz_amm52x40_x1_avxifma256(BN_ULONG *res, +# const BN_ULONG *a, +# const BN_ULONG *b, +# const BN_ULONG *m, +# BN_ULONG k0); +############################################################################### +{ +# input parameters ("%rdi","%rsi","%rdx","%rcx","%r8") +my ($res,$a,$b,$m,$k0) = @_6_args_universal_ABI; + +my $mask52 = "%rax"; +my $acc0_0 = "%r9"; +my $acc0_0_low = "%r9d"; +my $acc0_1 = "%r15"; +my $acc0_1_low = "%r15d"; +my $b_ptr = "%r11"; + +my $iter = "%ebx"; + +my $zero = "%ymm0"; +my $zero_xmm = "%xmm0"; +my $Bi = "%ymm1"; +my $Bi_xmm = "%xmm1"; +my $Yi = "%ymm2"; +my $Yi_xmm = "%xmm2"; +my $tmp = "%ymm13"; +my $tmp2 = "%ymm14"; +my ($R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,$R4_0,$R4_0h) = map("%ymm$_",(3..12)); + +sub amm52x40_x1() { +# _data_offset - offset in the |a| or |m| arrays pointing to the beginning +# of data for corresponding AMM operation; +# _b_offset - offset in the |b| array pointing to the next qword digit; +my ($_data_offset,$_b_offset,$_acc,$_R0,$_R0h,$_R1,$_R1h,$_R2,$_R2h,$_R3,$_R3h,$_R4,$_R4h,$_k0) = @_; +$code.=<<___; + movq $_b_offset($b_ptr), %r13 # b[i] + + vpbroadcastq $_b_offset($b_ptr),$Bi # broadcast b[i] + movq $_data_offset($a), %rdx + mulx %r13, %r13, %r12 # a[0]*b[i] = (t0,t2) + addq %r13, $_acc # acc += t0 + movq %r12, %r10 + adcq \$0, %r10 # t2 += CF + + movq $_k0, %r13 + imulq $_acc, %r13 # acc * k0 + andq $mask52, %r13 # yi = (acc * k0) & mask52 + + vmovq %r13, $Yi_xmm + vpbroadcastq $Yi_xmm, $Yi # broadcast y[i] + movq $_data_offset($m), %rdx + mulx %r13, %r13, %r12 # yi * m[0] = (t0,t1) + addq %r13, $_acc # acc += t0 + adcq %r12, %r10 # t2 += (t1 + CF) + + shrq \$52, $_acc + salq \$12, %r10 + or %r10, $_acc # acc = ((acc >> 52) | (t2 << 12)) + + lea -328(%rsp), %rsp + + {vex} vpmadd52luq `$_data_offset`($a), $Bi, $_R0 + {vex} vpmadd52luq `$_data_offset+32`($a), $Bi, $_R0h + {vex} vpmadd52luq `$_data_offset+64*1`($a), $Bi, $_R1 + {vex} vpmadd52luq `$_data_offset+64*1+32`($a), $Bi, $_R1h + {vex} vpmadd52luq `$_data_offset+64*2`($a), $Bi, $_R2 + {vex} vpmadd52luq `$_data_offset+64*2+32`($a), $Bi, $_R2h + {vex} vpmadd52luq `$_data_offset+64*3`($a), $Bi, $_R3 + {vex} vpmadd52luq `$_data_offset+64*3+32`($a), $Bi, $_R3h + {vex} vpmadd52luq `$_data_offset+64*4`($a), $Bi, $_R4 + {vex} vpmadd52luq `$_data_offset+64*4+32`($a), $Bi, $_R4h + + {vex} vpmadd52luq `$_data_offset`($m), $Yi, $_R0 + {vex} vpmadd52luq `$_data_offset+32`($m), $Yi, $_R0h + {vex} vpmadd52luq `$_data_offset+64*1`($m), $Yi, $_R1 + {vex} vpmadd52luq `$_data_offset+64*1+32`($m), $Yi, $_R1h + {vex} vpmadd52luq `$_data_offset+64*2`($m), $Yi, $_R2 + {vex} vpmadd52luq `$_data_offset+64*2+32`($m), $Yi, $_R2h + {vex} vpmadd52luq `$_data_offset+64*3`($m), $Yi, $_R3 + {vex} vpmadd52luq `$_data_offset+64*3+32`($m), $Yi, $_R3h + {vex} vpmadd52luq `$_data_offset+64*4`($m), $Yi, $_R4 + {vex} vpmadd52luq `$_data_offset+64*4+32`($m), $Yi, $_R4h + vmovdqu $_R0, `32*0`(%rsp) + vmovdqu $_R0h, `32*1`(%rsp) + vmovdqu $_R1, `32*2`(%rsp) + vmovdqu $_R1h, `32*3`(%rsp) + vmovdqu $_R2, `32*4`(%rsp) + vmovdqu $_R2h, `32*5`(%rsp) + vmovdqu $_R3, `32*6`(%rsp) + vmovdqu $_R3h, `32*7`(%rsp) + vmovdqu $_R4, `32*8`(%rsp) + vmovdqu $_R4h, `32*9`(%rsp) + movq \$0, `32*10`(%rsp) + + vmovdqu `32*0 + 8`(%rsp), $_R0 + vmovdqu `32*1 + 8`(%rsp), $_R0h + vmovdqu `32*2 + 8`(%rsp), $_R1 + vmovdqu `32*3 + 8`(%rsp), $_R1h + vmovdqu `32*4 + 8`(%rsp), $_R2 + vmovdqu `32*5 + 8`(%rsp), $_R2h + vmovdqu `32*6 + 8`(%rsp), $_R3 + vmovdqu `32*7 + 8`(%rsp), $_R3h + vmovdqu `32*8 + 8`(%rsp), $_R4 + vmovdqu `32*9 + 8`(%rsp), $_R4h + + addq 8(%rsp), $_acc # acc += R0[0] + + {vex} vpmadd52huq `$_data_offset`($a), $Bi, $_R0 + {vex} vpmadd52huq `$_data_offset+32`($a), $Bi, $_R0h + {vex} vpmadd52huq `$_data_offset+64*1`($a), $Bi, $_R1 + {vex} vpmadd52huq `$_data_offset+64*1+32`($a), $Bi, $_R1h + {vex} vpmadd52huq `$_data_offset+64*2`($a), $Bi, $_R2 + {vex} vpmadd52huq `$_data_offset+64*2+32`($a), $Bi, $_R2h + {vex} vpmadd52huq `$_data_offset+64*3`($a), $Bi, $_R3 + {vex} vpmadd52huq `$_data_offset+64*3+32`($a), $Bi, $_R3h + {vex} vpmadd52huq `$_data_offset+64*4`($a), $Bi, $_R4 + {vex} vpmadd52huq `$_data_offset+64*4+32`($a), $Bi, $_R4h + + {vex} vpmadd52huq `$_data_offset`($m), $Yi, $_R0 + {vex} vpmadd52huq `$_data_offset+32`($m), $Yi, $_R0h + {vex} vpmadd52huq `$_data_offset+64*1`($m), $Yi, $_R1 + {vex} vpmadd52huq `$_data_offset+64*1+32`($m), $Yi, $_R1h + {vex} vpmadd52huq `$_data_offset+64*2`($m), $Yi, $_R2 + {vex} vpmadd52huq `$_data_offset+64*2+32`($m), $Yi, $_R2h + {vex} vpmadd52huq `$_data_offset+64*3`($m), $Yi, $_R3 + {vex} vpmadd52huq `$_data_offset+64*3+32`($m), $Yi, $_R3h + {vex} vpmadd52huq `$_data_offset+64*4`($m), $Yi, $_R4 + {vex} vpmadd52huq `$_data_offset+64*4+32`($m), $Yi, $_R4h + lea 328(%rsp),%rsp +___ +} + +# Normalization routine: handles carry bits and gets bignum qwords to normalized +# 2^52 representation. +# +# Uses %r8-14,%e[abcd]x +sub amm52x40_x1_norm { +my ($_acc,$_R0,$_R0h,$_R1,$_R1h,$_R2,$_R2h,$_R3,$_R3h,$_R4,$_R4h) = @_; +$code.=<<___; + # Put accumulator to low qword in R0 + vmovq $_acc, $zero_xmm + vpbroadcastq $zero_xmm, $zero + vpblendd \$3, $zero, $_R0, $_R0 + + lea -640(%rsp),%rsp + vmovupd $_R0, `0*32`(%rsp) + vmovupd $_R0h, `1*32`(%rsp) + vmovupd $_R1, `2*32`(%rsp) + vmovupd $_R1h, `3*32`(%rsp) + vmovupd $_R2, `4*32`(%rsp) + vmovupd $_R2h, `5*32`(%rsp) + vmovupd $_R3, `6*32`(%rsp) + vmovupd $_R3h, `7*32`(%rsp) + vmovupd $_R4, `8*32`(%rsp) + vmovupd $_R4h, `9*32`(%rsp) + + # Extract "carries" (12 high bits) from each QW of the bignum + # Save them to LSB of QWs in T0..Tn + vpsrlq \$52, $_R0, $_R0 + vpsrlq \$52, $_R0h, $_R0h + vpsrlq \$52, $_R1, $_R1 + vpsrlq \$52, $_R1h, $_R1h + vpsrlq \$52, $_R2, $_R2 + vpsrlq \$52, $_R2h, $_R2h + vpsrlq \$52, $_R3, $_R3 + vpsrlq \$52, $_R3h, $_R3h + vpsrlq \$52, $_R4, $_R4 + vpsrlq \$52, $_R4h, $_R4h + + # "Shift left" _R0.._Rn by 1 QW + vpermq \$144, $_R4h, $_R4h + vpermq \$3, $_R4, $tmp + vblendpd \$1, $tmp, $_R4h, $_R4h + + vpermq \$144, $_R4, $_R4 + vpermq \$3, $_R3h, $tmp + vblendpd \$1, $tmp, $_R4, $_R4 + + vpermq \$144, $_R3h, $_R3h + vpermq \$3, $_R3, $tmp + vblendpd \$1, $tmp, $_R3h, $_R3h + + vpermq \$144, $_R3, $_R3 + vpermq \$3, $_R2h, $tmp + vblendpd \$1, $tmp, $_R3, $_R3 + + vpermq \$144, $_R2h, $_R2h + vpermq \$3, $_R2, $tmp + vblendpd \$1, $tmp, $_R2h, $_R2h + + vpermq \$144, $_R2, $_R2 + vpermq \$3, $_R1h, $tmp + vblendpd \$1, $tmp, $_R2, $_R2 + + vpermq \$144, $_R1h, $_R1h + vpermq \$3, $_R1, $tmp + vblendpd \$1, $tmp, $_R1h, $_R1h + + vpermq \$144, $_R1, $_R1 + vpermq \$3, $_R0h, $tmp + vblendpd \$1, $tmp, $_R1, $_R1 + + vpermq \$144, $_R0h, $_R0h + vpermq \$3, $_R0, $tmp + vblendpd \$1, $tmp, $_R0h, $_R0h + + vpermq \$144, $_R0, $_R0 + vpand .Lhigh64x3(%rip), $_R0, $_R0 + + vmovupd $_R0, `10*32`(%rsp) + vmovupd $_R0h, `11*32`(%rsp) + vmovupd $_R1, `12*32`(%rsp) + vmovupd $_R1h, `13*32`(%rsp) + vmovupd $_R2, `14*32`(%rsp) + vmovupd $_R2h, `15*32`(%rsp) + vmovupd $_R3, `16*32`(%rsp) + vmovupd $_R3h, `17*32`(%rsp) + vmovupd $_R4, `18*32`(%rsp) + vmovupd $_R4h, `19*32`(%rsp) + + vmovupd `0*32`(%rsp), $_R0 + vmovupd `1*32`(%rsp), $_R0h + vmovupd `2*32`(%rsp), $_R1 + vmovupd `3*32`(%rsp), $_R1h + vmovupd `4*32`(%rsp), $_R2 + vmovupd `5*32`(%rsp), $_R2h + vmovupd `6*32`(%rsp), $_R3 + vmovupd `7*32`(%rsp), $_R3h + vmovupd `8*32`(%rsp), $_R4 + vmovupd `9*32`(%rsp), $_R4h + + # Drop "carries" from R0..Rn QWs + vpand .Lmask52x4(%rip), $_R0, $_R0 + vpand .Lmask52x4(%rip), $_R0h, $_R0h + vpand .Lmask52x4(%rip), $_R1, $_R1 + vpand .Lmask52x4(%rip), $_R1h, $_R1h + vpand .Lmask52x4(%rip), $_R2, $_R2 + vpand .Lmask52x4(%rip), $_R2h, $_R2h + vpand .Lmask52x4(%rip), $_R3, $_R3 + vpand .Lmask52x4(%rip), $_R3h, $_R3h + vpand .Lmask52x4(%rip), $_R4, $_R4 + vpand .Lmask52x4(%rip), $_R4h, $_R4h + + # Sum R0..Rn with corresponding adjusted carries + vpaddq `10*32`(%rsp), $_R0, $_R0 + vpaddq `11*32`(%rsp), $_R0h, $_R0h + vpaddq `12*32`(%rsp), $_R1, $_R1 + vpaddq `13*32`(%rsp), $_R1h, $_R1h + vpaddq `14*32`(%rsp), $_R2, $_R2 + vpaddq `15*32`(%rsp), $_R2h, $_R2h + vpaddq `16*32`(%rsp), $_R3, $_R3 + vpaddq `17*32`(%rsp), $_R3h, $_R3h + vpaddq `18*32`(%rsp), $_R4, $_R4 + vpaddq `19*32`(%rsp), $_R4h, $_R4h + + lea 640(%rsp),%rsp + + # Now handle carry bits from this addition + # Get mask of QWs whose 52-bit parts overflow + vpcmpgtq .Lmask52x4(%rip),${_R0},$tmp # OP=nle (i.e. gt) + vmovmskpd $tmp,%r14d + vpcmpgtq .Lmask52x4(%rip),${_R0h},$tmp + vmovmskpd $tmp,%r13d + shl \$4,%r13b + or %r13b,%r14b + + vpcmpgtq .Lmask52x4(%rip),${_R1},$tmp # OP=nle (i.e. gt) + vmovmskpd $tmp,%r13d + vpcmpgtq .Lmask52x4(%rip),${_R1h},$tmp + vmovmskpd $tmp,%r12d + shl \$4,%r12b + or %r12b,%r13b + + vpcmpgtq .Lmask52x4(%rip),${_R2},$tmp # OP=nle (i.e. gt) + vmovmskpd $tmp,%r12d + vpcmpgtq .Lmask52x4(%rip),${_R2h},$tmp + vmovmskpd $tmp,%r11d + shl \$4,%r11b + or %r11b,%r12b + + vpcmpgtq .Lmask52x4(%rip),${_R3},$tmp # OP=nle (i.e. gt) + vmovmskpd $tmp,%r11d + vpcmpgtq .Lmask52x4(%rip),${_R3h},$tmp + vmovmskpd $tmp,%r10d + shl \$4,%r10b + or %r10b,%r11b + + vpcmpgtq .Lmask52x4(%rip),${_R4},$tmp # OP=nle (i.e. gt) + vmovmskpd $tmp,%r10d + vpcmpgtq .Lmask52x4(%rip),${_R4h},$tmp + vmovmskpd $tmp,%r9d + shl \$4,%r9b + or %r9b,%r10b + + addb %r14b,%r14b + adcb %r13b,%r13b + adcb %r12b,%r12b + adcb %r11b,%r11b + adcb %r10b,%r10b + + # Get mask of QWs whose 52-bit parts saturated + vpcmpeqq .Lmask52x4(%rip),${_R0},$tmp # OP=nle (i.e. gt) + vmovmskpd $tmp,%r9d + vpcmpeqq .Lmask52x4(%rip),${_R0h},$tmp + vmovmskpd $tmp,%r8d + shl \$4,%r8b + or %r8b,%r9b + + vpcmpeqq .Lmask52x4(%rip),${_R1},$tmp # OP=nle (i.e. gt) + vmovmskpd $tmp,%r8d + vpcmpeqq .Lmask52x4(%rip),${_R1h},$tmp + vmovmskpd $tmp,%edx + shl \$4,%dl + or %dl,%r8b + + vpcmpeqq .Lmask52x4(%rip),${_R2},$tmp # OP=nle (i.e. gt) + vmovmskpd $tmp,%edx + vpcmpeqq .Lmask52x4(%rip),${_R2h},$tmp + vmovmskpd $tmp,%ecx + shl \$4,%cl + or %cl,%dl + + vpcmpeqq .Lmask52x4(%rip),${_R3},$tmp # OP=nle (i.e. gt) + vmovmskpd $tmp,%ecx + vpcmpeqq .Lmask52x4(%rip),${_R3h},$tmp + vmovmskpd $tmp,%ebx + shl \$4,%bl + or %bl,%cl + + vpcmpeqq .Lmask52x4(%rip),${_R4},$tmp # OP=nle (i.e. gt) + vmovmskpd $tmp,%ebx + vpcmpeqq .Lmask52x4(%rip),${_R4h},$tmp + vmovmskpd $tmp,%eax + shl \$4,%al + or %al,%bl + + addb %r9b,%r14b + adcb %r8b,%r13b + adcb %dl,%r12b + adcb %cl,%r11b + adcb %bl,%r10b + + xor %r9b,%r14b + xor %r8b,%r13b + xor %dl,%r12b + xor %cl,%r11b + xor %bl,%r10b + + push %r9 + push %r8 + + lea .Lkmasklut(%rip), %r8 + + movb %r14b,%r9b + and \$0xf, %r14 + vpsubq .Lmask52x4(%rip), $_R0, $tmp + shl \$5, %r14 + vmovapd (%r8,%r14), $tmp2 + vblendvpd $tmp2, $tmp, $_R0, $_R0 + + shr \$4, %r9b + and \$0xf, %r9 + vpsubq .Lmask52x4(%rip), $_R0h, $tmp + shl \$5, %r9 + vmovapd (%r8,%r9), $tmp2 + vblendvpd $tmp2, $tmp, $_R0h, $_R0h + + movb %r13b,%r9b + and \$0xf, %r13 + vpsubq .Lmask52x4(%rip), $_R1, $tmp + shl \$5, %r13 + vmovapd (%r8,%r13), $tmp2 + vblendvpd $tmp2, $tmp, $_R1, $_R1 + + shr \$4, %r9b + and \$0xf, %r9 + vpsubq .Lmask52x4(%rip), $_R1h, $tmp + shl \$5, %r9 + vmovapd (%r8,%r9), $tmp2 + vblendvpd $tmp2, $tmp, $_R1h, $_R1h + + movb %r12b,%r9b + and \$0xf, %r12 + vpsubq .Lmask52x4(%rip), $_R2, $tmp + shl \$5, %r12 + vmovapd (%r8,%r12), $tmp2 + vblendvpd $tmp2, $tmp, $_R2, $_R2 + + shr \$4, %r9b + and \$0xf, %r9 + vpsubq .Lmask52x4(%rip), $_R2h, $tmp + shl \$5, %r9 + vmovapd (%r8,%r9), $tmp2 + vblendvpd $tmp2, $tmp, $_R2h, $_R2h + + movb %r11b,%r9b + and \$0xf, %r11 + vpsubq .Lmask52x4(%rip), $_R3, $tmp + shl \$5, %r11 + vmovapd (%r8,%r11), $tmp2 + vblendvpd $tmp2, $tmp, $_R3, $_R3 + + shr \$4, %r9b + and \$0xf, %r9 + vpsubq .Lmask52x4(%rip), $_R3h, $tmp + shl \$5, %r9 + vmovapd (%r8,%r9), $tmp2 + vblendvpd $tmp2, $tmp, $_R3h, $_R3h + + movb %r10b,%r9b + and \$0xf, %r10 + vpsubq .Lmask52x4(%rip), $_R4, $tmp + shl \$5, %r10 + vmovapd (%r8,%r10), $tmp2 + vblendvpd $tmp2, $tmp, $_R4, $_R4 + + shr \$4, %r9b + and \$0xf, %r9 + vpsubq .Lmask52x4(%rip), $_R4h, $tmp + shl \$5, %r9 + vmovapd (%r8,%r9), $tmp2 + vblendvpd $tmp2, $tmp, $_R4h, $_R4h + + pop %r8 + pop %r9 + + vpand .Lmask52x4(%rip), $_R0, $_R0 + vpand .Lmask52x4(%rip), $_R0h, $_R0h + vpand .Lmask52x4(%rip), $_R1, $_R1 + vpand .Lmask52x4(%rip), $_R1h, $_R1h + vpand .Lmask52x4(%rip), $_R2, $_R2 + vpand .Lmask52x4(%rip), $_R2h, $_R2h + vpand .Lmask52x4(%rip), $_R3, $_R3 + + vpand .Lmask52x4(%rip), $_R3h, $_R3h + vpand .Lmask52x4(%rip), $_R4, $_R4 + vpand .Lmask52x4(%rip), $_R4h, $_R4h +___ +} + +$code.=<<___; +.text + +.globl ossl_rsaz_amm52x40_x1_avxifma256 +.type ossl_rsaz_amm52x40_x1_avxifma256,\@function,5 +.align 32 +ossl_rsaz_amm52x40_x1_avxifma256: +.cfi_startproc + endbranch + push %rbx +.cfi_push %rbx + push %rbp +.cfi_push %rbp + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +___ +$code.=<<___ if ($win64); + lea -168(%rsp),%rsp # 16*10 + (8 bytes to get correct 16-byte SIMD alignment) + vmovapd %xmm6, `0*16`(%rsp) # save non-volatile registers + vmovapd %xmm7, `1*16`(%rsp) + vmovapd %xmm8, `2*16`(%rsp) + vmovapd %xmm9, `3*16`(%rsp) + vmovapd %xmm10,`4*16`(%rsp) + vmovapd %xmm11,`5*16`(%rsp) + vmovapd %xmm12,`6*16`(%rsp) + vmovapd %xmm13,`7*16`(%rsp) + vmovapd %xmm14,`8*16`(%rsp) + vmovapd %xmm15,`9*16`(%rsp) +.Lossl_rsaz_amm52x40_x1_avxifma256_body: +___ +$code.=<<___; + # Zeroing accumulators + vpxor $zero, $zero, $zero + vmovapd $zero, $R0_0 + vmovapd $zero, $R0_0h + vmovapd $zero, $R1_0 + vmovapd $zero, $R1_0h + vmovapd $zero, $R2_0 + vmovapd $zero, $R2_0h + vmovapd $zero, $R3_0 + vmovapd $zero, $R3_0h + vmovapd $zero, $R4_0 + vmovapd $zero, $R4_0h + + xorl $acc0_0_low, $acc0_0_low + + movq $b, $b_ptr # backup address of b + movq \$0xfffffffffffff, $mask52 # 52-bit mask + + # Loop over 40 digits unrolled by 4 + mov \$10, $iter + +.align 32 +.Lloop10: +___ + foreach my $idx (0..3) { + &amm52x40_x1(0,8*$idx,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,$R4_0,$R4_0h,$k0); + } +$code.=<<___; + lea `4*8`($b_ptr), $b_ptr + dec $iter + jne .Lloop10 +___ + &amm52x40_x1_norm($acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,$R4_0,$R4_0h); +$code.=<<___; + + vmovdqu $R0_0, `0*32`($res) + vmovdqu $R0_0h, `1*32`($res) + vmovdqu $R1_0, `2*32`($res) + vmovdqu $R1_0h, `3*32`($res) + vmovdqu $R2_0, `4*32`($res) + vmovdqu $R2_0h, `5*32`($res) + vmovdqu $R3_0, `6*32`($res) + vmovdqu $R3_0h, `7*32`($res) + vmovdqu $R4_0, `8*32`($res) + vmovdqu $R4_0h, `9*32`($res) + + vzeroupper + lea (%rsp),%rax +.cfi_def_cfa_register %rax +___ +$code.=<<___ if ($win64); + vmovapd `0*16`(%rax),%xmm6 + vmovapd `1*16`(%rax),%xmm7 + vmovapd `2*16`(%rax),%xmm8 + vmovapd `3*16`(%rax),%xmm9 + vmovapd `4*16`(%rax),%xmm10 + vmovapd `5*16`(%rax),%xmm11 + vmovapd `6*16`(%rax),%xmm12 + vmovapd `7*16`(%rax),%xmm13 + vmovapd `8*16`(%rax),%xmm14 + vmovapd `9*16`(%rax),%xmm15 + lea 168(%rsp),%rax +___ +$code.=<<___; + mov 0(%rax),%r15 +.cfi_restore %r15 + mov 8(%rax),%r14 +.cfi_restore %r14 + mov 16(%rax),%r13 +.cfi_restore %r13 + mov 24(%rax),%r12 +.cfi_restore %r12 + mov 32(%rax),%rbp +.cfi_restore %rbp + mov 40(%rax),%rbx +.cfi_restore %rbx + lea 48(%rax),%rsp # restore rsp +.cfi_def_cfa %rsp,8 +.Lossl_rsaz_amm52x40_x1_avxifma256_epilogue: + + ret +.cfi_endproc +.size ossl_rsaz_amm52x40_x1_avxifma256, .-ossl_rsaz_amm52x40_x1_avxifma256 +___ + +$code.=<<___; +.section .rodata align=32 +.align 32 +.Lmask52x4: + .quad 0xfffffffffffff + .quad 0xfffffffffffff + .quad 0xfffffffffffff + .quad 0xfffffffffffff +.Lhigh64x3: + .quad 0x0 + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff +.Lkmasklut: + #0000 + .quad 0x0 + .quad 0x0 + .quad 0x0 + .quad 0x0 + #0001 + .quad 0xffffffffffffffff + .quad 0x0 + .quad 0x0 + .quad 0x0 + #0010 + .quad 0x0 + .quad 0xffffffffffffffff + .quad 0x0 + .quad 0x0 + #0011 + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + .quad 0x0 + .quad 0x0 + #0100 + .quad 0x0 + .quad 0x0 + .quad 0xffffffffffffffff + .quad 0x0 + #0101 + .quad 0xffffffffffffffff + .quad 0x0 + .quad 0xffffffffffffffff + .quad 0x0 + #0110 + .quad 0x0 + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + .quad 0x0 + #0111 + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + .quad 0x0 + #1000 + .quad 0x0 + .quad 0x0 + .quad 0x0 + .quad 0xffffffffffffffff + #1001 + .quad 0xffffffffffffffff + .quad 0x0 + .quad 0x0 + .quad 0xffffffffffffffff + #1010 + .quad 0x0 + .quad 0xffffffffffffffff + .quad 0x0 + .quad 0xffffffffffffffff + #1011 + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + .quad 0x0 + .quad 0xffffffffffffffff + #1100 + .quad 0x0 + .quad 0x0 + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + #1101 + .quad 0xffffffffffffffff + .quad 0x0 + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + #1110 + .quad 0x0 + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + #1111 + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff + .quad 0xffffffffffffffff +___ + +############################################################################### +# Dual Almost Montgomery Multiplication for 40-digit number in radix 2^52 +# +# See description of ossl_rsaz_amm52x40_x1_ifma256() above for details about Almost +# Montgomery Multiplication algorithm and function input parameters description. +# +# This function does two AMMs for two independent inputs, hence dual. +# +# void ossl_rsaz_amm52x40_x2_avxifma256(BN_ULONG out[2][40], +# const BN_ULONG a[2][40], +# const BN_ULONG b[2][40], +# const BN_ULONG m[2][40], +# const BN_ULONG k0[2]); +############################################################################### + +$code.=<<___; +.text + +.globl ossl_rsaz_amm52x40_x2_avxifma256 +.type ossl_rsaz_amm52x40_x2_avxifma256,\@function,5 +.align 32 +ossl_rsaz_amm52x40_x2_avxifma256: +.cfi_startproc + endbranch + push %rbx +.cfi_push %rbx + push %rbp +.cfi_push %rbp + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +___ +$code.=<<___ if ($win64); + lea -168(%rsp),%rsp + vmovapd %xmm6, `0*16`(%rsp) # save non-volatile registers + vmovapd %xmm7, `1*16`(%rsp) + vmovapd %xmm8, `2*16`(%rsp) + vmovapd %xmm9, `3*16`(%rsp) + vmovapd %xmm10,`4*16`(%rsp) + vmovapd %xmm11,`5*16`(%rsp) + vmovapd %xmm12,`6*16`(%rsp) + vmovapd %xmm13,`7*16`(%rsp) + vmovapd %xmm14,`8*16`(%rsp) + vmovapd %xmm15,`9*16`(%rsp) +.Lossl_rsaz_amm52x40_x2_avxifma256_body: +___ +$code.=<<___; + # Zeroing accumulators + vpxor $zero, $zero, $zero + vmovapd $zero, $R0_0 + vmovapd $zero, $R0_0h + vmovapd $zero, $R1_0 + vmovapd $zero, $R1_0h + vmovapd $zero, $R2_0 + vmovapd $zero, $R2_0h + vmovapd $zero, $R3_0 + vmovapd $zero, $R3_0h + vmovapd $zero, $R4_0 + vmovapd $zero, $R4_0h + + xorl $acc0_0_low, $acc0_0_low + + movq $b, $b_ptr # backup address of b + movq \$0xfffffffffffff, $mask52 # 52-bit mask + + mov \$40, $iter + +.align 32 +.Lloop40: +___ + &amm52x40_x1( 0, 0,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,$R4_0,$R4_0h,"($k0)"); +$code.=<<___; + lea 8($b_ptr), $b_ptr + dec $iter + jne .Lloop40 + + push $b_ptr + push $a + push $m + push $k0 +___ + &amm52x40_x1_norm($acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,$R4_0,$R4_0h); +$code.=<<___; + + pop $k0 + pop $m + pop $a + pop $b_ptr + + vmovdqu $R0_0, `0*32`($res) + vmovdqu $R0_0h, `1*32`($res) + vmovdqu $R1_0, `2*32`($res) + vmovdqu $R1_0h, `3*32`($res) + vmovdqu $R2_0, `4*32`($res) + vmovdqu $R2_0h, `5*32`($res) + vmovdqu $R3_0, `6*32`($res) + vmovdqu $R3_0h, `7*32`($res) + vmovdqu $R4_0, `8*32`($res) + vmovdqu $R4_0h, `9*32`($res) + + xorl $acc0_1_low, $acc0_1_low + + movq \$0xfffffffffffff, $mask52 + + mov \$40, $iter + + vpxor $zero, $zero, $zero + vmovapd $zero, $R0_0 + vmovapd $zero, $R0_0h + vmovapd $zero, $R1_0 + vmovapd $zero, $R1_0h + vmovapd $zero, $R2_0 + vmovapd $zero, $R2_0h + vmovapd $zero, $R3_0 + vmovapd $zero, $R3_0h + vmovapd $zero, $R4_0 + vmovapd $zero, $R4_0h +.align 32 +.Lloop40_1: +___ + # 40*8 = offset of the next dimension in two-dimension array + &amm52x40_x1(40*8, 0,$acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,$R4_0,$R4_0h,"8($k0)"); +$code.=<<___; + lea 8($b_ptr), $b_ptr + dec $iter + jne .Lloop40_1 +___ + &amm52x40_x1_norm($acc0_0,$R0_0,$R0_0h,$R1_0,$R1_0h,$R2_0,$R2_0h,$R3_0,$R3_0h,$R4_0,$R4_0h); +$code.=<<___; + + vmovdqu $R0_0, `10*32`($res) + vmovdqu $R0_0h, `11*32`($res) + vmovdqu $R1_0, `12*32`($res) + vmovdqu $R1_0h, `13*32`($res) + vmovdqu $R2_0, `14*32`($res) + vmovdqu $R2_0h, `15*32`($res) + vmovdqu $R3_0, `16*32`($res) + vmovdqu $R3_0h, `17*32`($res) + vmovdqu $R4_0, `18*32`($res) + vmovdqu $R4_0h, `19*32`($res) + + vzeroupper + lea (%rsp),%rax +.cfi_def_cfa_register %rax +___ +$code.=<<___ if ($win64); + vmovapd `0*16`(%rax),%xmm6 + vmovapd `1*16`(%rax),%xmm7 + vmovapd `2*16`(%rax),%xmm8 + vmovapd `3*16`(%rax),%xmm9 + vmovapd `4*16`(%rax),%xmm10 + vmovapd `5*16`(%rax),%xmm11 + vmovapd `6*16`(%rax),%xmm12 + vmovapd `7*16`(%rax),%xmm13 + vmovapd `8*16`(%rax),%xmm14 + vmovapd `9*16`(%rax),%xmm15 + lea 168(%rsp),%rax +___ +$code.=<<___; + mov 0(%rax),%r15 +.cfi_restore %r15 + mov 8(%rax),%r14 +.cfi_restore %r14 + mov 16(%rax),%r13 +.cfi_restore %r13 + mov 24(%rax),%r12 +.cfi_restore %r12 + mov 32(%rax),%rbp +.cfi_restore %rbp + mov 40(%rax),%rbx +.cfi_restore %rbx + lea 48(%rax),%rsp +.cfi_def_cfa %rsp,8 +.Lossl_rsaz_amm52x40_x2_avxifma256_epilogue: + ret +.cfi_endproc +.size ossl_rsaz_amm52x40_x2_avxifma256, .-ossl_rsaz_amm52x40_x2_avxifma256 +___ +} + +############################################################################### +# Constant time extraction from the precomputed table of powers base^i, where +# i = 0..2^EXP_WIN_SIZE-1 +# +# The input |red_table| contains precomputations for two independent base values. +# |red_table_idx1| and |red_table_idx2| are corresponding power indexes. +# +# Extracted value (output) is 2 40 digits numbers in 2^52 radix. +# +# void ossl_extract_multiplier_2x40_win5_avx(BN_ULONG *red_Y, +# const BN_ULONG red_table[1 << EXP_WIN_SIZE][2][40], +# int red_table_idx1, int red_table_idx2); +# +# EXP_WIN_SIZE = 5 +############################################################################### +{ +# input parameters +my ($out,$red_tbl,$red_tbl_idx1,$red_tbl_idx2)=$win64 ? ("%rcx","%rdx","%r8", "%r9") : # Win64 order + ("%rdi","%rsi","%rdx","%rcx"); # Unix order + +my ($t0,$t1,$t2,$t3,$t4,$t5) = map("%ymm$_", (0..5)); +my ($t6,$t7,$t8,$t9) = map("%ymm$_", (6..9)); +my ($tmp,$cur_idx,$idx1,$idx2,$ones,$mask) = map("%ymm$_", (10..15)); +my $tmp_xmm = "%xmm10"; + +my @t = ($t0,$t1,$t2,$t3,$t4,$t5,$t6,$t7,$t8,$t9); +my $t0xmm = $t0; +$t0xmm =~ s/%y/%x/; + +sub get_table_value_consttime() { +my ($_idx,$_offset) = @_; +$code.=<<___; + vpxor $cur_idx, $cur_idx, $cur_idx +.align 32 +.Lloop_$_offset: + vpcmpeqq $cur_idx, $_idx, $mask # mask of (idx == cur_idx) +___ +foreach (0..9) { +$code.=<<___; + vmovdqu `$_offset+${_}*32`($red_tbl), $tmp # load data from red_tbl + #vpblendmq $tmp, $t[$_], ${t[$_]}{%k1} # extract data when mask is not zero + vblendvpd $mask, $tmp, $t[$_], ${t[$_]} # extract data when mask is not zero +___ +} +$code.=<<___; + vpaddq $ones, $cur_idx, $cur_idx # increment cur_idx + addq \$`2*40*8`, $red_tbl + cmpq $red_tbl, %rax + jne .Lloop_$_offset +___ +} + +$code.=<<___; +.text + +.align 32 +.globl ossl_extract_multiplier_2x40_win5_avx +.type ossl_extract_multiplier_2x40_win5_avx,\@abi-omnipotent +ossl_extract_multiplier_2x40_win5_avx: +.cfi_startproc + endbranch + vmovapd .Lones(%rip), $ones # broadcast ones + vmovq $red_tbl_idx1, $tmp_xmm + vpbroadcastq $tmp_xmm, $idx1 + vmovq $red_tbl_idx2, $tmp_xmm + vpbroadcastq $tmp_xmm, $idx2 + leaq `(1<<5)*2*40*8`($red_tbl), %rax # holds end of the tbl + + # backup red_tbl address + movq $red_tbl, %r10 + + # zeroing t0..n, cur_idx + vpxor $t0xmm, $t0xmm, $t0xmm +___ +foreach (1..9) { + $code.="vmovapd $t0, $t[$_] \n"; +} + +&get_table_value_consttime($idx1, 0); +foreach (0..9) { + $code.="vmovdqu $t[$_], `(0+$_)*32`($out) \n"; +} +$code.="movq %r10, $red_tbl \n"; +&get_table_value_consttime($idx2, 40*8); +foreach (0..9) { + $code.="vmovdqu $t[$_], `(10+$_)*32`($out) \n"; +} +$code.=<<___; + + ret +.cfi_endproc +.size ossl_extract_multiplier_2x40_win5_avx, .-ossl_extract_multiplier_2x40_win5_avx +___ +$code.=<<___; +.section .rodata align=32 +.align 32 +.Lones: + .quad 1,1,1,1 +.Lzeros: + .quad 0,0,0,0 +___ +} + +if ($win64) { +$rec="%rcx"; +$frame="%rdx"; +$context="%r8"; +$disp="%r9"; + +$code.=<<___; +.extern __imp_RtlVirtualUnwind +.type rsaz_avx_handler,\@abi-omnipotent +.align 16 +rsaz_avx_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + mov 8($disp),%rsi # disp->ImageBase + mov 56($disp),%r11 # disp->HandlerData + + mov 0(%r11),%r10d # HandlerData[0] + lea (%rsi,%r10),%r10 # prologue label + cmp %r10,%rbx # context->Rip<.Lprologue + jb .Lcommon_seh_tail + + mov 4(%r11),%r10d # HandlerData[1] + lea (%rsi,%r10),%r10 # epilogue label + cmp %r10,%rbx # context->Rip>=.Lepilogue + jae .Lcommon_seh_tail + + mov 152($context),%rax # pull context->Rsp + + lea (%rax),%rsi # %xmm save area + lea 512($context),%rdi # & context.Xmm6 + mov \$20,%ecx # 10*sizeof(%xmm0)/sizeof(%rax) + .long 0xa548f3fc # cld; rep movsq + + lea `48+168`(%rax),%rax + + mov -8(%rax),%rbx + mov -16(%rax),%rbp + mov -24(%rax),%r12 + mov -32(%rax),%r13 + mov -40(%rax),%r14 + mov -48(%rax),%r15 + mov %rbx,144($context) # restore context->Rbx + mov %rbp,160($context) # restore context->Rbp + mov %r12,216($context) # restore context->R12 + mov %r13,224($context) # restore context->R13 + mov %r14,232($context) # restore context->R14 + mov %r15,240($context) # restore context->R14 + +.Lcommon_seh_tail: + mov 8(%rax),%rdi + mov 16(%rax),%rsi + mov %rax,152($context) # restore context->Rsp + mov %rsi,168($context) # restore context->Rsi + mov %rdi,176($context) # restore context->Rdi + + mov 40($disp),%rdi # disp->ContextRecord + mov $context,%rsi # context + mov \$154,%ecx # sizeof(CONTEXT) + .long 0xa548f3fc # cld; rep movsq + + mov $disp,%rsi + xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER + mov 8(%rsi),%rdx # arg2, disp->ImageBase + mov 0(%rsi),%r8 # arg3, disp->ControlPc + mov 16(%rsi),%r9 # arg4, disp->FunctionEntry + mov 40(%rsi),%r10 # disp->ContextRecord + lea 56(%rsi),%r11 # &disp->HandlerData + lea 24(%rsi),%r12 # &disp->EstablisherFrame + mov %r10,32(%rsp) # arg5 + mov %r11,40(%rsp) # arg6 + mov %r12,48(%rsp) # arg7 + mov %rcx,56(%rsp) # arg8, (NULL) + call *__imp_RtlVirtualUnwind(%rip) + + mov \$1,%eax # ExceptionContinueSearch + add \$64,%rsp + popfq + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbp + pop %rbx + pop %rdi + pop %rsi + ret +.size rsaz_avx_handler,.-rsaz_avx_handler + +.section .pdata +.align 4 + .rva .LSEH_begin_ossl_rsaz_amm52x40_x1_avxifma256 + .rva .LSEH_end_ossl_rsaz_amm52x40_x1_avxifma256 + .rva .LSEH_info_ossl_rsaz_amm52x40_x1_avxifma256 + + .rva .LSEH_begin_ossl_rsaz_amm52x40_x2_avxifma256 + .rva .LSEH_end_ossl_rsaz_amm52x40_x2_avxifma256 + .rva .LSEH_info_ossl_rsaz_amm52x40_x2_avxifma256 + +.section .xdata +.align 8 +.LSEH_info_ossl_rsaz_amm52x40_x1_avxifma256: + .byte 9,0,0,0 + .rva rsaz_avx_handler + .rva .Lossl_rsaz_amm52x40_x1_avxifma256_body,.Lossl_rsaz_amm52x40_x1_avxifma256_epilogue +.LSEH_info_ossl_rsaz_amm52x40_x2_avxifma256: + .byte 9,0,0,0 + .rva rsaz_avx_handler + .rva .Lossl_rsaz_amm52x40_x2_avxifma256_body,.Lossl_rsaz_amm52x40_x2_avxifma256_epilogue +___ +} +}}} else {{{ # fallback for old assembler +$code.=<<___; +.text + +.globl ossl_rsaz_amm52x40_x1_avxifma256 +.globl ossl_rsaz_amm52x40_x2_avxifma256 +.globl ossl_extract_multiplier_2x40_win5_avx +.type ossl_rsaz_amm52x40_x1_avxifma256,\@abi-omnipotent +ossl_rsaz_amm52x40_x1_avxifma256: +ossl_rsaz_amm52x40_x2_avxifma256: +ossl_extract_multiplier_2x40_win5_avx: + .byte 0x0f,0x0b # ud2 + ret +.size ossl_rsaz_amm52x40_x1_avxifma256, .-ossl_rsaz_amm52x40_x1_avxifma256 +___ +}}} + +$code =~ s/\`([^\`]*)\`/eval $1/gem; +print $code; +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/bn/asm/rsaz-avx2.pl b/crypto/openssl/crypto/bn/asm/rsaz-avx2.pl index 3d0e342a6b8c..59b9c89b5be2 100755 --- a/crypto/openssl/crypto/bn/asm/rsaz-avx2.pl +++ b/crypto/openssl/crypto/bn/asm/rsaz-avx2.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2013-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2013-2024 The OpenSSL Project Authors. All Rights Reserved. # Copyright (c) 2012, Intel Corporation. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use @@ -1779,6 +1779,7 @@ $code.=<<___; ret .size rsaz_avx2_eligible,.-rsaz_avx2_eligible +.section .rodata align=64 .align 64 .Land_mask: .quad 0x1fffffff,0x1fffffff,0x1fffffff,0x1fffffff @@ -1790,6 +1791,7 @@ $code.=<<___; .long 0,0,0,0, 1,1,1,1 .long 2,2,2,2, 3,3,3,3 .long 4,4,4,4, 4,4,4,4 +.previous .align 64 ___ diff --git a/crypto/openssl/crypto/bn/asm/rsaz-x86_64.pl b/crypto/openssl/crypto/bn/asm/rsaz-x86_64.pl index 5c7d526fa37a..36bf229477aa 100755 --- a/crypto/openssl/crypto/bn/asm/rsaz-x86_64.pl +++ b/crypto/openssl/crypto/bn/asm/rsaz-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2013-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2013-2024 The OpenSSL Project Authors. All Rights Reserved. # Copyright (c) 2012, Intel Corporation. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use @@ -2248,10 +2248,12 @@ $code.=<<___; .cfi_endproc .size rsaz_512_gather4,.-rsaz_512_gather4 +.section .rodata align=64 .align 64 .Linc: .long 0,0, 1,1 .long 2,2, 2,2 +.previous ___ } diff --git a/crypto/openssl/crypto/bn/asm/sparcv8.S b/crypto/openssl/crypto/bn/asm/sparcv8.S index 94487008443e..e5402cc4e26f 100644 --- a/crypto/openssl/crypto/bn/asm/sparcv8.S +++ b/crypto/openssl/crypto/bn/asm/sparcv8.S @@ -381,7 +381,7 @@ bn_add_words: inc 16,%o1 addxcc %g3,%g4,%g3 st %g3,[%o0+4] - + ld [%o1-4],%g3 ld [%o2+12],%g4 inc 16,%o2 @@ -469,7 +469,7 @@ bn_sub_words: inc 16,%o1 subxcc %g3,%g4,%g4 st %g4,[%o0+4] - + ld [%o1-4],%g3 ld [%o2+12],%g4 inc 16,%o2 @@ -1040,7 +1040,7 @@ bn_mul_comba4: addxcc c_2,t_2,c_2 != st c_1,rp(6) !r[6]=c1; st c_2,rp(7) !r[7]=c2; - + ret restore %g0,%g0,%o0 @@ -1448,7 +1448,7 @@ bn_sqr_comba4: addxcc c_2,t_2,c_2 != st c_1,rp(6) !r[6]=c1; st c_2,rp(7) !r[7]=c2; - + ret restore %g0,%g0,%o0 diff --git a/crypto/openssl/crypto/bn/asm/sparcv8plus.S b/crypto/openssl/crypto/bn/asm/sparcv8plus.S index 696dc7b5fe09..8a936f1c72e4 100644 --- a/crypto/openssl/crypto/bn/asm/sparcv8plus.S +++ b/crypto/openssl/crypto/bn/asm/sparcv8plus.S @@ -496,7 +496,7 @@ bn_add_words: inc 16,%o1 addccc %g1,%g2,%g1 stuw %g1,[%o0+4] - + inc 16,%o2 addccc %g3,%g4,%g3 stuw %g3,[%o0+8] @@ -1126,7 +1126,7 @@ bn_mul_comba4: srlx t_1,32,c_12 != stuw t_1,rp(6) !r[6]=c1; stuw c_12,rp(7) !r[7]=c2; - + ret restore %g0,%g0,%o0 @@ -1548,7 +1548,7 @@ bn_sqr_comba4: srlx t_1,32,c_12 stuw t_1,rp(6) !r[6]=c1; stuw c_12,rp(7) !r[7]=c2; - + ret restore %g0,%g0,%o0 diff --git a/crypto/openssl/crypto/bn/asm/x86_64-mont5.pl b/crypto/openssl/crypto/bn/asm/x86_64-mont5.pl index 185d9e76ce23..07dd40d9228c 100755 --- a/crypto/openssl/crypto/bn/asm/x86_64-mont5.pl +++ b/crypto/openssl/crypto/bn/asm/x86_64-mont5.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2011-2022 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2011-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -3577,11 +3577,13 @@ $code.=<<___; ___ } $code.=<<___; +.section .rodata align=64 .align 64 .Linc: .long 0,0, 1,1 .long 2,2, 2,2 .asciz "Montgomery Multiplication with scatter/gather for x86_64, CRYPTOGAMS by " +.previous ___ # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, diff --git a/crypto/openssl/crypto/camellia/asm/cmll-x86_64.pl b/crypto/openssl/crypto/camellia/asm/cmll-x86_64.pl index 7e069614d915..43023a6986b5 100755 --- a/crypto/openssl/crypto/camellia/asm/cmll-x86_64.pl +++ b/crypto/openssl/crypto/camellia/asm/cmll-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2008-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2008-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -657,6 +657,7 @@ sub S0222 { my $i=shift; $i=@SBOX[$i]; $i=($i<<1|$i>>7)&0xff; $i=$i<<16|$i<<8|$i sub S3033 { my $i=shift; $i=@SBOX[$i]; $i=($i>>1|$i<<7)&0xff; $i=$i<<24|$i<<8|$i; sprintf("0x%08x",$i); } $code.=<<___; +.section .rodata align=64 .align 64 .LCamellia_SIGMA: .long 0x3bcc908b, 0xa09e667f, 0x4caa73b2, 0xb67ae858 @@ -682,6 +683,7 @@ $_ivp="40(%rsp)"; $_rsp="48(%rsp)"; $code.=<<___; +.text .globl Camellia_cbc_encrypt .type Camellia_cbc_encrypt,\@function,6 .align 16 diff --git a/crypto/openssl/crypto/chacha/asm/chacha-armv4.pl b/crypto/openssl/crypto/chacha/asm/chacha-armv4.pl index 0aa13519eb83..bb48b6eeb99e 100755 --- a/crypto/openssl/crypto/chacha/asm/chacha-armv4.pl +++ b/crypto/openssl/crypto/chacha/asm/chacha-armv4.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2016-2023 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -1154,7 +1154,8 @@ $code.=<<___; add sp,sp,#4*(16+3) ldmia sp!,{r4-r11,pc} .size ChaCha20_neon,.-ChaCha20_neon -.comm OPENSSL_armcap_P,4,4 +.extern OPENSSL_armcap_P +.hidden OPENSSL_armcap_P #endif ___ }}} diff --git a/crypto/openssl/crypto/chacha/asm/chacha-armv8-sve.pl b/crypto/openssl/crypto/chacha/asm/chacha-armv8-sve.pl new file mode 100755 index 000000000000..62a8be6fe12c --- /dev/null +++ b/crypto/openssl/crypto/chacha/asm/chacha-armv8-sve.pl @@ -0,0 +1,1178 @@ +#! /usr/bin/env perl +# Copyright 2022-2025 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html +# +# +# ChaCha20 for ARMv8 via SVE +# +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or +die "can't locate arm-xlate.pl"; + +open OUT,"| \"$^X\" $xlate $flavour \"$output\"" + or die "can't call $xlate: $!"; +*STDOUT=*OUT; + +sub AUTOLOAD() # thunk [simplified] x86-style perlasm +{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./; + my $arg = pop; + $arg = "#$arg" if ($arg*1 eq $arg); + $code .= "\t$opcode\t".join(',',@_,$arg)."\n"; +} + +$prefix="chacha_sve"; +my ($outp,$inp,$len,$key,$ctr) = map("x$_",(0..4)); +my ($veclen) = ("x5"); +my ($counter) = ("x6"); +my ($counter_w) = ("w6"); +my @xx=(7..22); +my @sxx=map("x$_",@xx); +my @sx=map("w$_",@xx); +my @K=map("x$_",(23..30)); +my @elem=(0,4,8,12,1,5,9,13,2,6,10,14,3,7,11,15); +my @KL=map("w$_",(23..30)); +my @mx=map("z$_",@elem); +my @vx=map("v$_",@elem); +my ($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3, + $xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3) = @mx; +my ($zctr) = ("z16"); +my @tt=(17..24); +my @xt=map("z$_",@tt); +my @vt=map("v$_",@tt); +my @perm=map("z$_",(25..30)); +my ($rot8) = ("z31"); +my @bak=(@perm[0],@perm[1],@perm[2],@perm[3],@perm[4],@perm[5],@xt[4],@xt[5],@xt[6],@xt[7],@xt[0],@xt[1],$zctr,@xt[2],@xt[3],$rot8); +my $debug_encoder=0; + +sub SVE_ADD() { + my $x = shift; + my $y = shift; + +$code.=<<___; + add @mx[$x].s,@mx[$x].s,@mx[$y].s + .if mixin == 1 + add @sx[$x],@sx[$x],@sx[$y] + .endif +___ + if (@_) { + &SVE_ADD(@_); + } +} + +sub SVE_EOR() { + my $x = shift; + my $y = shift; + +$code.=<<___; + eor @mx[$x].d,@mx[$x].d,@mx[$y].d + .if mixin == 1 + eor @sx[$x],@sx[$x],@sx[$y] + .endif +___ + if (@_) { + &SVE_EOR(@_); + } +} + +sub SVE_LSL() { + my $bits = shift; + my $x = shift; + my $y = shift; + my $next = $x + 1; + +$code.=<<___; + lsl @xt[$x].s,@mx[$y].s,$bits +___ + if (@_) { + &SVE_LSL($bits,$next,@_); + } +} + +sub SVE_LSR() { + my $bits = shift; + my $x = shift; + +$code.=<<___; + lsr @mx[$x].s,@mx[$x].s,$bits + .if mixin == 1 + ror @sx[$x],@sx[$x],$bits + .endif +___ + if (@_) { + &SVE_LSR($bits,@_); + } +} + +sub SVE_ORR() { + my $x = shift; + my $y = shift; + my $next = $x + 1; + +$code.=<<___; + orr @mx[$y].d,@mx[$y].d,@xt[$x].d +___ + if (@_) { + &SVE_ORR($next,@_); + } +} + +sub SVE_REV16() { + my $x = shift; + +$code.=<<___; + revh @mx[$x].s,p0/m,@mx[$x].s + .if mixin == 1 + ror @sx[$x],@sx[$x],#16 + .endif +___ + if (@_) { + &SVE_REV16(@_); + } +} + +sub SVE_ROT8() { + my $x = shift; + +$code.=<<___; + tbl @mx[$x].b,{@mx[$x].b},$rot8.b + .if mixin == 1 + ror @sx[$x],@sx[$x],#24 + .endif +___ + if (@_) { + &SVE_ROT8(@_); + } +} + +sub SVE2_XAR() { + my $bits = shift; + my $x = shift; + my $y = shift; + my $rbits = 32-$bits; + +$code.=<<___; + .if mixin == 1 + eor @sx[$x],@sx[$x],@sx[$y] + .endif + xar @mx[$x].s,@mx[$x].s,@mx[$y].s,$rbits + .if mixin == 1 + ror @sx[$x],@sx[$x],$rbits + .endif +___ + if (@_) { + &SVE2_XAR($bits,@_); + } +} + +sub SVE2_QR_GROUP() { + my ($a0,$b0,$c0,$d0,$a1,$b1,$c1,$d1,$a2,$b2,$c2,$d2,$a3,$b3,$c3,$d3) = @_; + + &SVE_ADD($a0,$b0,$a1,$b1,$a2,$b2,$a3,$b3); + &SVE2_XAR(16,$d0,$a0,$d1,$a1,$d2,$a2,$d3,$a3); + + &SVE_ADD($c0,$d0,$c1,$d1,$c2,$d2,$c3,$d3); + &SVE2_XAR(12,$b0,$c0,$b1,$c1,$b2,$c2,$b3,$c3); + + &SVE_ADD($a0,$b0,$a1,$b1,$a2,$b2,$a3,$b3); + &SVE2_XAR(8,$d0,$a0,$d1,$a1,$d2,$a2,$d3,$a3); + + &SVE_ADD($c0,$d0,$c1,$d1,$c2,$d2,$c3,$d3); + &SVE2_XAR(7,$b0,$c0,$b1,$c1,$b2,$c2,$b3,$c3); +} + +sub SVE_QR_GROUP() { + my ($a0,$b0,$c0,$d0,$a1,$b1,$c1,$d1,$a2,$b2,$c2,$d2,$a3,$b3,$c3,$d3) = @_; + + &SVE_ADD($a0,$b0,$a1,$b1,$a2,$b2,$a3,$b3); + &SVE_EOR($d0,$a0,$d1,$a1,$d2,$a2,$d3,$a3); + &SVE_REV16($d0,$d1,$d2,$d3); + + &SVE_ADD($c0,$d0,$c1,$d1,$c2,$d2,$c3,$d3); + &SVE_EOR($b0,$c0,$b1,$c1,$b2,$c2,$b3,$c3); + &SVE_LSL(12,0,$b0,$b1,$b2,$b3); + &SVE_LSR(20,$b0,$b1,$b2,$b3); + &SVE_ORR(0,$b0,$b1,$b2,$b3); + + &SVE_ADD($a0,$b0,$a1,$b1,$a2,$b2,$a3,$b3); + &SVE_EOR($d0,$a0,$d1,$a1,$d2,$a2,$d3,$a3); + &SVE_ROT8($d0,$d1,$d2,$d3); + + &SVE_ADD($c0,$d0,$c1,$d1,$c2,$d2,$c3,$d3); + &SVE_EOR($b0,$c0,$b1,$c1,$b2,$c2,$b3,$c3); + &SVE_LSL(7,0,$b0,$b1,$b2,$b3); + &SVE_LSR(25,$b0,$b1,$b2,$b3); + &SVE_ORR(0,$b0,$b1,$b2,$b3); +} + +sub SVE_INNER_BLOCK() { +$code.=<<___; + mov $counter,#10 +10: +.align 5 +___ + &SVE_QR_GROUP(0,4,8,12,1,5,9,13,2,6,10,14,3,7,11,15); + &SVE_QR_GROUP(0,5,10,15,1,6,11,12,2,7,8,13,3,4,9,14); +$code.=<<___; + sub $counter,$counter,1 + cbnz $counter,10b +___ +} + +sub SVE2_INNER_BLOCK() { +$code.=<<___; + mov $counter,#10 +10: +.align 5 +___ + &SVE2_QR_GROUP(0,4,8,12,1,5,9,13,2,6,10,14,3,7,11,15); + &SVE2_QR_GROUP(0,5,10,15,1,6,11,12,2,7,8,13,3,4,9,14); +$code.=<<___; + sub $counter,$counter,1 + cbnz $counter,10b +___ +} + +sub load_regs() { + my $offset = shift; + my $reg = shift; + my $next_offset = $offset + 1; +$code.=<<___; + ld1w {$reg.s},p0/z,[$inp,#$offset,MUL VL] +___ + if (@_) { + &load_regs($next_offset, @_); + } else { +$code.=<<___; + addvl $inp,$inp,$next_offset +___ + } +} + +sub load() { + if (@_) { + &load_regs(0, @_); + } +} + +sub store_regs() { + my $offset = shift; + my $reg = shift; + my $next_offset = $offset + 1; +$code.=<<___; + st1w {$reg.s},p0,[$outp,#$offset,MUL VL] +___ + if (@_) { + &store_regs($next_offset, @_); + } else { +$code.=<<___; + addvl $outp,$outp,$next_offset +___ + } +} + +sub store() { + if (@_) { + &store_regs(0, @_); + } +} + +sub transpose() { + my $xa = shift; + my $xb = shift; + my $xc = shift; + my $xd = shift; + my $xa1 = shift; + my $xb1 = shift; + my $xc1 = shift; + my $xd1 = shift; +$code.=<<___; + zip1 @xt[0].s,$xa.s,$xb.s + zip2 @xt[1].s,$xa.s,$xb.s + zip1 @xt[2].s,$xc.s,$xd.s + zip2 @xt[3].s,$xc.s,$xd.s + + zip1 @xt[4].s,$xa1.s,$xb1.s + zip2 @xt[5].s,$xa1.s,$xb1.s + zip1 @xt[6].s,$xc1.s,$xd1.s + zip2 @xt[7].s,$xc1.s,$xd1.s + + zip1 $xa.d,@xt[0].d,@xt[2].d + zip2 $xb.d,@xt[0].d,@xt[2].d + zip1 $xc.d,@xt[1].d,@xt[3].d + zip2 $xd.d,@xt[1].d,@xt[3].d + + zip1 $xa1.d,@xt[4].d,@xt[6].d + zip2 $xb1.d,@xt[4].d,@xt[6].d + zip1 $xc1.d,@xt[5].d,@xt[7].d + zip2 $xd1.d,@xt[5].d,@xt[7].d +___ +} + +sub ACCUM() { + my $idx0 = shift; + my $idx1 = $idx0 + 1; + my $x0 = @sx[$idx0]; + my $xx0 = @sxx[$idx0]; + my $x1 = @sx[$idx1]; + my $xx1 = @sxx[$idx1]; + my $d = $idx0/2; + my ($tmp,$tmpw) = ($counter,$counter_w); + my $bk0 = @_ ? shift : @bak[$idx0]; + my $bk1 = @_ ? shift : @bak[$idx1]; + +$code.=<<___; + .if mixin == 1 + add @sx[$idx0],@sx[$idx0],@KL[$d] + .endif + add @mx[$idx0].s,@mx[$idx0].s,$bk0.s + .if mixin == 1 + add @sxx[$idx1],@sxx[$idx1],@K[$d],lsr #32 + .endif + add @mx[$idx1].s,@mx[$idx1].s,$bk1.s + .if mixin == 1 + add @sxx[$idx0],@sxx[$idx0],$sxx[$idx1],lsl #32 // pack + .endif +___ +} + +sub SCA_INP() { + my $idx0 = shift; + my $idx1 = $idx0 + 2; +$code.=<<___; + .if mixin == 1 + ldp @sxx[$idx0],@sxx[$idx1],[$inp],#16 + .endif +___ +} + +sub SVE_ACCUM_STATES() { + my ($tmp,$tmpw) = ($counter,$counter_w); + +$code.=<<___; + lsr $tmp,@K[5],#32 + dup @bak[10].s,@KL[5] + dup @bak[11].s,$tmpw + lsr $tmp,@K[6],#32 + dup @bak[13].s,$tmpw + lsr $tmp,@K[7],#32 +___ + &ACCUM(0); + &ACCUM(2); + &SCA_INP(1); + &ACCUM(4); + &ACCUM(6); + &SCA_INP(5); + &ACCUM(8); + &ACCUM(10); + &SCA_INP(9); +$code.=<<___; + dup @bak[14].s,@KL[7] + dup @bak[0].s,$tmpw // bak[15] not available for SVE +___ + &ACCUM(12); + &ACCUM(14, @bak[14],@bak[0]); + &SCA_INP(13); +} + +sub SVE2_ACCUM_STATES() { + &ACCUM(0); + &ACCUM(2); + &SCA_INP(1); + &ACCUM(4); + &ACCUM(6); + &SCA_INP(5); + &ACCUM(8); + &ACCUM(10); + &SCA_INP(9); + &ACCUM(12); + &ACCUM(14); + &SCA_INP(13); +} + +sub SCA_EOR() { + my $idx0 = shift; + my $idx1 = $idx0 + 1; +$code.=<<___; + .if mixin == 1 + eor @sxx[$idx0],@sxx[$idx0],@sxx[$idx1] + .endif +___ +} + +sub SCA_SAVE() { + my $idx0 = shift; + my $idx1 = shift; +$code.=<<___; + .if mixin == 1 + stp @sxx[$idx0],@sxx[$idx1],[$outp],#16 + .endif +___ +} + +sub SVE_VL128_TRANSFORMS() { + &SCA_EOR(0); + &SCA_EOR(2); + &SCA_EOR(4); + &transpose($xa0,$xa1,$xa2,$xa3,$xb0,$xb1,$xb2,$xb3); + &SCA_EOR(6); + &SCA_EOR(8); + &SCA_EOR(10); + &transpose($xc0,$xc1,$xc2,$xc3,$xd0,$xd1,$xd2,$xd3); + &SCA_EOR(12); + &SCA_EOR(14); +$code.=<<___; + ld1 {@vt[0].4s-@vt[3].4s},[$inp],#64 + ld1 {@vt[4].4s-@vt[7].4s},[$inp],#64 + eor $xa0.d,$xa0.d,@xt[0].d + eor $xb0.d,$xb0.d,@xt[1].d + eor $xc0.d,$xc0.d,@xt[2].d + eor $xd0.d,$xd0.d,@xt[3].d + eor $xa1.d,$xa1.d,@xt[4].d + eor $xb1.d,$xb1.d,@xt[5].d + eor $xc1.d,$xc1.d,@xt[6].d + eor $xd1.d,$xd1.d,@xt[7].d + ld1 {@vt[0].4s-@vt[3].4s},[$inp],#64 + ld1 {@vt[4].4s-@vt[7].4s},[$inp],#64 +___ + &SCA_SAVE(0,2); +$code.=<<___; + eor $xa2.d,$xa2.d,@xt[0].d + eor $xb2.d,$xb2.d,@xt[1].d +___ + &SCA_SAVE(4,6); +$code.=<<___; + eor $xc2.d,$xc2.d,@xt[2].d + eor $xd2.d,$xd2.d,@xt[3].d +___ + &SCA_SAVE(8,10); +$code.=<<___; + eor $xa3.d,$xa3.d,@xt[4].d + eor $xb3.d,$xb3.d,@xt[5].d +___ + &SCA_SAVE(12,14); +$code.=<<___; + eor $xc3.d,$xc3.d,@xt[6].d + eor $xd3.d,$xd3.d,@xt[7].d + st1 {@vx[0].4s-@vx[12].4s},[$outp],#64 + st1 {@vx[1].4s-@vx[13].4s},[$outp],#64 + st1 {@vx[2].4s-@vx[14].4s},[$outp],#64 + st1 {@vx[3].4s-@vx[15].4s},[$outp],#64 +___ +} + +sub SVE_TRANSFORMS() { +$code.=<<___; +#ifdef __AARCH64EB__ + rev @sxx[0],@sxx[0] + revb @mx[0].s,p0/m,@mx[0].s + revb @mx[1].s,p0/m,@mx[1].s + rev @sxx[2],@sxx[2] + revb @mx[2].s,p0/m,@mx[2].s + revb @mx[3].s,p0/m,@mx[3].s + rev @sxx[4],@sxx[4] + revb @mx[4].s,p0/m,@mx[4].s + revb @mx[5].s,p0/m,@mx[5].s + rev @sxx[6],@sxx[6] + revb @mx[6].s,p0/m,@mx[6].s + revb @mx[7].s,p0/m,@mx[7].s + rev @sxx[8],@sxx[8] + revb @mx[8].s,p0/m,@mx[8].s + revb @mx[9].s,p0/m,@mx[9].s + rev @sxx[10],@sxx[10] + revb @mx[10].s,p0/m,@mx[10].s + revb @mx[11].s,p0/m,@mx[11].s + rev @sxx[12],@sxx[12] + revb @mx[12].s,p0/m,@mx[12].s + revb @mx[13].s,p0/m,@mx[13].s + rev @sxx[14],@sxx[14] + revb @mx[14].s,p0/m,@mx[14].s + revb @mx[15].s,p0/m,@mx[15].s +#endif + .if mixin == 1 + add @K[6],@K[6],#1 + .endif + cmp $veclen,4 + b.ne 200f +___ + &SVE_VL128_TRANSFORMS(); +$code.=<<___; + b 210f +200: +___ + &transpose($xa0,$xb0,$xc0,$xd0,$xa1,$xb1,$xc1,$xd1); + &SCA_EOR(0); + &SCA_EOR(2); + &transpose($xa2,$xb2,$xc2,$xd2,$xa3,$xb3,$xc3,$xd3); + &SCA_EOR(4); + &SCA_EOR(6); + &transpose($xa0,$xa1,$xa2,$xa3,$xb0,$xb1,$xb2,$xb3); + &SCA_EOR(8); + &SCA_EOR(10); + &transpose($xc0,$xc1,$xc2,$xc3,$xd0,$xd1,$xd2,$xd3); + &SCA_EOR(12); + &SCA_EOR(14); + &load(@xt[0],@xt[1],@xt[2],@xt[3],@xt[4],@xt[5],@xt[6],@xt[7]); +$code.=<<___; + eor $xa0.d,$xa0.d,@xt[0].d + eor $xa1.d,$xa1.d,@xt[1].d + eor $xa2.d,$xa2.d,@xt[2].d + eor $xa3.d,$xa3.d,@xt[3].d + eor $xb0.d,$xb0.d,@xt[4].d + eor $xb1.d,$xb1.d,@xt[5].d + eor $xb2.d,$xb2.d,@xt[6].d + eor $xb3.d,$xb3.d,@xt[7].d +___ + &load(@xt[0],@xt[1],@xt[2],@xt[3],@xt[4],@xt[5],@xt[6],@xt[7]); + &SCA_SAVE(0,2); +$code.=<<___; + eor $xc0.d,$xc0.d,@xt[0].d + eor $xc1.d,$xc1.d,@xt[1].d +___ + &SCA_SAVE(4,6); +$code.=<<___; + eor $xc2.d,$xc2.d,@xt[2].d + eor $xc3.d,$xc3.d,@xt[3].d +___ + &SCA_SAVE(8,10); +$code.=<<___; + eor $xd0.d,$xd0.d,@xt[4].d + eor $xd1.d,$xd1.d,@xt[5].d +___ + &SCA_SAVE(12,14); +$code.=<<___; + eor $xd2.d,$xd2.d,@xt[6].d + eor $xd3.d,$xd3.d,@xt[7].d +___ + &store($xa0,$xa1,$xa2,$xa3,$xb0,$xb1,$xb2,$xb3); + &store($xc0,$xc1,$xc2,$xc3,$xd0,$xd1,$xd2,$xd3); +$code.=<<___; +210: + incw @K[6], ALL, MUL #1 +___ +} + +sub SET_STATE_BAK() { + my $idx0 = shift; + my $idx1 = $idx0 + 1; + my $x0 = @sx[$idx0]; + my $xx0 = @sxx[$idx0]; + my $x1 = @sx[$idx1]; + my $xx1 = @sxx[$idx1]; + my $d = $idx0/2; + +$code.=<<___; + lsr $xx1,@K[$d],#32 + dup @mx[$idx0].s,@KL[$d] + dup @bak[$idx0].s,@KL[$d] + .if mixin == 1 + mov $x0,@KL[$d] + .endif + dup @mx[$idx1].s,$x1 + dup @bak[$idx1].s,$x1 +___ +} + +sub SET_STATE() { + my $idx0 = shift; + my $idx1 = $idx0 + 1; + my $x0 = @sx[$idx0]; + my $xx0 = @sxx[$idx0]; + my $x1 = @sx[$idx1]; + my $xx1 = @sxx[$idx1]; + my $d = $idx0/2; + +$code.=<<___; + lsr $xx1,@K[$d],#32 + dup @mx[$idx0].s,@KL[$d] + .if mixin == 1 + mov $x0,@KL[$d] + .endif + dup @mx[$idx1].s,$x1 +___ +} + +sub SVE_LOAD_STATES() { + &SET_STATE_BAK(0); + &SET_STATE_BAK(2); + &SET_STATE_BAK(4); + &SET_STATE_BAK(6); + &SET_STATE_BAK(8); + &SET_STATE(10); + &SET_STATE(14); +$code.=<<___; + .if mixin == 1 + add @sx[13],@KL[6],#1 + mov @sx[12],@KL[6] + index $zctr.s,@sx[13],1 + index @mx[12].s,@sx[13],1 + .else + index $zctr.s,@KL[6],1 + index @mx[12].s,@KL[6],1 + .endif + lsr @sxx[13],@K[6],#32 + dup @mx[13].s,@sx[13] +___ +} + +sub SVE2_LOAD_STATES() { + &SET_STATE_BAK(0); + &SET_STATE_BAK(2); + &SET_STATE_BAK(4); + &SET_STATE_BAK(6); + &SET_STATE_BAK(8); + &SET_STATE_BAK(10); + &SET_STATE_BAK(14); + +$code.=<<___; + .if mixin == 1 + add @sx[13],@KL[6],#1 + mov @sx[12],@KL[6] + index $zctr.s,@sx[13],1 + index @mx[12].s,@sx[13],1 + .else + index $zctr.s,@KL[6],1 + index @mx[12].s,@KL[6],1 + .endif + lsr @sxx[13],@K[6],#32 + dup @mx[13].s,@sx[13] + dup @bak[13].s,@sx[13] +___ +} + +sub chacha20_sve() { + my ($tmp) = (@sxx[0]); + +$code.=<<___; +.align 5 +100: + subs $tmp,$len,$veclen,lsl #6 + b.lt 110f + mov $len,$tmp + b.eq 101f + cmp $len,64 + b.lt 101f + mixin=1 +___ + &SVE_LOAD_STATES(); + &SVE_INNER_BLOCK(); + &SVE_ACCUM_STATES(); + &SVE_TRANSFORMS(); +$code.=<<___; + subs $len,$len,64 + b.gt 100b + b 110f +101: + mixin=0 +___ + &SVE_LOAD_STATES(); + &SVE_INNER_BLOCK(); + &SVE_ACCUM_STATES(); + &SVE_TRANSFORMS(); +$code.=<<___; +110: +___ +} + +sub chacha20_sve2() { + my ($tmp) = (@sxx[0]); + +$code.=<<___; +.align 5 +100: + subs $tmp,$len,$veclen,lsl #6 + b.lt 110f + mov $len,$tmp + b.eq 101f + cmp $len,64 + b.lt 101f + mixin=1 +___ + &SVE2_LOAD_STATES(); + &SVE2_INNER_BLOCK(); + &SVE2_ACCUM_STATES(); + &SVE_TRANSFORMS(); +$code.=<<___; + subs $len,$len,64 + b.gt 100b + b 110f +101: + mixin=0 +___ + &SVE2_LOAD_STATES(); + &SVE2_INNER_BLOCK(); + &SVE2_ACCUM_STATES(); + &SVE_TRANSFORMS(); +$code.=<<___; +110: +___ +} + + +{{{ + my ($tmp,$tmpw) = ("x6", "w6"); + my ($tmpw0,$tmp0,$tmpw1,$tmp1) = ("w9","x9", "w10","x10"); + my ($sve2flag) = ("x7"); + +$code.=<<___; +#include "arm_arch.h" + +.arch armv8-a + +.extern OPENSSL_armcap_P +.hidden OPENSSL_armcap_P + +.text + +.rodata +.align 5 +.type _${prefix}_consts,%object +_${prefix}_consts: +.Lchacha20_consts: +.quad 0x3320646e61707865,0x6b20657479622d32 // endian-neutral +.Lrot8: + .word 0x02010003,0x04040404,0x02010003,0x04040404 +.size _${prefix}_consts,.-_${prefix}_consts + +.previous + +.globl ChaCha20_ctr32_sve +.type ChaCha20_ctr32_sve,%function +.align 5 +ChaCha20_ctr32_sve: + AARCH64_VALID_CALL_TARGET + cntw $veclen, ALL, MUL #1 + cmp $len,$veclen,lsl #6 + b.lt .Lreturn + mov $sve2flag,0 + adrp $tmp,OPENSSL_armcap_P + ldr $tmpw,[$tmp,#:lo12:OPENSSL_armcap_P] + tst $tmpw,#ARMV8_SVE2 + b.eq 1f + mov $sve2flag,1 + b 2f +1: + cmp $veclen,4 + b.le .Lreturn + adrp $tmp,.Lrot8 + add $tmp,$tmp,#:lo12:.Lrot8 + ldp $tmpw0,$tmpw1,[$tmp] + index $rot8.s,$tmpw0,$tmpw1 +2: + AARCH64_SIGN_LINK_REGISTER + stp d8,d9,[sp,-192]! + stp d10,d11,[sp,16] + stp d12,d13,[sp,32] + stp d14,d15,[sp,48] + stp x16,x17,[sp,64] + stp x18,x19,[sp,80] + stp x20,x21,[sp,96] + stp x22,x23,[sp,112] + stp x24,x25,[sp,128] + stp x26,x27,[sp,144] + stp x28,x29,[sp,160] + str x30,[sp,176] + + adrp $tmp,.Lchacha20_consts + add $tmp,$tmp,#:lo12:.Lchacha20_consts + ldp @K[0],@K[1],[$tmp] + ldp @K[2],@K[3],[$key] + ldp @K[4],@K[5],[$key, 16] + ldp @K[6],@K[7],[$ctr] + ptrues p0.s,ALL +#ifdef __AARCH64EB__ + ror @K[2],@K[2],#32 + ror @K[3],@K[3],#32 + ror @K[4],@K[4],#32 + ror @K[5],@K[5],#32 + ror @K[6],@K[6],#32 + ror @K[7],@K[7],#32 +#endif + cbz $sve2flag, 1f +___ + &chacha20_sve2(); +$code.=<<___; + b 2f +1: +___ + &chacha20_sve(); +$code.=<<___; +2: + str @KL[6],[$ctr] + ldp d10,d11,[sp,16] + ldp d12,d13,[sp,32] + ldp d14,d15,[sp,48] + ldp x16,x17,[sp,64] + ldp x18,x19,[sp,80] + ldp x20,x21,[sp,96] + ldp x22,x23,[sp,112] + ldp x24,x25,[sp,128] + ldp x26,x27,[sp,144] + ldp x28,x29,[sp,160] + ldr x30,[sp,176] + ldp d8,d9,[sp],192 + AARCH64_VALIDATE_LINK_REGISTER +.Lreturn: + ret +.size ChaCha20_ctr32_sve,.-ChaCha20_ctr32_sve +___ + +}}} + +######################################## +{ +my %opcode_unpred = ( + "movprfx" => 0x0420BC00, + "eor" => 0x04a03000, + "add" => 0x04200000, + "orr" => 0x04603000, + "lsl" => 0x04209C00, + "lsr" => 0x04209400, + "incw" => 0x04B00000, + "xar" => 0x04203400, + "zip1" => 0x05206000, + "zip2" => 0x05206400, + "uzp1" => 0x05206800, + "uzp2" => 0x05206C00, + "index" => 0x04204C00, + "mov" => 0x05203800, + "dup" => 0x05203800, + "cntw" => 0x04A0E000, + "tbl" => 0x05203000); + +my %opcode_imm_unpred = ( + "dup" => 0x2538C000, + "index" => 0x04204400); + +my %opcode_scalar_pred = ( + "mov" => 0x0528A000, + "cpy" => 0x0528A000, + "st4w" => 0xE5606000, + "st1w" => 0xE5004000, + "ld1w" => 0xA5404000); + +my %opcode_gather_pred = ( + "ld1w" => 0x85204000); + +my %opcode_pred = ( + "eor" => 0x04190000, + "add" => 0x04000000, + "orr" => 0x04180000, + "whilelo" => 0x25200C00, + "whilelt" => 0x25200400, + "cntp" => 0x25208000, + "addvl" => 0x04205000, + "lsl" => 0x04038000, + "lsr" => 0x04018000, + "sel" => 0x0520C000, + "mov" => 0x0520C000, + "ptrue" => 0x2518E000, + "pfalse" => 0x2518E400, + "ptrues" => 0x2519E000, + "pnext" => 0x2519C400, + "ld4w" => 0xA560E000, + "st4w" => 0xE570E000, + "st1w" => 0xE500E000, + "ld1w" => 0xA540A000, + "ld1rw" => 0x8540C000, + "lasta" => 0x0520A000, + "revh" => 0x05258000, + "revb" => 0x05248000); + +my %tsize = ( + 'b' => 0, + 'h' => 1, + 's' => 2, + 'd' => 3); + +my %sf = ( + "w" => 0, + "x" => 1); + +my %pattern = ( + "POW2" => 0, + "VL1" => 1, + "VL2" => 2, + "VL3" => 3, + "VL4" => 4, + "VL5" => 5, + "VL6" => 6, + "VL7" => 7, + "VL8" => 8, + "VL16" => 9, + "VL32" => 10, + "VL64" => 11, + "VL128" => 12, + "VL256" => 13, + "MUL4" => 29, + "MUL3" => 30, + "ALL" => 31); + +sub create_verifier { + my $filename="./compile_sve.sh"; + +$scripts = <<___; +#! /bin/bash +set -e +CROSS_COMPILE=\${CROSS_COMPILE:-'aarch64-none-linux-gnu-'} + +[ -z "\$1" ] && exit 1 +ARCH=`uname -p | xargs echo -n` + +# need gcc-10 and above to compile SVE code +# change this according to your system during debugging +if [ \$ARCH == 'aarch64' ]; then + CC=gcc-11 + OBJDUMP=objdump +else + CC=\${CROSS_COMPILE}gcc + OBJDUMP=\${CROSS_COMPILE}objdump +fi +TMPFILE=/tmp/\$\$ +cat > \$TMPFILE.c << EOF +extern __attribute__((noinline, section("disasm_output"))) void dummy_func() +{ + asm("\$@\\t\\n"); +} +int main(int argc, char *argv[]) +{ +} +EOF +\$CC -march=armv8.2-a+sve+sve2 -o \$TMPFILE.out \$TMPFILE.c +\$OBJDUMP -d \$TMPFILE.out | awk -F"\\n" -v RS="\\n\\n" '\$1 ~ /dummy_func/' | awk 'FNR == 2 {printf "%s",\$2}' +rm \$TMPFILE.c \$TMPFILE.out +___ + open(FH, '>', $filename) or die $!; + print FH $scripts; + close(FH); + system("chmod a+x ./compile_sve.sh"); +} + +sub compile_sve { + return `./compile_sve.sh '@_'` +} + +sub verify_inst { + my ($code,$inst)=@_; + my $hexcode = (sprintf "%08x", $code); + + if ($debug_encoder == 1) { + my $expect=&compile_sve($inst); + if ($expect ne $hexcode) { + return (sprintf "%s // Encode Error! expect [%s] actual [%s]", $inst, $expect, $hexcode); + } + } + return (sprintf ".inst\t0x%s\t//%s", $hexcode, $inst); +} + +sub reg_code { + my $code = shift; + + if ($code == "zr") { + return "31"; + } + return $code; +} + +sub encode_size_imm() { + my ($mnemonic, $isize, $const)=@_; + my $esize = (8<<$tsize{$isize}); + my $tsize_imm = $esize + $const; + + if ($mnemonic eq "lsr" || $mnemonic eq "xar") { + $tsize_imm = 2*$esize - $const; + } + return (($tsize_imm>>5)<<22)|(($tsize_imm&0x1f)<<16); +} + +sub encode_shift_pred() { + my ($mnemonic, $isize, $const)=@_; + my $esize = (8<<$tsize{$isize}); + my $tsize_imm = $esize + $const; + + if ($mnemonic eq "lsr") { + $tsize_imm = 2*$esize - $const; + } + return (($tsize_imm>>5)<<22)|(($tsize_imm&0x1f)<<5); +} + +sub sve_unpred { + my ($mnemonic,$arg)=@_; + my $inst = (sprintf "%s %s", $mnemonic,$arg); + + if ($arg =~ m/z([0-9]+)\.([bhsd]),\s*\{\s*z([0-9]+)\.[bhsd].*\},\s*z([0-9]+)\.[bhsd].*/o) { + return &verify_inst($opcode_unpred{$mnemonic}|$1|($3<<5)|($tsize{$2}<<22)|($4<<16), + $inst) + } elsif ($arg =~ m/z([0-9]+)\.([bhsd]),\s*([zwx][0-9]+.*)/o) { + my $regd = $1; + my $isize = $2; + my $regs=$3; + + if (($mnemonic eq "lsl") || ($mnemonic eq "lsr")) { + if ($regs =~ m/z([0-9]+)[^,]*(?:,\s*#?([0-9]+))?/o + && ((8<<$tsize{$isize}) > $2)) { + return &verify_inst($opcode_unpred{$mnemonic}|$regd|($1<<5)|&encode_size_imm($mnemonic,$isize,$2), + $inst); + } + } elsif($regs =~ m/[wx]([0-9]+),\s*[wx]([0-9]+)/o) { + return &verify_inst($opcode_unpred{$mnemonic}|$regd|($tsize{$isize}<<22)|($1<<5)|($2<<16), $inst); + } elsif ($regs =~ m/[wx]([0-9]+),\s*#?([0-9]+)/o) { + return &verify_inst($opcode_imm_unpred{$mnemonic}|$regd|($tsize{$isize}<<22)|($1<<5)|($2<<16), $inst); + } elsif ($regs =~ m/[wx]([0-9]+)/o) { + return &verify_inst($opcode_unpred{$mnemonic}|$regd|($tsize{$isize}<<22)|($1<<5), $inst); + } else { + my $encoded_size = 0; + if (($mnemonic eq "add") || ($mnemonic =~ /zip./) || ($mnemonic =~ /uzp./) ) { + $encoded_size = ($tsize{$isize}<<22); + } + if ($regs =~ m/z([0-9]+)\.[bhsd],\s*z([0-9]+)\.[bhsd],\s*([0-9]+)/o && + $1 == $regd) { + return &verify_inst($opcode_unpred{$mnemonic}|$regd|($2<<5)|&encode_size_imm($mnemonic,$isize,$3), $inst); + } elsif ($regs =~ m/z([0-9]+)\.[bhsd],\s*z([0-9]+)\.[bhsd]/o) { + return &verify_inst($opcode_unpred{$mnemonic}|$regd|$encoded_size|($1<<5)|($2<<16), $inst); + } + } + } elsif ($arg =~ m/z([0-9]+)\.([bhsd]),\s*#?([0-9]+)/o) { + return &verify_inst($opcode_imm_unpred{$mnemonic}|$1|($3<<5)|($tsize{$2}<<22), + $inst) + } + sprintf "%s // fail to parse", $inst; +} + +sub sve_pred { + my ($mnemonic,,$arg)=@_; + my $inst = (sprintf "%s %s", $mnemonic,$arg); + + if ($arg =~ m/\{\s*z([0-9]+)\.([bhsd]).*\},\s*p([0-9])+(\/z)?,\s*\[(\s*[xs].*)\]/o) { + my $zt = $1; + my $size = $tsize{$2}; + my $pg = $3; + my $addr = $5; + my $xn = 31; + + if ($addr =~ m/x([0-9]+)\s*/o) { + $xn = $1; + } + + if ($mnemonic =~m/ld1r[bhwd]/o) { + $size = 0; + } + if ($addr =~ m/\w+\s*,\s*x([0-9]+),.*/o) { + return &verify_inst($opcode_scalar_pred{$mnemonic}|($size<<21)|$zt|($pg<<10)|($1<<16)|($xn<<5),$inst); + } elsif ($addr =~ m/\w+\s*,\s*z([0-9]+)\.s,\s*([US]\w+)/o) { + my $xs = ($2 eq "SXTW") ? 1 : 0; + return &verify_inst($opcode_gather_pred{$mnemonic}|($xs<<22)|$zt|($pg<<10)|($1<<16)|($xn<<5),$inst); + } elsif($addr =~ m/\w+\s*,\s*#?([0-9]+)/o) { + return &verify_inst($opcode_pred{$mnemonic}|($size<<21)|$zt|($pg<<10)|($1<<16)|($xn<<5),$inst); + } else { + return &verify_inst($opcode_pred{$mnemonic}|($size<<21)|$zt|($pg<<10)|($xn<<5),$inst); + } + } elsif ($arg =~ m/z([0-9]+)\.([bhsd]),\s*p([0-9]+)\/([mz]),\s*([zwx][0-9]+.*)/o) { + my $regd = $1; + my $isize = $2; + my $pg = $3; + my $mod = $4; + my $regs = $5; + + if (($mnemonic eq "lsl") || ($mnemonic eq "lsr")) { + if ($regs =~ m/z([0-9]+)[^,]*(?:,\s*#?([0-9]+))?/o + && $regd == $1 + && $mode == 'm' + && ((8<<$tsize{$isize}) > $2)) { + return &verify_inst($opcode_pred{$mnemonic}|$regd|($pg<<10)|&encode_shift_pred($mnemonic,$isize,$2), $inst); + } + } elsif($regs =~ m/[wx]([0-9]+)/o) { + return &verify_inst($opcode_scalar_pred{$mnemonic}|$regd|($tsize{$isize}<<22)|($pg<<10)|($1<<5), $inst); + } elsif ($regs =~ m/z([0-9]+)[^,]*(?:,\s*z([0-9]+))?/o) { + if ($mnemonic eq "sel") { + return &verify_inst($opcode_pred{$mnemonic}|$regd|($tsize{$isize}<<22)|($pg<<10)|($1<<5)|($2<<16), $inst); + } elsif ($mnemonic eq "mov") { + return &verify_inst($opcode_pred{$mnemonic}|$regd|($tsize{$isize}<<22)|($pg<<10)|($1<<5)|($regd<<16), $inst); + } elsif (length $2 > 0) { + return &verify_inst($opcode_pred{$mnemonic}|$regd|($tsize{$isize}<<22)|($pg<<10)|($2<<5), $inst); + } else { + return &verify_inst($opcode_pred{$mnemonic}|$regd|($tsize{$isize}<<22)|($pg<<10)|($1<<5), $inst); + } + } + } elsif ($arg =~ m/p([0-9]+)\.([bhsd]),\s*(\w+.*)/o) { + my $pg = $1; + my $isize = $2; + my $regs = $3; + + if ($regs =~ m/([wx])(zr|[0-9]+),\s*[wx](zr|[0-9]+)/o) { + return &verify_inst($opcode_pred{$mnemonic}|($tsize{$isize}<<22)|$pg|($sf{$1}<<12)|(®_code($2)<<5)|(®_code($3)<<16), $inst); + } elsif ($regs =~ m/p([0-9]+),\s*p([0-9]+)\.[bhsd]/o) { + return &verify_inst($opcode_pred{$mnemonic}|($tsize{$isize}<<22)|$pg|($1<<5), $inst); + } else { + return &verify_inst($opcode_pred{$mnemonic}|($tsize{$isize}<<22)|$pg|($pattern{$regs}<<5), $inst); + } + } elsif ($arg =~ m/p([0-9]+)\.([bhsd])/o) { + return &verify_inst($opcode_pred{$mnemonic}|$1, $inst); + } + + sprintf "%s // fail to parse", $inst; +} + +sub sve_other { + my ($mnemonic,$arg)=@_; + my $inst = (sprintf "%s %s", $mnemonic,$arg); + + if ($arg =~ m/x([0-9]+)[^,]*,\s*p([0-9]+)[^,]*,\s*p([0-9]+)\.([bhsd])/o) { + return &verify_inst($opcode_pred{$mnemonic}|($tsize{$4}<<22)|$1|($2<<10)|($3<<5), $inst); + } elsif ($arg =~ m/(x|w)([0-9]+)[^,]*,\s*p([0-9]+)[^,]*,\s*z([0-9]+)\.([bhsd])/o) { + return &verify_inst($opcode_pred{$mnemonic}|($tsize{$5}<<22)|$1|($3<<10)|($4<<5)|$2, $inst); + }elsif ($mnemonic =~ /inc[bhdw]/) { + if ($arg =~ m/x([0-9]+)[^,]*,\s*(\w+)[^,]*,\s*MUL\s*#?([0-9]+)/o) { + return &verify_inst($opcode_unpred{$mnemonic}|$1|($pattern{$2}<<5)|(2<<12)|(($3 - 1)<<16)|0xE000, $inst); + } elsif ($arg =~ m/z([0-9]+)[^,]*,\s*(\w+)[^,]*,\s*MUL\s*#?([0-9]+)/o) { + return &verify_inst($opcode_unpred{$mnemonic}|$1|($pattern{$2}<<5)|(($3 - 1)<<16)|0xC000, $inst); + } elsif ($arg =~ m/x([0-9]+)/o) { + return &verify_inst($opcode_unpred{$mnemonic}|$1|(31<<5)|(0<<16)|0xE000, $inst); + } + } elsif ($mnemonic =~ /cnt[bhdw]/) { + if ($arg =~ m/x([0-9]+)[^,]*,\s*(\w+)[^,]*,\s*MUL\s*#?([0-9]+)/o) { + return &verify_inst($opcode_unpred{$mnemonic}|$1|($pattern{$2}<<5)|(($3 - 1)<<16), $inst); + } + } elsif ($arg =~ m/x([0-9]+)[^,]*,\s*x([0-9]+)[^,]*,\s*#?([0-9]+)/o) { + return &verify_inst($opcode_pred{$mnemonic}|$1|($2<<16)|($3<<5), $inst); + } elsif ($arg =~ m/z([0-9]+)[^,]*,\s*z([0-9]+)/o) { + return &verify_inst($opcode_unpred{$mnemonic}|$1|($2<<5), $inst); + } + sprintf "%s // fail to parse", $inst; +} +} + +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/\/\// and !/^$/); + print; +} +close SELF; + +if ($debug_encoder == 1) { + &create_verifier(); +} + +foreach(split("\n",$code)) { + s/\`([^\`]*)\`/eval($1)/ge; + s/\b(\w+)\s+(z[0-9]+\.[bhsd],\s*[#zwx]?[0-9]+.*)/sve_unpred($1,$2)/ge; + s/\b(\w+)\s+(z[0-9]+\.[bhsd],\s*\{.*\},\s*z[0-9]+.*)/sve_unpred($1,$2)/ge; + s/\b(\w+)\s+(z[0-9]+\.[bhsd],\s*p[0-9].*)/sve_pred($1,$2)/ge; + s/\b(\w+[1-4]r[bhwd])\s+(\{\s*z[0-9]+.*\},\s*p[0-9]+.*)/sve_pred($1,$2)/ge; + s/\b(\w+[1-4][bhwd])\s+(\{\s*z[0-9]+.*\},\s*p[0-9]+.*)/sve_pred($1,$2)/ge; + s/\b(\w+)\s+(p[0-9]+\.[bhsd].*)/sve_pred($1,$2)/ge; + s/\b(movprfx|lasta|cntp|cnt[bhdw]|addvl|inc[bhdw])\s+((x|z|w).*)/sve_other($1,$2)/ge; + print $_,"\n"; +} + +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/chacha/asm/chacha-armv8.pl b/crypto/openssl/crypto/chacha/asm/chacha-armv8.pl index b109dd63f24c..c695d6c35c8b 100755 --- a/crypto/openssl/crypto/chacha/asm/chacha-armv8.pl +++ b/crypto/openssl/crypto/chacha/asm/chacha-armv8.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2016-2025 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -138,9 +138,11 @@ $code.=<<___; #ifndef __KERNEL__ .extern OPENSSL_armcap_P .hidden OPENSSL_armcap_P + +.extern ChaCha20_ctr32_sve #endif -.text +.rodata .align 5 .Lsigma: @@ -151,15 +153,15 @@ $code.=<<___; .long 0x02010003,0x06050407,0x0a09080b,0x0e0d0c0f .asciz "ChaCha20 for ARMv8, CRYPTOGAMS by \@dot-asm" -.globl ChaCha20_ctr32 -.type ChaCha20_ctr32,%function +.text + +.globl ChaCha20_ctr32_dflt +.type ChaCha20_ctr32_dflt,%function .align 5 -ChaCha20_ctr32: +ChaCha20_ctr32_dflt: AARCH64_SIGN_LINK_REGISTER - cbz $len,.Labort cmp $len,#192 b.lo .Lshort - #ifndef __KERNEL__ #ifdef __CHERI_PURE_CAPABILITY__ adrp c17,:got:OPENSSL_armcap_P @@ -169,6 +171,7 @@ ChaCha20_ctr32: adrp x17,OPENSSL_armcap_P ldr w17,[x17,#:lo12:OPENSSL_armcap_P] #endif +.Lcheck_neon: tst w17,#ARMV7_NEON b.ne .LChaCha20_neon #endif @@ -177,7 +180,8 @@ ChaCha20_ctr32: stp PTR(29),PTR(30),[PTRN(sp),#-(12*PTR_WIDTH)]! add PTR(29),PTRN(sp),#0 - adr @p[0],.Lsigma + adrp @p[0],.Lsigma + add @p[0],@p[0],:lo12:.Lsigma stp PTR(19),PTR(20),[PTRN(sp),#(2*PTR_WIDTH)] stp PTR(21),PTR(22),[PTRN(sp),#(4*PTR_WIDTH)] stp PTR(23),PTR(24),[PTRN(sp),#(6*PTR_WIDTH)] @@ -352,6 +356,41 @@ $code.=<<___; ldp PTR(29),PTR(30),[PTRN(sp)],#(12*PTR_WIDTH) AARCH64_VALIDATE_LINK_REGISTER ret +.size ChaCha20_ctr32_dflt,.-ChaCha20_ctr32_dflt + +.globl ChaCha20_ctr32 +.type ChaCha20_ctr32,%function +.align 5 +ChaCha20_ctr32: + AARCH64_SIGN_LINK_REGISTER + cbz $len,.Labort + cmp $len,#192 + b.lo .Lshort +#ifndef __KERNEL__ + adrp x17,OPENSSL_armcap_P + ldr w17,[x17,#:lo12:OPENSSL_armcap_P] + tst w17,#ARMV8_SVE + b.eq .Lcheck_neon + stp x29,x30,[sp,#-16]! + sub sp,sp,#16 + // SVE handling will inevitably increment the counter + // Neon/Scalar code that follows to process tail data needs to + // use new counter, unfortunately the input counter buffer + // pointed to by ctr is meant to be read-only per API contract + // we have to copy the buffer to stack to be writable by SVE + ldp x5,x6,[$ctr] + stp x5,x6,[sp] + mov $ctr,sp + bl ChaCha20_ctr32_sve + cbz $len,1f + bl ChaCha20_ctr32_dflt +1: + add sp,sp,#16 + ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER + ret +#endif + b .Lshort .size ChaCha20_ctr32,.-ChaCha20_ctr32 ___ @@ -445,7 +484,8 @@ ChaCha20_neon: stp PTR(29),PTR(30),[PTRN(sp),#-(12*PTR_WIDTH)]! add PTR(29),PTRN(sp),#0 - adr @p[0],.Lsigma + adrp @p[0],.Lsigma + add @p[0],@p[0],:lo12:.Lsigma stp PTR(19),PTR(20),[PTRN(sp),#(2*PTR_WIDTH)] stp PTR(21),PTR(22),[PTRN(sp),#(4*PTR_WIDTH)] stp PTR(23),PTR(24),[PTRN(sp),#(6*PTR_WIDTH)] @@ -856,7 +896,8 @@ ChaCha20_512_neon: stp PTR(29),PTR(30),[PTRN(sp),#-(12*PTR_WIDTH)]! add PTR(29),PTRN(sp),#0 - adr @p[0],.Lsigma + adrp @p[0],.Lsigma + add @p[0],@p[0],:lo12:.Lsigma stp PTR(19),PTR(20),[PTRN(sp),#(2*PTR_WIDTH)] stp PTR(21),PTR(22),[PTRN(sp),#(4*PTR_WIDTH)] stp PTR(23),PTR(24),[PTRN(sp),#(6*PTR_WIDTH)] diff --git a/crypto/openssl/crypto/chacha/asm/chacha-loongarch64.pl b/crypto/openssl/crypto/chacha/asm/chacha-loongarch64.pl new file mode 100644 index 000000000000..d608c909a6fc --- /dev/null +++ b/crypto/openssl/crypto/chacha/asm/chacha-loongarch64.pl @@ -0,0 +1,1439 @@ +#! /usr/bin/env perl +# Author: Min Zhou +# Copyright 2023-2025 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +use strict; + +my $code; + +# Here is the scalar register layout for LoongArch. +my ($zero,$ra,$tp,$sp,$fp)=map("\$r$_",(0..3,22)); +my ($a0,$a1,$a2,$a3,$a4,$a5,$a6,$a7)=map("\$r$_",(4..11)); +my ($t0,$t1,$t2,$t3,$t4,$t5,$t6,$t7,$t8,$x)=map("\$r$_",(12..21)); +my ($s0,$s1,$s2,$s3,$s4,$s5,$s6,$s7,$s8)=map("\$r$_",(23..31)); + +# The saved floating-point registers in the LP64D ABI. In LoongArch +# with vector extension, the low 64 bits of a vector register alias with +# the corresponding FPR. So we must save and restore the corresponding +# FPR if we'll write into a vector register. The ABI only requires +# saving and restoring the FPR (i.e. 64 bits of the corresponding vector +# register), not the entire vector register. +my ($fs0,$fs1,$fs2,$fs3,$fs4,$fs5,$fs6,$fs7)=map("\$f$_",(24..31)); + +# Here is the 128-bit vector register layout for LSX extension. +my ($vr0,$vr1,$vr2,$vr3,$vr4,$vr5,$vr6,$vr7,$vr8,$vr9,$vr10, + $vr11,$vr12,$vr13,$vr14,$vr15,$vr16,$vr17,$vr18,$vr19, + $vr20,$vr21,$vr22,$vr23,$vr24,$vr25,$vr26,$vr27,$vr28, + $vr29,$vr30,$vr31)=map("\$vr$_",(0..31)); + +# Here is the 256-bit vector register layout for LASX extension. +my ($xr0,$xr1,$xr2,$xr3,$xr4,$xr5,$xr6,$xr7,$xr8,$xr9,$xr10, + $xr11,$xr12,$xr13,$xr14,$xr15,$xr16,$xr17,$xr18,$xr19, + $xr20,$xr21,$xr22,$xr23,$xr24,$xr25,$xr26,$xr27,$xr28, + $xr29,$xr30,$xr31)=map("\$xr$_",(0..31)); + +# $output is the last argument if it looks like a file (it has an extension) +my $output; +$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +open STDOUT,">$output"; + +# Input parameter block +my ($out, $inp, $len, $key, $counter) = ($a0, $a1, $a2, $a3, $a4); + +$code .= < +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# - RV64I +# - RISC-V Vector ('V') with VLEN >= 128 +# - RISC-V Basic Bit-manipulation extension ('Zbb') +# - RISC-V Zicclsm(Main memory supports misaligned loads/stores) +# Optional: +# - RISC-V Vector Cryptography Bit-manipulation extension ('Zvkb') + +use strict; +use warnings; + +use FindBin qw($Bin); +use lib "$Bin"; +use lib "$Bin/../../perlasm"; +use riscv; + +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +my $use_zvkb = $flavour && $flavour =~ /zvkb/i ? 1 : 0; +my $isaext = "_v_zbb" . ( $use_zvkb ? "_zvkb" : "" ); + +$output and open STDOUT, ">$output"; + +my $code = <<___; +.text +___ + +# void ChaCha20_ctr32@{[$isaext]}(unsigned char *out, const unsigned char *inp, +# size_t len, const unsigned int key[8], +# const unsigned int counter[4]); +################################################################################ +my ( $OUTPUT, $INPUT, $LEN, $KEY, $COUNTER ) = ( "a0", "a1", "a2", "a3", "a4" ); +my ( $CONST_DATA0, $CONST_DATA1, $CONST_DATA2, $CONST_DATA3 ) = ( "a5", "a6", + "a7", "s0" ); +my ( $KEY0, $KEY1, $KEY2, $KEY3, $KEY4, $KEY5, $KEY6, $KEY7, $COUNTER0, + $COUNTER1, $NONCE0, $NONCE1) = ( "s1", "s2", "s3", "s4", "s5", "s6", "s7", + "s8", "s9", "s10", "s11", "t0" ); +my ( $STATE0, $STATE1, $STATE2, $STATE3, + $STATE4, $STATE5, $STATE6, $STATE7, + $STATE8, $STATE9, $STATE10, $STATE11, + $STATE12, $STATE13, $STATE14, $STATE15) = ( + $CONST_DATA0, $CONST_DATA1, $CONST_DATA2, $CONST_DATA3, + $KEY0, $KEY1, $KEY2, $KEY3, + $KEY4, $KEY5, $KEY6, $KEY7, + $COUNTER0, $COUNTER1, $NONCE0, $NONCE1 ); +my ( $VL ) = ( "t1" ); +my ( $CURRENT_COUNTER ) = ( "t2" ); +my ( $T0 ) = ( "t3" ); +my ( $T1 ) = ( "t4" ); +my ( $T2 ) = ( "t5" ); +my ( $T3 ) = ( "t6" ); +my ( + $V0, $V1, $V2, $V3, $V4, $V5, $V6, $V7, $V8, $V9, $V10, + $V11, $V12, $V13, $V14, $V15, $V16, $V17, $V18, $V19, $V20, $V21, + $V22, $V23, $V24, $V25, $V26, $V27, $V28, $V29, $V30, $V31, +) = map( "v$_", ( 0 .. 31 ) ); + +sub chacha_sub_round { + my ( + $A0, $B0, $C0, + $A1, $B1, $C1, + $A2, $B2, $C2, + $A3, $B3, $C3, + + $S_A0, $S_B0, $S_C0, + $S_A1, $S_B1, $S_C1, + $S_A2, $S_B2, $S_C2, + $S_A3, $S_B3, $S_C3, + + $ROL_SHIFT, + + $V_T0, $V_T1, $V_T2, $V_T3, + ) = @_; + + # a += b; c ^= a; + my $code = <<___; + @{[vadd_vv $A0, $A0, $B0]} + add $S_A0, $S_A0, $S_B0 + @{[vadd_vv $A1, $A1, $B1]} + add $S_A1, $S_A1, $S_B1 + @{[vadd_vv $A2, $A2, $B2]} + add $S_A2, $S_A2, $S_B2 + @{[vadd_vv $A3, $A3, $B3]} + add $S_A3, $S_A3, $S_B3 + @{[vxor_vv $C0, $C0, $A0]} + xor $S_C0, $S_C0, $S_A0 + @{[vxor_vv $C1, $C1, $A1]} + xor $S_C1, $S_C1, $S_A1 + @{[vxor_vv $C2, $C2, $A2]} + xor $S_C2, $S_C2, $S_A2 + @{[vxor_vv $C3, $C3, $A3]} + xor $S_C3, $S_C3, $S_A3 +___ + + # c <<<= $ROL_SHIFT; + if ($use_zvkb) { + my $ror_part = <<___; + @{[vror_vi $C0, $C0, 32 - $ROL_SHIFT]} + @{[roriw $S_C0, $S_C0, 32 - $ROL_SHIFT]} + @{[vror_vi $C1, $C1, 32 - $ROL_SHIFT]} + @{[roriw $S_C1, $S_C1, 32 - $ROL_SHIFT]} + @{[vror_vi $C2, $C2, 32 - $ROL_SHIFT]} + @{[roriw $S_C2, $S_C2, 32 - $ROL_SHIFT]} + @{[vror_vi $C3, $C3, 32 - $ROL_SHIFT]} + @{[roriw $S_C3, $S_C3, 32 - $ROL_SHIFT]} +___ + + $code .= $ror_part; + } else { + my $ror_part = <<___; + @{[vsll_vi $V_T0, $C0, $ROL_SHIFT]} + @{[vsll_vi $V_T1, $C1, $ROL_SHIFT]} + @{[vsll_vi $V_T2, $C2, $ROL_SHIFT]} + @{[vsll_vi $V_T3, $C3, $ROL_SHIFT]} + @{[vsrl_vi $C0, $C0, 32 - $ROL_SHIFT]} + @{[vsrl_vi $C1, $C1, 32 - $ROL_SHIFT]} + @{[vsrl_vi $C2, $C2, 32 - $ROL_SHIFT]} + @{[vsrl_vi $C3, $C3, 32 - $ROL_SHIFT]} + @{[vor_vv $C0, $C0, $V_T0]} + @{[roriw $S_C0, $S_C0, 32 - $ROL_SHIFT]} + @{[vor_vv $C1, $C1, $V_T1]} + @{[roriw $S_C1, $S_C1, 32 - $ROL_SHIFT]} + @{[vor_vv $C2, $C2, $V_T2]} + @{[roriw $S_C2, $S_C2, 32 - $ROL_SHIFT]} + @{[vor_vv $C3, $C3, $V_T3]} + @{[roriw $S_C3, $S_C3, 32 - $ROL_SHIFT]} +___ + + $code .= $ror_part; + } + + return $code; +} + +sub chacha_quad_round_group { + my ( + $A0, $B0, $C0, $D0, + $A1, $B1, $C1, $D1, + $A2, $B2, $C2, $D2, + $A3, $B3, $C3, $D3, + + $S_A0, $S_B0, $S_C0, $S_D0, + $S_A1, $S_B1, $S_C1, $S_D1, + $S_A2, $S_B2, $S_C2, $S_D2, + $S_A3, $S_B3, $S_C3, $S_D3, + + $V_T0, $V_T1, $V_T2, $V_T3, + ) = @_; + + my $code = <<___; + # a += b; d ^= a; d <<<= 16; + @{[chacha_sub_round + $A0, $B0, $D0, + $A1, $B1, $D1, + $A2, $B2, $D2, + $A3, $B3, $D3, + $S_A0, $S_B0, $S_D0, + $S_A1, $S_B1, $S_D1, + $S_A2, $S_B2, $S_D2, + $S_A3, $S_B3, $S_D3, + 16, + $V_T0, $V_T1, $V_T2, $V_T3]} + # c += d; b ^= c; b <<<= 12; + @{[chacha_sub_round + $C0, $D0, $B0, + $C1, $D1, $B1, + $C2, $D2, $B2, + $C3, $D3, $B3, + $S_C0, $S_D0, $S_B0, + $S_C1, $S_D1, $S_B1, + $S_C2, $S_D2, $S_B2, + $S_C3, $S_D3, $S_B3, + 12, + $V_T0, $V_T1, $V_T2, $V_T3]} + # a += b; d ^= a; d <<<= 8; + @{[chacha_sub_round + $A0, $B0, $D0, + $A1, $B1, $D1, + $A2, $B2, $D2, + $A3, $B3, $D3, + $S_A0, $S_B0, $S_D0, + $S_A1, $S_B1, $S_D1, + $S_A2, $S_B2, $S_D2, + $S_A3, $S_B3, $S_D3, + 8, + $V_T0, $V_T1, $V_T2, $V_T3]} + # c += d; b ^= c; b <<<= 7; + @{[chacha_sub_round + $C0, $D0, $B0, + $C1, $D1, $B1, + $C2, $D2, $B2, + $C3, $D3, $B3, + $S_C0, $S_D0, $S_B0, + $S_C1, $S_D1, $S_B1, + $S_C2, $S_D2, $S_B2, + $S_C3, $S_D3, $S_B3, + 7, + $V_T0, $V_T1, $V_T2, $V_T3]} +___ + + return $code; +} + +$code .= <<___; +.p2align 3 +.globl ChaCha20_ctr32@{[$isaext]} +.type ChaCha20_ctr32@{[$isaext]},\@function +ChaCha20_ctr32@{[$isaext]}: + addi sp, sp, -96 + sd s0, 0(sp) + sd s1, 8(sp) + sd s2, 16(sp) + sd s3, 24(sp) + sd s4, 32(sp) + sd s5, 40(sp) + sd s6, 48(sp) + sd s7, 56(sp) + sd s8, 64(sp) + sd s9, 72(sp) + sd s10, 80(sp) + sd s11, 88(sp) + addi sp, sp, -64 + + lw $CURRENT_COUNTER, 0($COUNTER) + +.Lblock_loop: + # We will use the scalar ALU for 1 chacha block. + srli $T0, $LEN, 6 + @{[vsetvli $VL, $T0, "e32", "m1", "ta", "ma"]} + slli $T1, $VL, 6 + bltu $T1, $LEN, 1f + # Since there is no more chacha block existed, we need to split 1 block + # from vector ALU. + addi $T1, $VL, -1 + @{[vsetvli $VL, $T1, "e32", "m1", "ta", "ma"]} +1: + + #### chacha block data + # init chacha const states into $V0~$V3 + # "expa" little endian + li $CONST_DATA0, 0x61707865 + @{[vmv_v_x $V0, $CONST_DATA0]} + # "nd 3" little endian + li $CONST_DATA1, 0x3320646e + @{[vmv_v_x $V1, $CONST_DATA1]} + # "2-by" little endian + li $CONST_DATA2, 0x79622d32 + @{[vmv_v_x $V2, $CONST_DATA2]} + # "te k" little endian + li $CONST_DATA3, 0x6b206574 + lw $KEY0, 0($KEY) + @{[vmv_v_x $V3, $CONST_DATA3]} + + # init chacha key states into $V4~$V11 + lw $KEY1, 4($KEY) + @{[vmv_v_x $V4, $KEY0]} + lw $KEY2, 8($KEY) + @{[vmv_v_x $V5, $KEY1]} + lw $KEY3, 12($KEY) + @{[vmv_v_x $V6, $KEY2]} + lw $KEY4, 16($KEY) + @{[vmv_v_x $V7, $KEY3]} + lw $KEY5, 20($KEY) + @{[vmv_v_x $V8, $KEY4]} + lw $KEY6, 24($KEY) + @{[vmv_v_x $V9, $KEY5]} + lw $KEY7, 28($KEY) + @{[vmv_v_x $V10, $KEY6]} + @{[vmv_v_x $V11, $KEY7]} + + # init chacha key states into $V12~$V13 + lw $COUNTER1, 4($COUNTER) + @{[vid_v $V12]} + lw $NONCE0, 8($COUNTER) + @{[vadd_vx $V12, $V12, $CURRENT_COUNTER]} + lw $NONCE1, 12($COUNTER) + @{[vmv_v_x $V13, $COUNTER1]} + add $COUNTER0, $CURRENT_COUNTER, $VL + + # init chacha nonce states into $V14~$V15 + @{[vmv_v_x $V14, $NONCE0]} + @{[vmv_v_x $V15, $NONCE1]} + + li $T0, 64 + # load the top-half of input data into $V16~$V23 + @{[vlsseg_nf_e32_v 8, $V16, $INPUT, $T0]} + + # till now in block_loop, we used: + # - $V0~$V15 for chacha states. + # - $V16~$V23 for top-half of input data. + # - $V24~$V31 haven't been used yet. + + # 20 round groups + li $T0, 10 +.Lround_loop: + # we can use $V24~$V31 as temporary registers in round_loop. + addi $T0, $T0, -1 + @{[chacha_quad_round_group + $V0, $V4, $V8, $V12, + $V1, $V5, $V9, $V13, + $V2, $V6, $V10, $V14, + $V3, $V7, $V11, $V15, + $STATE0, $STATE4, $STATE8, $STATE12, + $STATE1, $STATE5, $STATE9, $STATE13, + $STATE2, $STATE6, $STATE10, $STATE14, + $STATE3, $STATE7, $STATE11, $STATE15, + $V24, $V25, $V26, $V27]} + @{[chacha_quad_round_group + $V3, $V4, $V9, $V14, + $V0, $V5, $V10, $V15, + $V1, $V6, $V11, $V12, + $V2, $V7, $V8, $V13, + $STATE3, $STATE4, $STATE9, $STATE14, + $STATE0, $STATE5, $STATE10, $STATE15, + $STATE1, $STATE6, $STATE11, $STATE12, + $STATE2, $STATE7, $STATE8, $STATE13, + $V24, $V25, $V26, $V27]} + bnez $T0, .Lround_loop + + li $T0, 64 + # load the bottom-half of input data into $V24~$V31 + addi $T1, $INPUT, 32 + @{[vlsseg_nf_e32_v 8, $V24, $T1, $T0]} + + # now, there are no free vector registers until the round_loop exits. + + # add chacha top-half initial block states + # "expa" little endian + li $T0, 0x61707865 + @{[vadd_vx $V0, $V0, $T0]} + add $STATE0, $STATE0, $T0 + # "nd 3" little endian + li $T1, 0x3320646e + @{[vadd_vx $V1, $V1, $T1]} + add $STATE1, $STATE1, $T1 + lw $T0, 0($KEY) + # "2-by" little endian + li $T2, 0x79622d32 + @{[vadd_vx $V2, $V2, $T2]} + add $STATE2, $STATE2, $T2 + lw $T1, 4($KEY) + # "te k" little endian + li $T3, 0x6b206574 + @{[vadd_vx $V3, $V3, $T3]} + add $STATE3, $STATE3, $T3 + lw $T2, 8($KEY) + @{[vadd_vx $V4, $V4, $T0]} + add $STATE4, $STATE4, $T0 + lw $T3, 12($KEY) + @{[vadd_vx $V5, $V5, $T1]} + add $STATE5, $STATE5, $T1 + @{[vadd_vx $V6, $V6, $T2]} + add $STATE6, $STATE6, $T2 + @{[vadd_vx $V7, $V7, $T3]} + add $STATE7, $STATE7, $T3 + + # xor with the top-half input + @{[vxor_vv $V16, $V16, $V0]} + sw $STATE0, 0(sp) + sw $STATE1, 4(sp) + @{[vxor_vv $V17, $V17, $V1]} + sw $STATE2, 8(sp) + sw $STATE3, 12(sp) + @{[vxor_vv $V18, $V18, $V2]} + sw $STATE4, 16(sp) + sw $STATE5, 20(sp) + @{[vxor_vv $V19, $V19, $V3]} + sw $STATE6, 24(sp) + sw $STATE7, 28(sp) + @{[vxor_vv $V20, $V20, $V4]} + lw $T0, 16($KEY) + @{[vxor_vv $V21, $V21, $V5]} + lw $T1, 20($KEY) + @{[vxor_vv $V22, $V22, $V6]} + lw $T2, 24($KEY) + @{[vxor_vv $V23, $V23, $V7]} + + # save the top-half of output from $V16~$V23 + li $T3, 64 + @{[vssseg_nf_e32_v 8, $V16, $OUTPUT, $T3]} + + # add chacha bottom-half initial block states + @{[vadd_vx $V8, $V8, $T0]} + add $STATE8, $STATE8, $T0 + lw $T3, 28($KEY) + @{[vadd_vx $V9, $V9, $T1]} + add $STATE9, $STATE9, $T1 + lw $T0, 4($COUNTER) + @{[vadd_vx $V10, $V10, $T2]} + add $STATE10, $STATE10, $T2 + lw $T1, 8($COUNTER) + @{[vadd_vx $V11, $V11, $T3]} + add $STATE11, $STATE11, $T3 + lw $T2, 12($COUNTER) + @{[vid_v $V0]} + add $STATE12, $STATE12, $CURRENT_COUNTER + @{[vadd_vx $V12, $V12, $CURRENT_COUNTER]} + add $STATE12, $STATE12, $VL + @{[vadd_vx $V13, $V13, $T0]} + add $STATE13, $STATE13, $T0 + @{[vadd_vx $V14, $V14, $T1]} + add $STATE14, $STATE14, $T1 + @{[vadd_vx $V15, $V15, $T2]} + add $STATE15, $STATE15, $T2 + @{[vadd_vv $V12, $V12, $V0]} + # xor with the bottom-half input + @{[vxor_vv $V24, $V24, $V8]} + sw $STATE8, 32(sp) + @{[vxor_vv $V25, $V25, $V9]} + sw $STATE9, 36(sp) + @{[vxor_vv $V26, $V26, $V10]} + sw $STATE10, 40(sp) + @{[vxor_vv $V27, $V27, $V11]} + sw $STATE11, 44(sp) + @{[vxor_vv $V29, $V29, $V13]} + sw $STATE12, 48(sp) + @{[vxor_vv $V28, $V28, $V12]} + sw $STATE13, 52(sp) + @{[vxor_vv $V30, $V30, $V14]} + sw $STATE14, 56(sp) + @{[vxor_vv $V31, $V31, $V15]} + sw $STATE15, 60(sp) + + # save the bottom-half of output from $V24~$V31 + li $T0, 64 + addi $T1, $OUTPUT, 32 + @{[vssseg_nf_e32_v 8, $V24, $T1, $T0]} + + # the computed vector parts: `64 * VL` + slli $T0, $VL, 6 + + add $INPUT, $INPUT, $T0 + add $OUTPUT, $OUTPUT, $T0 + sub $LEN, $LEN, $T0 + add $CURRENT_COUNTER, $CURRENT_COUNTER, $VL + + # process the scalar data block + addi $CURRENT_COUNTER, $CURRENT_COUNTER, 1 + li $T0, 64 + @{[minu $T1, $LEN, $T0]} + sub $LEN, $LEN, $T1 + mv $T2, sp +.Lscalar_data_loop: + @{[vsetvli $VL, $T1, "e8", "m8", "ta", "ma"]} + # from this on, vector registers are grouped with lmul = 8 + @{[vle8_v $V8, $INPUT]} + @{[vle8_v $V16, $T2]} + @{[vxor_vv $V8, $V8, $V16]} + @{[vse8_v $V8, $OUTPUT]} + add $INPUT, $INPUT, $VL + add $OUTPUT, $OUTPUT, $VL + add $T2, $T2, $VL + sub $T1, $T1, $VL + bnez $T1, .Lscalar_data_loop + + bnez $LEN, .Lblock_loop + + addi sp, sp, 64 + ld s0, 0(sp) + ld s1, 8(sp) + ld s2, 16(sp) + ld s3, 24(sp) + ld s4, 32(sp) + ld s5, 40(sp) + ld s6, 48(sp) + ld s7, 56(sp) + ld s8, 64(sp) + ld s9, 72(sp) + ld s10, 80(sp) + ld s11, 88(sp) + addi sp, sp, 96 + + ret +.size ChaCha20_ctr32@{[$isaext]},.-ChaCha20_ctr32@{[$isaext]} +___ + +print $code; + +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/chacha/asm/chacha-x86_64.pl b/crypto/openssl/crypto/chacha/asm/chacha-x86_64.pl index cdb900c037fd..5506e33fa2d0 100755 --- a/crypto/openssl/crypto/chacha/asm/chacha-x86_64.pl +++ b/crypto/openssl/crypto/chacha/asm/chacha-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2016-2021 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2016-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -102,6 +102,7 @@ $code.=<<___; .extern OPENSSL_ia32cap_P +.section .rodata align=64 .align 64 .Lzero: .long 0,0,0,0 @@ -133,6 +134,7 @@ $code.=<<___; .Lsigma: .asciz "expand 32-byte k" .asciz "ChaCha20 for x86_64, CRYPTOGAMS by " +.previous ___ sub AUTOLOAD() # thunk [simplified] 32-bit style perlasm diff --git a/crypto/openssl/crypto/chacha/asm/chachap10-ppc.pl b/crypto/openssl/crypto/chacha/asm/chachap10-ppc.pl new file mode 100755 index 000000000000..a6bff8db38ad --- /dev/null +++ b/crypto/openssl/crypto/chacha/asm/chachap10-ppc.pl @@ -0,0 +1,1290 @@ +#! /usr/bin/env perl +# Copyright 2016-2024 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +# +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# October 2015 +# +# ChaCha20 for PowerPC/AltiVec. +# +# June 2018 +# +# Add VSX 2.07 code path. Original 3xAltiVec+1xIALU is well-suited for +# processors that can't issue more than one vector instruction per +# cycle. But POWER8 (and POWER9) can issue a pair, and vector-only 4x +# interleave would perform better. Incidentally PowerISA 2.07 (first +# implemented by POWER8) defined new usable instructions, hence 4xVSX +# code path... +# +# Performance in cycles per byte out of large buffer. +# +# IALU/gcc-4.x 3xAltiVec+1xIALU 4xVSX +# +# Freescale e300 13.6/+115% - - +# PPC74x0/G4e 6.81/+310% 3.81 - +# PPC970/G5 9.29/+160% ? - +# POWER7 8.62/+61% 3.35 - +# POWER8 8.70/+51% 2.91 2.09 +# POWER9 8.80/+29% 4.44(*) 2.45(**) +# +# (*) this is trade-off result, it's possible to improve it, but +# then it would negatively affect all others; +# (**) POWER9 seems to be "allergic" to mixing vector and integer +# instructions, which is why switch to vector-only code pays +# off that much; + +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +if ($flavour =~ /64/) { + $SIZE_T =8; + $LRSAVE =2*$SIZE_T; + $STU ="stdu"; + $POP ="ld"; + $PUSH ="std"; + $UCMP ="cmpld"; +} elsif ($flavour =~ /32/) { + $SIZE_T =4; + $LRSAVE =$SIZE_T; + $STU ="stwu"; + $POP ="lwz"; + $PUSH ="stw"; + $UCMP ="cmplw"; +} else { die "nonsense $flavour"; } + +$LITTLE_ENDIAN = ($flavour=~/le$/) ? 1 : 0; + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or +die "can't locate ppc-xlate.pl"; + +open STDOUT,"| $^X $xlate $flavour \"$output\"" + or die "can't call $xlate: $!"; + +$LOCALS=6*$SIZE_T; +$FRAME=$LOCALS+64+18*$SIZE_T; # 64 is for local variables + +sub AUTOLOAD() # thunk [simplified] x86-style perlasm +{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./; + $code .= "\t$opcode\t".join(',',@_)."\n"; +} + +my $sp = "r1"; + +my ($out,$inp,$len,$key,$ctr) = map("r$_",(3..7)); + + +{{{ +my ($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3, + $xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3) = map("v$_",(0..15)); +my @K = map("v$_",(16..19)); +my $CTR = "v26"; +my ($xt0,$xt1,$xt2,$xt3) = map("v$_",(27..30)); +my ($sixteen,$twelve,$eight,$seven) = ($xt0,$xt1,$xt2,$xt3); +my $beperm = "v31"; + +my ($x00,$x10,$x20,$x30) = (0, map("r$_",(8..10))); + +my $FRAME=$LOCALS+64+7*16; # 7*16 is for v26-v31 offload + + +sub VSX_lane_ROUND_4x { +my ($a0,$b0,$c0,$d0)=@_; +my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0)); +my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1)); +my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2)); +my @x=map("\"v$_\"",(0..15)); + + ( + "&vadduwm (@x[$a0],@x[$a0],@x[$b0])", # Q1 + "&vadduwm (@x[$a1],@x[$a1],@x[$b1])", # Q2 + "&vadduwm (@x[$a2],@x[$a2],@x[$b2])", # Q3 + "&vadduwm (@x[$a3],@x[$a3],@x[$b3])", # Q4 + "&vxor (@x[$d0],@x[$d0],@x[$a0])", + "&vxor (@x[$d1],@x[$d1],@x[$a1])", + "&vxor (@x[$d2],@x[$d2],@x[$a2])", + "&vxor (@x[$d3],@x[$d3],@x[$a3])", + "&vrlw (@x[$d0],@x[$d0],'$sixteen')", + "&vrlw (@x[$d1],@x[$d1],'$sixteen')", + "&vrlw (@x[$d2],@x[$d2],'$sixteen')", + "&vrlw (@x[$d3],@x[$d3],'$sixteen')", + + "&vadduwm (@x[$c0],@x[$c0],@x[$d0])", + "&vadduwm (@x[$c1],@x[$c1],@x[$d1])", + "&vadduwm (@x[$c2],@x[$c2],@x[$d2])", + "&vadduwm (@x[$c3],@x[$c3],@x[$d3])", + "&vxor (@x[$b0],@x[$b0],@x[$c0])", + "&vxor (@x[$b1],@x[$b1],@x[$c1])", + "&vxor (@x[$b2],@x[$b2],@x[$c2])", + "&vxor (@x[$b3],@x[$b3],@x[$c3])", + "&vrlw (@x[$b0],@x[$b0],'$twelve')", + "&vrlw (@x[$b1],@x[$b1],'$twelve')", + "&vrlw (@x[$b2],@x[$b2],'$twelve')", + "&vrlw (@x[$b3],@x[$b3],'$twelve')", + + "&vadduwm (@x[$a0],@x[$a0],@x[$b0])", + "&vadduwm (@x[$a1],@x[$a1],@x[$b1])", + "&vadduwm (@x[$a2],@x[$a2],@x[$b2])", + "&vadduwm (@x[$a3],@x[$a3],@x[$b3])", + "&vxor (@x[$d0],@x[$d0],@x[$a0])", + "&vxor (@x[$d1],@x[$d1],@x[$a1])", + "&vxor (@x[$d2],@x[$d2],@x[$a2])", + "&vxor (@x[$d3],@x[$d3],@x[$a3])", + "&vrlw (@x[$d0],@x[$d0],'$eight')", + "&vrlw (@x[$d1],@x[$d1],'$eight')", + "&vrlw (@x[$d2],@x[$d2],'$eight')", + "&vrlw (@x[$d3],@x[$d3],'$eight')", + + "&vadduwm (@x[$c0],@x[$c0],@x[$d0])", + "&vadduwm (@x[$c1],@x[$c1],@x[$d1])", + "&vadduwm (@x[$c2],@x[$c2],@x[$d2])", + "&vadduwm (@x[$c3],@x[$c3],@x[$d3])", + "&vxor (@x[$b0],@x[$b0],@x[$c0])", + "&vxor (@x[$b1],@x[$b1],@x[$c1])", + "&vxor (@x[$b2],@x[$b2],@x[$c2])", + "&vxor (@x[$b3],@x[$b3],@x[$c3])", + "&vrlw (@x[$b0],@x[$b0],'$seven')", + "&vrlw (@x[$b1],@x[$b1],'$seven')", + "&vrlw (@x[$b2],@x[$b2],'$seven')", + "&vrlw (@x[$b3],@x[$b3],'$seven')" + ); +} + +$code.=<<___; + +.globl .ChaCha20_ctr32_vsx_p10 +.align 5 +.ChaCha20_ctr32_vsx_p10: + ${UCMP}i $len,255 + ble .Not_greater_than_8x + b ChaCha20_ctr32_vsx_8x +.Not_greater_than_8x: + $STU $sp,-$FRAME($sp) + mflr r0 + li r10,`15+$LOCALS+64` + li r11,`31+$LOCALS+64` + mfspr r12,256 + stvx v26,r10,$sp + addi r10,r10,32 + stvx v27,r11,$sp + addi r11,r11,32 + stvx v28,r10,$sp + addi r10,r10,32 + stvx v29,r11,$sp + addi r11,r11,32 + stvx v30,r10,$sp + stvx v31,r11,$sp + stw r12,`$FRAME-4`($sp) # save vrsave + li r12,-4096+63 + $PUSH r0, `$FRAME+$LRSAVE`($sp) + mtspr 256,r12 # preserve 29 AltiVec registers + + bl Lconsts # returns pointer Lsigma in r12 + lvx_4w @K[0],0,r12 # load sigma + addi r12,r12,0x70 + li $x10,16 + li $x20,32 + li $x30,48 + li r11,64 + + lvx_4w @K[1],0,$key # load key + lvx_4w @K[2],$x10,$key + lvx_4w @K[3],0,$ctr # load counter + + vxor $xt0,$xt0,$xt0 + lvx_4w $xt1,r11,r12 + vspltw $CTR,@K[3],0 + vsldoi @K[3],@K[3],$xt0,4 + vsldoi @K[3],$xt0,@K[3],12 # clear @K[3].word[0] + vadduwm $CTR,$CTR,$xt1 + + be?lvsl $beperm,0,$x10 # 0x00..0f + be?vspltisb $xt0,3 # 0x03..03 + be?vxor $beperm,$beperm,$xt0 # swap bytes within words + + li r0,10 # inner loop counter + mtctr r0 + b Loop_outer_vsx + +.align 5 +Loop_outer_vsx: + lvx $xa0,$x00,r12 # load [smashed] sigma + lvx $xa1,$x10,r12 + lvx $xa2,$x20,r12 + lvx $xa3,$x30,r12 + + vspltw $xb0,@K[1],0 # smash the key + vspltw $xb1,@K[1],1 + vspltw $xb2,@K[1],2 + vspltw $xb3,@K[1],3 + + vspltw $xc0,@K[2],0 + vspltw $xc1,@K[2],1 + vspltw $xc2,@K[2],2 + vspltw $xc3,@K[2],3 + + vmr $xd0,$CTR # smash the counter + vspltw $xd1,@K[3],1 + vspltw $xd2,@K[3],2 + vspltw $xd3,@K[3],3 + + vspltisw $sixteen,-16 # synthesize constants + vspltisw $twelve,12 + vspltisw $eight,8 + vspltisw $seven,7 + +Loop_vsx_4x: +___ + foreach (&VSX_lane_ROUND_4x(0, 4, 8,12)) { eval; } + foreach (&VSX_lane_ROUND_4x(0, 5,10,15)) { eval; } +$code.=<<___; + + bdnz Loop_vsx_4x + + vadduwm $xd0,$xd0,$CTR + + vmrgew $xt0,$xa0,$xa1 # transpose data + vmrgew $xt1,$xa2,$xa3 + vmrgow $xa0,$xa0,$xa1 + vmrgow $xa2,$xa2,$xa3 + vmrgew $xt2,$xb0,$xb1 + vmrgew $xt3,$xb2,$xb3 + vpermdi $xa1,$xa0,$xa2,0b00 + vpermdi $xa3,$xa0,$xa2,0b11 + vpermdi $xa0,$xt0,$xt1,0b00 + vpermdi $xa2,$xt0,$xt1,0b11 + + vmrgow $xb0,$xb0,$xb1 + vmrgow $xb2,$xb2,$xb3 + vmrgew $xt0,$xc0,$xc1 + vmrgew $xt1,$xc2,$xc3 + vpermdi $xb1,$xb0,$xb2,0b00 + vpermdi $xb3,$xb0,$xb2,0b11 + vpermdi $xb0,$xt2,$xt3,0b00 + vpermdi $xb2,$xt2,$xt3,0b11 + + vmrgow $xc0,$xc0,$xc1 + vmrgow $xc2,$xc2,$xc3 + vmrgew $xt2,$xd0,$xd1 + vmrgew $xt3,$xd2,$xd3 + vpermdi $xc1,$xc0,$xc2,0b00 + vpermdi $xc3,$xc0,$xc2,0b11 + vpermdi $xc0,$xt0,$xt1,0b00 + vpermdi $xc2,$xt0,$xt1,0b11 + + vmrgow $xd0,$xd0,$xd1 + vmrgow $xd2,$xd2,$xd3 + vspltisw $xt0,4 + vadduwm $CTR,$CTR,$xt0 # next counter value + vpermdi $xd1,$xd0,$xd2,0b00 + vpermdi $xd3,$xd0,$xd2,0b11 + vpermdi $xd0,$xt2,$xt3,0b00 + vpermdi $xd2,$xt2,$xt3,0b11 + + vadduwm $xa0,$xa0,@K[0] + vadduwm $xb0,$xb0,@K[1] + vadduwm $xc0,$xc0,@K[2] + vadduwm $xd0,$xd0,@K[3] + + be?vperm $xa0,$xa0,$xa0,$beperm + be?vperm $xb0,$xb0,$xb0,$beperm + be?vperm $xc0,$xc0,$xc0,$beperm + be?vperm $xd0,$xd0,$xd0,$beperm + + ${UCMP}i $len,0x40 + blt Ltail_vsx + + lvx_4w $xt0,$x00,$inp + lvx_4w $xt1,$x10,$inp + lvx_4w $xt2,$x20,$inp + lvx_4w $xt3,$x30,$inp + + vxor $xt0,$xt0,$xa0 + vxor $xt1,$xt1,$xb0 + vxor $xt2,$xt2,$xc0 + vxor $xt3,$xt3,$xd0 + + stvx_4w $xt0,$x00,$out + stvx_4w $xt1,$x10,$out + addi $inp,$inp,0x40 + stvx_4w $xt2,$x20,$out + subi $len,$len,0x40 + stvx_4w $xt3,$x30,$out + addi $out,$out,0x40 + beq Ldone_vsx + + vadduwm $xa0,$xa1,@K[0] + vadduwm $xb0,$xb1,@K[1] + vadduwm $xc0,$xc1,@K[2] + vadduwm $xd0,$xd1,@K[3] + + be?vperm $xa0,$xa0,$xa0,$beperm + be?vperm $xb0,$xb0,$xb0,$beperm + be?vperm $xc0,$xc0,$xc0,$beperm + be?vperm $xd0,$xd0,$xd0,$beperm + + ${UCMP}i $len,0x40 + blt Ltail_vsx + + lvx_4w $xt0,$x00,$inp + lvx_4w $xt1,$x10,$inp + lvx_4w $xt2,$x20,$inp + lvx_4w $xt3,$x30,$inp + + vxor $xt0,$xt0,$xa0 + vxor $xt1,$xt1,$xb0 + vxor $xt2,$xt2,$xc0 + vxor $xt3,$xt3,$xd0 + + stvx_4w $xt0,$x00,$out + stvx_4w $xt1,$x10,$out + addi $inp,$inp,0x40 + stvx_4w $xt2,$x20,$out + subi $len,$len,0x40 + stvx_4w $xt3,$x30,$out + addi $out,$out,0x40 + beq Ldone_vsx + + vadduwm $xa0,$xa2,@K[0] + vadduwm $xb0,$xb2,@K[1] + vadduwm $xc0,$xc2,@K[2] + vadduwm $xd0,$xd2,@K[3] + + be?vperm $xa0,$xa0,$xa0,$beperm + be?vperm $xb0,$xb0,$xb0,$beperm + be?vperm $xc0,$xc0,$xc0,$beperm + be?vperm $xd0,$xd0,$xd0,$beperm + + ${UCMP}i $len,0x40 + blt Ltail_vsx + + lvx_4w $xt0,$x00,$inp + lvx_4w $xt1,$x10,$inp + lvx_4w $xt2,$x20,$inp + lvx_4w $xt3,$x30,$inp + + vxor $xt0,$xt0,$xa0 + vxor $xt1,$xt1,$xb0 + vxor $xt2,$xt2,$xc0 + vxor $xt3,$xt3,$xd0 + + stvx_4w $xt0,$x00,$out + stvx_4w $xt1,$x10,$out + addi $inp,$inp,0x40 + stvx_4w $xt2,$x20,$out + subi $len,$len,0x40 + stvx_4w $xt3,$x30,$out + addi $out,$out,0x40 + beq Ldone_vsx + + vadduwm $xa0,$xa3,@K[0] + vadduwm $xb0,$xb3,@K[1] + vadduwm $xc0,$xc3,@K[2] + vadduwm $xd0,$xd3,@K[3] + + be?vperm $xa0,$xa0,$xa0,$beperm + be?vperm $xb0,$xb0,$xb0,$beperm + be?vperm $xc0,$xc0,$xc0,$beperm + be?vperm $xd0,$xd0,$xd0,$beperm + + ${UCMP}i $len,0x40 + blt Ltail_vsx + + lvx_4w $xt0,$x00,$inp + lvx_4w $xt1,$x10,$inp + lvx_4w $xt2,$x20,$inp + lvx_4w $xt3,$x30,$inp + + vxor $xt0,$xt0,$xa0 + vxor $xt1,$xt1,$xb0 + vxor $xt2,$xt2,$xc0 + vxor $xt3,$xt3,$xd0 + + stvx_4w $xt0,$x00,$out + stvx_4w $xt1,$x10,$out + addi $inp,$inp,0x40 + stvx_4w $xt2,$x20,$out + subi $len,$len,0x40 + stvx_4w $xt3,$x30,$out + addi $out,$out,0x40 + mtctr r0 + bne Loop_outer_vsx + +Ldone_vsx: + lwz r12,`$FRAME-4`($sp) # pull vrsave + li r10,`15+$LOCALS+64` + li r11,`31+$LOCALS+64` + $POP r0, `$FRAME+$LRSAVE`($sp) + mtspr 256,r12 # restore vrsave + lvx v26,r10,$sp + addi r10,r10,32 + lvx v27,r11,$sp + addi r11,r11,32 + lvx v28,r10,$sp + addi r10,r10,32 + lvx v29,r11,$sp + addi r11,r11,32 + lvx v30,r10,$sp + lvx v31,r11,$sp + mtlr r0 + addi $sp,$sp,$FRAME + blr + +.align 4 +Ltail_vsx: + addi r11,$sp,$LOCALS + mtctr $len + stvx_4w $xa0,$x00,r11 # offload block to stack + stvx_4w $xb0,$x10,r11 + stvx_4w $xc0,$x20,r11 + stvx_4w $xd0,$x30,r11 + subi r12,r11,1 # prepare for *++ptr + subi $inp,$inp,1 + subi $out,$out,1 + +Loop_tail_vsx: + lbzu r6,1(r12) + lbzu r7,1($inp) + xor r6,r6,r7 + stbu r6,1($out) + bdnz Loop_tail_vsx + + stvx_4w $K[0],$x00,r11 # wipe copy of the block + stvx_4w $K[0],$x10,r11 + stvx_4w $K[0],$x20,r11 + stvx_4w $K[0],$x30,r11 + + b Ldone_vsx + .long 0 + .byte 0,12,0x04,1,0x80,0,5,0 + .long 0 +.size .ChaCha20_ctr32_vsx_p10,.-.ChaCha20_ctr32_vsx_p10 +___ +}}} + +##This is 8 block in parallel implementation. The heart of chacha round uses vector instruction that has access to +# vsr[32+X]. To perform the 8 parallel block we tend to use all 32 register to hold the 8 block info. +# WE need to store few register value on side, so we can use VSR{32+X} for few vector instructions used in round op and hold intermediate value. +# WE use the VSR[0]-VSR[31] for holding intermediate value and perform 8 block in parallel. +# +{{{ +#### ($out,$inp,$len,$key,$ctr) = map("r$_",(3..7)); +my ($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3, + $xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3, + $xa4,$xa5,$xa6,$xa7, $xb4,$xb5,$xb6,$xb7, + $xc4,$xc5,$xc6,$xc7, $xd4,$xd5,$xd6,$xd7) = map("v$_",(0..31)); +my ($xcn4,$xcn5,$xcn6,$xcn7, $xdn4,$xdn5,$xdn6,$xdn7) = map("v$_",(8..15)); +my ($xan0,$xbn0,$xcn0,$xdn0) = map("v$_",(0..3)); +my @K = map("v$_",27,(24..26)); +my ($xt0,$xt1,$xt2,$xt3,$xt4) = map("v$_",23,(28..31)); +my $xr0 = "v4"; +my $CTR0 = "v22"; +my $CTR1 = "v5"; +my $beperm = "v31"; +my ($x00,$x10,$x20,$x30) = (0, map("r$_",(8..10))); +my ($xv0,$xv1,$xv2,$xv3,$xv4,$xv5,$xv6,$xv7) = map("v$_",(0..7)); +my ($xv8,$xv9,$xv10,$xv11,$xv12,$xv13,$xv14,$xv15,$xv16,$xv17) = map("v$_",(8..17)); +my ($xv18,$xv19,$xv20,$xv21) = map("v$_",(18..21)); +my ($xv22,$xv23,$xv24,$xv25,$xv26) = map("v$_",(22..26)); + +my $FRAME=$LOCALS+64+9*16; # 8*16 is for v24-v31 offload + +sub VSX_lane_ROUND_8x { +my ($a0,$b0,$c0,$d0,$a4,$b4,$c4,$d4)=@_; +my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0)); +my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1)); +my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2)); +my ($a5,$b5,$c5,$d5)=map(($_&~3)+(($_+1)&3),($a4,$b4,$c4,$d4)); +my ($a6,$b6,$c6,$d6)=map(($_&~3)+(($_+1)&3),($a5,$b5,$c5,$d5)); +my ($a7,$b7,$c7,$d7)=map(($_&~3)+(($_+1)&3),($a6,$b6,$c6,$d6)); +my ($xv8,$xv9,$xv10,$xv11,$xv12,$xv13,$xv14,$xv15,$xv16,$xv17) = map("\"v$_\"",(8..17)); +my @x=map("\"v$_\"",(0..31)); + + ( + "&vxxlor ($xv15 ,@x[$c7],@x[$c7])", #copy v30 to v13 + "&vxxlorc (@x[$c7], $xv9,$xv9)", + + "&vadduwm (@x[$a0],@x[$a0],@x[$b0])", # Q1 + "&vadduwm (@x[$a1],@x[$a1],@x[$b1])", # Q2 + "&vadduwm (@x[$a2],@x[$a2],@x[$b2])", # Q3 + "&vadduwm (@x[$a3],@x[$a3],@x[$b3])", # Q4 + "&vadduwm (@x[$a4],@x[$a4],@x[$b4])", # Q1 + "&vadduwm (@x[$a5],@x[$a5],@x[$b5])", # Q2 + "&vadduwm (@x[$a6],@x[$a6],@x[$b6])", # Q3 + "&vadduwm (@x[$a7],@x[$a7],@x[$b7])", # Q4 + + "&vxor (@x[$d0],@x[$d0],@x[$a0])", + "&vxor (@x[$d1],@x[$d1],@x[$a1])", + "&vxor (@x[$d2],@x[$d2],@x[$a2])", + "&vxor (@x[$d3],@x[$d3],@x[$a3])", + "&vxor (@x[$d4],@x[$d4],@x[$a4])", + "&vxor (@x[$d5],@x[$d5],@x[$a5])", + "&vxor (@x[$d6],@x[$d6],@x[$a6])", + "&vxor (@x[$d7],@x[$d7],@x[$a7])", + + "&vrlw (@x[$d0],@x[$d0],@x[$c7])", + "&vrlw (@x[$d1],@x[$d1],@x[$c7])", + "&vrlw (@x[$d2],@x[$d2],@x[$c7])", + "&vrlw (@x[$d3],@x[$d3],@x[$c7])", + "&vrlw (@x[$d4],@x[$d4],@x[$c7])", + "&vrlw (@x[$d5],@x[$d5],@x[$c7])", + "&vrlw (@x[$d6],@x[$d6],@x[$c7])", + "&vrlw (@x[$d7],@x[$d7],@x[$c7])", + + "&vxxlor ($xv13 ,@x[$a7],@x[$a7])", + "&vxxlorc (@x[$c7], $xv15,$xv15)", + "&vxxlorc (@x[$a7], $xv10,$xv10)", + + "&vadduwm (@x[$c0],@x[$c0],@x[$d0])", + "&vadduwm (@x[$c1],@x[$c1],@x[$d1])", + "&vadduwm (@x[$c2],@x[$c2],@x[$d2])", + "&vadduwm (@x[$c3],@x[$c3],@x[$d3])", + "&vadduwm (@x[$c4],@x[$c4],@x[$d4])", + "&vadduwm (@x[$c5],@x[$c5],@x[$d5])", + "&vadduwm (@x[$c6],@x[$c6],@x[$d6])", + "&vadduwm (@x[$c7],@x[$c7],@x[$d7])", + + "&vxor (@x[$b0],@x[$b0],@x[$c0])", + "&vxor (@x[$b1],@x[$b1],@x[$c1])", + "&vxor (@x[$b2],@x[$b2],@x[$c2])", + "&vxor (@x[$b3],@x[$b3],@x[$c3])", + "&vxor (@x[$b4],@x[$b4],@x[$c4])", + "&vxor (@x[$b5],@x[$b5],@x[$c5])", + "&vxor (@x[$b6],@x[$b6],@x[$c6])", + "&vxor (@x[$b7],@x[$b7],@x[$c7])", + + "&vrlw (@x[$b0],@x[$b0],@x[$a7])", + "&vrlw (@x[$b1],@x[$b1],@x[$a7])", + "&vrlw (@x[$b2],@x[$b2],@x[$a7])", + "&vrlw (@x[$b3],@x[$b3],@x[$a7])", + "&vrlw (@x[$b4],@x[$b4],@x[$a7])", + "&vrlw (@x[$b5],@x[$b5],@x[$a7])", + "&vrlw (@x[$b6],@x[$b6],@x[$a7])", + "&vrlw (@x[$b7],@x[$b7],@x[$a7])", + + "&vxxlorc (@x[$a7], $xv13,$xv13)", + "&vxxlor ($xv15 ,@x[$c7],@x[$c7])", + "&vxxlorc (@x[$c7], $xv11,$xv11)", + + + "&vadduwm (@x[$a0],@x[$a0],@x[$b0])", + "&vadduwm (@x[$a1],@x[$a1],@x[$b1])", + "&vadduwm (@x[$a2],@x[$a2],@x[$b2])", + "&vadduwm (@x[$a3],@x[$a3],@x[$b3])", + "&vadduwm (@x[$a4],@x[$a4],@x[$b4])", + "&vadduwm (@x[$a5],@x[$a5],@x[$b5])", + "&vadduwm (@x[$a6],@x[$a6],@x[$b6])", + "&vadduwm (@x[$a7],@x[$a7],@x[$b7])", + + "&vxor (@x[$d0],@x[$d0],@x[$a0])", + "&vxor (@x[$d1],@x[$d1],@x[$a1])", + "&vxor (@x[$d2],@x[$d2],@x[$a2])", + "&vxor (@x[$d3],@x[$d3],@x[$a3])", + "&vxor (@x[$d4],@x[$d4],@x[$a4])", + "&vxor (@x[$d5],@x[$d5],@x[$a5])", + "&vxor (@x[$d6],@x[$d6],@x[$a6])", + "&vxor (@x[$d7],@x[$d7],@x[$a7])", + + "&vrlw (@x[$d0],@x[$d0],@x[$c7])", + "&vrlw (@x[$d1],@x[$d1],@x[$c7])", + "&vrlw (@x[$d2],@x[$d2],@x[$c7])", + "&vrlw (@x[$d3],@x[$d3],@x[$c7])", + "&vrlw (@x[$d4],@x[$d4],@x[$c7])", + "&vrlw (@x[$d5],@x[$d5],@x[$c7])", + "&vrlw (@x[$d6],@x[$d6],@x[$c7])", + "&vrlw (@x[$d7],@x[$d7],@x[$c7])", + + "&vxxlorc (@x[$c7], $xv15,$xv15)", + "&vxxlor ($xv13 ,@x[$a7],@x[$a7])", + "&vxxlorc (@x[$a7], $xv12,$xv12)", + + "&vadduwm (@x[$c0],@x[$c0],@x[$d0])", + "&vadduwm (@x[$c1],@x[$c1],@x[$d1])", + "&vadduwm (@x[$c2],@x[$c2],@x[$d2])", + "&vadduwm (@x[$c3],@x[$c3],@x[$d3])", + "&vadduwm (@x[$c4],@x[$c4],@x[$d4])", + "&vadduwm (@x[$c5],@x[$c5],@x[$d5])", + "&vadduwm (@x[$c6],@x[$c6],@x[$d6])", + "&vadduwm (@x[$c7],@x[$c7],@x[$d7])", + "&vxor (@x[$b0],@x[$b0],@x[$c0])", + "&vxor (@x[$b1],@x[$b1],@x[$c1])", + "&vxor (@x[$b2],@x[$b2],@x[$c2])", + "&vxor (@x[$b3],@x[$b3],@x[$c3])", + "&vxor (@x[$b4],@x[$b4],@x[$c4])", + "&vxor (@x[$b5],@x[$b5],@x[$c5])", + "&vxor (@x[$b6],@x[$b6],@x[$c6])", + "&vxor (@x[$b7],@x[$b7],@x[$c7])", + "&vrlw (@x[$b0],@x[$b0],@x[$a7])", + "&vrlw (@x[$b1],@x[$b1],@x[$a7])", + "&vrlw (@x[$b2],@x[$b2],@x[$a7])", + "&vrlw (@x[$b3],@x[$b3],@x[$a7])", + "&vrlw (@x[$b4],@x[$b4],@x[$a7])", + "&vrlw (@x[$b5],@x[$b5],@x[$a7])", + "&vrlw (@x[$b6],@x[$b6],@x[$a7])", + "&vrlw (@x[$b7],@x[$b7],@x[$a7])", + + "&vxxlorc (@x[$a7], $xv13,$xv13)", + ); +} + +$code.=<<___; + +.globl .ChaCha20_ctr32_vsx_8x +.align 5 +.ChaCha20_ctr32_vsx_8x: + $STU $sp,-$FRAME($sp) + mflr r0 + li r10,`15+$LOCALS+64` + li r11,`31+$LOCALS+64` + mfspr r12,256 + stvx v24,r10,$sp + addi r10,r10,32 + stvx v25,r11,$sp + addi r11,r11,32 + stvx v26,r10,$sp + addi r10,r10,32 + stvx v27,r11,$sp + addi r11,r11,32 + stvx v28,r10,$sp + addi r10,r10,32 + stvx v29,r11,$sp + addi r11,r11,32 + stvx v30,r10,$sp + stvx v31,r11,$sp + stw r12,`$FRAME-4`($sp) # save vrsave + li r12,-4096+63 + $PUSH r0, `$FRAME+$LRSAVE`($sp) + mtspr 256,r12 # preserve 29 AltiVec registers + + bl Lconsts # returns pointer Lsigma in r12 + + lvx_4w @K[0],0,r12 # load sigma + addi r12,r12,0x70 + li $x10,16 + li $x20,32 + li $x30,48 + li r11,64 + + vspltisw $xa4,-16 # synthesize constants + vspltisw $xb4,12 # synthesize constants + vspltisw $xc4,8 # synthesize constants + vspltisw $xd4,7 # synthesize constants + + lvx $xa0,$x00,r12 # load [smashed] sigma + lvx $xa1,$x10,r12 + lvx $xa2,$x20,r12 + lvx $xa3,$x30,r12 + + vxxlor $xv9 ,$xa4,$xa4 #save shift val in vr9-12 + vxxlor $xv10 ,$xb4,$xb4 + vxxlor $xv11 ,$xc4,$xc4 + vxxlor $xv12 ,$xd4,$xd4 + vxxlor $xv22 ,$xa0,$xa0 #save sigma in vr22-25 + vxxlor $xv23 ,$xa1,$xa1 + vxxlor $xv24 ,$xa2,$xa2 + vxxlor $xv25 ,$xa3,$xa3 + + lvx_4w @K[1],0,$key # load key + lvx_4w @K[2],$x10,$key + lvx_4w @K[3],0,$ctr # load counter + vspltisw $xt3,4 + + + vxor $xt2,$xt2,$xt2 + lvx_4w $xt1,r11,r12 + vspltw $xa2,@K[3],0 #save the original count after spltw + vsldoi @K[3],@K[3],$xt2,4 + vsldoi @K[3],$xt2,@K[3],12 # clear @K[3].word[0] + vadduwm $xt1,$xa2,$xt1 + vadduwm $xt3,$xt1,$xt3 # next counter value + vspltw $xa0,@K[2],2 # save the K[2] spltw 2 and save v8. + + be?lvsl $beperm,0,$x10 # 0x00..0f + be?vspltisb $xt0,3 # 0x03..03 + be?vxor $beperm,$beperm,$xt0 # swap bytes within words + be?vxxlor $xv26 ,$beperm,$beperm + + vxxlor $xv0 ,@K[0],@K[0] # K0,k1,k2 to vr0,1,2 + vxxlor $xv1 ,@K[1],@K[1] + vxxlor $xv2 ,@K[2],@K[2] + vxxlor $xv3 ,@K[3],@K[3] + vxxlor $xv4 ,$xt1,$xt1 #CTR ->4, CTR+4-> 5 + vxxlor $xv5 ,$xt3,$xt3 + vxxlor $xv8 ,$xa0,$xa0 + + li r0,10 # inner loop counter + mtctr r0 + b Loop_outer_vsx_8x + +.align 5 +Loop_outer_vsx_8x: + vxxlorc $xa0,$xv22,$xv22 # load [smashed] sigma + vxxlorc $xa1,$xv23,$xv23 + vxxlorc $xa2,$xv24,$xv24 + vxxlorc $xa3,$xv25,$xv25 + vxxlorc $xa4,$xv22,$xv22 + vxxlorc $xa5,$xv23,$xv23 + vxxlorc $xa6,$xv24,$xv24 + vxxlorc $xa7,$xv25,$xv25 + + vspltw $xb0,@K[1],0 # smash the key + vspltw $xb1,@K[1],1 + vspltw $xb2,@K[1],2 + vspltw $xb3,@K[1],3 + vspltw $xb4,@K[1],0 # smash the key + vspltw $xb5,@K[1],1 + vspltw $xb6,@K[1],2 + vspltw $xb7,@K[1],3 + + vspltw $xc0,@K[2],0 + vspltw $xc1,@K[2],1 + vspltw $xc2,@K[2],2 + vspltw $xc3,@K[2],3 + vspltw $xc4,@K[2],0 + vspltw $xc7,@K[2],3 + vspltw $xc5,@K[2],1 + + vxxlorc $xd0,$xv4,$xv4 # smash the counter + vspltw $xd1,@K[3],1 + vspltw $xd2,@K[3],2 + vspltw $xd3,@K[3],3 + vxxlorc $xd4,$xv5,$xv5 # smash the counter + vspltw $xd5,@K[3],1 + vspltw $xd6,@K[3],2 + vspltw $xd7,@K[3],3 + vxxlorc $xc6,$xv8,$xv8 #copy of vlspt k[2],2 is in v8.v26 ->k[3] so need to wait until k3 is done + +Loop_vsx_8x: +___ + foreach (&VSX_lane_ROUND_8x(0,4, 8,12,16,20,24,28)) { eval; } + foreach (&VSX_lane_ROUND_8x(0,5,10,15,16,21,26,31)) { eval; } +$code.=<<___; + + bdnz Loop_vsx_8x + vxxlor $xv13 ,$xd4,$xd4 # save the register vr24-31 + vxxlor $xv14 ,$xd5,$xd5 # + vxxlor $xv15 ,$xd6,$xd6 # + vxxlor $xv16 ,$xd7,$xd7 # + + vxxlor $xv18 ,$xc4,$xc4 # + vxxlor $xv19 ,$xc5,$xc5 # + vxxlor $xv20 ,$xc6,$xc6 # + vxxlor $xv21 ,$xc7,$xc7 # + + vxxlor $xv6 ,$xb6,$xb6 # save vr23, so we get 8 regs + vxxlor $xv7 ,$xb7,$xb7 # save vr23, so we get 8 regs + be?vxxlorc $beperm,$xv26,$xv26 # copy back the beperm. + + vxxlorc @K[0],$xv0,$xv0 #27 + vxxlorc @K[1],$xv1,$xv1 #24 + vxxlorc @K[2],$xv2,$xv2 #25 + vxxlorc @K[3],$xv3,$xv3 #26 + vxxlorc $CTR0,$xv4,$xv4 +###changing to vertical + + vmrgew $xt0,$xa0,$xa1 # transpose data + vmrgew $xt1,$xa2,$xa3 + vmrgow $xa0,$xa0,$xa1 + vmrgow $xa2,$xa2,$xa3 + + vmrgew $xt2,$xb0,$xb1 + vmrgew $xt3,$xb2,$xb3 + vmrgow $xb0,$xb0,$xb1 + vmrgow $xb2,$xb2,$xb3 + + vadduwm $xd0,$xd0,$CTR0 + + vpermdi $xa1,$xa0,$xa2,0b00 + vpermdi $xa3,$xa0,$xa2,0b11 + vpermdi $xa0,$xt0,$xt1,0b00 + vpermdi $xa2,$xt0,$xt1,0b11 + vpermdi $xb1,$xb0,$xb2,0b00 + vpermdi $xb3,$xb0,$xb2,0b11 + vpermdi $xb0,$xt2,$xt3,0b00 + vpermdi $xb2,$xt2,$xt3,0b11 + + vmrgew $xt0,$xc0,$xc1 + vmrgew $xt1,$xc2,$xc3 + vmrgow $xc0,$xc0,$xc1 + vmrgow $xc2,$xc2,$xc3 + vmrgew $xt2,$xd0,$xd1 + vmrgew $xt3,$xd2,$xd3 + vmrgow $xd0,$xd0,$xd1 + vmrgow $xd2,$xd2,$xd3 + + vpermdi $xc1,$xc0,$xc2,0b00 + vpermdi $xc3,$xc0,$xc2,0b11 + vpermdi $xc0,$xt0,$xt1,0b00 + vpermdi $xc2,$xt0,$xt1,0b11 + vpermdi $xd1,$xd0,$xd2,0b00 + vpermdi $xd3,$xd0,$xd2,0b11 + vpermdi $xd0,$xt2,$xt3,0b00 + vpermdi $xd2,$xt2,$xt3,0b11 + + vspltisw $xt0,8 + vadduwm $CTR0,$CTR0,$xt0 # next counter value + vxxlor $xv4 ,$CTR0,$CTR0 #CTR+4-> 5 + + vadduwm $xa0,$xa0,@K[0] + vadduwm $xb0,$xb0,@K[1] + vadduwm $xc0,$xc0,@K[2] + vadduwm $xd0,$xd0,@K[3] + + be?vperm $xa0,$xa0,$xa0,$beperm + be?vperm $xb0,$xb0,$xb0,$beperm + be?vperm $xc0,$xc0,$xc0,$beperm + be?vperm $xd0,$xd0,$xd0,$beperm + + ${UCMP}i $len,0x40 + blt Ltail_vsx_8x + + lvx_4w $xt0,$x00,$inp + lvx_4w $xt1,$x10,$inp + lvx_4w $xt2,$x20,$inp + lvx_4w $xt3,$x30,$inp + + vxor $xt0,$xt0,$xa0 + vxor $xt1,$xt1,$xb0 + vxor $xt2,$xt2,$xc0 + vxor $xt3,$xt3,$xd0 + + stvx_4w $xt0,$x00,$out + stvx_4w $xt1,$x10,$out + addi $inp,$inp,0x40 + stvx_4w $xt2,$x20,$out + subi $len,$len,0x40 + stvx_4w $xt3,$x30,$out + addi $out,$out,0x40 + beq Ldone_vsx_8x + + vadduwm $xa0,$xa1,@K[0] + vadduwm $xb0,$xb1,@K[1] + vadduwm $xc0,$xc1,@K[2] + vadduwm $xd0,$xd1,@K[3] + + be?vperm $xa0,$xa0,$xa0,$beperm + be?vperm $xb0,$xb0,$xb0,$beperm + be?vperm $xc0,$xc0,$xc0,$beperm + be?vperm $xd0,$xd0,$xd0,$beperm + + ${UCMP}i $len,0x40 + blt Ltail_vsx_8x + + lvx_4w $xt0,$x00,$inp + lvx_4w $xt1,$x10,$inp + lvx_4w $xt2,$x20,$inp + lvx_4w $xt3,$x30,$inp + + vxor $xt0,$xt0,$xa0 + vxor $xt1,$xt1,$xb0 + vxor $xt2,$xt2,$xc0 + vxor $xt3,$xt3,$xd0 + + stvx_4w $xt0,$x00,$out + stvx_4w $xt1,$x10,$out + addi $inp,$inp,0x40 + stvx_4w $xt2,$x20,$out + subi $len,$len,0x40 + stvx_4w $xt3,$x30,$out + addi $out,$out,0x40 + beq Ldone_vsx_8x + + vadduwm $xa0,$xa2,@K[0] + vadduwm $xb0,$xb2,@K[1] + vadduwm $xc0,$xc2,@K[2] + vadduwm $xd0,$xd2,@K[3] + + be?vperm $xa0,$xa0,$xa0,$beperm + be?vperm $xb0,$xb0,$xb0,$beperm + be?vperm $xc0,$xc0,$xc0,$beperm + be?vperm $xd0,$xd0,$xd0,$beperm + + ${UCMP}i $len,0x40 + blt Ltail_vsx_8x + + lvx_4w $xt0,$x00,$inp + lvx_4w $xt1,$x10,$inp + lvx_4w $xt2,$x20,$inp + lvx_4w $xt3,$x30,$inp + + vxor $xt0,$xt0,$xa0 + vxor $xt1,$xt1,$xb0 + vxor $xt2,$xt2,$xc0 + vxor $xt3,$xt3,$xd0 + + stvx_4w $xt0,$x00,$out + stvx_4w $xt1,$x10,$out + addi $inp,$inp,0x40 + stvx_4w $xt2,$x20,$out + subi $len,$len,0x40 + stvx_4w $xt3,$x30,$out + addi $out,$out,0x40 + beq Ldone_vsx_8x + + vadduwm $xa0,$xa3,@K[0] + vadduwm $xb0,$xb3,@K[1] + vadduwm $xc0,$xc3,@K[2] + vadduwm $xd0,$xd3,@K[3] + + be?vperm $xa0,$xa0,$xa0,$beperm + be?vperm $xb0,$xb0,$xb0,$beperm + be?vperm $xc0,$xc0,$xc0,$beperm + be?vperm $xd0,$xd0,$xd0,$beperm + + ${UCMP}i $len,0x40 + blt Ltail_vsx_8x + + lvx_4w $xt0,$x00,$inp + lvx_4w $xt1,$x10,$inp + lvx_4w $xt2,$x20,$inp + lvx_4w $xt3,$x30,$inp + + vxor $xt0,$xt0,$xa0 + vxor $xt1,$xt1,$xb0 + vxor $xt2,$xt2,$xc0 + vxor $xt3,$xt3,$xd0 + + stvx_4w $xt0,$x00,$out + stvx_4w $xt1,$x10,$out + addi $inp,$inp,0x40 + stvx_4w $xt2,$x20,$out + subi $len,$len,0x40 + stvx_4w $xt3,$x30,$out + addi $out,$out,0x40 + beq Ldone_vsx_8x + +#blk4-7: 24:31 remain the same as we can use the same logic above . Reg a4-b7 remain same.Load c4,d7--> position 8-15.we can reuse vr24-31. +#VR0-3 : are used to load temp value, vr4 --> as xr0 instead of xt0. + + vxxlorc $CTR1 ,$xv5,$xv5 + + vxxlorc $xcn4 ,$xv18,$xv18 + vxxlorc $xcn5 ,$xv19,$xv19 + vxxlorc $xcn6 ,$xv20,$xv20 + vxxlorc $xcn7 ,$xv21,$xv21 + + vxxlorc $xdn4 ,$xv13,$xv13 + vxxlorc $xdn5 ,$xv14,$xv14 + vxxlorc $xdn6 ,$xv15,$xv15 + vxxlorc $xdn7 ,$xv16,$xv16 + vadduwm $xdn4,$xdn4,$CTR1 + + vxxlorc $xb6 ,$xv6,$xv6 + vxxlorc $xb7 ,$xv7,$xv7 +#use xa1->xr0, as xt0...in the block 4-7 + + vmrgew $xr0,$xa4,$xa5 # transpose data + vmrgew $xt1,$xa6,$xa7 + vmrgow $xa4,$xa4,$xa5 + vmrgow $xa6,$xa6,$xa7 + vmrgew $xt2,$xb4,$xb5 + vmrgew $xt3,$xb6,$xb7 + vmrgow $xb4,$xb4,$xb5 + vmrgow $xb6,$xb6,$xb7 + + vpermdi $xa5,$xa4,$xa6,0b00 + vpermdi $xa7,$xa4,$xa6,0b11 + vpermdi $xa4,$xr0,$xt1,0b00 + vpermdi $xa6,$xr0,$xt1,0b11 + vpermdi $xb5,$xb4,$xb6,0b00 + vpermdi $xb7,$xb4,$xb6,0b11 + vpermdi $xb4,$xt2,$xt3,0b00 + vpermdi $xb6,$xt2,$xt3,0b11 + + vmrgew $xr0,$xcn4,$xcn5 + vmrgew $xt1,$xcn6,$xcn7 + vmrgow $xcn4,$xcn4,$xcn5 + vmrgow $xcn6,$xcn6,$xcn7 + vmrgew $xt2,$xdn4,$xdn5 + vmrgew $xt3,$xdn6,$xdn7 + vmrgow $xdn4,$xdn4,$xdn5 + vmrgow $xdn6,$xdn6,$xdn7 + + vpermdi $xcn5,$xcn4,$xcn6,0b00 + vpermdi $xcn7,$xcn4,$xcn6,0b11 + vpermdi $xcn4,$xr0,$xt1,0b00 + vpermdi $xcn6,$xr0,$xt1,0b11 + vpermdi $xdn5,$xdn4,$xdn6,0b00 + vpermdi $xdn7,$xdn4,$xdn6,0b11 + vpermdi $xdn4,$xt2,$xt3,0b00 + vpermdi $xdn6,$xt2,$xt3,0b11 + + vspltisw $xr0,8 + vadduwm $CTR1,$CTR1,$xr0 # next counter value + vxxlor $xv5 ,$CTR1,$CTR1 #CTR+4-> 5 + + vadduwm $xan0,$xa4,@K[0] + vadduwm $xbn0,$xb4,@K[1] + vadduwm $xcn0,$xcn4,@K[2] + vadduwm $xdn0,$xdn4,@K[3] + + be?vperm $xan0,$xan0,$xan0,$beperm + be?vperm $xbn0,$xbn0,$xbn0,$beperm + be?vperm $xcn0,$xcn0,$xcn0,$beperm + be?vperm $xdn0,$xdn0,$xdn0,$beperm + + ${UCMP}i $len,0x40 + blt Ltail_vsx_8x_1 + + lvx_4w $xr0,$x00,$inp + lvx_4w $xt1,$x10,$inp + lvx_4w $xt2,$x20,$inp + lvx_4w $xt3,$x30,$inp + + vxor $xr0,$xr0,$xan0 + vxor $xt1,$xt1,$xbn0 + vxor $xt2,$xt2,$xcn0 + vxor $xt3,$xt3,$xdn0 + + stvx_4w $xr0,$x00,$out + stvx_4w $xt1,$x10,$out + addi $inp,$inp,0x40 + stvx_4w $xt2,$x20,$out + subi $len,$len,0x40 + stvx_4w $xt3,$x30,$out + addi $out,$out,0x40 + beq Ldone_vsx_8x + + vadduwm $xan0,$xa5,@K[0] + vadduwm $xbn0,$xb5,@K[1] + vadduwm $xcn0,$xcn5,@K[2] + vadduwm $xdn0,$xdn5,@K[3] + + be?vperm $xan0,$xan0,$xan0,$beperm + be?vperm $xbn0,$xbn0,$xbn0,$beperm + be?vperm $xcn0,$xcn0,$xcn0,$beperm + be?vperm $xdn0,$xdn0,$xdn0,$beperm + + ${UCMP}i $len,0x40 + blt Ltail_vsx_8x_1 + + lvx_4w $xr0,$x00,$inp + lvx_4w $xt1,$x10,$inp + lvx_4w $xt2,$x20,$inp + lvx_4w $xt3,$x30,$inp + + vxor $xr0,$xr0,$xan0 + vxor $xt1,$xt1,$xbn0 + vxor $xt2,$xt2,$xcn0 + vxor $xt3,$xt3,$xdn0 + + stvx_4w $xr0,$x00,$out + stvx_4w $xt1,$x10,$out + addi $inp,$inp,0x40 + stvx_4w $xt2,$x20,$out + subi $len,$len,0x40 + stvx_4w $xt3,$x30,$out + addi $out,$out,0x40 + beq Ldone_vsx_8x + + vadduwm $xan0,$xa6,@K[0] + vadduwm $xbn0,$xb6,@K[1] + vadduwm $xcn0,$xcn6,@K[2] + vadduwm $xdn0,$xdn6,@K[3] + + be?vperm $xan0,$xan0,$xan0,$beperm + be?vperm $xbn0,$xbn0,$xbn0,$beperm + be?vperm $xcn0,$xcn0,$xcn0,$beperm + be?vperm $xdn0,$xdn0,$xdn0,$beperm + + ${UCMP}i $len,0x40 + blt Ltail_vsx_8x_1 + + lvx_4w $xr0,$x00,$inp + lvx_4w $xt1,$x10,$inp + lvx_4w $xt2,$x20,$inp + lvx_4w $xt3,$x30,$inp + + vxor $xr0,$xr0,$xan0 + vxor $xt1,$xt1,$xbn0 + vxor $xt2,$xt2,$xcn0 + vxor $xt3,$xt3,$xdn0 + + stvx_4w $xr0,$x00,$out + stvx_4w $xt1,$x10,$out + addi $inp,$inp,0x40 + stvx_4w $xt2,$x20,$out + subi $len,$len,0x40 + stvx_4w $xt3,$x30,$out + addi $out,$out,0x40 + beq Ldone_vsx_8x + + vadduwm $xan0,$xa7,@K[0] + vadduwm $xbn0,$xb7,@K[1] + vadduwm $xcn0,$xcn7,@K[2] + vadduwm $xdn0,$xdn7,@K[3] + + be?vperm $xan0,$xan0,$xan0,$beperm + be?vperm $xbn0,$xbn0,$xbn0,$beperm + be?vperm $xcn0,$xcn0,$xcn0,$beperm + be?vperm $xdn0,$xdn0,$xdn0,$beperm + + ${UCMP}i $len,0x40 + blt Ltail_vsx_8x_1 + + lvx_4w $xr0,$x00,$inp + lvx_4w $xt1,$x10,$inp + lvx_4w $xt2,$x20,$inp + lvx_4w $xt3,$x30,$inp + + vxor $xr0,$xr0,$xan0 + vxor $xt1,$xt1,$xbn0 + vxor $xt2,$xt2,$xcn0 + vxor $xt3,$xt3,$xdn0 + + stvx_4w $xr0,$x00,$out + stvx_4w $xt1,$x10,$out + addi $inp,$inp,0x40 + stvx_4w $xt2,$x20,$out + subi $len,$len,0x40 + stvx_4w $xt3,$x30,$out + addi $out,$out,0x40 + beq Ldone_vsx_8x + + mtctr r0 + bne Loop_outer_vsx_8x + +Ldone_vsx_8x: + lwz r12,`$FRAME-4`($sp) # pull vrsave + li r10,`15+$LOCALS+64` + li r11,`31+$LOCALS+64` + $POP r0, `$FRAME+$LRSAVE`($sp) + mtspr 256,r12 # restore vrsave + lvx v24,r10,$sp + addi r10,r10,32 + lvx v25,r11,$sp + addi r11,r11,32 + lvx v26,r10,$sp + addi r10,r10,32 + lvx v27,r11,$sp + addi r11,r11,32 + lvx v28,r10,$sp + addi r10,r10,32 + lvx v29,r11,$sp + addi r11,r11,32 + lvx v30,r10,$sp + lvx v31,r11,$sp + mtlr r0 + addi $sp,$sp,$FRAME + blr + +.align 4 +Ltail_vsx_8x: + addi r11,$sp,$LOCALS + mtctr $len + stvx_4w $xa0,$x00,r11 # offload block to stack + stvx_4w $xb0,$x10,r11 + stvx_4w $xc0,$x20,r11 + stvx_4w $xd0,$x30,r11 + subi r12,r11,1 # prepare for *++ptr + subi $inp,$inp,1 + subi $out,$out,1 + bl Loop_tail_vsx_8x +Ltail_vsx_8x_1: + addi r11,$sp,$LOCALS + mtctr $len + stvx_4w $xan0,$x00,r11 # offload block to stack + stvx_4w $xbn0,$x10,r11 + stvx_4w $xcn0,$x20,r11 + stvx_4w $xdn0,$x30,r11 + subi r12,r11,1 # prepare for *++ptr + subi $inp,$inp,1 + subi $out,$out,1 + bl Loop_tail_vsx_8x + +Loop_tail_vsx_8x: + lbzu r6,1(r12) + lbzu r7,1($inp) + xor r6,r6,r7 + stbu r6,1($out) + bdnz Loop_tail_vsx_8x + + stvx_4w $K[0],$x00,r11 # wipe copy of the block + stvx_4w $K[0],$x10,r11 + stvx_4w $K[0],$x20,r11 + stvx_4w $K[0],$x30,r11 + + b Ldone_vsx_8x + .long 0 + .byte 0,12,0x04,1,0x80,0,5,0 + .long 0 +.size .ChaCha20_ctr32_vsx_8x,.-.ChaCha20_ctr32_vsx_8x +___ +}}} + + +$code.=<<___; +.align 5 +Lconsts: + mflr r0 + bcl 20,31,\$+4 + mflr r12 #vvvvv "distance between . and Lsigma + addi r12,r12,`64-8` + mtlr r0 + blr + .long 0 + .byte 0,12,0x14,0,0,0,0,0 + .space `64-9*4` +Lsigma: + .long 0x61707865,0x3320646e,0x79622d32,0x6b206574 + .long 1,0,0,0 + .long 2,0,0,0 + .long 3,0,0,0 + .long 4,0,0,0 +___ +$code.=<<___ if ($LITTLE_ENDIAN); + .long 0x0e0f0c0d,0x0a0b0809,0x06070405,0x02030001 + .long 0x0d0e0f0c,0x090a0b08,0x05060704,0x01020300 +___ +$code.=<<___ if (!$LITTLE_ENDIAN); # flipped words + .long 0x02030001,0x06070405,0x0a0b0809,0x0e0f0c0d + .long 0x01020300,0x05060704,0x090a0b08,0x0d0e0f0c +___ +$code.=<<___; + .long 0x61707865,0x61707865,0x61707865,0x61707865 + .long 0x3320646e,0x3320646e,0x3320646e,0x3320646e + .long 0x79622d32,0x79622d32,0x79622d32,0x79622d32 + .long 0x6b206574,0x6b206574,0x6b206574,0x6b206574 + .long 0,1,2,3 + .long 0x03020100,0x07060504,0x0b0a0908,0x0f0e0d0c +.asciz "ChaCha20 for PowerPC/AltiVec, CRYPTOGAMS by " +.align 2 +___ + +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval $1/ge; + + # instructions prefixed with '?' are endian-specific and need + # to be adjusted accordingly... + if ($flavour !~ /le$/) { # big-endian + s/be\?// or + s/le\?/#le#/ or + s/\?lvsr/lvsl/ or + s/\?lvsl/lvsr/ or + s/\?(vperm\s+v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+)/$1$3$2$4/ or + s/vrldoi(\s+v[0-9]+,\s*)(v[0-9]+,)\s*([0-9]+)/vsldoi$1$2$2 16-$3/; + } else { # little-endian + s/le\?// or + s/be\?/#be#/ or + s/\?([a-z]+)/$1/ or + s/vrldoi(\s+v[0-9]+,\s*)(v[0-9]+,)\s*([0-9]+)/vsldoi$1$2$2 $3/; + } + + print $_,"\n"; +} + +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/ec/asm/ecp_nistp384-ppc64.pl b/crypto/openssl/crypto/ec/asm/ecp_nistp384-ppc64.pl new file mode 100755 index 000000000000..43028d047d08 --- /dev/null +++ b/crypto/openssl/crypto/ec/asm/ecp_nistp384-ppc64.pl @@ -0,0 +1,1532 @@ +#! /usr/bin/env perl +# Copyright 2023-2025 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html +# +# ==================================================================== +# Written by Danny Tsen # for the OpenSSL project. +# +# Copyright 2025- IBM Corp. +# ==================================================================== +# +# p384 lower-level primitives for PPC64. +# + + +use strict; +use warnings; + +my $flavour = shift; +my $output = ""; +while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} +if (!$output) { + $output = "-"; +} + +my ($xlate, $dir); +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or +die "can't locate ppc-xlate.pl"; + +open OUT,"| \"$^X\" $xlate $flavour $output"; +*STDOUT=*OUT; + +my $code = ""; + +$code.=<<___; +.machine "any" +.text + +.globl p384_felem_mul +.type p384_felem_mul,\@function +.align 4 +p384_felem_mul: + + stdu 1, -176(1) + mflr 0 + std 14, 56(1) + std 15, 64(1) + std 16, 72(1) + std 17, 80(1) + std 18, 88(1) + std 19, 96(1) + std 20, 104(1) + std 21, 112(1) + std 22, 120(1) + + bl _p384_felem_mul_core + + mtlr 0 + ld 14, 56(1) + ld 15, 64(1) + ld 16, 72(1) + ld 17, 80(1) + ld 18, 88(1) + ld 19, 96(1) + ld 20, 104(1) + ld 21, 112(1) + ld 22, 120(1) + addi 1, 1, 176 + blr +.size p384_felem_mul,.-p384_felem_mul + +.globl p384_felem_square +.type p384_felem_square,\@function +.align 4 +p384_felem_square: + + stdu 1, -176(1) + mflr 0 + std 14, 56(1) + std 15, 64(1) + std 16, 72(1) + std 17, 80(1) + + bl _p384_felem_square_core + + mtlr 0 + ld 14, 56(1) + ld 15, 64(1) + ld 16, 72(1) + ld 17, 80(1) + addi 1, 1, 176 + blr +.size p384_felem_square,.-p384_felem_square + +# +# Felem mul core function - +# r3, r4 and r5 need to pre-loaded. +# +.type _p384_felem_mul_core,\@function +.align 4 +_p384_felem_mul_core: + + ld 6,0(4) + ld 14,0(5) + ld 7,8(4) + ld 15,8(5) + ld 8,16(4) + ld 16,16(5) + ld 9,24(4) + ld 17,24(5) + ld 10,32(4) + ld 18,32(5) + ld 11,40(4) + ld 19,40(5) + ld 12,48(4) + ld 20,48(5) + + # out0 + mulld 21, 14, 6 + mulhdu 22, 14, 6 + std 21, 0(3) + std 22, 8(3) + + vxor 0, 0, 0 + + # out1 + mtvsrdd 32+13, 14, 6 + mtvsrdd 32+14, 7, 15 + vmsumudm 1, 13, 14, 0 + + # out2 + mtvsrdd 32+15, 15, 6 + mtvsrdd 32+16, 7, 16 + mtvsrdd 32+17, 0, 8 + mtvsrdd 32+18, 0, 14 + vmsumudm 19, 15, 16, 0 + vmsumudm 2, 17, 18, 19 + + # out3 + mtvsrdd 32+13, 16, 6 + mtvsrdd 32+14, 7, 17 + mtvsrdd 32+15, 14, 8 + mtvsrdd 32+16, 9, 15 + vmsumudm 19, 13, 14, 0 + vmsumudm 3, 15, 16, 19 + + # out4 + mtvsrdd 32+13, 17, 6 + mtvsrdd 32+14, 7, 18 + mtvsrdd 32+15, 15, 8 + mtvsrdd 32+16, 9, 16 + mtvsrdd 32+17, 0, 10 + mtvsrdd 32+18, 0, 14 + vmsumudm 19, 13, 14, 0 + vmsumudm 4, 15, 16, 19 + vmsumudm 4, 17, 18, 4 + + # out5 + mtvsrdd 32+13, 18, 6 + mtvsrdd 32+14, 7, 19 + mtvsrdd 32+15, 16, 8 + mtvsrdd 32+16, 9, 17 + mtvsrdd 32+17, 14, 10 + mtvsrdd 32+18, 11, 15 + vmsumudm 19, 13, 14, 0 + vmsumudm 5, 15, 16, 19 + vmsumudm 5, 17, 18, 5 + + stxv 32+1, 16(3) + stxv 32+2, 32(3) + stxv 32+3, 48(3) + stxv 32+4, 64(3) + stxv 32+5, 80(3) + + # out6 + mtvsrdd 32+13, 19, 6 + mtvsrdd 32+14, 7, 20 + mtvsrdd 32+15, 17, 8 + mtvsrdd 32+16, 9, 18 + mtvsrdd 32+17, 15, 10 + mtvsrdd 32+18, 11, 16 + vmsumudm 19, 13, 14, 0 + vmsumudm 6, 15, 16, 19 + mtvsrdd 32+13, 0, 12 + mtvsrdd 32+14, 0, 14 + vmsumudm 19, 17, 18, 6 + vmsumudm 6, 13, 14, 19 + + # out7 + mtvsrdd 32+13, 19, 7 + mtvsrdd 32+14, 8, 20 + mtvsrdd 32+15, 17, 9 + mtvsrdd 32+16, 10, 18 + mtvsrdd 32+17, 15, 11 + mtvsrdd 32+18, 12, 16 + vmsumudm 19, 13, 14, 0 + vmsumudm 7, 15, 16, 19 + vmsumudm 7, 17, 18, 7 + + # out8 + mtvsrdd 32+13, 19, 8 + mtvsrdd 32+14, 9, 20 + mtvsrdd 32+15, 17, 10 + mtvsrdd 32+16, 11, 18 + mtvsrdd 32+17, 0, 12 + mtvsrdd 32+18, 0, 16 + vmsumudm 19, 13, 14, 0 + vmsumudm 8, 15, 16, 19 + vmsumudm 8, 17, 18, 8 + + # out9 + mtvsrdd 32+13, 19, 9 + mtvsrdd 32+14, 10, 20 + mtvsrdd 32+15, 17, 11 + mtvsrdd 32+16, 12, 18 + vmsumudm 19, 13, 14, 0 + vmsumudm 9, 15, 16, 19 + + # out10 + mtvsrdd 32+13, 19, 10 + mtvsrdd 32+14, 11, 20 + mtvsrdd 32+15, 0, 12 + mtvsrdd 32+16, 0, 18 + vmsumudm 19, 13, 14, 0 + vmsumudm 10, 15, 16, 19 + + # out11 + mtvsrdd 32+17, 19, 11 + mtvsrdd 32+18, 12, 20 + vmsumudm 11, 17, 18, 0 + + stxv 32+6, 96(3) + stxv 32+7, 112(3) + stxv 32+8, 128(3) + stxv 32+9, 144(3) + stxv 32+10, 160(3) + stxv 32+11, 176(3) + + # out12 + mulld 21, 20, 12 + mulhdu 22, 20, 12 # out12 + + std 21, 192(3) + std 22, 200(3) + + blr +.size _p384_felem_mul_core,.-_p384_felem_mul_core + +# +# Felem square core function - +# r3 and r4 need to pre-loaded. +# +.type _p384_felem_square_core,\@function +.align 4 +_p384_felem_square_core: + + ld 6, 0(4) + ld 7, 8(4) + ld 8, 16(4) + ld 9, 24(4) + ld 10, 32(4) + ld 11, 40(4) + ld 12, 48(4) + + vxor 0, 0, 0 + + # out0 + mulld 14, 6, 6 + mulhdu 15, 6, 6 + std 14, 0(3) + std 15, 8(3) + + # out1 + add 14, 6, 6 + mtvsrdd 32+13, 0, 14 + mtvsrdd 32+14, 0, 7 + vmsumudm 1, 13, 14, 0 + + # out2 + mtvsrdd 32+15, 7, 14 + mtvsrdd 32+16, 7, 8 + vmsumudm 2, 15, 16, 0 + + # out3 + add 15, 7, 7 + mtvsrdd 32+13, 8, 14 + mtvsrdd 32+14, 15, 9 + vmsumudm 3, 13, 14, 0 + + # out4 + mtvsrdd 32+13, 9, 14 + mtvsrdd 32+14, 15, 10 + mtvsrdd 32+15, 0, 8 + vmsumudm 4, 13, 14, 0 + vmsumudm 4, 15, 15, 4 + + # out5 + mtvsrdd 32+13, 10, 14 + mtvsrdd 32+14, 15, 11 + add 16, 8, 8 + mtvsrdd 32+15, 0, 16 + mtvsrdd 32+16, 0, 9 + vmsumudm 5, 13, 14, 0 + vmsumudm 5, 15, 16, 5 + + stxv 32+1, 16(3) + stxv 32+2, 32(3) + stxv 32+3, 48(3) + stxv 32+4, 64(3) + + # out6 + mtvsrdd 32+13, 11, 14 + mtvsrdd 32+14, 15, 12 + mtvsrdd 32+15, 9, 16 + mtvsrdd 32+16, 9, 10 + stxv 32+5, 80(3) + vmsumudm 19, 13, 14, 0 + vmsumudm 6, 15, 16, 19 + + # out7 + add 17, 9, 9 + mtvsrdd 32+13, 11, 15 + mtvsrdd 32+14, 16, 12 + mtvsrdd 32+15, 0, 17 + mtvsrdd 32+16, 0, 10 + vmsumudm 19, 13, 14, 0 + vmsumudm 7, 15, 16, 19 + + # out8 + mtvsrdd 32+13, 11, 16 + mtvsrdd 32+14, 17, 12 + mtvsrdd 32+15, 0, 10 + vmsumudm 19, 13, 14, 0 + vmsumudm 8, 15, 15, 19 + + # out9 + add 14, 10, 10 + mtvsrdd 32+13, 11, 17 + mtvsrdd 32+14, 14, 12 + vmsumudm 9, 13, 14, 0 + + # out10 + mtvsrdd 32+13, 11, 14 + mtvsrdd 32+14, 11, 12 + vmsumudm 10, 13, 14, 0 + + stxv 32+6, 96(3) + stxv 32+7, 112(3) + + # out11 + #add 14, 11, 11 + #mtvsrdd 32+13, 0, 14 + #mtvsrdd 32+14, 0, 12 + #vmsumudm 11, 13, 14, 0 + + mulld 6, 12, 11 + mulhdu 7, 12, 11 + addc 8, 6, 6 + adde 9, 7, 7 + + stxv 32+8, 128(3) + stxv 32+9, 144(3) + stxv 32+10, 160(3) + #stxv 32+11, 176(3) + + # out12 + mulld 14, 12, 12 + mulhdu 15, 12, 12 + + std 8, 176(3) + std 9, 184(3) + std 14, 192(3) + std 15, 200(3) + + blr +.size _p384_felem_square_core,.-_p384_felem_square_core + +# +# widefelem (128 bits) * 8 +# +.macro F128_X_8 _off1 _off2 + ld 9,\\_off1(3) + ld 8,\\_off2(3) + srdi 10,9,61 + rldimi 10,8,3,0 + sldi 9,9,3 + std 9,\\_off1(3) + std 10,\\_off2(3) +.endm + +.globl p384_felem128_mul_by_8 +.type p384_felem128_mul_by_8, \@function +.align 4 +p384_felem128_mul_by_8: + + F128_X_8 0, 8 + + F128_X_8 16, 24 + + F128_X_8 32, 40 + + F128_X_8 48, 56 + + F128_X_8 64, 72 + + F128_X_8 80, 88 + + F128_X_8 96, 104 + + F128_X_8 112, 120 + + F128_X_8 128, 136 + + F128_X_8 144, 152 + + F128_X_8 160, 168 + + F128_X_8 176, 184 + + F128_X_8 192, 200 + + blr +.size p384_felem128_mul_by_8,.-p384_felem128_mul_by_8 + +# +# widefelem (128 bits) * 2 +# +.macro F128_X_2 _off1 _off2 + ld 9,\\_off1(3) + ld 8,\\_off2(3) + srdi 10,9,63 + rldimi 10,8,1,0 + sldi 9,9,1 + std 9,\\_off1(3) + std 10,\\_off2(3) +.endm + +.globl p384_felem128_mul_by_2 +.type p384_felem128_mul_by_2, \@function +.align 4 +p384_felem128_mul_by_2: + + F128_X_2 0, 8 + + F128_X_2 16, 24 + + F128_X_2 32, 40 + + F128_X_2 48, 56 + + F128_X_2 64, 72 + + F128_X_2 80, 88 + + F128_X_2 96, 104 + + F128_X_2 112, 120 + + F128_X_2 128, 136 + + F128_X_2 144, 152 + + F128_X_2 160, 168 + + F128_X_2 176, 184 + + F128_X_2 192, 200 + + blr +.size p384_felem128_mul_by_2,.-p384_felem128_mul_by_2 + +.globl p384_felem_diff128 +.type p384_felem_diff128, \@function +.align 4 +p384_felem_diff128: + + addis 5, 2, .LConst_two127\@toc\@ha + addi 5, 5, .LConst_two127\@toc\@l + + ld 10, 0(3) + ld 8, 8(3) + li 9, 0 + addc 10, 10, 9 + li 7, -1 + rldicr 7, 7, 0, 0 # two127 + adde 8, 8, 7 + ld 11, 0(4) + ld 12, 8(4) + subfc 11, 11, 10 + subfe 12, 12, 8 + std 11, 0(3) # out0 + std 12, 8(3) + + # two127m71 = (r10, r9) + ld 8, 16(3) + ld 7, 24(3) + ld 10, 24(5) # two127m71 + addc 8, 8, 9 + adde 7, 7, 10 + ld 11, 16(4) + ld 12, 24(4) + subfc 11, 11, 8 + subfe 12, 12, 7 + std 11, 16(3) # out1 + std 12, 24(3) + + ld 8, 32(3) + ld 7, 40(3) + addc 8, 8, 9 + adde 7, 7, 10 + ld 11, 32(4) + ld 12, 40(4) + subfc 11, 11, 8 + subfe 12, 12, 7 + std 11, 32(3) # out2 + std 12, 40(3) + + ld 8, 48(3) + ld 7, 56(3) + addc 8, 8, 9 + adde 7, 7, 10 + ld 11, 48(4) + ld 12, 56(4) + subfc 11, 11, 8 + subfe 12, 12, 7 + std 11, 48(3) # out3 + std 12, 56(3) + + ld 8, 64(3) + ld 7, 72(3) + addc 8, 8, 9 + adde 7, 7, 10 + ld 11, 64(4) + ld 12, 72(4) + subfc 11, 11, 8 + subfe 12, 12, 7 + std 11, 64(3) # out4 + std 12, 72(3) + + ld 8, 80(3) + ld 7, 88(3) + addc 8, 8, 9 + adde 7, 7, 10 + ld 11, 80(4) + ld 12, 88(4) + subfc 11, 11, 8 + subfe 12, 12, 7 + std 11, 80(3) # out5 + std 12, 88(3) + + ld 8, 96(3) + ld 7, 104(3) + ld 6, 40(5) # two127p111m79m71 + addc 8, 8, 9 + adde 7, 7, 6 + ld 11, 96(4) + ld 12, 104(4) + subfc 11, 11, 8 + subfe 12, 12, 7 + std 11, 96(3) # out6 + std 12, 104(3) + + ld 8, 112(3) + ld 7, 120(3) + ld 6, 56(5) # two127m119m71 + addc 8, 8, 9 + adde 7, 7, 6 + ld 11, 112(4) + ld 12, 120(4) + subfc 11, 11, 8 + subfe 12, 12, 7 + std 11, 112(3) # out7 + std 12, 120(3) + + ld 8, 128(3) + ld 7, 136(3) + ld 6, 72(5) # two127m95m71 + addc 8, 8, 9 + adde 7, 7, 6 + ld 11, 128(4) + ld 12, 136(4) + subfc 11, 11, 8 + subfe 12, 12, 7 + std 11, 128(3) # out8 + std 12, 136(3) + + ld 8, 144(3) + ld 7, 152(3) + addc 8, 8, 9 + adde 7, 7, 10 + ld 11, 144(4) + ld 12, 152(4) + subfc 11, 11, 8 + subfe 12, 12, 7 + std 11, 144(3) # out9 + std 12, 152(3) + + ld 8, 160(3) + ld 7, 168(3) + addc 8, 8, 9 + adde 7, 7, 10 + ld 11, 160(4) + ld 12, 168(4) + subfc 11, 11, 8 + subfe 12, 12, 7 + std 11, 160(3) # out10 + std 12, 168(3) + + ld 8, 176(3) + ld 7, 184(3) + addc 8, 8, 9 + adde 7, 7, 10 + ld 11, 176(4) + ld 12, 184(4) + subfc 11, 11, 8 + subfe 12, 12, 7 + std 11, 176(3) # out11 + std 12, 184(3) + + ld 8, 192(3) + ld 7, 200(3) + addc 8, 8, 9 + adde 7, 7, 10 + ld 11, 192(4) + ld 12, 200(4) + subfc 11, 11, 8 + subfe 12, 12, 7 + std 11, 192(3) # out12 + std 12, 200(3) + + blr +.size p384_felem_diff128,.-p384_felem_diff128 + +.data +.align 4 +.LConst_two127: +#two127 +.long 0x00000000, 0x00000000, 0x00000000, 0x80000000 +#two127m71 +.long 0x00000000, 0x00000000, 0xffffff80, 0x7fffffff +#two127p111m79m71 +.long 0x00000000, 0x00000000, 0xffff7f80, 0x80007fff +#two127m119m71 +.long 0x00000000, 0x00000000, 0xffffff80, 0x7f7fffff +#two127m95m71 +.long 0x00000000, 0x00000000, 0x7fffff80, 0x7fffffff + +.text + +.globl p384_felem_diff_128_64 +.type p384_felem_diff_128_64, \@function +.align 4 +p384_felem_diff_128_64: + addis 5, 2, .LConst_128_two64\@toc\@ha + addi 5, 5, .LConst_128_two64\@toc\@l + + ld 9, 0(3) + ld 10, 8(3) + ld 8, 48(5) # two64p48m16 + li 7, 0 + addc 9, 9, 8 + li 6, 1 + adde 10, 10, 6 + ld 11, 0(4) + subfc 8, 11, 9 + subfe 12, 7, 10 + std 8, 0(3) # out0 + std 12, 8(3) + + ld 9, 16(3) + ld 10, 24(3) + ld 8, 0(5) # two64m56m8 + addc 9, 9, 8 + addze 10, 10 + ld 11, 8(4) + subfc 11, 11, 9 + subfe 12, 7, 10 + std 11, 16(3) # out1 + std 12, 24(3) + + ld 9, 32(3) + ld 10, 40(3) + ld 8, 16(5) # two64m32m8 + addc 9, 9, 8 + addze 10, 10 + ld 11, 16(4) + subfc 11, 11, 9 + subfe 12, 7, 10 + std 11, 32(3) # out2 + std 12, 40(3) + + ld 10, 48(3) + ld 8, 56(3) + #ld 9, 32(5) # two64m8 + li 9, -256 # two64m8 + addc 10, 10, 9 + addze 8, 8 + ld 11, 24(4) + subfc 11, 11, 10 + subfe 12, 7, 8 + std 11, 48(3) # out3 + std 12, 56(3) + + ld 10, 64(3) + ld 8, 72(3) + addc 10, 10, 9 + addze 8, 8 + ld 11, 32(4) + subfc 11, 11, 10 + subfe 12, 7, 8 + std 11, 64(3) # out4 + std 12, 72(3) + + ld 10, 80(3) + ld 8, 88(3) + addc 10, 10, 9 + addze 8, 8 + ld 11, 40(4) + subfc 11, 11, 10 + subfe 12, 7, 8 + std 11, 80(3) # out5 + std 12, 88(3) + + ld 10, 96(3) + ld 8, 104(3) + addc 10, 10, 9 + addze 9, 8 + ld 11, 48(4) + subfc 11, 11, 10 + subfe 12, 7, 9 + std 11, 96(3) # out6 + std 12, 104(3) + + blr +.size p384_felem_diff_128_64,.-p384_felem_diff_128_64 + +.data +.align 4 +.LConst_128_two64: +#two64m56m8 +.long 0xffffff00, 0xfeffffff, 0x00000000, 0x00000000 +#two64m32m8 +.long 0xffffff00, 0xfffffffe, 0x00000000, 0x00000000 +#two64m8 +.long 0xffffff00, 0xffffffff, 0x00000000, 0x00000000 +#two64p48m16 +.long 0xffff0000, 0x0000ffff, 0x00000001, 0x00000000 + +.LConst_two60: +#two60m52m4 +.long 0xfffffff0, 0x0fefffff, 0x0, 0x0 +#two60p44m12 +.long 0xfffff000, 0x10000fff, 0x0, 0x0 +#two60m28m4 +.long 0xeffffff0, 0x0fffffff, 0x0, 0x0 +#two60m4 +.long 0xfffffff0, 0x0fffffff, 0x0, 0x0 + +.text +# +# static void felem_diff64(felem out, const felem in) +# +.globl p384_felem_diff64 +.type p384_felem_diff64, \@function +.align 4 +p384_felem_diff64: + addis 5, 2, .LConst_two60\@toc\@ha + addi 5, 5, .LConst_two60\@toc\@l + + ld 9, 0(3) + ld 8, 16(5) # two60p44m12 + li 7, 0 + add 9, 9, 8 + ld 11, 0(4) + subf 8, 11, 9 + std 8, 0(3) # out0 + + ld 9, 8(3) + ld 8, 0(5) # two60m52m4 + add 9, 9, 8 + ld 11, 8(4) + subf 11, 11, 9 + std 11, 8(3) # out1 + + ld 9, 16(3) + ld 8, 32(5) # two60m28m4 + add 9, 9, 8 + ld 11, 16(4) + subf 11, 11, 9 + std 11, 16(3) # out2 + + ld 10, 24(3) + ld 9, 48(5) # two60m4 + add 10, 10, 9 + ld 12, 24(4) + subf 12, 12, 10 + std 12, 24(3) # out3 + + ld 10, 32(3) + add 10, 10, 9 + ld 11, 32(4) + subf 11, 11, 10 + std 11, 32(3) # out4 + + ld 10, 40(3) + add 10, 10, 9 + ld 12, 40(4) + subf 12, 12, 10 + std 12, 40(3) # out5 + + ld 10, 48(3) + add 10, 10, 9 + ld 11, 48(4) + subf 11, 11, 10 + std 11, 48(3) # out6 + + blr +.size p384_felem_diff64,.-p384_felem_diff64 + +.text +# +# Shift 128 bits right +# +.macro SHR o_h o_l in_h in_l nbits + srdi \\o_l, \\in_l, \\nbits # shift lower right + rldimi \\o_l, \\in_h, 64-\\nbits, 0 # insert <64-nbits> from hi + srdi \\o_h, \\in_h, \\nbits # shift higher right +.endm + +# +# static void felem_reduce(felem out, const widefelem in) +# +.global p384_felem_reduce +.type p384_felem_reduce,\@function +.align 4 +p384_felem_reduce: + + stdu 1, -208(1) + mflr 0 + std 14, 56(1) + std 15, 64(1) + std 16, 72(1) + std 17, 80(1) + std 18, 88(1) + std 19, 96(1) + std 20, 104(1) + std 21, 112(1) + std 22, 120(1) + std 23, 128(1) + std 24, 136(1) + std 25, 144(1) + std 26, 152(1) + std 27, 160(1) + std 28, 168(1) + std 29, 176(1) + std 30, 184(1) + std 31, 192(1) + + bl _p384_felem_reduce_core + + mtlr 0 + ld 14, 56(1) + ld 15, 64(1) + ld 16, 72(1) + ld 17, 80(1) + ld 18, 88(1) + ld 19, 96(1) + ld 20, 104(1) + ld 21, 112(1) + ld 22, 120(1) + ld 23, 128(1) + ld 24, 136(1) + ld 25, 144(1) + ld 26, 152(1) + ld 27, 160(1) + ld 28, 168(1) + ld 29, 176(1) + ld 30, 184(1) + ld 31, 192(1) + addi 1, 1, 208 + blr +.size p384_felem_reduce,.-p384_felem_reduce + +# +# Felem reduction core function - +# r3 and r4 need to pre-loaded. +# +.type _p384_felem_reduce_core,\@function +.align 4 +_p384_felem_reduce_core: + addis 12, 2, .LConst\@toc\@ha + addi 12, 12, .LConst\@toc\@l + + # load constat p + ld 11, 8(12) # hi - two124m68 + + # acc[6] = in[6] + two124m68; + ld 26, 96(4) # in[6].l + ld 27, 96+8(4) # in[6].h + add 27, 27, 11 + + # acc[5] = in[5] + two124m68; + ld 24, 80(4) # in[5].l + ld 25, 80+8(4) # in[5].h + add 25, 25, 11 + + # acc[4] = in[4] + two124m68; + ld 22, 64(4) # in[4].l + ld 23, 64+8(4) # in[4].h + add 23, 23, 11 + + # acc[3] = in[3] + two124m68; + ld 20, 48(4) # in[3].l + ld 21, 48+8(4) # in[3].h + add 21, 21, 11 + + ld 11, 48+8(12) # hi - two124m92m68 + + # acc[2] = in[2] + two124m92m68; + ld 18, 32(4) # in[2].l + ld 19, 32+8(4) # in[2].h + add 19, 19, 11 + + ld 11, 16+8(12) # high - two124m116m68 + + # acc[1] = in[1] + two124m116m68; + ld 16, 16(4) # in[1].l + ld 17, 16+8(4) # in[1].h + add 17, 17, 11 + + ld 11, 32+8(12) # high - two124p108m76 + + # acc[0] = in[0] + two124p108m76; + ld 14, 0(4) # in[0].l + ld 15, 0+8(4) # in[0].h + add 15, 15, 11 + + # compute mask + li 7, -1 + + # Eliminate in[12] + + # acc[8] += in[12] >> 32; + ld 5, 192(4) # in[12].l + ld 6, 192+8(4) # in[12].h + SHR 9, 10, 6, 5, 32 + ld 30, 128(4) # in[8].l + ld 31, 136(4) # in[8].h + addc 30, 30, 10 + adde 31, 31, 9 + + # acc[7] += (in[12] & 0xffffffff) << 24; + srdi 11, 7, 32 # 0xffffffff + and 11, 11, 5 + sldi 11, 11, 24 # << 24 + ld 28, 112(4) # in[7].l + ld 29, 120(4) # in[7].h + addc 28, 28, 11 + addze 29, 29 + + # acc[7] += in[12] >> 8; + SHR 9, 10, 6, 5, 8 + addc 28, 28, 10 + adde 29, 29, 9 + + # acc[6] += (in[12] & 0xff) << 48; + andi. 11, 5, 0xff + sldi 11, 11, 48 + addc 26, 26, 11 + addze 27, 27 + + # acc[6] -= in[12] >> 16; + SHR 9, 10, 6, 5, 16 + subfc 26, 10, 26 + subfe 27, 9, 27 + + # acc[5] -= (in[12] & 0xffff) << 40; + srdi 11, 7, 48 # 0xffff + and 11, 11, 5 + sldi 11, 11, 40 # << 40 + li 9, 0 + subfc 24, 11, 24 + subfe 25, 9, 25 + + # acc[6] += in[12] >> 48; + SHR 9, 10, 6, 5, 48 + addc 26, 26, 10 + adde 27, 27, 9 + + # acc[5] += (in[12] & 0xffffffffffff) << 8; + srdi 11, 7, 16 # 0xffffffffffff + and 11, 11, 5 + sldi 11, 11, 8 # << 8 + addc 24, 24, 11 + addze 25, 25 + + # Eliminate in[11] + + # acc[7] += in[11] >> 32; + ld 5, 176(4) # in[11].l + ld 6, 176+8(4) # in[11].h + SHR 9, 10, 6, 5, 32 + addc 28, 28, 10 + adde 29, 29, 9 + + # acc[6] += (in[11] & 0xffffffff) << 24; + srdi 11, 7, 32 # 0xffffffff + and 11, 11, 5 + sldi 11, 11, 24 # << 24 + addc 26, 26, 11 + addze 27, 27 + + # acc[6] += in[11] >> 8; + SHR 9, 10, 6, 5, 8 + addc 26, 26, 10 + adde 27, 27, 9 + + # acc[5] += (in[11] & 0xff) << 48; + andi. 11, 5, 0xff + sldi 11, 11, 48 + addc 24, 24, 11 + addze 25, 25 + + # acc[5] -= in[11] >> 16; + SHR 9, 10, 6, 5, 16 + subfc 24, 10, 24 + subfe 25, 9, 25 + + # acc[4] -= (in[11] & 0xffff) << 40; + srdi 11, 7, 48 # 0xffff + and 11, 11, 5 + sldi 11, 11, 40 # << 40 + li 9, 0 + subfc 22, 11, 22 + subfe 23, 9, 23 + + # acc[5] += in[11] >> 48; + SHR 9, 10, 6, 5, 48 + addc 24, 24, 10 + adde 25, 25, 9 + + # acc[4] += (in[11] & 0xffffffffffff) << 8; + srdi 11, 7, 16 # 0xffffffffffff + and 11, 11, 5 + sldi 11, 11, 8 # << 8 + addc 22, 22, 11 + addze 23, 23 + + # Eliminate in[10] + + # acc[6] += in[10] >> 32; + ld 5, 160(4) # in[10].l + ld 6, 160+8(4) # in[10].h + SHR 9, 10, 6, 5, 32 + addc 26, 26, 10 + adde 27, 27, 9 + + # acc[5] += (in[10] & 0xffffffff) << 24; + srdi 11, 7, 32 # 0xffffffff + and 11, 11, 5 + sldi 11, 11, 24 # << 24 + addc 24, 24, 11 + addze 25, 25 + + # acc[5] += in[10] >> 8; + SHR 9, 10, 6, 5, 8 + addc 24, 24, 10 + adde 25, 25, 9 + + # acc[4] += (in[10] & 0xff) << 48; + andi. 11, 5, 0xff + sldi 11, 11, 48 + addc 22, 22, 11 + addze 23, 23 + + # acc[4] -= in[10] >> 16; + SHR 9, 10, 6, 5, 16 + subfc 22, 10, 22 + subfe 23, 9, 23 + + # acc[3] -= (in[10] & 0xffff) << 40; + srdi 11, 7, 48 # 0xffff + and 11, 11, 5 + sldi 11, 11, 40 # << 40 + li 9, 0 + subfc 20, 11, 20 + subfe 21, 9, 21 + + # acc[4] += in[10] >> 48; + SHR 9, 10, 6, 5, 48 + addc 22, 22, 10 + adde 23, 23, 9 + + # acc[3] += (in[10] & 0xffffffffffff) << 8; + srdi 11, 7, 16 # 0xffffffffffff + and 11, 11, 5 + sldi 11, 11, 8 # << 8 + addc 20, 20, 11 + addze 21, 21 + + # Eliminate in[9] + + # acc[5] += in[9] >> 32; + ld 5, 144(4) # in[9].l + ld 6, 144+8(4) # in[9].h + SHR 9, 10, 6, 5, 32 + addc 24, 24, 10 + adde 25, 25, 9 + + # acc[4] += (in[9] & 0xffffffff) << 24; + srdi 11, 7, 32 # 0xffffffff + and 11, 11, 5 + sldi 11, 11, 24 # << 24 + addc 22, 22, 11 + addze 23, 23 + + # acc[4] += in[9] >> 8; + SHR 9, 10, 6, 5, 8 + addc 22, 22, 10 + adde 23, 23, 9 + + # acc[3] += (in[9] & 0xff) << 48; + andi. 11, 5, 0xff + sldi 11, 11, 48 + addc 20, 20, 11 + addze 21, 21 + + # acc[3] -= in[9] >> 16; + SHR 9, 10, 6, 5, 16 + subfc 20, 10, 20 + subfe 21, 9, 21 + + # acc[2] -= (in[9] & 0xffff) << 40; + srdi 11, 7, 48 # 0xffff + and 11, 11, 5 + sldi 11, 11, 40 # << 40 + li 9, 0 + subfc 18, 11, 18 + subfe 19, 9, 19 + + # acc[3] += in[9] >> 48; + SHR 9, 10, 6, 5, 48 + addc 20, 20, 10 + adde 21, 21, 9 + + # acc[2] += (in[9] & 0xffffffffffff) << 8; + srdi 11, 7, 16 # 0xffffffffffff + and 11, 11, 5 + sldi 11, 11, 8 # << 8 + addc 18, 18, 11 + addze 19, 19 + + # Eliminate acc[8] + + # acc[4] += acc[8] >> 32; + mr 5, 30 # acc[8].l + mr 6, 31 # acc[8].h + SHR 9, 10, 6, 5, 32 + addc 22, 22, 10 + adde 23, 23, 9 + + # acc[3] += (acc[8] & 0xffffffff) << 24; + srdi 11, 7, 32 # 0xffffffff + and 11, 11, 5 + sldi 11, 11, 24 # << 24 + addc 20, 20, 11 + addze 21, 21 + + # acc[3] += acc[8] >> 8; + SHR 9, 10, 6, 5, 8 + addc 20, 20, 10 + adde 21, 21, 9 + + # acc[2] += (acc[8] & 0xff) << 48; + andi. 11, 5, 0xff + sldi 11, 11, 48 + addc 18, 18, 11 + addze 19, 19 + + # acc[2] -= acc[8] >> 16; + SHR 9, 10, 6, 5, 16 + subfc 18, 10, 18 + subfe 19, 9, 19 + + # acc[1] -= (acc[8] & 0xffff) << 40; + srdi 11, 7, 48 # 0xffff + and 11, 11, 5 + sldi 11, 11, 40 # << 40 + li 9, 0 + subfc 16, 11, 16 + subfe 17, 9, 17 + + #acc[2] += acc[8] >> 48; + SHR 9, 10, 6, 5, 48 + addc 18, 18, 10 + adde 19, 19, 9 + + # acc[1] += (acc[8] & 0xffffffffffff) << 8; + srdi 11, 7, 16 # 0xffffffffffff + and 11, 11, 5 + sldi 11, 11, 8 # << 8 + addc 16, 16, 11 + addze 17, 17 + + # Eliminate acc[7] + + # acc[3] += acc[7] >> 32; + mr 5, 28 # acc[7].l + mr 6, 29 # acc[7].h + SHR 9, 10, 6, 5, 32 + addc 20, 20, 10 + adde 21, 21, 9 + + # acc[2] += (acc[7] & 0xffffffff) << 24; + srdi 11, 7, 32 # 0xffffffff + and 11, 11, 5 + sldi 11, 11, 24 # << 24 + addc 18, 18, 11 + addze 19, 19 + + # acc[2] += acc[7] >> 8; + SHR 9, 10, 6, 5, 8 + addc 18, 18, 10 + adde 19, 19, 9 + + # acc[1] += (acc[7] & 0xff) << 48; + andi. 11, 5, 0xff + sldi 11, 11, 48 + addc 16, 16, 11 + addze 17, 17 + + # acc[1] -= acc[7] >> 16; + SHR 9, 10, 6, 5, 16 + subfc 16, 10, 16 + subfe 17, 9, 17 + + # acc[0] -= (acc[7] & 0xffff) << 40; + srdi 11, 7, 48 # 0xffff + and 11, 11, 5 + sldi 11, 11, 40 # << 40 + li 9, 0 + subfc 14, 11, 14 + subfe 15, 9, 15 + + # acc[1] += acc[7] >> 48; + SHR 9, 10, 6, 5, 48 + addc 16, 16, 10 + adde 17, 17, 9 + + # acc[0] += (acc[7] & 0xffffffffffff) << 8; + srdi 11, 7, 16 # 0xffffffffffff + and 11, 11, 5 + sldi 11, 11, 8 # << 8 + addc 14, 14, 11 + addze 15, 15 + + # + # Carry 4 -> 5 -> 6 + # + # acc[5] += acc[4] >> 56; + # acc[4] &= 0x00ffffffffffffff; + SHR 9, 10, 23, 22, 56 + addc 24, 24, 10 + adde 25, 25, 9 + srdi 11, 7, 8 # 0x00ffffffffffffff + and 22, 22, 11 + li 23, 0 + + # acc[6] += acc[5] >> 56; + # acc[5] &= 0x00ffffffffffffff; + SHR 9, 10, 25, 24, 56 + addc 26, 26, 10 + adde 27, 27, 9 + and 24, 24, 11 + li 25, 0 + + # [3]: Eliminate high bits of acc[6] */ + # temp = acc[6] >> 48; + # acc[6] &= 0x0000ffffffffffff; + SHR 31, 30, 27, 26, 48 # temp = acc[6] >> 48 + srdi 11, 7, 16 # 0x0000ffffffffffff + and 26, 26, 11 + li 27, 0 + + # temp < 2^80 + # acc[3] += temp >> 40; + SHR 9, 10, 31, 30, 40 + addc 20, 20, 10 + adde 21, 21, 9 + + # acc[2] += (temp & 0xffffffffff) << 16; + srdi 11, 7, 24 # 0xffffffffff + and 10, 30, 11 + sldi 10, 10, 16 + addc 18, 18, 10 + addze 19, 19 + + # acc[2] += temp >> 16; + SHR 9, 10, 31, 30, 16 + addc 18, 18, 10 + adde 19, 19, 9 + + # acc[1] += (temp & 0xffff) << 40; + srdi 11, 7, 48 # 0xffff + and 10, 30, 11 + sldi 10, 10, 40 + addc 16, 16, 10 + addze 17, 17 + + # acc[1] -= temp >> 24; + SHR 9, 10, 31, 30, 24 + subfc 16, 10, 16 + subfe 17, 9, 17 + + # acc[0] -= (temp & 0xffffff) << 32; + srdi 11, 7, 40 # 0xffffff + and 10, 30, 11 + sldi 10, 10, 32 + li 9, 0 + subfc 14, 10, 14 + subfe 15, 9, 15 + + # acc[0] += temp; + addc 14, 14, 30 + adde 15, 15, 31 + + # Carry 0 -> 1 -> 2 -> 3 -> 4 -> 5 -> 6 + # + # acc[1] += acc[0] >> 56; /* acc[1] < acc_old[1] + 2^72 */ + SHR 9, 10, 15, 14, 56 + addc 16, 16, 10 + adde 17, 17, 9 + + # acc[0] &= 0x00ffffffffffffff; + srdi 11, 7, 8 # 0x00ffffffffffffff + and 14, 14, 11 + li 15, 0 + + # acc[2] += acc[1] >> 56; /* acc[2] < acc_old[2] + 2^72 + 2^16 */ + SHR 9, 10, 17, 16, 56 + addc 18, 18, 10 + adde 19, 19, 9 + + # acc[1] &= 0x00ffffffffffffff; + and 16, 16, 11 + li 17, 0 + + # acc[3] += acc[2] >> 56; /* acc[3] < acc_old[3] + 2^72 + 2^16 */ + SHR 9, 10, 19, 18, 56 + addc 20, 20, 10 + adde 21, 21, 9 + + # acc[2] &= 0x00ffffffffffffff; + and 18, 18, 11 + li 19, 0 + + # acc[4] += acc[3] >> 56; + SHR 9, 10, 21, 20, 56 + addc 22, 22, 10 + adde 23, 23, 9 + + # acc[3] &= 0x00ffffffffffffff; + and 20, 20, 11 + li 21, 0 + + # acc[5] += acc[4] >> 56; + SHR 9, 10, 23, 22, 56 + addc 24, 24, 10 + adde 25, 25, 9 + + # acc[4] &= 0x00ffffffffffffff; + and 22, 22, 11 + + # acc[6] += acc[5] >> 56; + SHR 9, 10, 25, 24, 56 + addc 26, 26, 10 + adde 27, 27, 9 + + # acc[5] &= 0x00ffffffffffffff; + and 24, 24, 11 + + std 14, 0(3) + std 16, 8(3) + std 18, 16(3) + std 20, 24(3) + std 22, 32(3) + std 24, 40(3) + std 26, 48(3) + blr +.size _p384_felem_reduce_core,.-_p384_felem_reduce_core + +.data +.align 4 +.LConst: +# two124m68: +.long 0x0, 0x0, 0xfffffff0, 0xfffffff +# two124m116m68: +.long 0x0, 0x0, 0xfffffff0, 0xfefffff +#two124p108m76: +.long 0x0, 0x0, 0xfffff000, 0x10000fff +#two124m92m68: +.long 0x0, 0x0, 0xeffffff0, 0xfffffff + +.text + +# +# void p384_felem_square_reduce(felem out, const felem in) +# +.global p384_felem_square_reduce +.type p384_felem_square_reduce,\@function +.align 4 +p384_felem_square_reduce: + stdu 1, -512(1) + mflr 0 + std 14, 56(1) + std 15, 64(1) + std 16, 72(1) + std 17, 80(1) + std 18, 88(1) + std 19, 96(1) + std 20, 104(1) + std 21, 112(1) + std 22, 120(1) + std 23, 128(1) + std 24, 136(1) + std 25, 144(1) + std 26, 152(1) + std 27, 160(1) + std 28, 168(1) + std 29, 176(1) + std 30, 184(1) + std 31, 192(1) + + std 3, 496(1) + addi 3, 1, 208 + bl _p384_felem_square_core + + mr 4, 3 + ld 3, 496(1) + bl _p384_felem_reduce_core + + ld 14, 56(1) + ld 15, 64(1) + ld 16, 72(1) + ld 17, 80(1) + ld 18, 88(1) + ld 19, 96(1) + ld 20, 104(1) + ld 21, 112(1) + ld 22, 120(1) + ld 23, 128(1) + ld 24, 136(1) + ld 25, 144(1) + ld 26, 152(1) + ld 27, 160(1) + ld 28, 168(1) + ld 29, 176(1) + ld 30, 184(1) + ld 31, 192(1) + addi 1, 1, 512 + mtlr 0 + blr +.size p384_felem_square_reduce,.-p384_felem_square_reduce + +# +# void p384_felem_mul_reduce(felem out, const felem in1, const felem in2) +# +.global p384_felem_mul_reduce +.type p384_felem_mul_reduce,\@function +.align 5 +p384_felem_mul_reduce: + stdu 1, -512(1) + mflr 0 + std 14, 56(1) + std 15, 64(1) + std 16, 72(1) + std 17, 80(1) + std 18, 88(1) + std 19, 96(1) + std 20, 104(1) + std 21, 112(1) + std 22, 120(1) + std 23, 128(1) + std 24, 136(1) + std 25, 144(1) + std 26, 152(1) + std 27, 160(1) + std 28, 168(1) + std 29, 176(1) + std 30, 184(1) + std 31, 192(1) + + std 3, 496(1) + addi 3, 1, 208 + bl _p384_felem_mul_core + + mr 4, 3 + ld 3, 496(1) + bl _p384_felem_reduce_core + + ld 14, 56(1) + ld 15, 64(1) + ld 16, 72(1) + ld 17, 80(1) + ld 18, 88(1) + ld 19, 96(1) + ld 20, 104(1) + ld 21, 112(1) + ld 22, 120(1) + ld 23, 128(1) + ld 24, 136(1) + ld 25, 144(1) + ld 26, 152(1) + ld 27, 160(1) + ld 28, 168(1) + ld 29, 176(1) + ld 30, 184(1) + ld 31, 192(1) + addi 1, 1, 512 + mtlr 0 + blr +.size p384_felem_mul_reduce,.-p384_felem_mul_reduce +___ + +$code =~ s/\`([^\`]*)\`/eval $1/gem; +print $code; +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/ec/asm/ecp_nistz256-armv8.pl b/crypto/openssl/crypto/ec/asm/ecp_nistz256-armv8.pl index 9a9114e37bbb..d81d39443ea6 100755 --- a/crypto/openssl/crypto/ec/asm/ecp_nistz256-armv8.pl +++ b/crypto/openssl/crypto/ec/asm/ecp_nistz256-armv8.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2015-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2015-2025 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -56,7 +56,7 @@ my ($acc6,$acc7)=($apx,$bpx); # used in __ecp_nistz256_sqr_mont $code.=<<___; #include "arm_arch.h" -.text +.rodata ___ ######################################################################## # Convert ecp_nistz256_table.c to layout expected by ecp_nistz_gather_w7 @@ -118,6 +118,8 @@ $code.=<<___; .quad 0xccd1c8aaee00bc4f .asciz "ECP_NISTZ256 for ARMv8, CRYPTOGAMS by " +.text + // void ecp_nistz256_to_mont(BN_ULONG x0[4],const BN_ULONG x1[4]); .globl ecp_nistz256_to_mont .type ecp_nistz256_to_mont,%function @@ -128,12 +130,16 @@ ecp_nistz256_to_mont: add PTR(29),PTRN(sp),#0 stp PTR(19),PTR(20),[PTRN(sp),#(2*PTR_WIDTH)] - ldr $bi,.LRR // bp[0] + adrp $bi,.LRR + ldr $bi,[$bi,:lo12:.LRR] // bp[0] ldp $a0,$a1,[$ap] ldp $a2,$a3,[$ap,#16] - ldr $poly1,.Lpoly+8 - ldr $poly3,.Lpoly+24 - adr $bp,.LRR // &bp[0] + adrp $poly3,.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] + adrp $bp,.LRR // &bp[0] + add $bp,$bp,:lo12:.LRR bl __ecp_nistz256_mul_mont @@ -156,9 +162,12 @@ ecp_nistz256_from_mont: mov $bi,#1 // bp[0] ldp $a0,$a1,[$ap] ldp $a2,$a3,[$ap,#16] - ldr $poly1,.Lpoly+8 - ldr $poly3,.Lpoly+24 - adr $bp,.Lone // &bp[0] + adrp $poly3,.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] + adrp $bp,.Lone // &bp[0] + add $bp,$bp,:lo12:.Lone bl __ecp_nistz256_mul_mont @@ -182,8 +191,10 @@ ecp_nistz256_mul_mont: ldr $bi,[$bp] // bp[0] ldp $a0,$a1,[$ap] ldp $a2,$a3,[$ap,#16] - ldr $poly1,.Lpoly+8 - ldr $poly3,.Lpoly+24 + adrp $poly3,.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] bl __ecp_nistz256_mul_mont @@ -205,8 +216,10 @@ ecp_nistz256_sqr_mont: ldp $a0,$a1,[$ap] ldp $a2,$a3,[$ap,#16] - ldr $poly1,.Lpoly+8 - ldr $poly3,.Lpoly+24 + adrp $poly3,.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] bl __ecp_nistz256_sqr_mont @@ -230,8 +243,10 @@ ecp_nistz256_add: ldp $t0,$t1,[$bp] ldp $acc2,$acc3,[$ap,#16] ldp $t2,$t3,[$bp,#16] - ldr $poly1,.Lpoly+8 - ldr $poly3,.Lpoly+24 + adrp $poly3,.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] bl __ecp_nistz256_add @@ -251,8 +266,10 @@ ecp_nistz256_div_by_2: ldp $acc0,$acc1,[$ap] ldp $acc2,$acc3,[$ap,#16] - ldr $poly1,.Lpoly+8 - ldr $poly3,.Lpoly+24 + adrp $poly3,.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] bl __ecp_nistz256_div_by_2 @@ -272,8 +289,10 @@ ecp_nistz256_mul_by_2: ldp $acc0,$acc1,[$ap] ldp $acc2,$acc3,[$ap,#16] - ldr $poly1,.Lpoly+8 - ldr $poly3,.Lpoly+24 + adrp $poly3,.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] mov $t0,$acc0 mov $t1,$acc1 mov $t2,$acc2 @@ -297,8 +316,10 @@ ecp_nistz256_mul_by_3: ldp $acc0,$acc1,[$ap] ldp $acc2,$acc3,[$ap,#16] - ldr $poly1,.Lpoly+8 - ldr $poly3,.Lpoly+24 + adrp $poly3,.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] mov $t0,$acc0 mov $t1,$acc1 mov $t2,$acc2 @@ -334,8 +355,10 @@ ecp_nistz256_sub: ldp $acc0,$acc1,[$ap] ldp $acc2,$acc3,[$ap,#16] - ldr $poly1,.Lpoly+8 - ldr $poly3,.Lpoly+24 + adrp $poly3,.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] bl __ecp_nistz256_sub_from @@ -358,8 +381,10 @@ ecp_nistz256_neg: mov $acc1,xzr mov $acc2,xzr mov $acc3,xzr - ldr $poly1,.Lpoly+8 - ldr $poly3,.Lpoly+24 + adrp $poly3,.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] bl __ecp_nistz256_sub_from @@ -737,9 +762,11 @@ ecp_nistz256_point_double: mov $rp_real,$rp ldp $acc2,$acc3,[$ap,#48] mov $ap_real,$ap - ldr $poly1,.Lpoly+8 + adrp $poly3,.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] mov $t0,$acc0 - ldr $poly3,.Lpoly+24 + ldr $poly3,[$poly3,#24] mov $t1,$acc1 ldp $a0,$a1,[$ap_real,#64] // forward load for p256_sqr_mont mov $t2,$acc2 @@ -899,8 +926,10 @@ ecp_nistz256_point_add: mov $rp_real,$rp mov $ap_real,$ap mov $bp_real,$bp - ldr $poly1,.Lpoly+8 - ldr $poly3,.Lpoly+24 + adrp $poly3,.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] orr $t0,$a0,$a1 orr $t2,$a2,$a3 orr $in2infty,$t0,$t2 @@ -1154,8 +1183,10 @@ ecp_nistz256_point_add_affine: mov $rp_real,$rp mov $ap_real,$ap mov $bp_real,$bp - ldr $poly1,.Lpoly+8 - ldr $poly3,.Lpoly+24 + adrp $poly3,.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] ldp $a0,$a1,[$ap,#64] // in1_z ldp $a2,$a3,[$ap,#64+16] @@ -1306,7 +1337,8 @@ $code.=<<___; stp $acc2,$acc3,[$rp_real,#$i+16] ___ $code.=<<___ if ($i == 0); - adr $bp_real,.Lone_mont-64 + adrp $bp_real,.Lone_mont-64 + add $bp_real,$bp_real,:lo12:.Lone_mont-64 ___ } $code.=<<___; @@ -1358,7 +1390,8 @@ ecp_nistz256_ord_mul_mont: stp PTR(21),PTR(22),[PTRN(sp),#(4*PTR_WIDTH)] stp PTR(23),PTR(24),[PTRN(sp),#(6*PTR_WIDTH)] - adr $ordp,.Lord + adrp $ordp,.Lord + add $ordp,$ordp,:lo12:.Lord ldr $bi,[$bp] // bp[0] ldp $a0,$a1,[$ap] ldp $a2,$a3,[$ap,#16] @@ -1501,7 +1534,8 @@ ecp_nistz256_ord_sqr_mont: stp PTR(21),PTR(22),[PTRN(sp),#(4*PTR_WIDTH)] stp PTR(23),PTR(24),[PTRN(sp),#(6*PTR_WIDTH)] - adr $ordp,.Lord + adrp $ordp,.Lord + add $ordp,$ordp,:lo12:.Lord ldp $a0,$a1,[$ap] ldp $a2,$a3,[$ap,#16] diff --git a/crypto/openssl/crypto/ec/asm/ecp_nistz256-x86_64.pl b/crypto/openssl/crypto/ec/asm/ecp_nistz256-x86_64.pl index 430b14c86d8d..4da9a149a99a 100755 --- a/crypto/openssl/crypto/ec/asm/ecp_nistz256-x86_64.pl +++ b/crypto/openssl/crypto/ec/asm/ecp_nistz256-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2014-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2014-2024 The OpenSSL Project Authors. All Rights Reserved. # Copyright (c) 2014, Intel Corporation. All Rights Reserved. # Copyright (c) 2015 CloudFlare, Inc. # @@ -85,6 +85,7 @@ $code.=<<___; .extern OPENSSL_ia32cap_P # The polynomial +.section .rodata align=4096 .align 64 .Lpoly: .quad 0xffffffffffffffff, 0x00000000ffffffff, 0x0000000000000000, 0xffffffff00000001 @@ -107,6 +108,7 @@ $code.=<<___; .quad 0xf3b9cac2fc632551, 0xbce6faada7179e84, 0xffffffffffffffff, 0xffffffff00000000 .LordK: .quad 0xccd1c8aaee00bc4f +.previous ___ { @@ -4723,7 +4725,7 @@ close TABLE; die "insane number of elements" if ($#arr != 64*16*37-1); print <<___; -.text +.section .rodata align=4096 .globl ecp_nistz256_precomputed .type ecp_nistz256_precomputed,\@object .align 4096 diff --git a/crypto/openssl/crypto/ec/asm/ecp_sm2p256-armv8.pl b/crypto/openssl/crypto/ec/asm/ecp_sm2p256-armv8.pl new file mode 100644 index 000000000000..6db0b6d7aa3f --- /dev/null +++ b/crypto/openssl/crypto/ec/asm/ecp_sm2p256-armv8.pl @@ -0,0 +1,831 @@ +#! /usr/bin/env perl +# Copyright 2023-2025 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or +die "can't locate arm-xlate.pl"; + +open OUT,"| \"$^X\" $xlate $flavour \"$output\"" + or die "can't call $xlate: $!"; +*STDOUT=*OUT; + +my ($s0,$s1,$s2,$s3,$s4,$s5,$s6,$s7)=map("x$_",(7..14)); +my ($a8,$a10,$a12,$a14,$a9,$a11,$a13,$a15)=map("x$_",(7..14)); +my ($t0,$t1,$t2,$t3)=map("x$_",(3..6)); +my ($t4,$t5,$t6,$t7,$t8)=map("x$_",(15..17,19,20)); + +sub bn_mod_add() { + my $mod = shift; +$code.=<<___; + // Load inputs + ldp $s0,$s1,[x1] + ldp $s2,$s3,[x1,#16] + ldp $s4,$s5,[x2] + ldp $s6,$s7,[x2,#16] + + // Addition + adds $s0,$s0,$s4 + adcs $s1,$s1,$s5 + adcs $s2,$s2,$s6 + adcs $s3,$s3,$s7 + adc $t4,xzr,xzr + + // Load polynomial + adrp x2,$mod + add x2,x2,:lo12:$mod + ldp $s4,$s5,[x2] + ldp $s6,$s7,[x2,#16] + + // Backup Addition + mov $t0,$s0 + mov $t1,$s1 + mov $t2,$s2 + mov $t3,$s3 + + // Sub polynomial + subs $t0,$t0,$s4 + sbcs $t1,$t1,$s5 + sbcs $t2,$t2,$s6 + sbcs $t3,$t3,$s7 + sbcs $t4,$t4,xzr + + // Select based on carry + csel $s0,$s0,$t0,cc + csel $s1,$s1,$t1,cc + csel $s2,$s2,$t2,cc + csel $s3,$s3,$t3,cc + + // Store results + stp $s0,$s1,[x0] + stp $s2,$s3,[x0,#16] +___ +} + +sub bn_mod_sub() { + my $mod = shift; +$code.=<<___; + // Load inputs + ldp $s0,$s1,[x1] + ldp $s2,$s3,[x1,#16] + ldp $s4,$s5,[x2] + ldp $s6,$s7,[x2,#16] + + // Subtraction + subs $s0,$s0,$s4 + sbcs $s1,$s1,$s5 + sbcs $s2,$s2,$s6 + sbcs $s3,$s3,$s7 + sbc $t4,xzr,xzr + + // Load polynomial + adrp x2,$mod + add x2,x2,:lo12:$mod + ldp $s4,$s5,[x2] + ldp $s6,$s7,[x2,#16] + + // Backup subtraction + mov $t0,$s0 + mov $t1,$s1 + mov $t2,$s2 + mov $t3,$s3 + + // Add polynomial + adds $t0,$t0,$s4 + adcs $t1,$t1,$s5 + adcs $t2,$t2,$s6 + adcs $t3,$t3,$s7 + tst $t4,$t4 + + // Select based on carry + csel $s0,$s0,$t0,eq + csel $s1,$s1,$t1,eq + csel $s2,$s2,$t2,eq + csel $s3,$s3,$t3,eq + + // Store results + stp $s0,$s1,[x0] + stp $s2,$s3,[x0,#16] +___ +} + +sub bn_mod_div_by_2() { + my $mod = shift; +$code.=<<___; + // Load inputs + ldp $s0,$s1,[x1] + ldp $s2,$s3,[x1,#16] + + // Save the least significant bit + mov $t0,$s0 + + // Right shift 1 + extr $s0,$s1,$s0,#1 + extr $s1,$s2,$s1,#1 + extr $s2,$s3,$s2,#1 + lsr $s3,$s3,#1 + + // Load mod + adrp x2,$mod + add x2,x2,:lo12:$mod + ldp $s4,$s5,[x2] + ldp $s6,$s7,[x2,#16] + + // Parity check + tst $t0,#1 + csel $s4,xzr,$s4,eq + csel $s5,xzr,$s5,eq + csel $s6,xzr,$s6,eq + csel $s7,xzr,$s7,eq + + // Add + adds $s0,$s0,$s4 + adcs $s1,$s1,$s5 + adcs $s2,$s2,$s6 + adc $s3,$s3,$s7 + + // Store results + stp $s0,$s1,[x0] + stp $s2,$s3,[x0,#16] +___ +} + +{ +$code.=<<___; +#include "arm_arch.h" +.arch armv8-a +.rodata + +.align 5 +// The polynomial p +.Lpoly: +.quad 0xffffffffffffffff,0xffffffff00000000,0xffffffffffffffff,0xfffffffeffffffff +// The order of polynomial n +.Lord: +.quad 0x53bbf40939d54123,0x7203df6b21c6052b,0xffffffffffffffff,0xfffffffeffffffff +// (p + 1) / 2 +.Lpoly_div_2: +.quad 0x8000000000000000,0xffffffff80000000,0xffffffffffffffff,0x7fffffff7fffffff +// (n + 1) / 2 +.Lord_div_2: +.quad 0xa9ddfa049ceaa092,0xb901efb590e30295,0xffffffffffffffff,0x7fffffff7fffffff + +.text + +// void bn_rshift1(BN_ULONG *a); +.globl bn_rshift1 +.type bn_rshift1,%function +.align 5 +bn_rshift1: + AARCH64_VALID_CALL_TARGET + // Load inputs + ldp $s0,$s1,[x0] + ldp $s2,$s3,[x0,#16] + + // Right shift + extr $s0,$s1,$s0,#1 + extr $s1,$s2,$s1,#1 + extr $s2,$s3,$s2,#1 + lsr $s3,$s3,#1 + + // Store results + stp $s0,$s1,[x0] + stp $s2,$s3,[x0,#16] + + ret +.size bn_rshift1,.-bn_rshift1 + +// void bn_sub(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b); +.globl bn_sub +.type bn_sub,%function +.align 5 +bn_sub: + AARCH64_VALID_CALL_TARGET + // Load inputs + ldp $s0,$s1,[x1] + ldp $s2,$s3,[x1,#16] + ldp $s4,$s5,[x2] + ldp $s6,$s7,[x2,#16] + + // Subtraction + subs $s0,$s0,$s4 + sbcs $s1,$s1,$s5 + sbcs $s2,$s2,$s6 + sbc $s3,$s3,$s7 + + // Store results + stp $s0,$s1,[x0] + stp $s2,$s3,[x0,#16] + + ret +.size bn_sub,.-bn_sub + +// void ecp_sm2p256_div_by_2(BN_ULONG *r,const BN_ULONG *a); +.globl ecp_sm2p256_div_by_2 +.type ecp_sm2p256_div_by_2,%function +.align 5 +ecp_sm2p256_div_by_2: + AARCH64_VALID_CALL_TARGET +___ + &bn_mod_div_by_2(".Lpoly_div_2"); +$code.=<<___; + ret +.size ecp_sm2p256_div_by_2,.-ecp_sm2p256_div_by_2 + +// void ecp_sm2p256_div_by_2_mod_ord(BN_ULONG *r,const BN_ULONG *a); +.globl ecp_sm2p256_div_by_2_mod_ord +.type ecp_sm2p256_div_by_2_mod_ord,%function +.align 5 +ecp_sm2p256_div_by_2_mod_ord: + AARCH64_VALID_CALL_TARGET +___ + &bn_mod_div_by_2(".Lord_div_2"); +$code.=<<___; + ret +.size ecp_sm2p256_div_by_2_mod_ord,.-ecp_sm2p256_div_by_2_mod_ord + +// void ecp_sm2p256_mul_by_3(BN_ULONG *r,const BN_ULONG *a); +.globl ecp_sm2p256_mul_by_3 +.type ecp_sm2p256_mul_by_3,%function +.align 5 +ecp_sm2p256_mul_by_3: + AARCH64_VALID_CALL_TARGET + // Load inputs + ldp $s0,$s1,[x1] + ldp $s2,$s3,[x1,#16] + + // 2*a + adds $s0,$s0,$s0 + adcs $s1,$s1,$s1 + adcs $s2,$s2,$s2 + adcs $s3,$s3,$s3 + adcs $t4,xzr,xzr + + mov $t0,$s0 + mov $t1,$s1 + mov $t2,$s2 + mov $t3,$s3 + + // Sub polynomial + adrp x2,.Lpoly + add x2,x2,:lo12:.Lpoly + ldp $s4,$s5,[x2] + ldp $s6,$s7,[x2,#16] + subs $s0,$s0,$s4 + sbcs $s1,$s1,$s5 + sbcs $s2,$s2,$s6 + sbcs $s3,$s3,$s7 + sbcs $t4,$t4,xzr + + csel $s0,$s0,$t0,cs + csel $s1,$s1,$t1,cs + csel $s2,$s2,$t2,cs + csel $s3,$s3,$t3,cs + eor $t4,$t4,$t4 + + // 3*a + ldp $s4,$s5,[x1] + ldp $s6,$s7,[x1,#16] + adds $s0,$s0,$s4 + adcs $s1,$s1,$s5 + adcs $s2,$s2,$s6 + adcs $s3,$s3,$s7 + adcs $t4,xzr,xzr + + mov $t0,$s0 + mov $t1,$s1 + mov $t2,$s2 + mov $t3,$s3 + + // Sub polynomial + adrp x2,.Lpoly + add x2,x2,:lo12:.Lpoly + ldp $s4,$s5,[x2] + ldp $s6,$s7,[x2,#16] + subs $s0,$s0,$s4 + sbcs $s1,$s1,$s5 + sbcs $s2,$s2,$s6 + sbcs $s3,$s3,$s7 + sbcs $t4,$t4,xzr + + csel $s0,$s0,$t0,cs + csel $s1,$s1,$t1,cs + csel $s2,$s2,$t2,cs + csel $s3,$s3,$t3,cs + + // Store results + stp $s0,$s1,[x0] + stp $s2,$s3,[x0,#16] + + ret +.size ecp_sm2p256_mul_by_3,.-ecp_sm2p256_mul_by_3 + +// void ecp_sm2p256_add(BN_ULONG *r,const BN_ULONG *a,const BN_ULONG *b); +.globl ecp_sm2p256_add +.type ecp_sm2p256_add,%function +.align 5 +ecp_sm2p256_add: + AARCH64_VALID_CALL_TARGET +___ + &bn_mod_add(".Lpoly"); +$code.=<<___; + ret +.size ecp_sm2p256_add,.-ecp_sm2p256_add + +// void ecp_sm2p256_sub(BN_ULONG *r,const BN_ULONG *a,const BN_ULONG *b); +.globl ecp_sm2p256_sub +.type ecp_sm2p256_sub,%function +.align 5 +ecp_sm2p256_sub: + AARCH64_VALID_CALL_TARGET +___ + &bn_mod_sub(".Lpoly"); +$code.=<<___; + ret +.size ecp_sm2p256_sub,.-ecp_sm2p256_sub + +// void ecp_sm2p256_sub_mod_ord(BN_ULONG *r,const BN_ULONG *a,const BN_ULONG *b); +.globl ecp_sm2p256_sub_mod_ord +.type ecp_sm2p256_sub_mod_ord,%function +.align 5 +ecp_sm2p256_sub_mod_ord: + AARCH64_VALID_CALL_TARGET +___ + &bn_mod_sub(".Lord"); +$code.=<<___; + ret +.size ecp_sm2p256_sub_mod_ord,.-ecp_sm2p256_sub_mod_ord + +.macro RDC + // a = | s7 | ... | s0 |, where si are 64-bit quantities + // = |a15|a14| ... |a1|a0|, where ai are 32-bit quantities + // | s7 | s6 | s5 | s4 | + // | a15 | a14 | a13 | a12 | a11 | a10 | a9 | a8 | + // | s3 | s2 | s1 | s0 | + // | a7 | a6 | a5 | a4 | a3 | a2 | a1 | a0 | + // ================================================= + // | a8 | a11 | a10 | a9 | a8 | 0 | s4 | (+) + // | a9 | a15 | s6 | a11 | 0 | a10 | a9 | (+) + // | a10 | 0 | a14 | a13 | a12 | 0 | s5 | (+) + // | a11 | 0 | s7 | a13 | 0 | a12 | a11 | (+) + // | a12 | 0 | s7 | a13 | 0 | s6 | (+) + // | a12 | 0 | 0 | a15 | a14 | 0 | a14 | a13 | (+) + // | a13 | 0 | 0 | 0 | a15 | 0 | a14 | a13 | (+) + // | a13 | 0 | 0 | 0 | 0 | 0 | s7 | (+) + // | a14 | 0 | 0 | 0 | 0 | 0 | s7 | (+) + // | a14 | 0 | 0 | 0 | 0 | 0 | 0 | a15 | (+) + // | a15 | 0 | 0 | 0 | 0 | 0 | 0 | a15 | (+) + // | a15 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | (+) + // | s7 | 0 | 0 | 0 | 0 | 0 | 0 | (+) + // | 0 | 0 | 0 | 0 | 0 | a8 | 0 | 0 | (-) + // | 0 | 0 | 0 | 0 | 0 | a9 | 0 | 0 | (-) + // | 0 | 0 | 0 | 0 | 0 | a13 | 0 | 0 | (-) + // | 0 | 0 | 0 | 0 | 0 | a14 | 0 | 0 | (-) + // | U[7]| U[6]| U[5]| U[4]| U[3]| U[2]| U[1]| U[0]| + // | V[3] | V[2] | V[1] | V[0] | + + // 1. 64-bit addition + // t2=s6+s7+s7 + adds $t2,$s6,$s7 + adcs $t1,xzr,xzr + adds $t2,$t2,$s7 + adcs $t1,$t1,xzr + // t3=s4+s5+t2 + adds $t3,$s4,$t2 + adcs $t4,$t1,xzr + adds $t3,$t3,$s5 + adcs $t4,$t4,xzr + // sum + adds $s0,$s0,$t3 + adcs $s1,$s1,$t4 + adcs $s2,$s2,$t2 + adcs $s3,$s3,$s7 + adcs $t0,xzr,xzr + adds $s3,$s3,$t1 + adcs $t0,$t0,xzr + + stp $s0,$s1,[sp,#32] + stp $s2,$s3,[sp,#48] + + // 2. 64-bit to 32-bit spread + mov $t1,#0xffffffff + mov $s0,$s4 + mov $s1,$s5 + mov $s2,$s6 + mov $s3,$s7 + and $s0,$s0,$t1 // a8 + and $s1,$s1,$t1 // a10 + and $s2,$s2,$t1 // a12 + and $s3,$s3,$t1 // a14 + lsr $s4,$s4,#32 // a9 + lsr $s5,$s5,#32 // a11 + lsr $s6,$s6,#32 // a13 + lsr $s7,$s7,#32 // a15 + + // 3. 32-bit addition + add $t1,$a14,$a12 // t1 <- a12 + a14 + add $t2,$a15,$a13 // t2 <- a13 + a15 + add $t3,$a8,$a9 // t3 <- a8 + a9 + add $t4,$a14,$a10 // t4 <- a10 + a14 + add $a15,$a15,$a11 // a15 <- a11 + a15 + add $a12,$t2,$t1 // a12 <- a12 + a13 + a14 + a15 + add $a10,$a10,$a12 // a10 <- a10 + a12 + a13 + a14 + a15 + add $a10,$a10,$a12 // a10 <- a10 + 2*(a12 + a13 + a14 + a15) + add $a10,$a10,$t3 // a10 <- a8 + a9 + a10 + 2*(a12 + a13 + a14 + a15) + add $a10,$a10,$a11 // a10 <- a8 + a9 + a10 + a11 + 2*(a12 + a13 + a14 + a15) + add $a12,$a12,$a13 // a12 <- a12 + 2*a13 + a14 + a15 + add $a12,$a12,$a11 // a12 <- a11 + a12 + 2*a13 + a14 + a15 + add $a12,$a12,$a8 // a12 <- a8 + a11 + a12 + 2*a13 + a14 + a15 + add $t3,$t3,$a14 // t3 <- a8 + a9 + a14 + add $t3,$t3,$a13 // t3 <- a8 + a9 + a13 + a14 + add $a9,$a9,$t2 // a9 <- a9 + a13 + a15 + add $a11,$a11,$a9 // a11 <- a9 + a11 + a13 + a15 + add $a11,$a11,$t2 // a11 <- a9 + a11 + 2*(a13 + a15) + add $t1,$t1,$t4 // t1 <- a10 + a12 + 2*a14 + + // U[0] s5 a9 + a11 + 2*(a13 + a15) + // U[1] t1 a10 + a12 + 2*a14 + // U[2] -t3 a8 + a9 + a13 + a14 + // U[3] s2 a8 + a11 + a12 + 2*a13 + a14 + a15 + // U[4] s4 a9 + a13 + a15 + // U[5] t4 a10 + a14 + // U[6] s7 a11 + a15 + // U[7] s1 a8 + a9 + a10 + a11 + 2*(a12 + a13 + a14 + a15) + + // 4. 32-bit to 64-bit + lsl $s0,$t1,#32 + extr $t1,$s2,$t1,#32 + extr $s2,$t4,$s2,#32 + extr $t4,$s1,$t4,#32 + lsr $s1,$s1,#32 + + // 5. 64-bit addition + adds $s5,$s5,$s0 + adcs $t1,$t1,xzr + adcs $s4,$s4,$s2 + adcs $s7,$s7,$t4 + adcs $t0,$t0,$s1 + + // V[0] s5 + // V[1] t1 + // V[2] s4 + // V[3] s7 + // carry t0 + // sub t3 + + // 5. Process s0-s3 + ldp $s0,$s1,[sp,#32] + ldp $s2,$s3,[sp,#48] + // add with V0-V3 + adds $s0,$s0,$s5 + adcs $s1,$s1,$t1 + adcs $s2,$s2,$s4 + adcs $s3,$s3,$s7 + adcs $t0,$t0,xzr + // sub with t3 + subs $s1,$s1,$t3 + sbcs $s2,$s2,xzr + sbcs $s3,$s3,xzr + sbcs $t0,$t0,xzr + + // 6. MOD + // First Mod + lsl $t1,$t0,#32 + subs $t2,$t1,$t0 + + adds $s0,$s0,$t0 + adcs $s1,$s1,$t2 + adcs $s2,$s2,xzr + adcs $s3,$s3,$t1 + + // Last Mod + // return y - p if y > p else y + mov $s4,$s0 + mov $s5,$s1 + mov $s6,$s2 + mov $s7,$s3 + + adrp $t0,.Lpoly + add $t0,$t0,:lo12:.Lpoly + ldp $t1,$t2,[$t0] + ldp $t3,$t4,[$t0,#16] + + adcs $t5,xzr,xzr + + subs $s0,$s0,$t1 + sbcs $s1,$s1,$t2 + sbcs $s2,$s2,$t3 + sbcs $s3,$s3,$t4 + sbcs $t5,$t5,xzr + + csel $s0,$s0,$s4,cs + csel $s1,$s1,$s5,cs + csel $s2,$s2,$s6,cs + csel $s3,$s3,$s7,cs + +.endm + +// void ecp_sm2p256_mul(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b); +.globl ecp_sm2p256_mul +.type ecp_sm2p256_mul,%function +.align 5 +ecp_sm2p256_mul: + AARCH64_SIGN_LINK_REGISTER + // Store scalar registers + stp x29,x30,[sp,#-80]! + add x29,sp,#0 + stp x16,x17,[sp,#16] + stp x19,x20,[sp,#64] + + // Load inputs + ldp $s0,$s1,[x1] + ldp $s2,$s3,[x1,#16] + ldp $s4,$s5,[x2] + ldp $s6,$s7,[x2,#16] + +// ### multiplication ### + // ======================== + // s3 s2 s1 s0 + // * s7 s6 s5 s4 + // ------------------------ + // + s0 s0 s0 s0 + // * * * * + // s7 s6 s5 s4 + // s1 s1 s1 s1 + // * * * * + // s7 s6 s5 s4 + // s2 s2 s2 s2 + // * * * * + // s7 s6 s5 s4 + // s3 s3 s3 s3 + // * * * * + // s7 s6 s5 s4 + // ------------------------ + // s7 s6 s5 s4 s3 s2 s1 s0 + // ======================== + +// ### s0*s4 ### + mul $t5,$s0,$s4 + umulh $t2,$s0,$s4 + +// ### s1*s4 + s0*s5 ### + mul $t0,$s1,$s4 + umulh $t1,$s1,$s4 + adds $t2,$t2,$t0 + adcs $t3,$t1,xzr + + mul $t0,$s0,$s5 + umulh $t1,$s0,$s5 + adds $t2,$t2,$t0 + adcs $t3,$t3,$t1 + adcs $t4,xzr,xzr + +// ### s2*s4 + s1*s5 + s0*s6 ### + mul $t0,$s2,$s4 + umulh $t1,$s2,$s4 + adds $t3,$t3,$t0 + adcs $t4,$t4,$t1 + + mul $t0,$s1,$s5 + umulh $t1,$s1,$s5 + adds $t3,$t3,$t0 + adcs $t4,$t4,$t1 + adcs $t6,xzr,xzr + + mul $t0,$s0,$s6 + umulh $t1,$s0,$s6 + adds $t3,$t3,$t0 + adcs $t4,$t4,$t1 + adcs $t6,$t6,xzr + +// ### s3*s4 + s2*s5 + s1*s6 + s0*s7 ### + mul $t0,$s3,$s4 + umulh $t1,$s3,$s4 + adds $t4,$t4,$t0 + adcs $t6,$t6,$t1 + adcs $t7,xzr,xzr + + mul $t0,$s2,$s5 + umulh $t1,$s2,$s5 + adds $t4,$t4,$t0 + adcs $t6,$t6,$t1 + adcs $t7,$t7,xzr + + mul $t0,$s1,$s6 + umulh $t1,$s1,$s6 + adds $t4,$t4,$t0 + adcs $t6,$t6,$t1 + adcs $t7,$t7,xzr + + mul $t0,$s0,$s7 + umulh $t1,$s0,$s7 + adds $t4,$t4,$t0 + adcs $t6,$t6,$t1 + adcs $t7,$t7,xzr + +// ### s3*s5 + s2*s6 + s1*s7 ### + mul $t0,$s3,$s5 + umulh $t1,$s3,$s5 + adds $t6,$t6,$t0 + adcs $t7,$t7,$t1 + adcs $t8,xzr,xzr + + mul $t0,$s2,$s6 + umulh $t1,$s2,$s6 + adds $t6,$t6,$t0 + adcs $t7,$t7,$t1 + adcs $t8,$t8,xzr + + mul $t0,$s1,$s7 + umulh $t1,$s1,$s7 + adds $s4,$t6,$t0 + adcs $t7,$t7,$t1 + adcs $t8,$t8,xzr + +// ### s3*s6 + s2*s7 ### + mul $t0,$s3,$s6 + umulh $t1,$s3,$s6 + adds $t7,$t7,$t0 + adcs $t8,$t8,$t1 + adcs $t6,xzr,xzr + + mul $t0,$s2,$s7 + umulh $t1,$s2,$s7 + adds $s5,$t7,$t0 + adcs $t8,$t8,$t1 + adcs $t6,$t6,xzr + +// ### s3*s7 ### + mul $t0,$s3,$s7 + umulh $t1,$s3,$s7 + adds $s6,$t8,$t0 + adcs $s7,$t6,$t1 + + mov $s0,$t5 + mov $s1,$t2 + mov $s2,$t3 + mov $s3,$t4 + + // result of mul: s7 s6 s5 s4 s3 s2 s1 s0 + +// ### Reduction ### + RDC + + stp $s0,$s1,[x0] + stp $s2,$s3,[x0,#16] + + // Restore scalar registers + ldp x16,x17,[sp,#16] + ldp x19,x20,[sp,#64] + ldp x29,x30,[sp],#80 + + AARCH64_VALIDATE_LINK_REGISTER + ret +.size ecp_sm2p256_mul,.-ecp_sm2p256_mul + +// void ecp_sm2p256_sqr(BN_ULONG *r, const BN_ULONG *a); +.globl ecp_sm2p256_sqr +.type ecp_sm2p256_sqr,%function +.align 5 + +ecp_sm2p256_sqr: + AARCH64_SIGN_LINK_REGISTER + // Store scalar registers + stp x29,x30,[sp,#-80]! + add x29,sp,#0 + stp x16,x17,[sp,#16] + stp x19,x20,[sp,#64] + + // Load inputs + ldp $s4,$s5,[x1] + ldp $s6,$s7,[x1,#16] + +// ### square ### + // ======================== + // s7 s6 s5 s4 + // * s7 s6 s5 s4 + // ------------------------ + // + s4 s4 s4 s4 + // * * * * + // s7 s6 s5 s4 + // s5 s5 s5 s5 + // * * * * + // s7 s6 s5 s4 + // s6 s6 s6 s6 + // * * * * + // s7 s6 s5 s4 + // s7 s7 s7 s7 + // * * * * + // s7 s6 s5 s4 + // ------------------------ + // s7 s6 s5 s4 s3 s2 s1 s0 + // ======================== + +// ### s4*s5 ### + mul $s1,$s4,$s5 + umulh $s2,$s4,$s5 + +// ### s4*s6 ### + mul $t0,$s6,$s4 + umulh $s3,$s6,$s4 + adds $s2,$s2,$t0 + adcs $s3,$s3,xzr + +// ### s4*s7 + s5*s6 ### + mul $t0,$s7,$s4 + umulh $t1,$s7,$s4 + adds $s3,$s3,$t0 + adcs $s0,$t1,xzr + + mul $t0,$s6,$s5 + umulh $t1,$s6,$s5 + adds $s3,$s3,$t0 + adcs $s0,$s0,$t1 + adcs $t2,xzr,xzr + +// ### s5*s7 ### + mul $t0,$s7,$s5 + umulh $t1,$s7,$s5 + adds $s0,$s0,$t0 + adcs $t2,$t2,$t1 + +// ### s6*s7 ### + mul $t0,$s7,$s6 + umulh $t1,$s7,$s6 + adds $t2,$t2,$t0 + adcs $t3,$t1,xzr + +// ### 2*(t3,t2,s0,s3,s2,s1) ### + adds $s1,$s1,$s1 + adcs $s2,$s2,$s2 + adcs $s3,$s3,$s3 + adcs $s0,$s0,$s0 + adcs $t2,$t2,$t2 + adcs $t3,$t3,$t3 + adcs $t4,xzr,xzr + +// ### s4*s4 ### + mul $t5,$s4,$s4 + umulh $t6,$s4,$s4 + +// ### s5*s5 ### + mul $s4,$s5,$s5 + umulh $s5,$s5,$s5 + +// ### s6*s6 ### + mul $t0,$s6,$s6 + umulh $t1,$s6,$s6 + +// ### s7*s7 ### + mul $t7,$s7,$s7 + umulh $t8,$s7,$s7 + + adds $s1,$s1,$t6 + adcs $s2,$s2,$s4 + adcs $s3,$s3,$s5 + adcs $s0,$s0,$t0 + adcs $t2,$t2,$t1 + adcs $t3,$t3,$t7 + adcs $t4,$t4,$t8 + + mov $s4,$s0 + mov $s0,$t5 + mov $s5,$t2 + mov $s6,$t3 + mov $s7,$t4 + + // result of mul: s7 s6 s5 s4 s3 s2 s1 s0 + +// ### Reduction ### + RDC + + stp $s0,$s1,[x0] + stp $s2,$s3,[x0,#16] + + // Restore scalar registers + ldp x16,x17,[sp,#16] + ldp x19,x20,[sp,#64] + ldp x29,x30,[sp],#80 + + AARCH64_VALIDATE_LINK_REGISTER + ret +.size ecp_sm2p256_sqr,.-ecp_sm2p256_sqr +___ +} + +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval $1/ge; + + print $_,"\n"; +} +close STDOUT or die "error closing STDOUT: $!"; # enforce flush diff --git a/crypto/openssl/crypto/md5/asm/md5-aarch64.pl b/crypto/openssl/crypto/md5/asm/md5-aarch64.pl new file mode 100755 index 000000000000..02bc05caaf51 --- /dev/null +++ b/crypto/openssl/crypto/md5/asm/md5-aarch64.pl @@ -0,0 +1,715 @@ +#! /usr/bin/env perl +# Copyright 2021-2025 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +# MD5 optimized for aarch64. + +use strict; + +my $code; + +#no warnings qw(uninitialized); +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; my $dir=$1; my $xlate; +( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or +die "can't locate arm-xlate.pl"; + +open OUT,"| \"$^X\" $xlate $flavour \"$output\"" + or die "can't call $xlate: $1"; +*STDOUT=*OUT; + +$code .= <A and state->B + ldp w12, w13, [x0, #8] // Load MD5 state->C and state->D +.align 5 +ossl_md5_blocks_loop: + eor x17, x12, x13 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) + and x16, x17, x11 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) + ldp w15, w20, [x1] // Load 2 words of input data0 M[0],M[1] + ldp w3, w21, [x1, #8] // Load 2 words of input data0 M[2],M[3] +#ifdef __AARCH64EB__ + rev w15, w15 + rev w20, w20 + rev w3, w3 + rev w21, w21 +#endif + eor x14, x16, x13 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) + movz x9, #0xa478 // Load lower half of constant 0xd76aa478 + movk x9, #0xd76a, lsl #16 // Load upper half of constant 0xd76aa478 + add w8, w10, w15 // Add dest value + add w7, w8, w9 // Add constant 0xd76aa478 + add w6, w7, w14 // Add aux function result + ror w6, w6, #25 // Rotate left s=7 bits + eor x5, x11, x12 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) + add w4, w11, w6 // Add X parameter round 1 A=FF(A, B, C, D, 0xd76aa478, s=7, M[0]) + and x8, x5, x4 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) + eor x17, x8, x12 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) + movz x16, #0xb756 // Load lower half of constant 0xe8c7b756 + movk x16, #0xe8c7, lsl #16 // Load upper half of constant 0xe8c7b756 + add w9, w13, w20 // Add dest value + add w7, w9, w16 // Add constant 0xe8c7b756 + add w14, w7, w17 // Add aux function result + ror w14, w14, #20 // Rotate left s=12 bits + eor x6, x4, x11 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) + add w5, w4, w14 // Add X parameter round 1 D=FF(D, A, B, C, 0xe8c7b756, s=12, M[1]) + and x8, x6, x5 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) + eor x9, x8, x11 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) + movz x16, #0x70db // Load lower half of constant 0x242070db + movk x16, #0x2420, lsl #16 // Load upper half of constant 0x242070db + add w7, w12, w3 // Add dest value + add w17, w7, w16 // Add constant 0x242070db + add w14, w17, w9 // Add aux function result + ror w14, w14, #15 // Rotate left s=17 bits + eor x6, x5, x4 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) + add w8, w5, w14 // Add X parameter round 1 C=FF(C, D, A, B, 0x242070db, s=17, M[2]) + and x7, x6, x8 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) + eor x16, x7, x4 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) + movz x9, #0xceee // Load lower half of constant 0xc1bdceee + movk x9, #0xc1bd, lsl #16 // Load upper half of constant 0xc1bdceee + add w14, w11, w21 // Add dest value + add w6, w14, w9 // Add constant 0xc1bdceee + add w7, w6, w16 // Add aux function result + ror w7, w7, #10 // Rotate left s=22 bits + eor x17, x8, x5 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) + add w9, w8, w7 // Add X parameter round 1 B=FF(B, C, D, A, 0xc1bdceee, s=22, M[3]) + ldp w14, w22, [x1, #16] // Load 2 words of input data0 M[4],M[5] + ldp w7, w23, [x1, #24] // Load 2 words of input data0 M[6],M[7] +#ifdef __AARCH64EB__ + rev w14, w14 + rev w22, w22 + rev w7, w7 + rev w23, w23 +#endif + and x16, x17, x9 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) + eor x6, x16, x5 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) + movz x16, #0xfaf // Load lower half of constant 0xf57c0faf + movk x16, #0xf57c, lsl #16 // Load upper half of constant 0xf57c0faf + add w17, w4, w14 // Add dest value + add w16, w17, w16 // Add constant 0xf57c0faf + add w4, w16, w6 // Add aux function result + ror w4, w4, #25 // Rotate left s=7 bits + eor x16, x9, x8 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) + add w17, w9, w4 // Add X parameter round 1 A=FF(A, B, C, D, 0xf57c0faf, s=7, M[4]) + and x16, x16, x17 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) + eor x6, x16, x8 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) + movz x4, #0xc62a // Load lower half of constant 0x4787c62a + movk x4, #0x4787, lsl #16 // Load upper half of constant 0x4787c62a + add w16, w5, w22 // Add dest value + add w16, w16, w4 // Add constant 0x4787c62a + add w5, w16, w6 // Add aux function result + ror w5, w5, #20 // Rotate left s=12 bits + eor x4, x17, x9 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) + add w19, w17, w5 // Add X parameter round 1 D=FF(D, A, B, C, 0x4787c62a, s=12, M[5]) + and x6, x4, x19 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) + eor x5, x6, x9 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) + movz x4, #0x4613 // Load lower half of constant 0xa8304613 + movk x4, #0xa830, lsl #16 // Load upper half of constant 0xa8304613 + add w6, w8, w7 // Add dest value + add w8, w6, w4 // Add constant 0xa8304613 + add w4, w8, w5 // Add aux function result + ror w4, w4, #15 // Rotate left s=17 bits + eor x6, x19, x17 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) + add w8, w19, w4 // Add X parameter round 1 C=FF(C, D, A, B, 0xa8304613, s=17, M[6]) + and x5, x6, x8 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) + eor x4, x5, x17 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) + movz x6, #0x9501 // Load lower half of constant 0xfd469501 + movk x6, #0xfd46, lsl #16 // Load upper half of constant 0xfd469501 + add w9, w9, w23 // Add dest value + add w5, w9, w6 // Add constant 0xfd469501 + add w9, w5, w4 // Add aux function result + ror w9, w9, #10 // Rotate left s=22 bits + eor x6, x8, x19 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) + add w4, w8, w9 // Add X parameter round 1 B=FF(B, C, D, A, 0xfd469501, s=22, M[7]) + ldp w5, w24, [x1, #32] // Load 2 words of input data0 M[8],M[9] + ldp w16, w25, [x1, #40] // Load 2 words of input data0 M[10],M[11] +#ifdef __AARCH64EB__ + rev w5, w5 + rev w24, w24 + rev w16, w16 + rev w25, w25 +#endif + and x9, x6, x4 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) + eor x6, x9, x19 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) + movz x9, #0x98d8 // Load lower half of constant 0x698098d8 + movk x9, #0x6980, lsl #16 // Load upper half of constant 0x698098d8 + add w17, w17, w5 // Add dest value + add w9, w17, w9 // Add constant 0x698098d8 + add w17, w9, w6 // Add aux function result + ror w17, w17, #25 // Rotate left s=7 bits + eor x9, x4, x8 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) + add w6, w4, w17 // Add X parameter round 1 A=FF(A, B, C, D, 0x698098d8, s=7, M[8]) + and x17, x9, x6 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) + eor x9, x17, x8 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) + movz x17, #0xf7af // Load lower half of constant 0x8b44f7af + movk x17, #0x8b44, lsl #16 // Load upper half of constant 0x8b44f7af + add w19, w19, w24 // Add dest value + add w17, w19, w17 // Add constant 0x8b44f7af + add w19, w17, w9 // Add aux function result + ror w19, w19, #20 // Rotate left s=12 bits + eor x9, x6, x4 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) + add w17, w6, w19 // Add X parameter round 1 D=FF(D, A, B, C, 0x8b44f7af, s=12, M[9]) + and x9, x9, x17 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) + eor x9, x9, x4 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) + movz x11, #0x5bb1 // Load lower half of constant 0xffff5bb1 + movk x11, #0xffff, lsl #16 // Load upper half of constant 0xffff5bb1 + add w8, w8, w16 // Add dest value + add w8, w8, w11 // Add constant 0xffff5bb1 + add w8, w8, w9 // Add aux function result + ror w8, w8, #15 // Rotate left s=17 bits + eor x9, x17, x6 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) + add w8, w17, w8 // Add X parameter round 1 C=FF(C, D, A, B, 0xffff5bb1, s=17, M[10]) + and x9, x9, x8 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) + eor x9, x9, x6 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) + movz x11, #0xd7be // Load lower half of constant 0x895cd7be + movk x11, #0x895c, lsl #16 // Load upper half of constant 0x895cd7be + add w4, w4, w25 // Add dest value + add w4, w4, w11 // Add constant 0x895cd7be + add w9, w4, w9 // Add aux function result + ror w9, w9, #10 // Rotate left s=22 bits + eor x4, x8, x17 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) + add w9, w8, w9 // Add X parameter round 1 B=FF(B, C, D, A, 0x895cd7be, s=22, M[11]) + ldp w11, w26, [x1, #48] // Load 2 words of input data0 M[12],M[13] + ldp w12, w27, [x1, #56] // Load 2 words of input data0 M[14],M[15] +#ifdef __AARCH64EB__ + rev w11, w11 + rev w26, w26 + rev w12, w12 + rev w27, w27 +#endif + and x4, x4, x9 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) + eor x4, x4, x17 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) + movz x19, #0x1122 // Load lower half of constant 0x6b901122 + movk x19, #0x6b90, lsl #16 // Load upper half of constant 0x6b901122 + add w6, w6, w11 // Add dest value + add w6, w6, w19 // Add constant 0x6b901122 + add w4, w6, w4 // Add aux function result + ror w4, w4, #25 // Rotate left s=7 bits + eor x6, x9, x8 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) + add w4, w9, w4 // Add X parameter round 1 A=FF(A, B, C, D, 0x6b901122, s=7, M[12]) + and x6, x6, x4 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) + eor x6, x6, x8 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) + movz x19, #0x7193 // Load lower half of constant 0xfd987193 + movk x19, #0xfd98, lsl #16 // Load upper half of constant 0xfd987193 + add w17, w17, w26 // Add dest value + add w17, w17, w19 // Add constant 0xfd987193 + add w17, w17, w6 // Add aux function result + ror w17, w17, #20 // Rotate left s=12 bits + eor x6, x4, x9 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) + add w17, w4, w17 // Add X parameter round 1 D=FF(D, A, B, C, 0xfd987193, s=12, M[13]) + and x6, x6, x17 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) + eor x6, x6, x9 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) + movz x13, #0x438e // Load lower half of constant 0xa679438e + movk x13, #0xa679, lsl #16 // Load upper half of constant 0xa679438e + add w8, w8, w12 // Add dest value + add w8, w8, w13 // Add constant 0xa679438e + add w8, w8, w6 // Add aux function result + ror w8, w8, #15 // Rotate left s=17 bits + eor x6, x17, x4 // Begin aux function round 1 F(x,y,z)=(((y^z)&x)^z) + add w8, w17, w8 // Add X parameter round 1 C=FF(C, D, A, B, 0xa679438e, s=17, M[14]) + and x6, x6, x8 // Continue aux function round 1 F(x,y,z)=(((y^z)&x)^z) + eor x6, x6, x4 // End aux function round 1 F(x,y,z)=(((y^z)&x)^z) + movz x13, #0x821 // Load lower half of constant 0x49b40821 + movk x13, #0x49b4, lsl #16 // Load upper half of constant 0x49b40821 + add w9, w9, w27 // Add dest value + add w9, w9, w13 // Add constant 0x49b40821 + add w9, w9, w6 // Add aux function result + ror w9, w9, #10 // Rotate left s=22 bits + bic x6, x8, x17 // Aux function round 2 (~z & y) + add w9, w8, w9 // Add X parameter round 1 B=FF(B, C, D, A, 0x49b40821, s=22, M[15]) + movz x13, #0x2562 // Load lower half of constant 0xf61e2562 + movk x13, #0xf61e, lsl #16 // Load upper half of constant 0xf61e2562 + add w4, w4, w20 // Add dest value + add w4, w4, w13 // Add constant 0xf61e2562 + and x13, x9, x17 // Aux function round 2 (x & z) + add w4, w4, w6 // Add (~z & y) + add w4, w4, w13 // Add (x & z) + ror w4, w4, #27 // Rotate left s=5 bits + bic x6, x9, x8 // Aux function round 2 (~z & y) + add w4, w9, w4 // Add X parameter round 2 A=GG(A, B, C, D, 0xf61e2562, s=5, M[1]) + movz x13, #0xb340 // Load lower half of constant 0xc040b340 + movk x13, #0xc040, lsl #16 // Load upper half of constant 0xc040b340 + add w17, w17, w7 // Add dest value + add w17, w17, w13 // Add constant 0xc040b340 + and x13, x4, x8 // Aux function round 2 (x & z) + add w17, w17, w6 // Add (~z & y) + add w17, w17, w13 // Add (x & z) + ror w17, w17, #23 // Rotate left s=9 bits + bic x6, x4, x9 // Aux function round 2 (~z & y) + add w17, w4, w17 // Add X parameter round 2 D=GG(D, A, B, C, 0xc040b340, s=9, M[6]) + movz x13, #0x5a51 // Load lower half of constant 0x265e5a51 + movk x13, #0x265e, lsl #16 // Load upper half of constant 0x265e5a51 + add w8, w8, w25 // Add dest value + add w8, w8, w13 // Add constant 0x265e5a51 + and x13, x17, x9 // Aux function round 2 (x & z) + add w8, w8, w6 // Add (~z & y) + add w8, w8, w13 // Add (x & z) + ror w8, w8, #18 // Rotate left s=14 bits + bic x6, x17, x4 // Aux function round 2 (~z & y) + add w8, w17, w8 // Add X parameter round 2 C=GG(C, D, A, B, 0x265e5a51, s=14, M[11]) + movz x13, #0xc7aa // Load lower half of constant 0xe9b6c7aa + movk x13, #0xe9b6, lsl #16 // Load upper half of constant 0xe9b6c7aa + add w9, w9, w15 // Add dest value + add w9, w9, w13 // Add constant 0xe9b6c7aa + and x13, x8, x4 // Aux function round 2 (x & z) + add w9, w9, w6 // Add (~z & y) + add w9, w9, w13 // Add (x & z) + ror w9, w9, #12 // Rotate left s=20 bits + bic x6, x8, x17 // Aux function round 2 (~z & y) + add w9, w8, w9 // Add X parameter round 2 B=GG(B, C, D, A, 0xe9b6c7aa, s=20, M[0]) + movz x13, #0x105d // Load lower half of constant 0xd62f105d + movk x13, #0xd62f, lsl #16 // Load upper half of constant 0xd62f105d + add w4, w4, w22 // Add dest value + add w4, w4, w13 // Add constant 0xd62f105d + and x13, x9, x17 // Aux function round 2 (x & z) + add w4, w4, w6 // Add (~z & y) + add w4, w4, w13 // Add (x & z) + ror w4, w4, #27 // Rotate left s=5 bits + bic x6, x9, x8 // Aux function round 2 (~z & y) + add w4, w9, w4 // Add X parameter round 2 A=GG(A, B, C, D, 0xd62f105d, s=5, M[5]) + movz x13, #0x1453 // Load lower half of constant 0x2441453 + movk x13, #0x244, lsl #16 // Load upper half of constant 0x2441453 + add w17, w17, w16 // Add dest value + add w17, w17, w13 // Add constant 0x2441453 + and x13, x4, x8 // Aux function round 2 (x & z) + add w17, w17, w6 // Add (~z & y) + add w17, w17, w13 // Add (x & z) + ror w17, w17, #23 // Rotate left s=9 bits + bic x6, x4, x9 // Aux function round 2 (~z & y) + add w17, w4, w17 // Add X parameter round 2 D=GG(D, A, B, C, 0x2441453, s=9, M[10]) + movz x13, #0xe681 // Load lower half of constant 0xd8a1e681 + movk x13, #0xd8a1, lsl #16 // Load upper half of constant 0xd8a1e681 + add w8, w8, w27 // Add dest value + add w8, w8, w13 // Add constant 0xd8a1e681 + and x13, x17, x9 // Aux function round 2 (x & z) + add w8, w8, w6 // Add (~z & y) + add w8, w8, w13 // Add (x & z) + ror w8, w8, #18 // Rotate left s=14 bits + bic x6, x17, x4 // Aux function round 2 (~z & y) + add w8, w17, w8 // Add X parameter round 2 C=GG(C, D, A, B, 0xd8a1e681, s=14, M[15]) + movz x13, #0xfbc8 // Load lower half of constant 0xe7d3fbc8 + movk x13, #0xe7d3, lsl #16 // Load upper half of constant 0xe7d3fbc8 + add w9, w9, w14 // Add dest value + add w9, w9, w13 // Add constant 0xe7d3fbc8 + and x13, x8, x4 // Aux function round 2 (x & z) + add w9, w9, w6 // Add (~z & y) + add w9, w9, w13 // Add (x & z) + ror w9, w9, #12 // Rotate left s=20 bits + bic x6, x8, x17 // Aux function round 2 (~z & y) + add w9, w8, w9 // Add X parameter round 2 B=GG(B, C, D, A, 0xe7d3fbc8, s=20, M[4]) + movz x13, #0xcde6 // Load lower half of constant 0x21e1cde6 + movk x13, #0x21e1, lsl #16 // Load upper half of constant 0x21e1cde6 + add w4, w4, w24 // Add dest value + add w4, w4, w13 // Add constant 0x21e1cde6 + and x13, x9, x17 // Aux function round 2 (x & z) + add w4, w4, w6 // Add (~z & y) + add w4, w4, w13 // Add (x & z) + ror w4, w4, #27 // Rotate left s=5 bits + bic x6, x9, x8 // Aux function round 2 (~z & y) + add w4, w9, w4 // Add X parameter round 2 A=GG(A, B, C, D, 0x21e1cde6, s=5, M[9]) + movz x13, #0x7d6 // Load lower half of constant 0xc33707d6 + movk x13, #0xc337, lsl #16 // Load upper half of constant 0xc33707d6 + add w17, w17, w12 // Add dest value + add w17, w17, w13 // Add constant 0xc33707d6 + and x13, x4, x8 // Aux function round 2 (x & z) + add w17, w17, w6 // Add (~z & y) + add w17, w17, w13 // Add (x & z) + ror w17, w17, #23 // Rotate left s=9 bits + bic x6, x4, x9 // Aux function round 2 (~z & y) + add w17, w4, w17 // Add X parameter round 2 D=GG(D, A, B, C, 0xc33707d6, s=9, M[14]) + movz x13, #0xd87 // Load lower half of constant 0xf4d50d87 + movk x13, #0xf4d5, lsl #16 // Load upper half of constant 0xf4d50d87 + add w8, w8, w21 // Add dest value + add w8, w8, w13 // Add constant 0xf4d50d87 + and x13, x17, x9 // Aux function round 2 (x & z) + add w8, w8, w6 // Add (~z & y) + add w8, w8, w13 // Add (x & z) + ror w8, w8, #18 // Rotate left s=14 bits + bic x6, x17, x4 // Aux function round 2 (~z & y) + add w8, w17, w8 // Add X parameter round 2 C=GG(C, D, A, B, 0xf4d50d87, s=14, M[3]) + movz x13, #0x14ed // Load lower half of constant 0x455a14ed + movk x13, #0x455a, lsl #16 // Load upper half of constant 0x455a14ed + add w9, w9, w5 // Add dest value + add w9, w9, w13 // Add constant 0x455a14ed + and x13, x8, x4 // Aux function round 2 (x & z) + add w9, w9, w6 // Add (~z & y) + add w9, w9, w13 // Add (x & z) + ror w9, w9, #12 // Rotate left s=20 bits + bic x6, x8, x17 // Aux function round 2 (~z & y) + add w9, w8, w9 // Add X parameter round 2 B=GG(B, C, D, A, 0x455a14ed, s=20, M[8]) + movz x13, #0xe905 // Load lower half of constant 0xa9e3e905 + movk x13, #0xa9e3, lsl #16 // Load upper half of constant 0xa9e3e905 + add w4, w4, w26 // Add dest value + add w4, w4, w13 // Add constant 0xa9e3e905 + and x13, x9, x17 // Aux function round 2 (x & z) + add w4, w4, w6 // Add (~z & y) + add w4, w4, w13 // Add (x & z) + ror w4, w4, #27 // Rotate left s=5 bits + bic x6, x9, x8 // Aux function round 2 (~z & y) + add w4, w9, w4 // Add X parameter round 2 A=GG(A, B, C, D, 0xa9e3e905, s=5, M[13]) + movz x13, #0xa3f8 // Load lower half of constant 0xfcefa3f8 + movk x13, #0xfcef, lsl #16 // Load upper half of constant 0xfcefa3f8 + add w17, w17, w3 // Add dest value + add w17, w17, w13 // Add constant 0xfcefa3f8 + and x13, x4, x8 // Aux function round 2 (x & z) + add w17, w17, w6 // Add (~z & y) + add w17, w17, w13 // Add (x & z) + ror w17, w17, #23 // Rotate left s=9 bits + bic x6, x4, x9 // Aux function round 2 (~z & y) + add w17, w4, w17 // Add X parameter round 2 D=GG(D, A, B, C, 0xfcefa3f8, s=9, M[2]) + movz x13, #0x2d9 // Load lower half of constant 0x676f02d9 + movk x13, #0x676f, lsl #16 // Load upper half of constant 0x676f02d9 + add w8, w8, w23 // Add dest value + add w8, w8, w13 // Add constant 0x676f02d9 + and x13, x17, x9 // Aux function round 2 (x & z) + add w8, w8, w6 // Add (~z & y) + add w8, w8, w13 // Add (x & z) + ror w8, w8, #18 // Rotate left s=14 bits + bic x6, x17, x4 // Aux function round 2 (~z & y) + add w8, w17, w8 // Add X parameter round 2 C=GG(C, D, A, B, 0x676f02d9, s=14, M[7]) + movz x13, #0x4c8a // Load lower half of constant 0x8d2a4c8a + movk x13, #0x8d2a, lsl #16 // Load upper half of constant 0x8d2a4c8a + add w9, w9, w11 // Add dest value + add w9, w9, w13 // Add constant 0x8d2a4c8a + and x13, x8, x4 // Aux function round 2 (x & z) + add w9, w9, w6 // Add (~z & y) + add w9, w9, w13 // Add (x & z) + eor x6, x8, x17 // Begin aux function round 3 H(x,y,z)=(x^y^z) + ror w9, w9, #12 // Rotate left s=20 bits + movz x10, #0x3942 // Load lower half of constant 0xfffa3942 + add w9, w8, w9 // Add X parameter round 2 B=GG(B, C, D, A, 0x8d2a4c8a, s=20, M[12]) + movk x10, #0xfffa, lsl #16 // Load upper half of constant 0xfffa3942 + add w4, w4, w22 // Add dest value + eor x6, x6, x9 // End aux function round 3 H(x,y,z)=(x^y^z) + add w4, w4, w10 // Add constant 0xfffa3942 + add w4, w4, w6 // Add aux function result + ror w4, w4, #28 // Rotate left s=4 bits + eor x6, x9, x8 // Begin aux function round 3 H(x,y,z)=(x^y^z) + movz x10, #0xf681 // Load lower half of constant 0x8771f681 + add w4, w9, w4 // Add X parameter round 3 A=HH(A, B, C, D, 0xfffa3942, s=4, M[5]) + movk x10, #0x8771, lsl #16 // Load upper half of constant 0x8771f681 + add w17, w17, w5 // Add dest value + eor x6, x6, x4 // End aux function round 3 H(x,y,z)=(x^y^z) + add w17, w17, w10 // Add constant 0x8771f681 + add w17, w17, w6 // Add aux function result + eor x6, x4, x9 // Begin aux function round 3 H(x,y,z)=(x^y^z) + ror w17, w17, #21 // Rotate left s=11 bits + movz x13, #0x6122 // Load lower half of constant 0x6d9d6122 + add w17, w4, w17 // Add X parameter round 3 D=HH(D, A, B, C, 0x8771f681, s=11, M[8]) + movk x13, #0x6d9d, lsl #16 // Load upper half of constant 0x6d9d6122 + add w8, w8, w25 // Add dest value + eor x6, x6, x17 // End aux function round 3 H(x,y,z)=(x^y^z) + add w8, w8, w13 // Add constant 0x6d9d6122 + add w8, w8, w6 // Add aux function result + ror w8, w8, #16 // Rotate left s=16 bits + eor x6, x17, x4 // Begin aux function round 3 H(x,y,z)=(x^y^z) + movz x13, #0x380c // Load lower half of constant 0xfde5380c + add w8, w17, w8 // Add X parameter round 3 C=HH(C, D, A, B, 0x6d9d6122, s=16, M[11]) + movk x13, #0xfde5, lsl #16 // Load upper half of constant 0xfde5380c + add w9, w9, w12 // Add dest value + eor x6, x6, x8 // End aux function round 3 H(x,y,z)=(x^y^z) + add w9, w9, w13 // Add constant 0xfde5380c + add w9, w9, w6 // Add aux function result + eor x6, x8, x17 // Begin aux function round 3 H(x,y,z)=(x^y^z) + ror w9, w9, #9 // Rotate left s=23 bits + movz x10, #0xea44 // Load lower half of constant 0xa4beea44 + add w9, w8, w9 // Add X parameter round 3 B=HH(B, C, D, A, 0xfde5380c, s=23, M[14]) + movk x10, #0xa4be, lsl #16 // Load upper half of constant 0xa4beea44 + add w4, w4, w20 // Add dest value + eor x6, x6, x9 // End aux function round 3 H(x,y,z)=(x^y^z) + add w4, w4, w10 // Add constant 0xa4beea44 + add w4, w4, w6 // Add aux function result + ror w4, w4, #28 // Rotate left s=4 bits + eor x6, x9, x8 // Begin aux function round 3 H(x,y,z)=(x^y^z) + movz x10, #0xcfa9 // Load lower half of constant 0x4bdecfa9 + add w4, w9, w4 // Add X parameter round 3 A=HH(A, B, C, D, 0xa4beea44, s=4, M[1]) + movk x10, #0x4bde, lsl #16 // Load upper half of constant 0x4bdecfa9 + add w17, w17, w14 // Add dest value + eor x6, x6, x4 // End aux function round 3 H(x,y,z)=(x^y^z) + add w17, w17, w10 // Add constant 0x4bdecfa9 + add w17, w17, w6 // Add aux function result + eor x6, x4, x9 // Begin aux function round 3 H(x,y,z)=(x^y^z) + ror w17, w17, #21 // Rotate left s=11 bits + movz x13, #0x4b60 // Load lower half of constant 0xf6bb4b60 + add w17, w4, w17 // Add X parameter round 3 D=HH(D, A, B, C, 0x4bdecfa9, s=11, M[4]) + movk x13, #0xf6bb, lsl #16 // Load upper half of constant 0xf6bb4b60 + add w8, w8, w23 // Add dest value + eor x6, x6, x17 // End aux function round 3 H(x,y,z)=(x^y^z) + add w8, w8, w13 // Add constant 0xf6bb4b60 + add w8, w8, w6 // Add aux function result + ror w8, w8, #16 // Rotate left s=16 bits + eor x6, x17, x4 // Begin aux function round 3 H(x,y,z)=(x^y^z) + movz x13, #0xbc70 // Load lower half of constant 0xbebfbc70 + add w8, w17, w8 // Add X parameter round 3 C=HH(C, D, A, B, 0xf6bb4b60, s=16, M[7]) + movk x13, #0xbebf, lsl #16 // Load upper half of constant 0xbebfbc70 + add w9, w9, w16 // Add dest value + eor x6, x6, x8 // End aux function round 3 H(x,y,z)=(x^y^z) + add w9, w9, w13 // Add constant 0xbebfbc70 + add w9, w9, w6 // Add aux function result + eor x6, x8, x17 // Begin aux function round 3 H(x,y,z)=(x^y^z) + ror w9, w9, #9 // Rotate left s=23 bits + movz x10, #0x7ec6 // Load lower half of constant 0x289b7ec6 + add w9, w8, w9 // Add X parameter round 3 B=HH(B, C, D, A, 0xbebfbc70, s=23, M[10]) + movk x10, #0x289b, lsl #16 // Load upper half of constant 0x289b7ec6 + add w4, w4, w26 // Add dest value + eor x6, x6, x9 // End aux function round 3 H(x,y,z)=(x^y^z) + add w4, w4, w10 // Add constant 0x289b7ec6 + add w4, w4, w6 // Add aux function result + ror w4, w4, #28 // Rotate left s=4 bits + eor x6, x9, x8 // Begin aux function round 3 H(x,y,z)=(x^y^z) + movz x10, #0x27fa // Load lower half of constant 0xeaa127fa + add w4, w9, w4 // Add X parameter round 3 A=HH(A, B, C, D, 0x289b7ec6, s=4, M[13]) + movk x10, #0xeaa1, lsl #16 // Load upper half of constant 0xeaa127fa + add w17, w17, w15 // Add dest value + eor x6, x6, x4 // End aux function round 3 H(x,y,z)=(x^y^z) + add w17, w17, w10 // Add constant 0xeaa127fa + add w17, w17, w6 // Add aux function result + eor x6, x4, x9 // Begin aux function round 3 H(x,y,z)=(x^y^z) + ror w17, w17, #21 // Rotate left s=11 bits + movz x13, #0x3085 // Load lower half of constant 0xd4ef3085 + add w17, w4, w17 // Add X parameter round 3 D=HH(D, A, B, C, 0xeaa127fa, s=11, M[0]) + movk x13, #0xd4ef, lsl #16 // Load upper half of constant 0xd4ef3085 + add w8, w8, w21 // Add dest value + eor x6, x6, x17 // End aux function round 3 H(x,y,z)=(x^y^z) + add w8, w8, w13 // Add constant 0xd4ef3085 + add w8, w8, w6 // Add aux function result + ror w8, w8, #16 // Rotate left s=16 bits + eor x6, x17, x4 // Begin aux function round 3 H(x,y,z)=(x^y^z) + movz x13, #0x1d05 // Load lower half of constant 0x4881d05 + add w8, w17, w8 // Add X parameter round 3 C=HH(C, D, A, B, 0xd4ef3085, s=16, M[3]) + movk x13, #0x488, lsl #16 // Load upper half of constant 0x4881d05 + add w9, w9, w7 // Add dest value + eor x6, x6, x8 // End aux function round 3 H(x,y,z)=(x^y^z) + add w9, w9, w13 // Add constant 0x4881d05 + add w9, w9, w6 // Add aux function result + eor x6, x8, x17 // Begin aux function round 3 H(x,y,z)=(x^y^z) + ror w9, w9, #9 // Rotate left s=23 bits + movz x10, #0xd039 // Load lower half of constant 0xd9d4d039 + add w9, w8, w9 // Add X parameter round 3 B=HH(B, C, D, A, 0x4881d05, s=23, M[6]) + movk x10, #0xd9d4, lsl #16 // Load upper half of constant 0xd9d4d039 + add w4, w4, w24 // Add dest value + eor x6, x6, x9 // End aux function round 3 H(x,y,z)=(x^y^z) + add w4, w4, w10 // Add constant 0xd9d4d039 + add w4, w4, w6 // Add aux function result + ror w4, w4, #28 // Rotate left s=4 bits + eor x6, x9, x8 // Begin aux function round 3 H(x,y,z)=(x^y^z) + movz x10, #0x99e5 // Load lower half of constant 0xe6db99e5 + add w4, w9, w4 // Add X parameter round 3 A=HH(A, B, C, D, 0xd9d4d039, s=4, M[9]) + movk x10, #0xe6db, lsl #16 // Load upper half of constant 0xe6db99e5 + add w17, w17, w11 // Add dest value + eor x6, x6, x4 // End aux function round 3 H(x,y,z)=(x^y^z) + add w17, w17, w10 // Add constant 0xe6db99e5 + add w17, w17, w6 // Add aux function result + eor x6, x4, x9 // Begin aux function round 3 H(x,y,z)=(x^y^z) + ror w17, w17, #21 // Rotate left s=11 bits + movz x13, #0x7cf8 // Load lower half of constant 0x1fa27cf8 + add w17, w4, w17 // Add X parameter round 3 D=HH(D, A, B, C, 0xe6db99e5, s=11, M[12]) + movk x13, #0x1fa2, lsl #16 // Load upper half of constant 0x1fa27cf8 + add w8, w8, w27 // Add dest value + eor x6, x6, x17 // End aux function round 3 H(x,y,z)=(x^y^z) + add w8, w8, w13 // Add constant 0x1fa27cf8 + add w8, w8, w6 // Add aux function result + ror w8, w8, #16 // Rotate left s=16 bits + eor x6, x17, x4 // Begin aux function round 3 H(x,y,z)=(x^y^z) + movz x13, #0x5665 // Load lower half of constant 0xc4ac5665 + add w8, w17, w8 // Add X parameter round 3 C=HH(C, D, A, B, 0x1fa27cf8, s=16, M[15]) + movk x13, #0xc4ac, lsl #16 // Load upper half of constant 0xc4ac5665 + add w9, w9, w3 // Add dest value + eor x6, x6, x8 // End aux function round 3 H(x,y,z)=(x^y^z) + add w9, w9, w13 // Add constant 0xc4ac5665 + add w9, w9, w6 // Add aux function result + ror w9, w9, #9 // Rotate left s=23 bits + movz x6, #0x2244 // Load lower half of constant 0xf4292244 + movk x6, #0xf429, lsl #16 // Load upper half of constant 0xf4292244 + add w9, w8, w9 // Add X parameter round 3 B=HH(B, C, D, A, 0xc4ac5665, s=23, M[2]) + add w4, w4, w15 // Add dest value + orn x13, x9, x17 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) + add w4, w4, w6 // Add constant 0xf4292244 + eor x6, x8, x13 // End aux function round 4 I(x,y,z)=((~z|x)^y) + add w4, w4, w6 // Add aux function result + ror w4, w4, #26 // Rotate left s=6 bits + movz x6, #0xff97 // Load lower half of constant 0x432aff97 + movk x6, #0x432a, lsl #16 // Load upper half of constant 0x432aff97 + add w4, w9, w4 // Add X parameter round 4 A=II(A, B, C, D, 0xf4292244, s=6, M[0]) + orn x10, x4, x8 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) + add w17, w17, w23 // Add dest value + eor x10, x9, x10 // End aux function round 4 I(x,y,z)=((~z|x)^y) + add w17, w17, w6 // Add constant 0x432aff97 + add w6, w17, w10 // Add aux function result + ror w6, w6, #22 // Rotate left s=10 bits + movz x17, #0x23a7 // Load lower half of constant 0xab9423a7 + movk x17, #0xab94, lsl #16 // Load upper half of constant 0xab9423a7 + add w6, w4, w6 // Add X parameter round 4 D=II(D, A, B, C, 0x432aff97, s=10, M[7]) + add w8, w8, w12 // Add dest value + orn x10, x6, x9 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) + add w8, w8, w17 // Add constant 0xab9423a7 + eor x17, x4, x10 // End aux function round 4 I(x,y,z)=((~z|x)^y) + add w8, w8, w17 // Add aux function result + ror w8, w8, #17 // Rotate left s=15 bits + movz x17, #0xa039 // Load lower half of constant 0xfc93a039 + movk x17, #0xfc93, lsl #16 // Load upper half of constant 0xfc93a039 + add w8, w6, w8 // Add X parameter round 4 C=II(C, D, A, B, 0xab9423a7, s=15, M[14]) + orn x13, x8, x4 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) + add w9, w9, w22 // Add dest value + eor x13, x6, x13 // End aux function round 4 I(x,y,z)=((~z|x)^y) + add w9, w9, w17 // Add constant 0xfc93a039 + add w17, w9, w13 // Add aux function result + ror w17, w17, #11 // Rotate left s=21 bits + movz x9, #0x59c3 // Load lower half of constant 0x655b59c3 + movk x9, #0x655b, lsl #16 // Load upper half of constant 0x655b59c3 + add w17, w8, w17 // Add X parameter round 4 B=II(B, C, D, A, 0xfc93a039, s=21, M[5]) + add w4, w4, w11 // Add dest value + orn x13, x17, x6 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) + add w9, w4, w9 // Add constant 0x655b59c3 + eor x4, x8, x13 // End aux function round 4 I(x,y,z)=((~z|x)^y) + add w9, w9, w4 // Add aux function result + ror w9, w9, #26 // Rotate left s=6 bits + movz x4, #0xcc92 // Load lower half of constant 0x8f0ccc92 + movk x4, #0x8f0c, lsl #16 // Load upper half of constant 0x8f0ccc92 + add w9, w17, w9 // Add X parameter round 4 A=II(A, B, C, D, 0x655b59c3, s=6, M[12]) + orn x10, x9, x8 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) + add w6, w6, w21 // Add dest value + eor x10, x17, x10 // End aux function round 4 I(x,y,z)=((~z|x)^y) + add w4, w6, w4 // Add constant 0x8f0ccc92 + add w6, w4, w10 // Add aux function result + ror w6, w6, #22 // Rotate left s=10 bits + movz x4, #0xf47d // Load lower half of constant 0xffeff47d + movk x4, #0xffef, lsl #16 // Load upper half of constant 0xffeff47d + add w6, w9, w6 // Add X parameter round 4 D=II(D, A, B, C, 0x8f0ccc92, s=10, M[3]) + add w8, w8, w16 // Add dest value + orn x10, x6, x17 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) + add w8, w8, w4 // Add constant 0xffeff47d + eor x4, x9, x10 // End aux function round 4 I(x,y,z)=((~z|x)^y) + add w8, w8, w4 // Add aux function result + ror w8, w8, #17 // Rotate left s=15 bits + movz x4, #0x5dd1 // Load lower half of constant 0x85845dd1 + movk x4, #0x8584, lsl #16 // Load upper half of constant 0x85845dd1 + add w8, w6, w8 // Add X parameter round 4 C=II(C, D, A, B, 0xffeff47d, s=15, M[10]) + orn x10, x8, x9 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) + add w15, w17, w20 // Add dest value + eor x17, x6, x10 // End aux function round 4 I(x,y,z)=((~z|x)^y) + add w15, w15, w4 // Add constant 0x85845dd1 + add w4, w15, w17 // Add aux function result + ror w4, w4, #11 // Rotate left s=21 bits + movz x15, #0x7e4f // Load lower half of constant 0x6fa87e4f + movk x15, #0x6fa8, lsl #16 // Load upper half of constant 0x6fa87e4f + add w17, w8, w4 // Add X parameter round 4 B=II(B, C, D, A, 0x85845dd1, s=21, M[1]) + add w4, w9, w5 // Add dest value + orn x9, x17, x6 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) + add w15, w4, w15 // Add constant 0x6fa87e4f + eor x4, x8, x9 // End aux function round 4 I(x,y,z)=((~z|x)^y) + add w9, w15, w4 // Add aux function result + ror w9, w9, #26 // Rotate left s=6 bits + movz x15, #0xe6e0 // Load lower half of constant 0xfe2ce6e0 + movk x15, #0xfe2c, lsl #16 // Load upper half of constant 0xfe2ce6e0 + add w4, w17, w9 // Add X parameter round 4 A=II(A, B, C, D, 0x6fa87e4f, s=6, M[8]) + orn x9, x4, x8 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) + add w6, w6, w27 // Add dest value + eor x9, x17, x9 // End aux function round 4 I(x,y,z)=((~z|x)^y) + add w15, w6, w15 // Add constant 0xfe2ce6e0 + add w6, w15, w9 // Add aux function result + ror w6, w6, #22 // Rotate left s=10 bits + movz x9, #0x4314 // Load lower half of constant 0xa3014314 + movk x9, #0xa301, lsl #16 // Load upper half of constant 0xa3014314 + add w15, w4, w6 // Add X parameter round 4 D=II(D, A, B, C, 0xfe2ce6e0, s=10, M[15]) + add w6, w8, w7 // Add dest value + orn x7, x15, x17 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) + add w8, w6, w9 // Add constant 0xa3014314 + eor x9, x4, x7 // End aux function round 4 I(x,y,z)=((~z|x)^y) + add w6, w8, w9 // Add aux function result + ror w6, w6, #17 // Rotate left s=15 bits + movz x7, #0x11a1 // Load lower half of constant 0x4e0811a1 + movk x7, #0x4e08, lsl #16 // Load upper half of constant 0x4e0811a1 + add w8, w15, w6 // Add X parameter round 4 C=II(C, D, A, B, 0xa3014314, s=15, M[6]) + orn x9, x8, x4 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) + add w6, w17, w26 // Add dest value + eor x17, x15, x9 // End aux function round 4 I(x,y,z)=((~z|x)^y) + add w9, w6, w7 // Add constant 0x4e0811a1 + add w7, w9, w17 // Add aux function result + ror w7, w7, #11 // Rotate left s=21 bits + movz x6, #0x7e82 // Load lower half of constant 0xf7537e82 + movk x6, #0xf753, lsl #16 // Load upper half of constant 0xf7537e82 + add w9, w8, w7 // Add X parameter round 4 B=II(B, C, D, A, 0x4e0811a1, s=21, M[13]) + add w17, w4, w14 // Add dest value + orn x7, x9, x15 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) + add w14, w17, w6 // Add constant 0xf7537e82 + eor x4, x8, x7 // End aux function round 4 I(x,y,z)=((~z|x)^y) + add w17, w14, w4 // Add aux function result + ror w17, w17, #26 // Rotate left s=6 bits + movz x6, #0xf235 // Load lower half of constant 0xbd3af235 + movk x6, #0xbd3a, lsl #16 // Load upper half of constant 0xbd3af235 + add w7, w9, w17 // Add X parameter round 4 A=II(A, B, C, D, 0xf7537e82, s=6, M[4]) + orn x14, x7, x8 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) + add w4, w15, w25 // Add dest value + eor x17, x9, x14 // End aux function round 4 I(x,y,z)=((~z|x)^y) + add w15, w4, w6 // Add constant 0xbd3af235 + add w16, w15, w17 // Add aux function result + ror w16, w16, #22 // Rotate left s=10 bits + movz x14, #0xd2bb // Load lower half of constant 0x2ad7d2bb + movk x14, #0x2ad7, lsl #16 // Load upper half of constant 0x2ad7d2bb + add w4, w7, w16 // Add X parameter round 4 D=II(D, A, B, C, 0xbd3af235, s=10, M[11]) + add w6, w8, w3 // Add dest value + orn x15, x4, x9 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) + add w17, w6, w14 // Add constant 0x2ad7d2bb + eor x16, x7, x15 // End aux function round 4 I(x,y,z)=((~z|x)^y) + add w8, w17, w16 // Add aux function result + ror w8, w8, #17 // Rotate left s=15 bits + movz x3, #0xd391 // Load lower half of constant 0xeb86d391 + movk x3, #0xeb86, lsl #16 // Load upper half of constant 0xeb86d391 + add w14, w4, w8 // Add X parameter round 4 C=II(C, D, A, B, 0x2ad7d2bb, s=15, M[2]) + orn x6, x14, x7 // Begin aux function round 4 I(x,y,z)=((~z|x)^y) + add w15, w9, w24 // Add dest value + eor x17, x4, x6 // End aux function round 4 I(x,y,z)=((~z|x)^y) + add w16, w15, w3 // Add constant 0xeb86d391 + add w8, w16, w17 // Add aux function result + ror w8, w8, #11 // Rotate left s=21 bits + ldp w6, w15, [x0] // Reload MD5 state->A and state->B + ldp w5, w9, [x0, #8] // Reload MD5 state->C and state->D + add w3, w14, w8 // Add X parameter round 4 B=II(B, C, D, A, 0xeb86d391, s=21, M[9]) + add w13, w4, w9 // Add result of MD5 rounds to state->D + add w12, w14, w5 // Add result of MD5 rounds to state->C + add w10, w7, w6 // Add result of MD5 rounds to state->A + add w11, w3, w15 // Add result of MD5 rounds to state->B + stp w12, w13, [x0, #8] // Store MD5 states C,D + stp w10, w11, [x0] // Store MD5 states A,B + add x1, x1, #64 // Increment data pointer + subs w2, w2, #1 // Decrement block counter + b.ne ossl_md5_blocks_loop + + ldp x21,x22,[sp,#16] + ldp x23,x24,[sp,#32] + ldp x25,x26,[sp,#48] + ldp x27,x28,[sp,#64] + ldp x19,x20,[sp],#80 + ret + +EOF + +# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, +# CONTEXT *context,DISPATCHER_CONTEXT *disp) + +print $code; + +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/md5/asm/md5-loongarch64.pl b/crypto/openssl/crypto/md5/asm/md5-loongarch64.pl new file mode 100755 index 000000000000..370efd08c41d --- /dev/null +++ b/crypto/openssl/crypto/md5/asm/md5-loongarch64.pl @@ -0,0 +1,298 @@ +#! /usr/bin/env perl +# Author: Min Zhou +# Copyright 2023-2025 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the OpenSSL license (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +# Reference to crypto/md5/asm/md5-x86_64.pl +# MD5 optimized for LoongArch. + +use strict; + +my $code; + +my ($zero,$ra,$tp,$sp,$fp)=map("\$r$_",(0..3,22)); +my ($a0,$a1,$a2,$a3,$a4,$a5,$a6,$a7)=map("\$r$_",(4..11)); +my ($t0,$t1,$t2,$t3,$t4,$t5,$t6,$t7,$t8,$x)=map("\$r$_",(12..21)); + +# $output is the last argument if it looks like a file (it has an extension) +my $output; +$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +open STDOUT,">$output"; + +# round1_step() does: +# dst = x + ((dst + F(x,y,z) + X[k] + T_i) <<< s) +# $t1 = y ^ z +# $t2 = dst + X[k_next] +sub round1_step +{ + my ($pos, $dst, $x, $y, $z, $k_next, $T_i, $s) = @_; + my $T_i_h = ($T_i & 0xfffff000) >> 12; + my $T_i_l = $T_i & 0xfff; + +# In LoongArch we have to use two instructions of lu12i.w and ori to load a +# 32-bit immediate into a general register. Meanwhile, the instruction lu12i.w +# treats the 20-bit immediate as a signed number. So if the T_i_h is greater +# than or equal to (1<<19), we need provide lu12i.w a corresponding negative +# number whose complement equals to the sign extension of T_i_h. + +# The details of the instruction lu12i.w can be found as following: +# https://loongson.github.io/LoongArch-Documentation/LoongArch-Vol1-EN.html#_lu12i_w_lu32i_d_lu52i_d + + $T_i_h = -((1<<32) - (0xfff00000 | $T_i_h)) if ($T_i_h >= (1<<19)); + + $code .= " ld.w $t0,$a1,0 /* (NEXT STEP) X[0] */\n" if ($pos == -1); + $code .= " xor $t1,$y,$z /* y ^ z */\n" if ($pos == -1); + $code .= " add.w $t2,$dst,$t0 /* dst + X[k] */\n" if ($pos == -1); + $code .= <> 12; + my $T_i_l = $T_i & 0xfff; + $T_i_h = -((1<<32) - (0xfff00000 | $T_i_h)) if ($T_i_h >= (1<<19)); + + $code .= <> 12; + my $T_i_l = $T_i & 0xfff; + $T_i_h = -((1<<32) - (0xfff00000 | $T_i_h)) if ($T_i_h >= (1<<19)); + + $code .= <> 12; + my $T_i_l = $T_i & 0xfff; + $T_i_h = -((1<<32) - (0xfff00000 | $T_i_h)) if ($T_i_h >= (1<<19)); + + $code .= <A + ld.w $a5,$a0,4 # a5 = ctx->B + ld.w $a6,$a0,8 # a6 = ctx->C + ld.w $a7,$a0,12 # a7 = ctx->D + +# BEGIN of loop over 16-word blocks +.align 6 +.Lloop: + # save old values of A, B, C, D + move $t3,$a4 + move $t4,$a5 + move $t5,$a6 + move $t6,$a7 + + preld 0,$a1,0 + preld 0,$a1,64 +EOF + +round1_step(-1, $a4, $a5, $a6, $a7, '1', 0xd76aa478, '7'); +round1_step(0, $a7, $a4, $a5, $a6, '2', 0xe8c7b756, '12'); +round1_step(0, $a6, $a7, $a4, $a5, '3', 0x242070db, '17'); +round1_step(0, $a5, $a6, $a7, $a4, '4', 0xc1bdceee, '22'); +round1_step(0, $a4, $a5, $a6, $a7, '5', 0xf57c0faf, '7'); +round1_step(0, $a7, $a4, $a5, $a6, '6', 0x4787c62a, '12'); +round1_step(0, $a6, $a7, $a4, $a5, '7', 0xa8304613, '17'); +round1_step(0, $a5, $a6, $a7, $a4, '8', 0xfd469501, '22'); +round1_step(0, $a4, $a5, $a6, $a7, '9', 0x698098d8, '7'); +round1_step(0, $a7, $a4, $a5, $a6, '10', 0x8b44f7af, '12'); +round1_step(0, $a6, $a7, $a4, $a5, '11', 0xffff5bb1, '17'); +round1_step(0, $a5, $a6, $a7, $a4, '12', 0x895cd7be, '22'); +round1_step(0, $a4, $a5, $a6, $a7, '13', 0x6b901122, '7'); +round1_step(0, $a7, $a4, $a5, $a6, '14', 0xfd987193, '12'); +round1_step(0, $a6, $a7, $a4, $a5, '15', 0xa679438e, '17'); +round1_step(1, $a5, $a6, $a7, $a4, '1', 0x49b40821, '22'); + +round2_step(-1, $a4, $a5, $a6, $a7, '6', 0xf61e2562, '5'); +round2_step(0, $a7, $a4, $a5, $a6, '11', 0xc040b340, '9'); +round2_step(0, $a6, $a7, $a4, $a5, '0', 0x265e5a51, '14'); +round2_step(0, $a5, $a6, $a7, $a4, '5', 0xe9b6c7aa, '20'); +round2_step(0, $a4, $a5, $a6, $a7, '10', 0xd62f105d, '5'); +round2_step(0, $a7, $a4, $a5, $a6, '15', 0x2441453, '9'); +round2_step(0, $a6, $a7, $a4, $a5, '4', 0xd8a1e681, '14'); +round2_step(0, $a5, $a6, $a7, $a4, '9', 0xe7d3fbc8, '20'); +round2_step(0, $a4, $a5, $a6, $a7, '14', 0x21e1cde6, '5'); +round2_step(0, $a7, $a4, $a5, $a6, '3', 0xc33707d6, '9'); +round2_step(0, $a6, $a7, $a4, $a5, '8', 0xf4d50d87, '14'); +round2_step(0, $a5, $a6, $a7, $a4, '13', 0x455a14ed, '20'); +round2_step(0, $a4, $a5, $a6, $a7, '2', 0xa9e3e905, '5'); +round2_step(0, $a7, $a4, $a5, $a6, '7', 0xfcefa3f8, '9'); +round2_step(0, $a6, $a7, $a4, $a5, '12', 0x676f02d9, '14'); +round2_step(1, $a5, $a6, $a7, $a4, '5', 0x8d2a4c8a, '20'); + +round3_step(-1, $a4, $a5, $a6, $a7, '8', 0xfffa3942, '4'); +round3_step(0, $a7, $a4, $a5, $a6, '11', 0x8771f681, '11'); +round3_step(0, $a6, $a7, $a4, $a5, '14', 0x6d9d6122, '16'); +round3_step(0, $a5, $a6, $a7, $a4, '1', 0xfde5380c, '23'); +round3_step(0, $a4, $a5, $a6, $a7, '4', 0xa4beea44, '4'); +round3_step(0, $a7, $a4, $a5, $a6, '7', 0x4bdecfa9, '11'); +round3_step(0, $a6, $a7, $a4, $a5, '10', 0xf6bb4b60, '16'); +round3_step(0, $a5, $a6, $a7, $a4, '13', 0xbebfbc70, '23'); +round3_step(0, $a4, $a5, $a6, $a7, '0', 0x289b7ec6, '4'); +round3_step(0, $a7, $a4, $a5, $a6, '3', 0xeaa127fa, '11'); +round3_step(0, $a6, $a7, $a4, $a5, '6', 0xd4ef3085, '16'); +round3_step(0, $a5, $a6, $a7, $a4, '9', 0x4881d05, '23'); +round3_step(0, $a4, $a5, $a6, $a7, '12', 0xd9d4d039, '4'); +round3_step(0, $a7, $a4, $a5, $a6, '15', 0xe6db99e5, '11'); +round3_step(0, $a6, $a7, $a4, $a5, '2', 0x1fa27cf8, '16'); +round3_step(1, $a5, $a6, $a7, $a4, '0', 0xc4ac5665, '23'); + +round4_step(-1, $a4, $a5, $a6, $a7, '7', 0xf4292244, '6'); +round4_step(0, $a7, $a4, $a5, $a6, '14', 0x432aff97, '10'); +round4_step(0, $a6, $a7, $a4, $a5, '5', 0xab9423a7, '15'); +round4_step(0, $a5, $a6, $a7, $a4, '12', 0xfc93a039, '21'); +round4_step(0, $a4, $a5, $a6, $a7, '3', 0x655b59c3, '6'); +round4_step(0, $a7, $a4, $a5, $a6, '10', 0x8f0ccc92, '10'); +round4_step(0, $a6, $a7, $a4, $a5, '1', 0xffeff47d, '15'); +round4_step(0, $a5, $a6, $a7, $a4, '8', 0x85845dd1, '21'); +round4_step(0, $a4, $a5, $a6, $a7, '15', 0x6fa87e4f, '6'); +round4_step(0, $a7, $a4, $a5, $a6, '6', 0xfe2ce6e0, '10'); +round4_step(0, $a6, $a7, $a4, $a5, '13', 0xa3014314, '15'); +round4_step(0, $a5, $a6, $a7, $a4, '4', 0x4e0811a1, '21'); +round4_step(0, $a4, $a5, $a6, $a7, '11', 0xf7537e82, '6'); +round4_step(0, $a7, $a4, $a5, $a6, '2', 0xbd3af235, '10'); +round4_step(0, $a6, $a7, $a4, $a5, '9', 0x2ad7d2bb, '15'); +round4_step(1, $a5, $a6, $a7, $a4, '0', 0xeb86d391, '21'); + +$code .= <A = A + st.w $a5,$a0,4 # ctx->B = B + st.w $a6,$a0,8 # ctx->C = C + st.w $a7,$a0,12 # ctx->D = D + +.Lend: + jr $ra +.size ossl_md5_block_asm_data_order,.-ossl_md5_block_asm_data_order +EOF + +$code =~ s/\`([^\`]*)\`/eval($1)/gem; + +print $code; + +close STDOUT; diff --git a/crypto/openssl/crypto/md5/asm/md5-x86_64.pl b/crypto/openssl/crypto/md5/asm/md5-x86_64.pl index 6625fb7d08aa..4fde03ecabc8 100755 --- a/crypto/openssl/crypto/md5/asm/md5-x86_64.pl +++ b/crypto/openssl/crypto/md5/asm/md5-x86_64.pl @@ -1,6 +1,6 @@ #! /usr/bin/env perl # Author: Marc Bevand -# Copyright 2005-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2005-2025 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -41,7 +41,6 @@ EOF # %r10d = X[k_next] # %r11d = z' (copy of z for the next step) # %r12d = z' (copy of z for the next step) -# Each round2_step() takes about 5.4 clocks (11 instructions, 2.0 IPC) sub round2_step { my ($pos, $dst, $x, $y, $z, $k_next, $T_i, $s) = @_; @@ -53,9 +52,9 @@ sub round2_step lea $T_i($dst,%r10d),$dst /* Const + dst + ... */ and $y, %r11d /* y & (not z) */ mov $k_next*4(%rsi),%r10d /* (NEXT STEP) X[$k_next] */ - or %r11d, %r12d /* (y & (not z)) | (x & z) */ + add %r11d, $dst /* dst += (y & (not z)) */ mov $y, %r11d /* (NEXT STEP) z' = $y */ - add %r12d, $dst /* dst += ... */ + add %r12d, $dst /* dst += (x & z) */ mov $y, %r12d /* (NEXT STEP) z' = $y */ rol \$$s, $dst /* dst <<< s */ add $x, $dst /* dst += x */ diff --git a/crypto/openssl/crypto/modes/asm/aes-gcm-armv8-unroll8_64.pl b/crypto/openssl/crypto/modes/asm/aes-gcm-armv8-unroll8_64.pl new file mode 100644 index 000000000000..d516359eb81b --- /dev/null +++ b/crypto/openssl/crypto/modes/asm/aes-gcm-armv8-unroll8_64.pl @@ -0,0 +1,7378 @@ +#! /usr/bin/env perl +# Copyright 2020-2023 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +# +#======================================================================== +# Written by Xiaokang Qian for the OpenSSL project, +# derived from https://github.com/ARM-software/AArch64cryptolib, original +# author Samuel Lee . The module is, however, dual +# licensed under OpenSSL and SPDX BSD-3-Clause licenses depending on where you +# obtain it. +#======================================================================== +# +# Approach - We want to reload constants as we have plenty of spare ASIMD slots around crypto units for loading +# Unroll x8 in main loop, main loop to act on 8 16B blocks per iteration, and then do modulo of the accumulated +# intermediate hashesfrom the 8 blocks. +# +# ____________________________________________________ +# | | +# | PRE | +# |____________________________________________________| +# | | | | +# | CTR block 8k+13| AES block 8k+8 | GHASH block 8k+0 | +# |________________|________________|__________________| +# | | | | +# | CTR block 8k+14| AES block 8k+9 | GHASH block 8k+1 | +# |________________|________________|__________________| +# | | | | +# | CTR block 8k+15| AES block 8k+10| GHASH block 8k+2 | +# |________________|________________|__________________| +# | | | | +# | CTR block 8k+16| AES block 8k+11| GHASH block 8k+3 | +# |________________|________________|__________________| +# | | | | +# | CTR block 8k+17| AES block 8k+12| GHASH block 8k+4 | +# |________________|________________|__________________| +# | | | | +# | CTR block 8k+18| AES block 8k+13| GHASH block 8k+5 | +# |________________|________________|__________________| +# | | | | +# | CTR block 8k+19| AES block 8k+14| GHASH block 8k+6 | +# |________________|________________|__________________| +# | | | | +# | CTR block 8k+20| AES block 8k+15| GHASH block 8k+7 | +# |________________|____(mostly)____|__________________| +# | | +# | MODULO | +# |____________________________________________________| +# +# PRE: +# Ensure previous generated intermediate hash is aligned and merged with result for GHASH 4k+0 +# EXT low_acc, low_acc, low_acc, #8 +# EOR res_curr (8k+0), res_curr (4k+0), low_acc +# +# CTR block: +# Increment and byte reverse counter in scalar registers and transfer to SIMD registers +# REV ctr32, rev_ctr32 +# ORR ctr64, constctr96_top32, ctr32, LSL #32 +# INS ctr_next.d[0], constctr96_bottom64 // Keeping this in scalar registers to free up space in SIMD RF +# INS ctr_next.d[1], ctr64X +# ADD rev_ctr32, #1 +# +# AES block: +# Do AES encryption/decryption on CTR block X and EOR it with input block X. Take 256 bytes key below for example. +# Doing small trick here of loading input in scalar registers, EORing with last key and then transferring +# Given we are very constrained in our ASIMD registers this is quite important +# +# Encrypt: +# LDR input_low, [ input_ptr ], #8 +# LDR input_high, [ input_ptr ], #8 +# EOR input_low, k14_low +# EOR input_high, k14_high +# INS res_curr.d[0], input_low +# INS res_curr.d[1], input_high +# AESE ctr_curr, k0; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k1; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k2; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k3; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k4; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k5; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k6; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k7; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k8; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k9; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k10; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k11; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k12; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k13 +# EOR res_curr, res_curr, ctr_curr +# ST1 { res_curr.16b }, [ output_ptr ], #16 +# +# Decrypt: +# AESE ctr_curr, k0; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k1; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k2; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k3; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k4; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k5; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k6; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k7; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k8; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k9; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k10; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k11; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k12; AESMC ctr_curr, ctr_curr +# AESE ctr_curr, k13 +# LDR res_curr, [ input_ptr ], #16 +# EOR res_curr, res_curr, ctr_curr +# MOV output_low, res_curr.d[0] +# MOV output_high, res_curr.d[1] +# EOR output_low, k14_low +# EOR output_high, k14_high +# STP output_low, output_high, [ output_ptr ], #16 + +# GHASH block X: +# Do 128b karatsuba polynomial multiplication on block +# We only have 64b->128b polynomial multipliers, naively that means we need to do 4 64b multiplies to generate a 128b +# +# multiplication: +# Pmull(A,B) == (Pmull(Ah,Bh)<<128 | Pmull(Al,Bl)) ^ (Pmull(Ah,Bl) ^ Pmull(Al,Bh))<<64 +# +# The idea behind Karatsuba multiplication is that we can do just 3 64b multiplies: +# Pmull(A,B) == (Pmull(Ah,Bh)<<128 | Pmull(Al,Bl)) ^ (Pmull(Ah^Al,Bh^Bl) ^ Pmull(Ah,Bh) ^ Pmull(Al,Bl))<<64 +# +# There is some complication here because the bit order of GHASH's PMULL is reversed compared to elsewhere, so we are +# multiplying with "twisted" powers of H +# +# Note: We can PMULL directly into the acc_x in first GHASH of the loop +# Note: For scheduling big cores we want to split the processing to happen over two loop iterations - otherwise the critical +# path latency dominates the performance +# +# This has a knock on effect on register pressure, so we have to be a bit more clever with our temporary registers +# than indicated here +# REV64 res_curr, res_curr +# INS t_m.d[0], res_curr.d[1] +# EOR t_m.8B, t_m.8B, res_curr.8B +# PMULL2 t_h, res_curr, HX +# PMULL t_l, res_curr, HX +# PMULL t_m, t_m, HX_k +# EOR acc_h, acc_h, t_h +# EOR acc_l, acc_l, t_l +# EOR acc_m, acc_m, t_m +# +# MODULO: take the partial accumulators (~representing sum of 256b multiplication results), from GHASH and do modulo reduction on them +# There is some complication here because the bit order of GHASH's PMULL is reversed compared to elsewhere, so we are doing modulo +# with a reversed constant +# EOR3 acc_m, acc_m, acc_l, acc_h // Finish off karatsuba processing +# PMULL t_mod, acc_h, mod_constant +# EXT acc_h, acc_h, acc_h, #8 +# EOR3 acc_m, acc_m, t_mod, acc_h +# PMULL acc_h, acc_m, mod_constant +# EXT acc_m, acc_m, acc_m, #8 +# EOR3 acc_l, acc_l, acc_m, acc_h + +$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate ) or +die "can't locate arm-xlate.pl"; + +die "only for 64 bit" if $flavour !~ /64/; + +open OUT,"| \"$^X\" $xlate $flavour $output"; +*STDOUT=*OUT; + +$code=<<___; +#include "arm_arch.h" + +#if __ARM_MAX_ARCH__>=8 +___ +$code.=".arch armv8-a+crypto\n.text\n"; + +$input_ptr="x0"; #argument block +$bit_length="x1"; +$byte_length="x9"; +$output_ptr="x2"; +$current_tag="x3"; +$counter="x16"; +$constant_temp="x15"; +$modulo_constant="x10"; +$cc="x8"; +{ +my ($end_input_ptr,$main_end_input_ptr,$temp0_x,$temp1_x)=map("x$_",(4..7)); +my ($temp2_x,$temp3_x)=map("x$_",(13..14)); +my ($ctr0b,$ctr1b,$ctr2b,$ctr3b,$ctr4b,$ctr5b,$ctr6b,$ctr7b,$res0b,$res1b,$res2b,$res3b,$res4b,$res5b,$res6b,$res7b)=map("v$_.16b",(0..15)); +my ($ctr0,$ctr1,$ctr2,$ctr3,$ctr4,$ctr5,$ctr6,$ctr7,$res0,$res1,$res2,$res3,$res4,$res5,$res6,$res7)=map("v$_",(0..15)); +my ($ctr0d,$ctr1d,$ctr2d,$ctr3d,$ctr4d,$ctr5d,$ctr6d,$ctr7d)=map("d$_",(0..7)); +my ($ctr0q,$ctr1q,$ctr2q,$ctr3q,$ctr4q,$ctr5q,$ctr6q,$ctr7q)=map("q$_",(0..7)); +my ($res0q,$res1q,$res2q,$res3q,$res4q,$res5q,$res6q,$res7q)=map("q$_",(8..15)); + +my ($ctr_t0,$ctr_t1,$ctr_t2,$ctr_t3,$ctr_t4,$ctr_t5,$ctr_t6,$ctr_t7)=map("v$_",(8..15)); +my ($ctr_t0b,$ctr_t1b,$ctr_t2b,$ctr_t3b,$ctr_t4b,$ctr_t5b,$ctr_t6b,$ctr_t7b)=map("v$_.16b",(8..15)); +my ($ctr_t0q,$ctr_t1q,$ctr_t2q,$ctr_t3q,$ctr_t4q,$ctr_t5q,$ctr_t6q,$ctr_t7q)=map("q$_",(8..15)); + +my ($acc_hb,$acc_mb,$acc_lb)=map("v$_.16b",(17..19)); +my ($acc_h,$acc_m,$acc_l)=map("v$_",(17..19)); + +my ($h1,$h12k,$h2,$h3,$h34k,$h4)=map("v$_",(20..25)); +my ($h5,$h56k,$h6,$h7,$h78k,$h8)=map("v$_",(20..25)); +my ($h1q,$h12kq,$h2q,$h3q,$h34kq,$h4q)=map("q$_",(20..25)); +my ($h5q,$h56kq,$h6q,$h7q,$h78kq,$h8q)=map("q$_",(20..25)); + +my $t0="v16"; +my $t0d="d16"; + +my $t1="v29"; +my $t2=$res1; +my $t3=$t1; + +my $t4=$res0; +my $t5=$res2; +my $t6=$t0; + +my $t7=$res3; +my $t8=$res4; +my $t9=$res5; + +my $t10=$res6; +my $t11="v21"; +my $t12=$t1; + +my $rtmp_ctr="v30"; +my $rtmp_ctrq="q30"; +my $rctr_inc="v31"; +my $rctr_incd="d31"; + +my $mod_constantd=$t0d; +my $mod_constant=$t0; + +my ($rk0,$rk1,$rk2)=map("v$_.16b",(26..28)); +my ($rk3,$rk4,$rk5)=map("v$_.16b",(26..28)); +my ($rk6,$rk7,$rk8)=map("v$_.16b",(26..28)); +my ($rk9,$rk10,$rk11)=map("v$_.16b",(26..28)); +my ($rk12,$rk13,$rk14)=map("v$_.16b",(26..28)); +my ($rk0q,$rk1q,$rk2q)=map("q$_",(26..28)); +my ($rk3q,$rk4q,$rk5q)=map("q$_",(26..28)); +my ($rk6q,$rk7q,$rk8q)=map("q$_",(26..28)); +my ($rk9q,$rk10q,$rk11q)=map("q$_",(26..28)); +my ($rk12q,$rk13q,$rk14q)=map("q$_",(26..28)); +my $rk2q1="v28.1q"; +my $rk3q1="v26.1q"; +my $rk4v="v27"; + + +######################################################################################### +# size_t unroll8_eor3_aes_gcm_enc_128_kernel(const uint8_t * plaintext, +# uint64_t plaintext_length, +# uint8_t * ciphertext, +# uint64_t *Xi, +# unsigned char ivec[16], +# const void *key); +# +$code.=<<___; +.global unroll8_eor3_aes_gcm_enc_128_kernel +.type unroll8_eor3_aes_gcm_enc_128_kernel,%function +.align 4 +unroll8_eor3_aes_gcm_enc_128_kernel: + AARCH64_VALID_CALL_TARGET + cbz x1, .L128_enc_ret + stp d8, d9, [sp, #-80]! + lsr $byte_length, $bit_length, #3 + mov $counter, x4 + mov $cc, x5 + stp d10, d11, [sp, #16] + stp d12, d13, [sp, #32] + stp d14, d15, [sp, #48] + mov x5, #0xc200000000000000 + stp x5, xzr, [sp, #64] + add $modulo_constant, sp, #64 + + mov $constant_temp, #0x100000000 @ set up counter increment + movi $rctr_inc.16b, #0x0 + mov $rctr_inc.d[1], $constant_temp + mov $main_end_input_ptr, $byte_length + ld1 { $ctr0b}, [$counter] @ CTR block 0 + + sub $main_end_input_ptr, $main_end_input_ptr, #1 @ byte_len - 1 + + and $main_end_input_ptr, $main_end_input_ptr, #0xffffffffffffff80 @ number of bytes to be processed in main loop (at least 1 byte must be handled by tail) + + rev32 $rtmp_ctr.16b, $ctr0.16b @ set up reversed counter + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 0 + + rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 1 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 1 + + rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 2 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 2 + + rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 3 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 3 + + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 4 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 4 + + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 5 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 5 + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 6 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 6 + + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 7 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 7 + + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 0 + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 0 + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 0 + + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 0 + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 0 + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 0 + + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 0 + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 0 + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 1 + + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 1 + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 1 + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 1 + + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 1 + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 1 + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 1 + + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 2 + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 1 + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 2 + + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 2 + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 2 + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 2 + + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 2 + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 2 + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 2 + + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 3 + + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 3 + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 3 + + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 3 + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 3 + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 3 + + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 3 + + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 4 + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 3 + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 4 + + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 4 + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 4 + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 4 + + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 4 + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 4 + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 4 + + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 5 + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 5 + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 5 + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 5 + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 5 + + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 5 + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 5 + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 5 + + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 6 + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 6 + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 6 + + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 6 + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 6 + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 6 + + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 6 + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 6 + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 7 + + ld1 { $acc_lb}, [$current_tag] + ext $acc_lb, $acc_lb, $acc_lb, #8 + rev64 $acc_lb, $acc_lb + + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 7 + + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 7 + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 7 + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 7 + + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 7 + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 7 + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 7 + + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8 + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8 + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8 + + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8 + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8 + ldr $rk10q, [$cc, #160] @ load rk10 + + aese $ctr3b, $rk9 @ AES block 8k+11 - round 9 + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8 + aese $ctr2b, $rk9 @ AES block 8k+10 - round 9 + + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8 + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8 + aese $ctr6b, $rk9 @ AES block 8k+14 - round 9 + + aese $ctr4b, $rk9 @ AES block 8k+12 - round 9 + add $main_end_input_ptr, $main_end_input_ptr, $input_ptr + aese $ctr0b, $rk9 @ AES block 8k+8 - round 9 + + aese $ctr7b, $rk9 @ AES block 8k+15 - round 9 + aese $ctr5b, $rk9 @ AES block 8k+13 - round 9 + aese $ctr1b, $rk9 @ AES block 8k+9 - round 9 + + add $end_input_ptr, $input_ptr, $bit_length, lsr #3 @ end_input_ptr + cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks + b.ge .L128_enc_tail @ handle tail + + ldp $ctr_t0q, $ctr_t1q, [$input_ptr], #32 @ AES block 0, 1 - load plaintext + + ldp $ctr_t2q, $ctr_t3q, [$input_ptr], #32 @ AES block 2, 3 - load plaintext + + ldp $ctr_t4q, $ctr_t5q, [$input_ptr], #32 @ AES block 4, 5 - load plaintext + + ldp $ctr_t6q, $ctr_t7q, [$input_ptr], #32 @ AES block 6, 7 - load plaintext + cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks + + eor3 $res0b, $ctr_t0b, $ctr0b, $rk10 @ AES block 0 - result + rev32 $ctr0.16b, $rtmp_ctr.16b @ CTR block 8 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8 + + eor3 $res1b, $ctr_t1b, $ctr1b, $rk10 @ AES block 1 - result + stp $res0q, $res1q, [$output_ptr], #32 @ AES block 0, 1 - store result + + rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 9 + eor3 $res5b, $ctr_t5b, $ctr5b, $rk10 @ AES block 5 - result + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 9 + + eor3 $res2b, $ctr_t2b, $ctr2b, $rk10 @ AES block 2 - result + eor3 $res6b, $ctr_t6b, $ctr6b, $rk10 @ AES block 6 - result + eor3 $res4b, $ctr_t4b, $ctr4b, $rk10 @ AES block 4 - result + + rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 10 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 10 + + eor3 $res3b, $ctr_t3b, $ctr3b, $rk10 @ AES block 3 - result + eor3 $res7b, $ctr_t7b, $ctr7b,$rk10 @ AES block 7 - result + stp $res2q, $res3q, [$output_ptr], #32 @ AES block 2, 3 - store result + + rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 11 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 11 + stp $res4q, $res5q, [$output_ptr], #32 @ AES block 4, 5 - store result + + stp $res6q, $res7q, [$output_ptr], #32 @ AES block 6, 7 - store result + + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 12 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 12 + b.ge .L128_enc_prepretail @ do prepretail + +.L128_enc_main_loop: @ main loop start + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13 + ldr $h5q, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + ldr $h6q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13 + + rev64 $res1b, $res1b @ GHASH block 8k+1 + rev64 $res0b, $res0b @ GHASH block 8k + ldr $h7q, [$current_tag, #176] @ load h7l | h7h + ext $h7.16b, $h7.16b, $h7.16b, #8 + ldr $h8q, [$current_tag, #208] @ load h8l | h8h + ext $h8.16b, $h8.16b, $h8.16b, #8 + + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14 + ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0 + + ldr $h56kq, [$current_tag, #144] @ load h6k | h5k + ldr $h78kq, [$current_tag, #192] @ load h8k | h7k + rev64 $res5b, $res5b @ GHASH block 8k+5 (t0, t1, t2 and t3 free) + rev64 $res3b, $res3b @ GHASH block 8k+3 + + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + eor $res0b, $res0b, $acc_lb @ PRE 1 + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15 + + rev64 $res7b, $res7b @ GHASH block 8k+7 (t0, t1, t2 and t3 free) + + pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high + rev64 $res2b, $res2b @ GHASH block 8k+2 + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high + + pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low + trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low + + trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high + pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high + + eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + ldr $h4q, [$current_tag, #112] @ load h3l | h3h + ext $h4.16b, $h4.16b, $h4.16b, #8 + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0 + + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0 + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0 + eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15 + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0 + eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid + + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0 + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1 + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0 + + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1 + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0 + pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low + + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1 + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0 + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1 + + eor3 $acc_hb, $acc_hb, $t1.16b,$t2.16b @ GHASH block 8k+2, 8k+3 - high + trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1 + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1 + + pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1 + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1 + + pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid + eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid + + rev64 $res6b, $res6b @ GHASH block 8k+6 (t0, t1, and t2 free) + eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low + + pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid + eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid + pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid + + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2 + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2 + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2 + + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2 + eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2 + + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2 + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2 + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2 + + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3 + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + rev64 $res4b, $res4b @ GHASH block 8k+4 (t0, t1, and t2 free) + + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3 + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3 + + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + ldr $h2q, [$current_tag, #64] @ load h1l | h1h + ext $h2.16b, $h2.16b, $h2.16b, #8 + pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high + pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low + + trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3 + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3 + + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3 + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3 + + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3 + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4 + + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4 + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4 + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4 + + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4 + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4 + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4 + + pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high + eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low + + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4 + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + + pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid + pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid + pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high + + pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5 + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5 + + pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low + eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high + trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + + eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5 + + eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5 + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5 + + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5 + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5 + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5 + + eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low + + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6 + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6 + + pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6 + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6 + + pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid + eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low + ldp $ctr_t0q, $ctr_t1q, [$input_ptr], #32 @ AES block 8k+8, 8k+9 - load plaintext + + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6 + rev32 $h1.16b, $rtmp_ctr.16b @ CTR block 8k+16 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+16 + + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6 + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6 + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6 + + eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high + + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7 + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7 + ldp $ctr_t2q, $ctr_t3q, [$input_ptr], #32 @ AES block 8k+10, 8k+11 - load plaintext + + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7 + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7 + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7 + + pmull $t11.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7 + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7 + + rev32 $h2.16b, $rtmp_ctr.16b @ CTR block 8k+17 + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7 + + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8 + ldp $ctr_t4q, $ctr_t5q, [$input_ptr], #32 @ AES block 8k+12, 8k+13 - load plaintext + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+17 + + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8 + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8 + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8 + + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8 + eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + ldr $rk10q, [$cc, #160] @ load rk10 + + ext $t12.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + rev32 $h3.16b, $rtmp_ctr.16b @ CTR block 8k+18 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+18 + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8 + + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8 + eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8 + + aese $ctr2b, $rk9 @ AES block 8k+10 - round 9 + aese $ctr4b, $rk9 @ AES block 8k+12 - round 9 + aese $ctr1b, $rk9 @ AES block 8k+9 - round 9 + + ldp $ctr_t6q, $ctr_t7q, [$input_ptr], #32 @ AES block 8k+14, 8k+15 - load plaintext + rev32 $h4.16b, $rtmp_ctr.16b @ CTR block 8k+19 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+19 + + cmp $input_ptr, $main_end_input_ptr @ LOOP CONTROL + eor3 $res4b, $ctr_t4b, $ctr4b, $rk10 @ AES block 4 - result + aese $ctr7b, $rk9 @ AES block 8k+15 - round 9 + + aese $ctr6b, $rk9 @ AES block 8k+14 - round 9 + aese $ctr3b, $rk9 @ AES block 8k+11 - round 9 + + eor3 $res2b, $ctr_t2b, $ctr2b, $rk10 @ AES block 8k+10 - result + + mov $ctr2.16b, $h3.16b @ CTR block 8k+18 + aese $ctr0b, $rk9 @ AES block 8k+8 - round 9 + + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 8k+20 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+20 + + eor3 $res7b, $ctr_t7b, $ctr7b, $rk10 @ AES block 7 - result + aese $ctr5b, $rk9 @ AES block 8k+13 - round 9 + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + + eor3 $res1b, $ctr_t1b, $ctr1b, $rk10 @ AES block 8k+9 - result + eor3 $res3b, $ctr_t3b, $ctr3b, $rk10 @ AES block 8k+11 - result + mov $ctr3.16b, $h4.16b @ CTR block 8k+19 + + ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + eor3 $res5b, $ctr_t5b, $ctr5b, $rk10 @ AES block 5 - result + mov $ctr1.16b, $h2.16b @ CTR block 8k+17 + + eor3 $res0b, $ctr_t0b, $ctr0b, $rk10 @ AES block 8k+8 - result + mov $ctr0.16b, $h1.16b @ CTR block 8k+16 + stp $res0q, $res1q, [$output_ptr], #32 @ AES block 8k+8, 8k+9 - store result + + stp $res2q, $res3q, [$output_ptr], #32 @ AES block 8k+10, 8k+11 - store result + eor3 $res6b, $ctr_t6b, $ctr6b, $rk10 @ AES block 6 - result + + stp $res4q, $res5q, [$output_ptr], #32 @ AES block 8k+12, 8k+13 - store result + eor3 $acc_lb, $acc_lb, $acc_hb, $t11.16b @ MODULO - fold into low + + stp $res6q, $res7q, [$output_ptr], #32 @ AES block 8k+14, 8k+15 - store result + b.lt .L128_enc_main_loop + +.L128_enc_prepretail: @ PREPRETAIL + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13 + ldr $h7q, [$current_tag, #176] @ load h7l | h7h + ext $h7.16b, $h7.16b, $h7.16b, #8 + ldr $h8q, [$current_tag, #208] @ load h8l | h8h + ext $h8.16b, $h8.16b, $h8.16b, #8 + ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0 + + ldr $h5q, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + ldr $h6q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + rev64 $res0b, $res0b @ GHASH block 8k + rev64 $res1b, $res1b @ GHASH block 8k+1 + + ldr $h56kq, [$current_tag, #144] @ load h6k | h5k + ldr $h78kq, [$current_tag, #192] @ load h6k | h5k + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13 + rev64 $res3b, $res3b @ GHASH block 8k+3 + + rev64 $res2b, $res2b @ GHASH block 8k+2 + eor $res0b, $res0b, $acc_lb @ PRE 1 + + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14 + + pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high + + rev64 $res5b, $res5b @ GHASH block 8k+5 (t0, t1, t2 and t3 free) + trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + + pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low + eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high + trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + + eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low + eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid + + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14 + + pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid + pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid + + rev64 $res4b, $res4b @ GHASH block 8k+4 (t0, t1, and t2 free) + rev64 $res7b, $res7b @ GHASH block 8k+7 (t0, t1, t2 and t3 free) + + eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid + + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15 + + rev64 $res6b, $res6b @ GHASH block 8k+6 (t0, t1, and t2 free) + + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0 + + pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high + pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high + + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0 + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0 + + pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0 + + eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high + trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0 + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0 + + eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0 + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0 + + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1 + pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low + + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1 + pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid + + eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low + pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid + + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1 + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1 + + eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1 + + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1 + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1 + + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2 + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2 + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2 + + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2 + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1 + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2 + + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3 + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2 + + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2 + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2 + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + ldr $h2q, [$current_tag, #64] @ load h1l | h1h + ext $h2.16b, $h2.16b, $h2.16b, #8 + trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3 + + pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3 + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3 + + pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low + trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high + + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15 + + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3 + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3 + eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + + pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3 + pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high + + trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low + trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4 + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4 + eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high + + eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low + eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid + + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5 + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4 + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4 + + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4 + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4 + + pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4 + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4 + + pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low + + eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid + pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid + + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5 + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5 + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5 + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5 + + eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5 + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5 + + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5 + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6 + + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6 + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6 + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6 + + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6 + eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low + eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6 + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6 + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6 + + pmull $t11.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7 + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7 + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7 + ext $t12.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7 + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7 + eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid + + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7 + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7 + + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7 + + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8 + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8 + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8 + ext $acc_mb, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8 + eor3 $acc_lb, $acc_lb, $acc_hb, $acc_mb @ MODULO - fold into low + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8 + + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8 + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8 + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8 + + ldr $rk10q, [$cc, #160] @ load rk10 + aese $ctr6b, $rk9 @ AES block 8k+14 - round 9 + aese $ctr2b, $rk9 @ AES block 8k+10 - round 9 + + aese $ctr0b, $rk9 @ AES block 8k+8 - round 9 + aese $ctr1b, $rk9 @ AES block 8k+9 - round 9 + + aese $ctr3b, $rk9 @ AES block 8k+11 - round 9 + aese $ctr5b, $rk9 @ AES block 8k+13 - round 9 + + aese $ctr4b, $rk9 @ AES block 8k+12 - round 9 + aese $ctr7b, $rk9 @ AES block 8k+15 - round 9 +.L128_enc_tail: @ TAIL + + sub $main_end_input_ptr, $end_input_ptr, $input_ptr @ main_end_input_ptr is number of bytes left to process + ldr $ctr_t0q, [$input_ptr], #16 @ AES block 8k+8 - load plaintext + + mov $t1.16b, $rk10 + ldp $h5q, $h56kq, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + + eor3 $res1b, $ctr_t0b, $ctr0b, $t1.16b @ AES block 8k+8 - result + ext $t0.16b, $acc_lb, $acc_lb, #8 @ prepare final partial tag + ldp $h6q, $h7q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + ext $h7.16b, $h7.16b, $h7.16b, #8 + + ldp $h78kq, $h8q, [$current_tag, #192] @ load h8k | h7k + ext $h8.16b, $h8.16b, $h8.16b, #8 + cmp $main_end_input_ptr, #112 + b.gt .L128_enc_blocks_more_than_7 + + mov $ctr7b, $ctr6b + mov $ctr6b, $ctr5b + movi $acc_h.8b, #0 + + cmp $main_end_input_ptr, #96 + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + mov $ctr5b, $ctr4b + + mov $ctr4b, $ctr3b + mov $ctr3b, $ctr2b + mov $ctr2b, $ctr1b + + movi $acc_l.8b, #0 + movi $acc_m.8b, #0 + b.gt .L128_enc_blocks_more_than_6 + + mov $ctr7b, $ctr6b + cmp $main_end_input_ptr, #80 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + mov $ctr6b, $ctr5b + mov $ctr5b, $ctr4b + + mov $ctr4b, $ctr3b + mov $ctr3b, $ctr1b + b.gt .L128_enc_blocks_more_than_5 + + cmp $main_end_input_ptr, #64 + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + + mov $ctr7b, $ctr6b + mov $ctr6b, $ctr5b + + mov $ctr5b, $ctr4b + mov $ctr4b, $ctr1b + b.gt .L128_enc_blocks_more_than_4 + + mov $ctr7b, $ctr6b + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + mov $ctr6b, $ctr5b + + mov $ctr5b, $ctr1b + cmp $main_end_input_ptr, #48 + b.gt .L128_enc_blocks_more_than_3 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + mov $ctr7b, $ctr6b + mov $ctr6b, $ctr1b + + cmp $main_end_input_ptr, #32 + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + b.gt .L128_enc_blocks_more_than_2 + + cmp $main_end_input_ptr, #16 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + mov $ctr7b, $ctr1b + b.gt .L128_enc_blocks_more_than_1 + + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + b .L128_enc_blocks_less_than_1 +.L128_enc_blocks_more_than_7: @ blocks left > 7 + st1 { $res1b}, [$output_ptr], #16 @ AES final-7 block - store result + + rev64 $res0b, $res1b @ GHASH final-7 block + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-6 block - load plaintext + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-7 block - mid + + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH final-7 block - high + + ins $acc_m.d[0], $h78k.d[1] @ GHASH final-7 block - mid + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-7 block - mid + movi $t0.8b, #0 @ suppress further partial tag feed in + + eor3 $res1b, $ctr_t1b, $ctr1b, $t1.16b @ AES final-6 block - result + + pmull $acc_m.1q, $rk4v.1d, $acc_m.1d @ GHASH final-7 block - mid + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH final-7 block - low +.L128_enc_blocks_more_than_6: @ blocks left > 6 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-6 block - store result + + rev64 $res0b, $res1b @ GHASH final-6 block + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-5 block - load plaintext + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-6 block - mid + + eor3 $res1b, $ctr_t1b, $ctr2b, $t1.16b @ AES final-5 block - result + pmull $rk3q1, $res0.1d, $h7.1d @ GHASH final-6 block - low + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-6 block - mid + movi $t0.8b, #0 @ suppress further partial tag feed in + + pmull $rk4v.1q, $rk4v.1d, $h78k.1d @ GHASH final-6 block - mid + pmull2 $rk2q1, $res0.2d, $h7.2d @ GHASH final-6 block - high + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-6 block - low + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-6 block - mid + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-6 block - high +.L128_enc_blocks_more_than_5: @ blocks left > 5 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-5 block - store result + + rev64 $res0b, $res1b @ GHASH final-5 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-5 block - mid + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-4 block - load plaintext + pmull2 $rk2q1, $res0.2d, $h6.2d @ GHASH final-5 block - high + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-5 block - high + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-5 block - mid + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-5 block - mid + + eor3 $res1b, $ctr_t1b, $ctr3b, $t1.16b @ AES final-4 block - result + pmull $rk3q1, $res0.1d, $h6.1d @ GHASH final-5 block - low + movi $t0.8b, #0 @ suppress further partial tag feed in + + pmull2 $rk4v.1q, $rk4v.2d, $h56k.2d @ GHASH final-5 block - mid + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-5 block - low + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-5 block - mid +.L128_enc_blocks_more_than_4: @ blocks left > 4 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-4 block - store result + + rev64 $res0b, $res1b @ GHASH final-4 block + + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-3 block - load plaintext + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-4 block - mid + movi $t0.8b, #0 @ suppress further partial tag feed in + pmull2 $rk2q1, $res0.2d, $h5.2d @ GHASH final-4 block - high + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-4 block - mid + + pmull $rk3q1, $res0.1d, $h5.1d @ GHASH final-4 block - low + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-4 block - high + pmull $rk4v.1q, $rk4v.1d, $h56k.1d @ GHASH final-4 block - mid + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-4 block - low + + eor3 $res1b, $ctr_t1b, $ctr4b, $t1.16b @ AES final-3 block - result + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-4 block - mid +.L128_enc_blocks_more_than_3: @ blocks left > 3 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-3 block - store result + + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + + rev64 $res0b, $res1b @ GHASH final-3 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + movi $t0.8b, #0 @ suppress further partial tag feed in + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-3 block - mid + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + pmull $rk3q1, $res0.1d, $h4.1d @ GHASH final-3 block - low + + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-2 block - load plaintext + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-3 block - mid + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-3 block - mid + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-3 block - low + + eor3 $res1b, $ctr_t1b, $ctr5b, $t1.16b @ AES final-2 block - result + + pmull2 $rk4v.1q, $rk4v.2d, $h34k.2d @ GHASH final-3 block - mid + pmull2 $rk2q1, $res0.2d, $h4.2d @ GHASH final-3 block - high + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-3 block - mid + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-3 block - high +.L128_enc_blocks_more_than_2: @ blocks left > 2 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-2 block - store result + + rev64 $res0b, $res1b @ GHASH final-2 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-1 block - load plaintext + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-2 block - mid + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + movi $t0.8b, #0 @ suppress further partial tag feed in + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-2 block - mid + eor3 $res1b, $ctr_t1b, $ctr6b, $t1.16b @ AES final-1 block - result + + pmull2 $rk2q1, $res0.2d, $h3.2d @ GHASH final-2 block - high + + pmull $rk3q1, $res0.1d, $h3.1d @ GHASH final-2 block - low + pmull $rk4v.1q, $rk4v.1d, $h34k.1d @ GHASH final-2 block - mid + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-2 block - high + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-2 block - mid + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-2 block - low +.L128_enc_blocks_more_than_1: @ blocks left > 1 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-1 block - store result + + ldr $h2q, [$current_tag, #64] @ load h2l | h2h + ext $h2.16b, $h2.16b, $h2.16b, #8 + rev64 $res0b, $res1b @ GHASH final-1 block + ldr $ctr_t1q, [$input_ptr], #16 @ AES final block - load plaintext + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + movi $t0.8b, #0 @ suppress further partial tag feed in + ins $rk4v.d[0], $res0.d[1] @ GHASH final-1 block - mid + eor3 $res1b, $ctr_t1b, $ctr7b, $t1.16b @ AES final block - result + + pmull2 $rk2q1, $res0.2d, $h2.2d @ GHASH final-1 block - high + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-1 block - mid + + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-1 block - mid + + pmull $rk3q1, $res0.1d, $h2.1d @ GHASH final-1 block - low + pmull2 $rk4v.1q, $rk4v.2d, $h12k.2d @ GHASH final-1 block - mid + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-1 block - high + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-1 block - mid + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-1 block - low +.L128_enc_blocks_less_than_1: @ blocks left <= 1 + + rev32 $rtmp_ctr.16b, $rtmp_ctr.16b + str $rtmp_ctrq, [$counter] @ store the updated counter + and $bit_length, $bit_length, #127 @ bit_length %= 128 + + sub $bit_length, $bit_length, #128 @ bit_length -= 128 + + neg $bit_length, $bit_length @ bit_length = 128 - #bits in input (in range [1,128]) + + mvn $temp0_x, xzr @ temp0_x = 0xffffffffffffffff + ld1 { $rk0}, [$output_ptr] @ load existing bytes where the possibly partial last block is to be stored + and $bit_length, $bit_length, #127 @ bit_length %= 128 + + lsr $temp0_x, $temp0_x, $bit_length @ temp0_x is mask for top 64b of last block + mvn $temp1_x, xzr @ temp1_x = 0xffffffffffffffff + cmp $bit_length, #64 + + csel $temp2_x, $temp1_x, $temp0_x, lt + csel $temp3_x, $temp0_x, xzr, lt + + mov $ctr0.d[1], $temp3_x + mov $ctr0.d[0], $temp2_x @ ctr0b is mask for last block + + and $res1b, $res1b, $ctr0b @ possibly partial last block has zeroes in highest bits + + rev64 $res0b, $res1b @ GHASH final block + + bif $res1b, $rk0, $ctr0b @ insert existing bytes in top end of result before storing + st1 { $res1b}, [$output_ptr] @ store all 16B + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $t0.d[0], $res0.d[1] @ GHASH final block - mid + + eor $t0.8b, $t0.8b, $res0.8b @ GHASH final block - mid + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + + pmull $t0.1q, $t0.1d, $h12k.1d @ GHASH final block - mid + + pmull2 $rk2q1, $res0.2d, $h1.2d @ GHASH final block - high + eor $acc_mb, $acc_mb, $t0.16b @ GHASH final block - mid + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + + pmull $rk3q1, $res0.1d, $h1.1d @ GHASH final block - low + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final block - high + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final block - low + + ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + + eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + + eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid + + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + + eor3 $acc_lb, $acc_lb, $acc_hb, $t11.16b @ MODULO - fold into low + ext $acc_lb, $acc_lb, $acc_lb, #8 + rev64 $acc_lb, $acc_lb + st1 { $acc_l.16b }, [$current_tag] + mov x0, $byte_length + + ldp d10, d11, [sp, #16] + ldp d12, d13, [sp, #32] + ldp d14, d15, [sp, #48] + ldp d8, d9, [sp], #80 + ret + +.L128_enc_ret: + mov w0, #0x0 + ret +.size unroll8_eor3_aes_gcm_enc_128_kernel,.-unroll8_eor3_aes_gcm_enc_128_kernel +___ + +######################################################################################### +# size_t unroll8_eor3_aes_gcm_dec_128_kernel(const uint8_t * ciphertext, +# uint64_t plaintext_length, +# uint8_t * plaintext, +# uint64_t *Xi, +# unsigned char ivec[16], +# const void *key); +# +$code.=<<___; +.global unroll8_eor3_aes_gcm_dec_128_kernel +.type unroll8_eor3_aes_gcm_dec_128_kernel,%function +.align 4 +unroll8_eor3_aes_gcm_dec_128_kernel: + AARCH64_VALID_CALL_TARGET + cbz x1, .L128_dec_ret + stp d8, d9, [sp, #-80]! + lsr $byte_length, $bit_length, #3 + mov $counter, x4 + mov $cc, x5 + stp d10, d11, [sp, #16] + stp d12, d13, [sp, #32] + stp d14, d15, [sp, #48] + mov x5, #0xc200000000000000 + stp x5, xzr, [sp, #64] + add $modulo_constant, sp, #64 + + mov $main_end_input_ptr, $byte_length + ld1 { $ctr0b}, [$counter] @ CTR block 0 + + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + sub $main_end_input_ptr, $main_end_input_ptr, #1 @ byte_len - 1 + + mov $constant_temp, #0x100000000 @ set up counter increment + movi $rctr_inc.16b, #0x0 + mov $rctr_inc.d[1], $constant_temp + ld1 { $acc_lb}, [$current_tag] + ext $acc_lb, $acc_lb, $acc_lb, #8 + rev64 $acc_lb, $acc_lb + + rev32 $rtmp_ctr.16b, $ctr0.16b @ set up reversed counter + + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 0 + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 0 + + rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 1 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 1 + + and $main_end_input_ptr, $main_end_input_ptr, #0xffffffffffffff80 @ number of bytes to be processed in main loop (at least 1 byte must be handled by tail) + + rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 2 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 2 + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 0 + + rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 3 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 3 + + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 1 + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 1 + + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 4 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 4 + + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 5 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 5 + + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 0 + + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 6 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 6 + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 0 + + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 0 + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 0 + + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 7 + + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 0 + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 1 + + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 0 + + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 1 + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 1 + + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 1 + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 1 + + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 2 + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 2 + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 1 + + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 2 + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 2 + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 2 + + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 2 + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 2 + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 2 + + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 3 + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 3 + + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 3 + + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 3 + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 3 + + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 3 + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 3 + + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 4 + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 4 + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 3 + + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 4 + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 4 + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 4 + + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 4 + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 4 + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 4 + + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 5 + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 5 + + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 5 + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 5 + + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 5 + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 5 + + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 5 + + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 6 + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 6 + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 5 + + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 6 + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 6 + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 6 + + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 6 + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 6 + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 6 + + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 7 + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 7 + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 7 + + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 7 + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 7 + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 7 + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 7 + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 7 + + add $main_end_input_ptr, $main_end_input_ptr, $input_ptr + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 7 + + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 8 + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 8 + + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 8 + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 8 + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 8 + + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 8 + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 8 + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 8 + + aese $ctr0b, $rk9 @ AES block 0 - round 9 + aese $ctr1b, $rk9 @ AES block 1 - round 9 + aese $ctr6b, $rk9 @ AES block 6 - round 9 + + ldr $rk10q, [$cc, #160] @ load rk10 + aese $ctr4b, $rk9 @ AES block 4 - round 9 + aese $ctr3b, $rk9 @ AES block 3 - round 9 + + aese $ctr2b, $rk9 @ AES block 2 - round 9 + aese $ctr5b, $rk9 @ AES block 5 - round 9 + aese $ctr7b, $rk9 @ AES block 7 - round 9 + + add $end_input_ptr, $input_ptr, $bit_length, lsr #3 @ end_input_ptr + cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks + b.ge .L128_dec_tail @ handle tail + + ldp $res0q, $res1q, [$input_ptr], #32 @ AES block 0, 1 - load ciphertext + + eor3 $ctr0b, $res0b, $ctr0b, $rk10 @ AES block 0 - result + eor3 $ctr1b, $res1b, $ctr1b, $rk10 @ AES block 1 - result + stp $ctr0q, $ctr1q, [$output_ptr], #32 @ AES block 0, 1 - store result + + rev32 $ctr0.16b, $rtmp_ctr.16b @ CTR block 8 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8 + ldp $res2q, $res3q, [$input_ptr], #32 @ AES block 2, 3 - load ciphertext + + ldp $res4q, $res5q, [$input_ptr], #32 @ AES block 4, 5 - load ciphertext + + rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 9 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 9 + ldp $res6q, $res7q, [$input_ptr], #32 @ AES block 6, 7 - load ciphertext + + eor3 $ctr3b, $res3b, $ctr3b, $rk10 @ AES block 3 - result + eor3 $ctr2b, $res2b, $ctr2b, $rk10 @ AES block 2 - result + stp $ctr2q, $ctr3q, [$output_ptr], #32 @ AES block 2, 3 - store result + + rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 10 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 10 + + eor3 $ctr6b, $res6b, $ctr6b, $rk10 @ AES block 6 - result + + rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 11 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 11 + + eor3 $ctr4b, $res4b, $ctr4b, $rk10 @ AES block 4 - result + eor3 $ctr5b, $res5b, $ctr5b, $rk10 @ AES block 5 - result + stp $ctr4q, $ctr5q, [$output_ptr], #32 @ AES block 4, 5 - store result + + eor3 $ctr7b, $res7b, $ctr7b, $rk10 @ AES block 7 - result + stp $ctr6q, $ctr7q, [$output_ptr], #32 @ AES block 6, 7 - store result + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 12 + + cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 12 + b.ge .L128_dec_prepretail @ do prepretail + +.L128_dec_main_loop: @ main loop start + ldr $h7q, [$current_tag, #176] @ load h7l | h7h + ext $h7.16b, $h7.16b, $h7.16b, #8 + ldr $h8q, [$current_tag, #208] @ load h8l | h8h + ext $h8.16b, $h8.16b, $h8.16b, #8 + + rev64 $res1b, $res1b @ GHASH block 8k+1 + rev64 $res0b, $res0b @ GHASH block 8k + ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0 + + rev64 $res6b, $res6b @ GHASH block 8k+6 + ldr $h5q, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + ldr $h6q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + + eor $res0b, $res0b, $acc_lb @ PRE 1 + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13 + + rev64 $res2b, $res2b @ GHASH block 8k+2 + rev64 $res4b, $res4b @ GHASH block 8k+4 + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14 + ldr $h56kq, [$current_tag, #144] @ load h6k | h5k + ldr $h78kq, [$current_tag, #192] @ load h8k | h7k + + pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high + rev64 $res3b, $res3b @ GHASH block 8k+3 + + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15 + trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + rev64 $res5b, $res5b @ GHASH block 8k+5 + + pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low + trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + + pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0 + pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high + + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0 + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0 + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0 + + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0 + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0 + eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high + + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0 + eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0 + + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1 + eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low + eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high + + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1 + + pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low + trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid + + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1 + + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1 + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1 + pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low + + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1 + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1 + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1 + + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2 + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2 + eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low + + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2 + eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + ldr $h2q, [$current_tag, #64] @ load h2l | h2h + ext $h2.16b, $h2.16b, $h2.16b, #8 + + eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2 + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2 + + trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2 + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2 + + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2 + pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid + pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid + + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3 + rev64 $res7b, $res7b @ GHASH block 8k+7 + pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high + + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low + eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3 + trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3 + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3 + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3 + + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3 + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3 + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3 + + pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high + pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low + pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high + + pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4 + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4 + + eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4 + + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4 + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4 + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4 + + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4 + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4 + trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5 + pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid + + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5 + eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5 + + pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5 + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5 + + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5 + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5 + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5 + + pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high + eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low + + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6 + eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6 + + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6 + pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6 + + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6 + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6 + pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low + + pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15 + + eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6 + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7 + + rev32 $h1.16b, $rtmp_ctr.16b @ CTR block 8k+16 + eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+16 + + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7 + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7 + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7 + + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7 + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7 + rev32 $h2.16b, $rtmp_ctr.16b @ CTR block 8k+17 + + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7 + ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + + eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+17 + + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8 + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8 + ldp $res0q, $res1q, [$input_ptr], #32 @ AES block 8k+8, 8k+9 - load ciphertext + + ldp $res2q, $res3q, [$input_ptr], #32 @ AES block 8k+10, 8k+11 - load ciphertext + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8 + rev32 $h3.16b, $rtmp_ctr.16b @ CTR block 8k+18 + + ldp $res4q, $res5q, [$input_ptr], #32 @ AES block 8k+12, 8k+13 - load ciphertext + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8 + eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid + + ldp $res6q, $res7q, [$input_ptr], #32 @ AES block 8k+14, 8k+15 - load ciphertext + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+18 + + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8 + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8 + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8 + + aese $ctr0b, $rk9 @ AES block 8k+8 - round 9 + aese $ctr1b, $rk9 @ AES block 8k+9 - round 9 + ldr $rk10q, [$cc, #160] @ load rk10 + + aese $ctr6b, $rk9 @ AES block 8k+14 - round 9 + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + aese $ctr2b, $rk9 @ AES block 8k+10 - round 9 + + aese $ctr7b, $rk9 @ AES block 8k+15 - round 9 + aese $ctr4b, $rk9 @ AES block 8k+12 - round 9 + ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + + rev32 $h4.16b, $rtmp_ctr.16b @ CTR block 8k+19 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+19 + + aese $ctr3b, $rk9 @ AES block 8k+11 - round 9 + aese $ctr5b, $rk9 @ AES block 8k+13 - round 9 + eor3 $ctr1b, $res1b, $ctr1b, $rk10 @ AES block 8k+9 - result + + eor3 $ctr0b, $res0b, $ctr0b, $rk10 @ AES block 8k+8 - result + eor3 $ctr7b, $res7b, $ctr7b, $rk10 @ AES block 8k+15 - result + eor3 $ctr6b, $res6b, $ctr6b, $rk10 @ AES block 8k+14 - result + + eor3 $ctr2b, $res2b, $ctr2b, $rk10 @ AES block 8k+10 - result + stp $ctr0q, $ctr1q, [$output_ptr], #32 @ AES block 8k+8, 8k+9 - store result + mov $ctr1.16b, $h2.16b @ CTR block 8k+17 + + eor3 $ctr4b, $res4b, $ctr4b, $rk10 @ AES block 8k+12 - result + eor3 $acc_lb, $acc_lb, $acc_hb, $t11.16b @ MODULO - fold into low + mov $ctr0.16b, $h1.16b @ CTR block 8k+16 + + eor3 $ctr3b, $res3b, $ctr3b, $rk10 @ AES block 8k+11 - result + cmp $input_ptr, $main_end_input_ptr @ LOOP CONTROL + stp $ctr2q, $ctr3q, [$output_ptr], #32 @ AES block 8k+10, 8k+11 - store result + + eor3 $ctr5b, $res5b, $ctr5b, $rk10 @ AES block 8k+13 - result + mov $ctr2.16b, $h3.16b @ CTR block 8k+18 + + stp $ctr4q, $ctr5q, [$output_ptr], #32 @ AES block 8k+12, 8k+13 - store result + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 8k+20 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+20 + + stp $ctr6q, $ctr7q, [$output_ptr], #32 @ AES block 8k+14, 8k+15 - store result + mov $ctr3.16b, $h4.16b @ CTR block 8k+19 + b.lt .L128_dec_main_loop + +.L128_dec_prepretail: @ PREPRETAIL + rev64 $res3b, $res3b @ GHASH block 8k+3 + ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0 + rev64 $res0b, $res0b @ GHASH block 8k + + rev64 $res2b, $res2b @ GHASH block 8k+2 + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13 + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + + ldr $h7q, [$current_tag, #176] @ load h7l | h7h + ext $h7.16b, $h7.16b, $h7.16b, #8 + ldr $h8q, [$current_tag, #208] @ load h8l | h8h + ext $h8.16b, $h8.16b, $h8.16b, #8 + eor $res0b, $res0b, $acc_lb @ PRE 1 + rev64 $res1b, $res1b @ GHASH block 8k+1 + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13 + ldr $h5q, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + ldr $h6q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + rev64 $res5b, $res5b @ GHASH block 8k+5 + + rev64 $res4b, $res4b @ GHASH block 8k+4 + + rev64 $res6b, $res6b @ GHASH block 8k+6 + + ldr $h56kq, [$current_tag, #144] @ load h6k | h5k + ldr $h78kq, [$current_tag, #192] @ load h8k | h7k + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14 + + pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high + + trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high + + pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low + pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0 + + eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0 + eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid + + pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15 + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0 + + eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high + trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0 + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0 + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0 + + pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid + pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid + pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low + + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1 + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0 + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0 + + eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low + eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid + + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1 + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1 + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1 + + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low + pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid + + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1 + pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid + + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1 + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1 + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1 + + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + ldr $h2q, [$current_tag, #64] @ load h2l | h2h + ext $h2.16b, $h2.16b, $h2.16b, #8 + eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2 + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2 + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2 + + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2 + trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2 + + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2 + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2 + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2 + + pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high + pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low + trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + rev64 $res7b, $res7b @ GHASH block 8k+7 + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3 + + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high + pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low + + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3 + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3 + trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + + pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high + pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low + trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3 + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3 + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3 + + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3 + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3 + eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + + eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4 + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4 + + eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4 + pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid + + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4 + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4 + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4 + + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4 + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4 + pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid + + pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high + pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid + pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid + + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5 + + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low + eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low + + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5 + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5 + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5 + + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5 + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5 + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5 + + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5 + eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low + + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6 + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6 + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6 + + eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6 + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6 + + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6 + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6 + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6 + + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7 + eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + + pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7 + ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7 + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7 + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7 + + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7 + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7 + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7 + + eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid + ldr $rk10q, [$cc, #160] @ load rk10 + + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8 + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8 + + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8 + ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8 + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8 + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8 + + aese $ctr6b, $rk9 @ AES block 8k+14 - round 9 + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8 + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8 + + eor3 $acc_lb, $acc_lb, $acc_hb, $t11.16b @ MODULO - fold into low + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15 + aese $ctr2b, $rk9 @ AES block 8k+10 - round 9 + + aese $ctr3b, $rk9 @ AES block 8k+11 - round 9 + aese $ctr5b, $rk9 @ AES block 8k+13 - round 9 + aese $ctr0b, $rk9 @ AES block 8k+8 - round 9 + + aese $ctr4b, $rk9 @ AES block 8k+12 - round 9 + aese $ctr1b, $rk9 @ AES block 8k+9 - round 9 + aese $ctr7b, $rk9 @ AES block 8k+15 - round 9 + +.L128_dec_tail: @ TAIL + + mov $t1.16b, $rk10 + sub $main_end_input_ptr, $end_input_ptr, $input_ptr @ main_end_input_ptr is number of bytes left to process + + cmp $main_end_input_ptr, #112 + + ldp $h78kq, $h8q, [$current_tag, #192] @ load h8k | h7k + ext $h8.16b, $h8.16b, $h8.16b, #8 + ldr $res1q, [$input_ptr], #16 @ AES block 8k+8 - load ciphertext + + ldp $h5q, $h56kq, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + ext $t0.16b, $acc_lb, $acc_lb, #8 @ prepare final partial tag + + ldp $h6q, $h7q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + ext $h7.16b, $h7.16b, $h7.16b, #8 + + eor3 $res4b, $res1b, $ctr0b, $t1.16b @ AES block 8k+8 - result + b.gt .L128_dec_blocks_more_than_7 + + cmp $main_end_input_ptr, #96 + mov $ctr7b, $ctr6b + movi $acc_l.8b, #0 + + movi $acc_h.8b, #0 + mov $ctr6b, $ctr5b + mov $ctr5b, $ctr4b + + mov $ctr4b, $ctr3b + mov $ctr3b, $ctr2b + mov $ctr2b, $ctr1b + + movi $acc_m.8b, #0 + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + b.gt .L128_dec_blocks_more_than_6 + + cmp $main_end_input_ptr, #80 + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + + mov $ctr7b, $ctr6b + mov $ctr6b, $ctr5b + mov $ctr5b, $ctr4b + + mov $ctr4b, $ctr3b + mov $ctr3b, $ctr1b + b.gt .L128_dec_blocks_more_than_5 + + cmp $main_end_input_ptr, #64 + + mov $ctr7b, $ctr6b + mov $ctr6b, $ctr5b + mov $ctr5b, $ctr4b + + mov $ctr4b, $ctr1b + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + b.gt .L128_dec_blocks_more_than_4 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + mov $ctr7b, $ctr6b + mov $ctr6b, $ctr5b + + mov $ctr5b, $ctr1b + cmp $main_end_input_ptr, #48 + b.gt .L128_dec_blocks_more_than_3 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + mov $ctr7b, $ctr6b + cmp $main_end_input_ptr, #32 + + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + mov $ctr6b, $ctr1b + b.gt .L128_dec_blocks_more_than_2 + + cmp $main_end_input_ptr, #16 + + mov $ctr7b, $ctr1b + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + b.gt L128_dec_blocks_more_than_1 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + b .L128_dec_blocks_less_than_1 +.L128_dec_blocks_more_than_7: @ blocks left > 7 + rev64 $res0b, $res1b @ GHASH final-7 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $acc_m.d[0], $h78k.d[1] @ GHASH final-7 block - mid + + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH final-7 block - low + ins $rk4v.d[0], $res0.d[1] @ GHASH final-7 block - mid + + movi $t0.8b, #0 @ suppress further partial tag feed in + ldr $res1q, [$input_ptr], #16 @ AES final-6 block - load ciphertext + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-7 block - mid + + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH final-7 block - high + st1 { $res4b}, [$output_ptr], #16 @ AES final-7 block - store result + eor3 $res4b, $res1b, $ctr1b, $t1.16b @ AES final-6 block - result + + pmull $acc_m.1q, $rk4v.1d, $acc_m.1d @ GHASH final-7 block - mid +.L128_dec_blocks_more_than_6: @ blocks left > 6 + + rev64 $res0b, $res1b @ GHASH final-6 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-6 block - mid + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-6 block - mid + + pmull $rk3q1, $res0.1d, $h7.1d @ GHASH final-6 block - low + ldr $res1q, [$input_ptr], #16 @ AES final-5 block - load ciphertext + movi $t0.8b, #0 @ suppress further partial tag feed in + + pmull $rk4v.1q, $rk4v.1d, $h78k.1d @ GHASH final-6 block - mid + st1 { $res4b}, [$output_ptr], #16 @ AES final-6 block - store result + pmull2 $rk2q1, $res0.2d, $h7.2d @ GHASH final-6 block - high + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-6 block - low + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-6 block - high + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-6 block - mid + eor3 $res4b, $res1b, $ctr2b, $t1.16b @ AES final-5 block - result +.L128_dec_blocks_more_than_5: @ blocks left > 5 + + rev64 $res0b, $res1b @ GHASH final-5 block + + ldr $res1q, [$input_ptr], #16 @ AES final-4 block - load ciphertext + st1 { $res4b}, [$output_ptr], #16 @ AES final-5 block - store result + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-5 block - mid + + eor3 $res4b, $res1b, $ctr3b, $t1.16b @ AES final-4 block - result + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-5 block - mid + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-5 block - mid + pmull $rk3q1, $res0.1d, $h6.1d @ GHASH final-5 block - low + movi $t0.8b, #0 @ suppress further partial tag feed in + + pmull2 $rk4v.1q, $rk4v.2d, $h56k.2d @ GHASH final-5 block - mid + pmull2 $rk2q1, $res0.2d, $h6.2d @ GHASH final-5 block - high + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-5 block - low + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-5 block - mid + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-5 block - high +.L128_dec_blocks_more_than_4: @ blocks left > 4 + + rev64 $res0b, $res1b @ GHASH final-4 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + ldr $res1q, [$input_ptr], #16 @ AES final-3 block - load ciphertext + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-4 block - mid + movi $t0.8b, #0 @ suppress further partial tag feed in + pmull2 $rk2q1, $res0.2d, $h5.2d @ GHASH final-4 block - high + + pmull $rk3q1, $res0.1d, $h5.1d @ GHASH final-4 block - low + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-4 block - high + + st1 { $res4b}, [$output_ptr], #16 @ AES final-4 block - store result + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-4 block - mid + + eor3 $res4b, $res1b, $ctr4b, $t1.16b @ AES final-3 block - result + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-4 block - low + + pmull $rk4v.1q, $rk4v.1d, $h56k.1d @ GHASH final-4 block - mid + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-4 block - mid +.L128_dec_blocks_more_than_3: @ blocks left > 3 + + st1 { $res4b}, [$output_ptr], #16 @ AES final-3 block - store result + rev64 $res0b, $res1b @ GHASH final-3 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-3 block - mid + + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-3 block - mid + + ldr $res1q, [$input_ptr], #16 @ AES final-2 block - load ciphertext + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-3 block - mid + pmull $rk3q1, $res0.1d, $h4.1d @ GHASH final-3 block - low + pmull2 $rk2q1, $res0.2d, $h4.2d @ GHASH final-3 block - high + + movi $t0.8b, #0 @ suppress further partial tag feed in + eor3 $res4b, $res1b, $ctr5b, $t1.16b @ AES final-2 block - result + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-3 block - low + + pmull2 $rk4v.1q, $rk4v.2d, $h34k.2d @ GHASH final-3 block - mid + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-3 block - high + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-3 block - mid +.L128_dec_blocks_more_than_2: @ blocks left > 2 + + rev64 $res0b, $res1b @ GHASH final-2 block + + st1 { $res4b}, [$output_ptr], #16 @ AES final-2 block - store result + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + movi $t0.8b, #0 @ suppress further partial tag feed in + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-2 block - mid + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-2 block - mid + + pmull $rk3q1, $res0.1d, $h3.1d @ GHASH final-2 block - low + + pmull2 $rk2q1, $res0.2d, $h3.2d @ GHASH final-2 block - high + pmull $rk4v.1q, $rk4v.1d, $h34k.1d @ GHASH final-2 block - mid + ldr $res1q, [$input_ptr], #16 @ AES final-1 block - load ciphertext + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-2 block - mid + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-2 block - low + + eor3 $res4b, $res1b, $ctr6b, $t1.16b @ AES final-1 block - result + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-2 block - high +.L128_dec_blocks_more_than_1: @ blocks left > 1 + + st1 { $res4b}, [$output_ptr], #16 @ AES final-1 block - store result + rev64 $res0b, $res1b @ GHASH final-1 block + + ldr $h2q, [$current_tag, #64] @ load h2l | h2h + ext $h2.16b, $h2.16b, $h2.16b, #8 + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + movi $t0.8b, #0 @ suppress further partial tag feed in + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-1 block - mid + + ldr $res1q, [$input_ptr], #16 @ AES final block - load ciphertext + pmull2 $rk2q1, $res0.2d, $h2.2d @ GHASH final-1 block - high + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-1 block - mid + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-1 block - high + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-1 block - mid + eor3 $res4b, $res1b, $ctr7b, $t1.16b @ AES final block - result + + pmull $rk3q1, $res0.1d, $h2.1d @ GHASH final-1 block - low + + pmull2 $rk4v.1q, $rk4v.2d, $h12k.2d @ GHASH final-1 block - mid + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-1 block - low + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-1 block - mid +.L128_dec_blocks_less_than_1: @ blocks left <= 1 + + and $bit_length, $bit_length, #127 @ bit_length %= 128 + + sub $bit_length, $bit_length, #128 @ bit_length -= 128 + + neg $bit_length, $bit_length @ bit_length = 128 - #bits in input (in range [1,128]) + + mvn $temp0_x, xzr @ temp0_x = 0xffffffffffffffff + and $bit_length, $bit_length, #127 @ bit_length %= 128 + + lsr $temp0_x, $temp0_x, $bit_length @ temp0_x is mask for top 64b of last block + cmp $bit_length, #64 + mvn $temp1_x, xzr @ temp1_x = 0xffffffffffffffff + + csel $temp2_x, $temp1_x, $temp0_x, lt + csel $temp3_x, $temp0_x, xzr, lt + + mov $ctr0.d[1], $temp3_x + mov $ctr0.d[0], $temp2_x @ ctr0b is mask for last block + + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + ld1 { $rk0}, [$output_ptr] @ load existing bytes where the possibly partial last block is to be stored + + and $res1b, $res1b, $ctr0b @ possibly partial last block has zeroes in highest bits + + rev64 $res0b, $res1b @ GHASH final block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + pmull2 $rk2q1, $res0.2d, $h1.2d @ GHASH final block - high + ins $t0.d[0], $res0.d[1] @ GHASH final block - mid + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final block - high + eor $t0.8b, $t0.8b, $res0.8b @ GHASH final block - mid + + bif $res4b, $rk0, $ctr0b @ insert existing bytes in top end of result before storing + + pmull $t0.1q, $t0.1d, $h12k.1d @ GHASH final block - mid + st1 { $res4b}, [$output_ptr] @ store all 16B + + pmull $rk3q1, $res0.1d, $h1.1d @ GHASH final block - low + + eor $acc_mb, $acc_mb, $t0.16b @ GHASH final block - mid + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final block - low + + eor $t10.16b, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + + pmull $t11.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + ext $acc_hb, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + + eor $acc_mb, $acc_mb, $t10.16b @ MODULO - karatsuba tidy up + + eor3 $acc_mb, $acc_mb, $acc_hb, $t11.16b @ MODULO - fold into mid + + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + ext $acc_mb, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + + eor3 $acc_lb, $acc_lb, $acc_mb, $acc_hb @ MODULO - fold into low + ext $acc_lb, $acc_lb, $acc_lb, #8 + rev64 $acc_lb, $acc_lb + st1 { $acc_l.16b }, [$current_tag] + rev32 $rtmp_ctr.16b, $rtmp_ctr.16b + + str $rtmp_ctrq, [$counter] @ store the updated counter + + mov x0, $byte_length + + ldp d10, d11, [sp, #16] + ldp d12, d13, [sp, #32] + ldp d14, d15, [sp, #48] + ldp d8, d9, [sp], #80 + ret +.L128_dec_ret: + mov w0, #0x0 + ret +.size unroll8_eor3_aes_gcm_dec_128_kernel,.-unroll8_eor3_aes_gcm_dec_128_kernel +___ +} + +{ +my ($end_input_ptr,$main_end_input_ptr,$temp0_x,$temp1_x)=map("x$_",(4..7)); +my ($temp2_x,$temp3_x)=map("x$_",(13..14)); +my ($ctr0b,$ctr1b,$ctr2b,$ctr3b,$ctr4b,$ctr5b,$ctr6b,$ctr7b,$res0b,$res1b,$res2b,$res3b,$res4b,$res5b,$res6b,$res7b)=map("v$_.16b",(0..15)); +my ($ctr0,$ctr1,$ctr2,$ctr3,$ctr4,$ctr5,$ctr6,$ctr7,$res0,$res1,$res2,$res3,$res4,$res5,$res6,$res7)=map("v$_",(0..15)); +my ($ctr0d,$ctr1d,$ctr2d,$ctr3d,$ctr4d,$ctr5d,$ctr6d,$ctr7d)=map("d$_",(0..7)); +my ($ctr0q,$ctr1q,$ctr2q,$ctr3q,$ctr4q,$ctr5q,$ctr6q,$ctr7q)=map("q$_",(0..7)); +my ($res0q,$res1q,$res2q,$res3q,$res4q,$res5q,$res6q,$res7q)=map("q$_",(8..15)); + +my ($ctr_t0,$ctr_t1,$ctr_t2,$ctr_t3,$ctr_t4,$ctr_t5,$ctr_t6,$ctr_t7)=map("v$_",(8..15)); +my ($ctr_t0b,$ctr_t1b,$ctr_t2b,$ctr_t3b,$ctr_t4b,$ctr_t5b,$ctr_t6b,$ctr_t7b)=map("v$_.16b",(8..15)); +my ($ctr_t0q,$ctr_t1q,$ctr_t2q,$ctr_t3q,$ctr_t4q,$ctr_t5q,$ctr_t6q,$ctr_t7q)=map("q$_",(8..15)); + +my ($acc_hb,$acc_mb,$acc_lb)=map("v$_.16b",(17..19)); +my ($acc_h,$acc_m,$acc_l)=map("v$_",(17..19)); + +my ($h1,$h12k,$h2,$h3,$h34k,$h4)=map("v$_",(20..25)); +my ($h5,$h56k,$h6,$h7,$h78k,$h8)=map("v$_",(20..25)); +my ($h1q,$h12kq,$h2q,$h3q,$h34kq,$h4q)=map("q$_",(20..25)); +my ($h5q,$h56kq,$h6q,$h7q,$h78kq,$h8q)=map("q$_",(20..25)); + +my $t0="v16"; +my $t0d="d16"; + +my $t1="v29"; +my $t2=$res1; +my $t3=$t1; + +my $t4=$res0; +my $t5=$res2; +my $t6=$t0; + +my $t7=$res3; +my $t8=$res4; +my $t9=$res5; + +my $t10=$res6; +my $t11="v21"; +my $t12=$t1; + +my $rtmp_ctr="v30"; +my $rtmp_ctrq="q30"; +my $rctr_inc="v31"; +my $rctr_incd="d31"; + +my $mod_constantd=$t0d; +my $mod_constant=$t0; + +my ($rk0,$rk1,$rk2)=map("v$_.16b",(26..28)); +my ($rk3,$rk4,$rk5)=map("v$_.16b",(26..28)); +my ($rk6,$rk7,$rk8)=map("v$_.16b",(26..28)); +my ($rk9,$rk10,$rk11)=map("v$_.16b",(26..28)); +my ($rk12,$rk13,$rk14)=map("v$_.16b",(26..28)); +my ($rk0q,$rk1q,$rk2q)=map("q$_",(26..28)); +my ($rk3q,$rk4q,$rk5q)=map("q$_",(26..28)); +my ($rk6q,$rk7q,$rk8q)=map("q$_",(26..28)); +my ($rk9q,$rk10q,$rk11q)=map("q$_",(26..28)); +my ($rk12q,$rk13q,$rk14q)=map("q$_",(26..28)); +my $rk2q1="v28.1q"; +my $rk3q1="v26.1q"; +my $rk4v="v27"; + +######################################################################################### +# size_t unroll8_eor3_aes_gcm_enc_192_kernel(const uint8_t * plaintext, +# uint64_t plaintext_length, +# uint8_t * ciphertext, +# uint64_t *Xi, +# unsigned char ivec[16], +# const void *key); +# +$code.=<<___; +.global unroll8_eor3_aes_gcm_enc_192_kernel +.type unroll8_eor3_aes_gcm_enc_192_kernel,%function +.align 4 +unroll8_eor3_aes_gcm_enc_192_kernel: + AARCH64_VALID_CALL_TARGET + cbz x1, .L192_enc_ret + stp d8, d9, [sp, #-80]! + lsr $byte_length, $bit_length, #3 + mov $counter, x4 + mov $cc, x5 + stp d10, d11, [sp, #16] + stp d12, d13, [sp, #32] + stp d14, d15, [sp, #48] + mov x5, #0xc200000000000000 + stp x5, xzr, [sp, #64] + add $modulo_constant, sp, #64 + + mov $main_end_input_ptr, $byte_length + ld1 { $ctr0b}, [$counter] @ CTR block 0 + + mov $constant_temp, #0x100000000 @ set up counter increment + movi $rctr_inc.16b, #0x0 + mov $rctr_inc.d[1], $constant_temp + + rev32 $rtmp_ctr.16b, $ctr0.16b @ set up reversed counter + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 0 + + rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 1 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 1 + + rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 2 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 2 + + rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 3 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 3 + + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 4 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 4 + sub $main_end_input_ptr, $main_end_input_ptr, #1 @ byte_len - 1 + + and $main_end_input_ptr, $main_end_input_ptr, #0xffffffffffffff80 @ number of bytes to be processed in main loop (at least 1 byte must be handled by tail) + + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 5 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 5 + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + + add $main_end_input_ptr, $main_end_input_ptr, $input_ptr + + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 6 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 6 + + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 7 + + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 0 + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 0 + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 0 + + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 0 + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 0 + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 0 + + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 0 + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 0 + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 1 + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 1 + + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 1 + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 1 + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 1 + + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 2 + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 1 + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 1 + + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 1 + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 2 + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 2 + + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 2 + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 2 + + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 2 + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 2 + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 2 + + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 3 + + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 3 + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 3 + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 3 + + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 3 + + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 3 + + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 3 + + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 4 + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 4 + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 3 + + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 4 + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 4 + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 4 + + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 4 + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 4 + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 4 + + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 5 + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 5 + + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 5 + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 5 + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 5 + + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 5 + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 5 + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 5 + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 7 + + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 6 + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 6 + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 6 + + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 6 + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 6 + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 6 + + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 6 + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 6 + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 7 + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 7 + + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 7 + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 7 + + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 7 + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 7 + + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 7 + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 7 + + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 8 + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 8 + + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 8 + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 8 + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 8 + + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 8 + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 8 + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 8 + + add $end_input_ptr, $input_ptr, $bit_length, lsr #3 @ end_input_ptr + cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks + aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 9 + + ld1 { $acc_lb}, [$current_tag] + ext $acc_lb, $acc_lb, $acc_lb, #8 + rev64 $acc_lb, $acc_lb + ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11 + + aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 9 + aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 9 + + aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 9 + aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 9 + + aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 9 + aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 9 + + aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 14 - round 10 + aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 9 + aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 11 - round 10 + + aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 9 - round 10 + aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 13 - round 10 + aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 12 - round 10 + + aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 8 - round 10 + aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 10 - round 10 + aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 15 - round 10 + + aese $ctr6b, $rk11 @ AES block 14 - round 11 + aese $ctr3b, $rk11 @ AES block 11 - round 11 + + aese $ctr4b, $rk11 @ AES block 12 - round 11 + aese $ctr7b, $rk11 @ AES block 15 - round 11 + ldr $rk12q, [$cc, #192] @ load rk12 + + aese $ctr1b, $rk11 @ AES block 9 - round 11 + aese $ctr5b, $rk11 @ AES block 13 - round 11 + + aese $ctr2b, $rk11 @ AES block 10 - round 11 + aese $ctr0b, $rk11 @ AES block 8 - round 11 + b.ge .L192_enc_tail @ handle tail + + ldp $ctr_t0q, $ctr_t1q, [$input_ptr], #32 @ AES block 0, 1 - load plaintext + + ldp $ctr_t2q, $ctr_t3q, [$input_ptr], #32 @ AES block 2, 3 - load plaintext + + ldp $ctr_t4q, $ctr_t5q, [$input_ptr], #32 @ AES block 4, 5 - load plaintext + + ldp $ctr_t6q, $ctr_t7q, [$input_ptr], #32 @ AES block 6, 7 - load plaintext + + eor3 $res0b, $ctr_t0b, $ctr0b, $rk12 @ AES block 0 - result + rev32 $ctr0.16b, $rtmp_ctr.16b @ CTR block 8 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8 + + eor3 $res3b, $ctr_t3b, $ctr3b, $rk12 @ AES block 3 - result + eor3 $res1b, $ctr_t1b, $ctr1b, $rk12 @ AES block 1 - result + + rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 9 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 9 + eor3 $res4b, $ctr_t4b, $ctr4b, $rk12 @ AES block 4 - result + + eor3 $res5b, $ctr_t5b, $ctr5b, $rk12 @ AES block 5 - result + eor3 $res7b, $ctr_t7b, $ctr7b, $rk12 @ AES block 7 - result + stp $res0q, $res1q, [$output_ptr], #32 @ AES block 0, 1 - store result + + eor3 $res2b, $ctr_t2b, $ctr2b, $rk12 @ AES block 2 - result + rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 10 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 10 + + stp $res2q, $res3q, [$output_ptr], #32 @ AES block 2, 3 - store result + cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks + + rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 11 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 11 + eor3 $res6b, $ctr_t6b, $ctr6b, $rk12 @ AES block 6 - result + + stp $res4q, $res5q, [$output_ptr], #32 @ AES block 4, 5 - store result + + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 12 + stp $res6q, $res7q, [$output_ptr], #32 @ AES block 6, 7 - store result + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 12 + + b.ge .L192_enc_prepretail @ do prepretail + +.L192_enc_main_loop: @ main loop start + rev64 $res4b, $res4b @ GHASH block 8k+4 (t0, t1, and t2 free) + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + rev64 $res2b, $res2b @ GHASH block 8k+2 + + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13 + ldr $h7q, [$current_tag, #176] @ load h7l | h7h + ext $h7.16b, $h7.16b, $h7.16b, #8 + ldr $h8q, [$current_tag, #208] @ load h8l | h8h + ext $h8.16b, $h8.16b, $h8.16b, #8 + + ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0 + rev64 $res0b, $res0b @ GHASH block 8k + ldr $h5q, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + ldr $h6q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + + rev64 $res1b, $res1b @ GHASH block 8k+1 + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14 + + eor $res0b, $res0b, $acc_lb @ PRE 1 + rev64 $res3b, $res3b @ GHASH block 8k+3 + rev64 $res5b, $res5b @ GHASH block 8k+5 (t0, t1, t2 and t3 free) + + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0 + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15 + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0 + + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0 + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0 + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0 + + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0 + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0 + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0 + + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1 + + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1 + pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high + pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low + + trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1 + ldr $h56kq, [$current_tag, #144] @ load h6k | h5k + ldr $h78kq, [$current_tag, #192] @ load h8k | h7k + + pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low + trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1 + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1 + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1 + + eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1 + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1 + + pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high + eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2 + + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2 + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2 + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2 + + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2 + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3 + eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high + + pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2 + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3 + + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2 + trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2 + + trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3 + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3 + eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + + pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid + pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid + pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low + + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3 + eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + + eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3 + eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low + + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4 + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4 + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3 + + pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4 + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3 + + pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4 + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4 + + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4 + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4 + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4 + + eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5 + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + ldr $h2q, [$current_tag, #64] @ load h2l | h2h + ext $h2.16b, $h2.16b, $h2.16b, #8 + + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5 + rev64 $res7b, $res7b @ GHASH block 8k+7 (t0, t1, t2 and t3 free) + + rev64 $res6b, $res6b @ GHASH block 8k+6 (t0, t1, and t2 free) + pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high + pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low + + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5 + trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5 + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5 + pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high + eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5 + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5 + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5 + + pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6 + trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6 + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6 + pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high + + pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low + trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6 + + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6 + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6 + + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6 + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7 + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6 + + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7 + eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + + pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid + + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7 + pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7 + + eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15 + + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7 + + pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid + pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7 + + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8 + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8 + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8 + + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8 + eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7 + + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8 + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8 + pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid + + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8 + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8 + ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11 + + eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low + rev32 $h1.16b, $rtmp_ctr.16b @ CTR block 8k+16 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+16 + + aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 9 + eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high + + aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 9 + aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 9 + ldp $ctr_t0q, $ctr_t1q, [$input_ptr], #32 @ AES block 8k+8, 8k+9 - load plaintext + + pmull $t11.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + rev32 $h2.16b, $rtmp_ctr.16b @ CTR block 8k+17 + aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 9 + + aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 9 + aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 9 + aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 9 + + eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 9 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+17 + + aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 10 + aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 10 + ldr $rk12q, [$cc, #192] @ load rk12 + ext $t12.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + + aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 10 + aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 10 + ldp $ctr_t2q, $ctr_t3q, [$input_ptr], #32 @ AES block 8k+10, 8k+11 - load plaintext + + aese $ctr4b, $rk11 @ AES block 8k+12 - round 11 + eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid + ldp $ctr_t4q, $ctr_t5q, [$input_ptr], #32 @ AES block 8k+12, 8k+13 - load plaintext + + ldp $ctr_t6q, $ctr_t7q, [$input_ptr], #32 @ AES block 8k+14, 8k+15 - load plaintext + aese $ctr2b, $rk11 @ AES block 8k+10 - round 11 + aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 10 + + rev32 $h3.16b, $rtmp_ctr.16b @ CTR block 8k+18 + aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 10 + + aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 10 + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + + aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 10 + aese $ctr5b, $rk11 @ AES block 8k+13 - round 11 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+18 + + aese $ctr7b, $rk11 @ AES block 8k+15 - round 11 + aese $ctr0b, $rk11 @ AES block 8k+8 - round 11 + eor3 $res4b, $ctr_t4b, $ctr4b, $rk12 @ AES block 4 - result + + aese $ctr6b, $rk11 @ AES block 8k+14 - round 11 + aese $ctr3b, $rk11 @ AES block 8k+11 - round 11 + aese $ctr1b, $rk11 @ AES block 8k+9 - round 11 + + rev32 $h4.16b, $rtmp_ctr.16b @ CTR block 8k+19 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+19 + eor3 $res7b, $ctr_t7b, $ctr7b, $rk12 @ AES block 7 - result + + eor3 $res2b, $ctr_t2b, $ctr2b, $rk12 @ AES block 8k+10 - result + eor3 $res0b, $ctr_t0b, $ctr0b, $rk12 @ AES block 8k+8 - result + mov $ctr2.16b, $h3.16b @ CTR block 8k+18 + + eor3 $res1b, $ctr_t1b, $ctr1b, $rk12 @ AES block 8k+9 - result + mov $ctr1.16b, $h2.16b @ CTR block 8k+17 + stp $res0q, $res1q, [$output_ptr], #32 @ AES block 8k+8, 8k+9 - store result + ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + + eor3 $res6b, $ctr_t6b, $ctr6b, $rk12 @ AES block 6 - result + mov $ctr0.16b, $h1.16b @ CTR block 8k+16 + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 8k+20 + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+20 + eor3 $res5b, $ctr_t5b, $ctr5b, $rk12 @ AES block 5 - result + eor3 $acc_lb, $acc_lb, $acc_hb, $t11.16b @ MODULO - fold into low + + eor3 $res3b, $ctr_t3b, $ctr3b, $rk12 @ AES block 8k+11 - result + mov $ctr3.16b, $h4.16b @ CTR block 8k+19 + + stp $res2q, $res3q, [$output_ptr], #32 @ AES block 8k+10, 8k+11 - store result + + stp $res4q, $res5q, [$output_ptr], #32 @ AES block 8k+12, 8k+13 - store result + + cmp $input_ptr, $main_end_input_ptr @ LOOP CONTROL + stp $res6q, $res7q, [$output_ptr], #32 @ AES block 8k+14, 8k+15 - store result + b.lt .L192_enc_main_loop + +.L192_enc_prepretail: @ PREPRETAIL + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13 + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13 + + ldr $h7q, [$current_tag, #176] @ load h7l | h7h + ext $h7.16b, $h7.16b, $h7.16b, #8 + ldr $h8q, [$current_tag, #208] @ load h8l | h8h + ext $h8.16b, $h8.16b, $h8.16b, #8 + rev64 $res0b, $res0b @ GHASH block 8k + ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0 + + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14 + ldr $h56kq, [$current_tag, #144] @ load h6k | h5k + ldr $h78kq, [$current_tag, #192] @ load h8k | h7k + + rev64 $res3b, $res3b @ GHASH block 8k+3 + rev64 $res2b, $res2b @ GHASH block 8k+2 + ldr $h5q, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + ldr $h6q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + + eor $res0b, $res0b, $acc_lb @ PRE 1 + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15 + rev64 $res1b, $res1b @ GHASH block 8k+1 + + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0 + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0 + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0 + + pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0 + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0 + + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0 + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0 + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high + + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1 + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low + trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + + trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0 + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + + pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low + eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1 + + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1 + eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1 + + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1 + pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high + pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high + + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1 + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1 + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1 + + pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2 + eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low + + pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2 + eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high + + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3 + trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2 + + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2 + pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid + trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2 + rev64 $res5b, $res5b @ GHASH block 8k+5 (t0, t1, t2 and t3 free) + rev64 $res6b, $res6b @ GHASH block 8k+6 (t0, t1, and t2 free) + + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2 + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2 + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2 + + eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3 + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3 + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3 + + eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid + eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3 + + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3 + pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid + + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + ldr $h2q, [$current_tag, #64] @ load h2l | h2h + ext $h2.16b, $h2.16b, $h2.16b, #8 + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3 + rev64 $res4b, $res4b @ GHASH block 8k+4 (t0, t1, and t2 free) + + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3 + pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4 + + trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4 + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4 + + eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4 + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4 + + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4 + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4 + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4 + + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5 + rev64 $res7b, $res7b @ GHASH block 8k+7 (t0, t1, t2 and t3 free) + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5 + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5 + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + + pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high + pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high + pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low + + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5 + trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + + pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high + pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low + pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low + + trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5 + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6 + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5 + + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5 + eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5 + + pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid + pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid + + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6 + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6 + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7 + + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6 + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6 + eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6 + eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7 + + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6 + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6 + + pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7 + eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low + + pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high + pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid + pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low + + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7 + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7 + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7 + eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + + eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low + eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high + + eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + ext $t12.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7 + pmull $t11.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8 + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8 + + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7 + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8 + eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid + + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8 + aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 9 + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8 + + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8 + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8 + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8 + + aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 9 + ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11 + aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 9 + + aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 9 + aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 9 + + ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 9 + aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 9 + aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 9 + + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + ldr $rk12q, [$cc, #192] @ load rk12 + + aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 10 + aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 10 + aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 10 + + eor3 $acc_lb, $acc_lb, $acc_hb, $t11.16b @ MODULO - fold into low + aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 10 + aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 10 + + aese $ctr1b, $rk11 @ AES block 8k+9 - round 11 + aese $ctr7b, $rk11 @ AES block 8k+15 - round 11 + + aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 10 + aese $ctr3b, $rk11 @ AES block 8k+11 - round 11 + + aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 10 + aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 10 + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15 + aese $ctr2b, $rk11 @ AES block 8k+10 - round 11 + aese $ctr0b, $rk11 @ AES block 8k+8 - round 11 + + aese $ctr6b, $rk11 @ AES block 8k+14 - round 11 + aese $ctr4b, $rk11 @ AES block 8k+12 - round 11 + aese $ctr5b, $rk11 @ AES block 8k+13 - round 11 + +.L192_enc_tail: @ TAIL + + ldp $h5q, $h56kq, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + sub $main_end_input_ptr, $end_input_ptr, $input_ptr @ main_end_input_ptr is number of bytes left to process + + ldr $ctr_t0q, [$input_ptr], #16 @ AES block 8k+8 - l3ad plaintext + + ldp $h78kq, $h8q, [$current_tag, #192] @ load h8k | h7k + ext $h8.16b, $h8.16b, $h8.16b, #8 + + mov $t1.16b, $rk12 + + ldp $h6q, $h7q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + ext $h7.16b, $h7.16b, $h7.16b, #8 + cmp $main_end_input_ptr, #112 + + eor3 $res1b, $ctr_t0b, $ctr0b, $t1.16b @ AES block 8k+8 - result + ext $t0.16b, $acc_lb, $acc_lb, #8 @ prepare final partial tag + b.gt .L192_enc_blocks_more_than_7 + + cmp $main_end_input_ptr, #96 + mov $ctr7b, $ctr6b + movi $acc_h.8b, #0 + + mov $ctr6b, $ctr5b + movi $acc_l.8b, #0 + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + + mov $ctr5b, $ctr4b + mov $ctr4b, $ctr3b + mov $ctr3b, $ctr2b + + mov $ctr2b, $ctr1b + movi $acc_m.8b, #0 + b.gt .L192_enc_blocks_more_than_6 + + mov $ctr7b, $ctr6b + cmp $main_end_input_ptr, #80 + + mov $ctr6b, $ctr5b + mov $ctr5b, $ctr4b + mov $ctr4b, $ctr3b + + mov $ctr3b, $ctr1b + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + b.gt .L192_enc_blocks_more_than_5 + + cmp $main_end_input_ptr, #64 + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + + mov $ctr7b, $ctr6b + mov $ctr6b, $ctr5b + mov $ctr5b, $ctr4b + + mov $ctr4b, $ctr1b + b.gt .L192_enc_blocks_more_than_4 + + mov $ctr7b, $ctr6b + mov $ctr6b, $ctr5b + mov $ctr5b, $ctr1b + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + cmp $main_end_input_ptr, #48 + b.gt .L192_enc_blocks_more_than_3 + + mov $ctr7b, $ctr6b + mov $ctr6b, $ctr1b + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + cmp $main_end_input_ptr, #32 + b.gt .L192_enc_blocks_more_than_2 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + + cmp $main_end_input_ptr, #16 + mov $ctr7b, $ctr1b + b.gt .L192_enc_blocks_more_than_1 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + b .L192_enc_blocks_less_than_1 +.L192_enc_blocks_more_than_7: @ blocks left > 7 + st1 { $res1b}, [$output_ptr], #16 @ AES final-7 block - store result + + rev64 $res0b, $res1b @ GHASH final-7 block + ins $acc_m.d[0], $h78k.d[1] @ GHASH final-7 block - mid + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-7 block - mid + + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-6 block - load plaintext + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-7 block - mid + movi $t0.8b, #0 @ suppress further partial tag feed in + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH final-7 block - low + + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH final-7 block - high + + pmull $acc_m.1q, $rk4v.1d, $acc_m.1d @ GHASH final-7 block - mid + eor3 $res1b, $ctr_t1b, $ctr1b, $t1.16b @ AES final-6 block - result +.L192_enc_blocks_more_than_6: @ blocks left > 6 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-6 block - store result + + rev64 $res0b, $res1b @ GHASH final-6 block + + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-5 block - load plaintext + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-6 block - mid + + pmull $rk3q1, $res0.1d, $h7.1d @ GHASH final-6 block - low + eor3 $res1b, $ctr_t1b, $ctr2b, $t1.16b @ AES final-5 block - result + + movi $t0.8b, #0 @ suppress further partial tag feed in + pmull2 $rk2q1, $res0.2d, $h7.2d @ GHASH final-6 block - high + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-6 block - mid + + pmull $rk4v.1q, $rk4v.1d, $h78k.1d @ GHASH final-6 block - mid + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-6 block - high + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-6 block - low + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-6 block - mid +.L192_enc_blocks_more_than_5: @ blocks left > 5 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-5 block - store result + + rev64 $res0b, $res1b @ GHASH final-5 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-5 block - mid + + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-4 block - load plaintext + pmull2 $rk2q1, $res0.2d, $h6.2d @ GHASH final-5 block - high + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-5 block - mid + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-5 block - high + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-5 block - mid + pmull $rk3q1, $res0.1d, $h6.1d @ GHASH final-5 block - low + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-5 block - low + pmull2 $rk4v.1q, $rk4v.2d, $h56k.2d @ GHASH final-5 block - mid + + eor3 $res1b, $ctr_t1b, $ctr3b, $t1.16b @ AES final-4 block - result + movi $t0.8b, #0 @ suppress further partial tag feed in + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-5 block - mid +.L192_enc_blocks_more_than_4: @ blocks left > 4 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-4 block - store result + + rev64 $res0b, $res1b @ GHASH final-4 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-3 block - load plaintext + pmull2 $rk2q1, $res0.2d, $h5.2d @ GHASH final-4 block - high + ins $rk4v.d[0], $res0.d[1] @ GHASH final-4 block - mid + + pmull $rk3q1, $res0.1d, $h5.1d @ GHASH final-4 block - low + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-4 block - high + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-4 block - mid + + movi $t0.8b, #0 @ suppress further partial tag feed in + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-4 block - low + + pmull $rk4v.1q, $rk4v.1d, $h56k.1d @ GHASH final-4 block - mid + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-4 block - mid + eor3 $res1b, $ctr_t1b, $ctr4b, $t1.16b @ AES final-3 block - result +.L192_enc_blocks_more_than_3: @ blocks left > 3 + + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + st1 { $res1b}, [$output_ptr], #16 @ AES final-3 block - store result + + rev64 $res0b, $res1b @ GHASH final-3 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + movi $t0.8b, #0 @ suppress further partial tag feed in + + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-2 block - load plaintext + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-3 block - mid + + eor3 $res1b, $ctr_t1b, $ctr5b, $t1.16b @ AES final-2 block - result + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-3 block - mid + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-3 block - mid + pmull $rk3q1, $res0.1d, $h4.1d @ GHASH final-3 block - low + + pmull2 $rk2q1, $res0.2d, $h4.2d @ GHASH final-3 block - high + pmull2 $rk4v.1q, $rk4v.2d, $h34k.2d @ GHASH final-3 block - mid + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-3 block - low + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-3 block - mid + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-3 block - high +.L192_enc_blocks_more_than_2: @ blocks left > 2 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-2 block - store result + + rev64 $res0b, $res1b @ GHASH final-2 block + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-1 block - load plaintext + ins $rk4v.d[0], $res0.d[1] @ GHASH final-2 block - mid + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-2 block - mid + + pmull $rk3q1, $res0.1d, $h3.1d @ GHASH final-2 block - low + pmull2 $rk2q1, $res0.2d, $h3.2d @ GHASH final-2 block - high + movi $t0.8b, #0 @ suppress further partial tag feed in + + pmull $rk4v.1q, $rk4v.1d, $h34k.1d @ GHASH final-2 block - mid + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-2 block - low + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-2 block - high + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-2 block - mid + eor3 $res1b, $ctr_t1b, $ctr6b, $t1.16b @ AES final-1 block - result +.L192_enc_blocks_more_than_1: @ blocks left > 1 + + ldr $h2q, [$current_tag, #64] @ load h1l | h1h + ext $h2.16b, $h2.16b, $h2.16b, #8 + st1 { $res1b}, [$output_ptr], #16 @ AES final-1 block - store result + + rev64 $res0b, $res1b @ GHASH final-1 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-1 block - mid + pmull $rk3q1, $res0.1d, $h2.1d @ GHASH final-1 block - low + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-1 block - low + pmull2 $rk2q1, $res0.2d, $h2.2d @ GHASH final-1 block - high + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-1 block - mid + + ldr $ctr_t1q, [$input_ptr], #16 @ AES final block - load plaintext + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-1 block - mid + + eor3 $res1b, $ctr_t1b, $ctr7b, $t1.16b @ AES final block - result + pmull2 $rk4v.1q, $rk4v.2d, $h12k.2d @ GHASH final-1 block - mid + + movi $t0.8b, #0 @ suppress further partial tag feed in + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-1 block - mid + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-1 block - high +.L192_enc_blocks_less_than_1: @ blocks left <= 1 + + mvn $temp0_x, xzr @ temp0_x = 0xffffffffffffffff + and $bit_length, $bit_length, #127 @ bit_length %= 128 + + sub $bit_length, $bit_length, #128 @ bit_length -= 128 + + neg $bit_length, $bit_length @ bit_length = 128 - #bits in input (in range [1,128]) + + and $bit_length, $bit_length, #127 @ bit_length %= 128 + + lsr $temp0_x, $temp0_x, $bit_length @ temp0_x is mask for top 64b of last block + cmp $bit_length, #64 + mvn $temp1_x, xzr @ temp1_x = 0xffffffffffffffff + + csel $temp2_x, $temp1_x, $temp0_x, lt + csel $temp3_x, $temp0_x, xzr, lt + + mov $ctr0.d[1], $temp3_x + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + + ld1 { $rk0}, [$output_ptr] @ load existing bytes where the possibly partial last block is to be stored + mov $ctr0.d[0], $temp2_x @ ctr0b is mask for last block + + and $res1b, $res1b, $ctr0b @ possibly partial last block has zeroes in highest bits + + rev64 $res0b, $res1b @ GHASH final block + bif $res1b, $rk0, $ctr0b @ insert existing bytes in top end of result before storing + + st1 { $res1b}, [$output_ptr] @ store all 16B + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $t0.d[0], $res0.d[1] @ GHASH final block - mid + pmull2 $rk2q1, $res0.2d, $h1.2d @ GHASH final block - high + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final block - high + pmull $rk3q1, $res0.1d, $h1.1d @ GHASH final block - low + + eor $t0.8b, $t0.8b, $res0.8b @ GHASH final block - mid + + pmull $t0.1q, $t0.1d, $h12k.1d @ GHASH final block - mid + + eor $acc_mb, $acc_mb, $t0.16b @ GHASH final block - mid + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final block - low + ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + + rev32 $rtmp_ctr.16b, $rtmp_ctr.16b + + str $rtmp_ctrq, [$counter] @ store the updated counter + eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + + pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + + eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid + + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + + eor3 $acc_lb, $acc_lb, $acc_hb, $t11.16b @ MODULO - fold into low + ext $acc_lb, $acc_lb, $acc_lb, #8 + rev64 $acc_lb, $acc_lb + st1 { $acc_l.16b }, [$current_tag] + + mov x0, $byte_length @ return sizes + + ldp d10, d11, [sp, #16] + ldp d12, d13, [sp, #32] + ldp d14, d15, [sp, #48] + ldp d8, d9, [sp], #80 + ret + +.L192_enc_ret: + mov w0, #0x0 + ret +.size unroll8_eor3_aes_gcm_enc_192_kernel,.-unroll8_eor3_aes_gcm_enc_192_kernel +___ + +######################################################################################### +# size_t unroll8_eor3_aes_gcm_dec_192_kernel(const uint8_t * ciphertext, +# uint64_t plaintext_length, +# uint8_t * plaintext, +# uint64_t *Xi, +# unsigned char ivec[16], +# const void *key); +# +$code.=<<___; +.global unroll8_eor3_aes_gcm_dec_192_kernel +.type unroll8_eor3_aes_gcm_dec_192_kernel,%function +.align 4 +unroll8_eor3_aes_gcm_dec_192_kernel: + AARCH64_VALID_CALL_TARGET + cbz x1, .L192_dec_ret + stp d8, d9, [sp, #-80]! + lsr $byte_length, $bit_length, #3 + mov $counter, x4 + mov $cc, x5 + stp d10, d11, [sp, #16] + stp d12, d13, [sp, #32] + stp d14, d15, [sp, #48] + mov x5, #0xc200000000000000 + stp x5, xzr, [sp, #64] + add $modulo_constant, sp, #64 + + mov $main_end_input_ptr, $byte_length + ld1 { $ctr0b}, [$counter] @ CTR block 0 + ld1 { $acc_lb}, [$current_tag] + + mov $constant_temp, #0x100000000 @ set up counter increment + movi $rctr_inc.16b, #0x0 + mov $rctr_inc.d[1], $constant_temp + + rev32 $rtmp_ctr.16b, $ctr0.16b @ set up reversed counter + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 0 + + rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 1 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 1 + + rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 2 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 2 + + rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 3 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 3 + + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 4 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 4 + + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 5 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 5 + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 6 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 6 + + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 7 + + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 0 + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 0 + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 0 + + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 0 + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 0 + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 0 + + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 0 + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 0 + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 1 + + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 1 + + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 1 + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 1 + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 1 + + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 1 + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 1 + + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 2 + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 2 + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 1 + + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 2 + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 2 + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 2 + + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 2 + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 2 + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 2 + + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 3 + + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 3 + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 3 + + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 3 + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 3 + + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 3 + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 3 + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 3 + + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 4 + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 4 + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 4 + + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 4 + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 4 + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 4 + + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 4 + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 5 + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 4 + + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 5 + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 5 + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 5 + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 5 + + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 5 + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 5 + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 5 + + sub $main_end_input_ptr, $main_end_input_ptr, #1 @ byte_len - 1 + + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 6 + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 6 + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 6 + + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 6 + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 6 + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 6 + + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 6 + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 6 + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 7 + + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 7 + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 7 + + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 7 + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 7 + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 7 + + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 7 + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 7 + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 7 + + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 8 + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 8 + and $main_end_input_ptr, $main_end_input_ptr, #0xffffffffffffff80 @ number of bytes to be processed in main loop (at least 1 byte must be handled by tail) + + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 8 + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 8 + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 8 + + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 8 + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 8 + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 8 + + add $end_input_ptr, $input_ptr, $bit_length, lsr #3 @ end_input_ptr + aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 9 + + ld1 { $acc_lb}, [$current_tag] + ext $acc_lb, $acc_lb, $acc_lb, #8 + rev64 $acc_lb, $acc_lb + + ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11 + + aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 9 + add $main_end_input_ptr, $main_end_input_ptr, $input_ptr + + aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 9 + aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 9 + aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 9 + + cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks + aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 9 + + aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 9 + aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 9 + + aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 10 + aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 10 + aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 10 + + aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 10 + aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 10 + aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 10 + + aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 10 + aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 10 + ldr $rk12q, [$cc, #192] @ load rk12 + + aese $ctr0b, $rk11 @ AES block 0 - round 11 + aese $ctr1b, $rk11 @ AES block 1 - round 11 + aese $ctr4b, $rk11 @ AES block 4 - round 11 + + aese $ctr6b, $rk11 @ AES block 6 - round 11 + aese $ctr5b, $rk11 @ AES block 5 - round 11 + aese $ctr7b, $rk11 @ AES block 7 - round 11 + + aese $ctr2b, $rk11 @ AES block 2 - round 11 + aese $ctr3b, $rk11 @ AES block 3 - round 11 + b.ge .L192_dec_tail @ handle tail + + ldp $res0q, $res1q, [$input_ptr], #32 @ AES block 0, 1 - load ciphertext + + ldp $res2q, $res3q, [$input_ptr], #32 @ AES block 2, 3 - load ciphertext + + ldp $res4q, $res5q, [$input_ptr], #32 @ AES block 4, 5 - load ciphertext + + eor3 $ctr1b, $res1b, $ctr1b, $rk12 @ AES block 1 - result + eor3 $ctr0b, $res0b, $ctr0b, $rk12 @ AES block 0 - result + stp $ctr0q, $ctr1q, [$output_ptr], #32 @ AES block 0, 1 - store result + + rev32 $ctr0.16b, $rtmp_ctr.16b @ CTR block 8 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8 + + rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 9 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 9 + eor3 $ctr3b, $res3b, $ctr3b, $rk12 @ AES block 3 - result + + eor3 $ctr2b, $res2b, $ctr2b, $rk12 @ AES block 2 - result + stp $ctr2q, $ctr3q, [$output_ptr], #32 @ AES block 2, 3 - store result + ldp $res6q, $res7q, [$input_ptr], #32 @ AES block 6, 7 - load ciphertext + + rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 10 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 10 + + eor3 $ctr4b, $res4b, $ctr4b, $rk12 @ AES block 4 - result + + rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 11 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 11 + + eor3 $ctr5b, $res5b, $ctr5b, $rk12 @ AES block 5 - result + stp $ctr4q, $ctr5q, [$output_ptr], #32 @ AES block 4, 5 - store result + cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks + + eor3 $ctr6b, $res6b, $ctr6b, $rk12 @ AES block 6 - result + eor3 $ctr7b, $res7b, $ctr7b, $rk12 @ AES block 7 - result + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 12 + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 12 + stp $ctr6q, $ctr7q, [$output_ptr], #32 @ AES block 6, 7 - store result + b.ge .L192_dec_prepretail @ do prepretail + +.L192_dec_main_loop: @ main loop start + rev64 $res1b, $res1b @ GHASH block 8k+1 + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0 + + rev64 $res0b, $res0b @ GHASH block 8k + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13 + + ldr $h7q, [$current_tag, #176] @ load h7l | h7h + ext $h7.16b, $h7.16b, $h7.16b, #8 + ldr $h8q, [$current_tag, #208] @ load h8l | h8h + ext $h8.16b, $h8.16b, $h8.16b, #8 + rev64 $res4b, $res4b @ GHASH block 8k+4 + rev64 $res3b, $res3b @ GHASH block 8k+3 + + eor $res0b, $res0b, $acc_lb @ PRE 1 + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14 + + rev64 $res5b, $res5b @ GHASH block 8k+5 + + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15 + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0 + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0 + + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0 + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0 + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0 + + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0 + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0 + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0 + + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low + pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1 + pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low + ldr $h5q, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + ldr $h6q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1 + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1 + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1 + + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1 + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1 + + trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + rev64 $res2b, $res2b @ GHASH block 8k+2 + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1 + + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1 + ldr $h56kq, [$current_tag, #144] @ load h6k | h5k + ldr $h78kq, [$current_tag, #192] @ load h8k | h7k + trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + + eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high + pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high + pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high + + eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid + eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2 + + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2 + pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low + eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high + + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2 + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3 + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2 + + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2 + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2 + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2 + + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2 + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3 + + pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low + trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3 + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3 + + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3 + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3 + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + + eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3 + + trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15 + + pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid + pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid + pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid + + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3 + pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid + pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high + + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4 + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4 + eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid + + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4 + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4 + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4 + + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4 + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4 + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4 + + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + ldr $h2q, [$current_tag, #64] @ load h2l | h2h + ext $h2.16b, $h2.16b, $h2.16b, #8 + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5 + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5 + + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5 + rev64 $res7b, $res7b @ GHASH block 8k+7 + + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5 + eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5 + + pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low + trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5 + + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5 + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5 + rev64 $res6b, $res6b @ GHASH block 8k+6 + + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high + pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low + + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6 + eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6 + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6 + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6 + + pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6 + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6 + + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7 + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7 + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6 + + pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid + eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high + eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low + + pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low + trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6 + + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7 + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7 + + eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7 + + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7 + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7 + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7 + + eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid + pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high + + pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low + + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8 + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8 + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8 + + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8 + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8 + eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low + + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8 + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8 + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8 + + eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high + rev32 $h1.16b, $rtmp_ctr.16b @ CTR block 8k+16 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+16 + + aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 9 + eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 9 + + aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 9 + aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 9 + ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11 + + eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + ldp $res0q, $res1q, [$input_ptr], #32 @ AES block 8k+8, 8k+9 - load ciphertext + + aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 9 + aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 9 + ldp $res2q, $res3q, [$input_ptr], #32 @ AES block 8k+10, 8k+11 - load ciphertext + + rev32 $h2.16b, $rtmp_ctr.16b @ CTR block 8k+17 + pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+17 + + aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 9 + aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 9 + ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + + aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 10 + aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 10 + ldp $res4q, $res5q, [$input_ptr], #32 @ AES block 8k+12, 8k+13 - load ciphertext + + rev32 $h3.16b, $rtmp_ctr.16b @ CTR block 8k+18 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+18 + eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid + + aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 10 + aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 10 + ldr $rk12q, [$cc, #192] @ load rk12 + + ldp $res6q, $res7q, [$input_ptr], #32 @ AES block 8k+14, 8k+15 - load ciphertext + aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 10 + aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 10 + + aese $ctr0b, $rk11 @ AES block 8k+8 - round 11 + ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + aese $ctr1b, $rk11 @ AES block 8k+9 - round 11 + + aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 10 + aese $ctr6b, $rk11 @ AES block 8k+14 - round 11 + aese $ctr3b, $rk11 @ AES block 8k+11 - round 11 + + eor3 $ctr0b, $res0b, $ctr0b, $rk12 @ AES block 8k+8 - result + rev32 $h4.16b, $rtmp_ctr.16b @ CTR block 8k+19 + aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 10 + + aese $ctr4b, $rk11 @ AES block 8k+12 - round 11 + aese $ctr2b, $rk11 @ AES block 8k+10 - round 11 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+19 + + aese $ctr7b, $rk11 @ AES block 8k+15 - round 11 + aese $ctr5b, $rk11 @ AES block 8k+13 - round 11 + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + + eor3 $ctr1b, $res1b, $ctr1b, $rk12 @ AES block 8k+9 - result + stp $ctr0q, $ctr1q, [$output_ptr], #32 @ AES block 8k+8, 8k+9 - store result + eor3 $ctr3b, $res3b, $ctr3b, $rk12 @ AES block 8k+11 - result + + eor3 $ctr2b, $res2b, $ctr2b, $rk12 @ AES block 8k+10 - result + eor3 $ctr7b, $res7b, $ctr7b, $rk12 @ AES block 8k+15 - result + stp $ctr2q, $ctr3q, [$output_ptr], #32 @ AES block 8k+10, 8k+11 - store result + + eor3 $ctr5b, $res5b, $ctr5b, $rk12 @ AES block 8k+13 - result + eor3 $acc_lb, $acc_lb, $acc_hb, $t11.16b @ MODULO - fold into low + mov $ctr3.16b, $h4.16b @ CTR block 8k+19 + + eor3 $ctr4b, $res4b, $ctr4b, $rk12 @ AES block 8k+12 - result + stp $ctr4q, $ctr5q, [$output_ptr], #32 @ AES block 8k+12, 8k+13 - store result + cmp $input_ptr, $main_end_input_ptr @ LOOP CONTROL + + eor3 $ctr6b, $res6b, $ctr6b, $rk12 @ AES block 8k+14 - result + stp $ctr6q, $ctr7q, [$output_ptr], #32 @ AES block 8k+14, 8k+15 - store result + mov $ctr0.16b, $h1.16b @ CTR block 8k+16 + + mov $ctr1.16b, $h2.16b @ CTR block 8k+17 + mov $ctr2.16b, $h3.16b @ CTR block 8k+18 + + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 8k+20 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+20 + b.lt .L192_dec_main_loop + +.L192_dec_prepretail: @ PREPRETAIL + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13 + + ldr $h7q, [$current_tag, #176] @ load h7l | h7h + ext $h7.16b, $h7.16b, $h7.16b, #8 + ldr $h8q, [$current_tag, #208] @ load h8l | h8h + ext $h8.16b, $h8.16b, $h8.16b, #8 + rev64 $res0b, $res0b @ GHASH block 8k + ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0 + + rev64 $res3b, $res3b @ GHASH block 8k+3 + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14 + + eor $res0b, $res0b, $acc_lb @ PRE 1 + rev64 $res2b, $res2b @ GHASH block 8k+2 + rev64 $res1b, $res1b @ GHASH block 8k+1 + + ldr $h5q, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + ldr $h6q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15 + + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0 + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0 + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0 + + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0 + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0 + pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high + + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0 + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0 + + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1 + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0 + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1 + pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high + pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low + + pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low + eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1 + + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1 + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1 + + trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high + + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1 + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1 + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1 + + ldr $h56kq, [$current_tag, #144] @ load h6k | h5k + ldr $h78kq, [$current_tag, #192] @ load h8k | h7k + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2 + eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid + + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2 + rev64 $res5b, $res5b @ GHASH block 8k+5 + pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low + + eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2 + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2 + + trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3 + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2 + + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2 + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2 + trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + + pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2 + pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid + + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3 + eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low + + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3 + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3 + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3 + + eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3 + + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid + pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid + + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + ldr $h2q, [$current_tag, #64] @ load h2l | h2h + ext $h2.16b, $h2.16b, $h2.16b, #8 + eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3 + + rev64 $res7b, $res7b @ GHASH block 8k+7 + + eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + rev64 $res4b, $res4b @ GHASH block 8k+4 + + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4 + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4 + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3 + + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4 + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4 + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4 + + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4 + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4 + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4 + + rev64 $res6b, $res6b @ GHASH block 8k+6 + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5 + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5 + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5 + + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5 + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5 + + pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high + pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high + pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low + + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5 + + pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low + trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high + + pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low + trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5 + + trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5 + eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6 + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6 + + eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6 + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6 + + pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid + pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6 + + pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6 + pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high + + eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7 + eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low + + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6 + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6 + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7 + + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7 + + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high + pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low + + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7 + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7 + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7 + + eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high + eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low + eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7 + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7 + + eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8 + + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8 + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8 + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8 + + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8 + pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8 + + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8 + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8 + ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11 + + eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid + aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 9 + aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 9 + + aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 9 + aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 9 + aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 9 + + aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 9 + aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 9 + aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 9 + + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + ldr $rk12q, [$cc, #192] @ load rk12 + ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + + aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 10 + aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 10 + aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 10 + + aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 10 + aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 10 + aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 10 + + aese $ctr0b, $rk11 @ AES block 8k+8 - round 11 + eor3 $acc_lb, $acc_lb, $acc_hb, $t11.16b @ MODULO - fold into low + aese $ctr5b, $rk11 @ AES block 8k+13 - round 11 + + aese $ctr2b, $rk11 @ AES block 8k+10 - round 11 + aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 10 + aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 10 + + aese $ctr6b, $rk11 @ AES block 8k+14 - round 11 + aese $ctr4b, $rk11 @ AES block 8k+12 - round 11 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15 + + aese $ctr3b, $rk11 @ AES block 8k+11 - round 11 + aese $ctr1b, $rk11 @ AES block 8k+9 - round 11 + aese $ctr7b, $rk11 @ AES block 8k+15 - round 11 + +.L192_dec_tail: @ TAIL + + sub $main_end_input_ptr, $end_input_ptr, $input_ptr @ main_end_input_ptr is number of bytes left to process + + ldp $h5q, $h56kq, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + ldr $res1q, [$input_ptr], #16 @ AES block 8k+8 - load ciphertext + + ldp $h78kq, $h8q, [$current_tag, #192] @ load h8k | h7k + ext $h8.16b, $h8.16b, $h8.16b, #8 + + mov $t1.16b, $rk12 + + ldp $h6q, $h7q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + ext $h7.16b, $h7.16b, $h7.16b, #8 + ext $t0.16b, $acc_lb, $acc_lb, #8 @ prepare final partial tag + + eor3 $res4b, $res1b, $ctr0b, $t1.16b @ AES block 8k+8 - result + cmp $main_end_input_ptr, #112 + b.gt .L192_dec_blocks_more_than_7 + + mov $ctr7b, $ctr6b + movi $acc_h.8b, #0 + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + + mov $ctr6b, $ctr5b + mov $ctr5b, $ctr4b + mov $ctr4b, $ctr3b + + cmp $main_end_input_ptr, #96 + movi $acc_l.8b, #0 + mov $ctr3b, $ctr2b + + mov $ctr2b, $ctr1b + movi $acc_m.8b, #0 + b.gt .L192_dec_blocks_more_than_6 + + mov $ctr7b, $ctr6b + mov $ctr6b, $ctr5b + mov $ctr5b, $ctr4b + + mov $ctr4b, $ctr3b + mov $ctr3b, $ctr1b + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + cmp $main_end_input_ptr, #80 + b.gt .L192_dec_blocks_more_than_5 + + mov $ctr7b, $ctr6b + mov $ctr6b, $ctr5b + + mov $ctr5b, $ctr4b + mov $ctr4b, $ctr1b + cmp $main_end_input_ptr, #64 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + b.gt .L192_dec_blocks_more_than_4 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + mov $ctr7b, $ctr6b + mov $ctr6b, $ctr5b + + mov $ctr5b, $ctr1b + cmp $main_end_input_ptr, #48 + b.gt .L192_dec_blocks_more_than_3 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + mov $ctr7b, $ctr6b + cmp $main_end_input_ptr, #32 + + mov $ctr6b, $ctr1b + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + b.gt .L192_dec_blocks_more_than_2 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + + mov $ctr7b, $ctr1b + cmp $main_end_input_ptr, #16 + b.gt .L192_dec_blocks_more_than_1 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + b .L192_dec_blocks_less_than_1 +.L192_dec_blocks_more_than_7: @ blocks left > 7 + rev64 $res0b, $res1b @ GHASH final-7 block + + ins $acc_m.d[0], $h78k.d[1] @ GHASH final-7 block - mid + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH final-7 block - high + ins $rk4v.d[0], $res0.d[1] @ GHASH final-7 block - mid + ldr $res1q, [$input_ptr], #16 @ AES final-6 block - load ciphertext + + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH final-7 block - low + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-7 block - mid + st1 { $res4b}, [$output_ptr], #16 @ AES final-7 block - store result + + eor3 $res4b, $res1b, $ctr1b, $t1.16b @ AES final-6 block - result + + pmull $acc_m.1q, $rk4v.1d, $acc_m.1d @ GHASH final-7 block - mid + movi $t0.8b, #0 @ suppress further partial tag feed in +.L192_dec_blocks_more_than_6: @ blocks left > 6 + + rev64 $res0b, $res1b @ GHASH final-6 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ldr $res1q, [$input_ptr], #16 @ AES final-5 block - load ciphertext + ins $rk4v.d[0], $res0.d[1] @ GHASH final-6 block - mid + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-6 block - mid + movi $t0.8b, #0 @ suppress further partial tag feed in + pmull2 $rk2q1, $res0.2d, $h7.2d @ GHASH final-6 block - high + + st1 { $res4b}, [$output_ptr], #16 @ AES final-6 block - store result + eor3 $res4b, $res1b, $ctr2b, $t1.16b @ AES final-5 block - result + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-6 block - high + pmull $rk4v.1q, $rk4v.1d, $h78k.1d @ GHASH final-6 block - mid + pmull $rk3q1, $res0.1d, $h7.1d @ GHASH final-6 block - low + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-6 block - mid + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-6 block - low +.L192_dec_blocks_more_than_5: @ blocks left > 5 + + rev64 $res0b, $res1b @ GHASH final-5 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-5 block - mid + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-5 block - mid + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-5 block - mid + pmull2 $rk2q1, $res0.2d, $h6.2d @ GHASH final-5 block - high + + ldr $res1q, [$input_ptr], #16 @ AES final-4 block - load ciphertext + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-5 block - high + pmull $rk3q1, $res0.1d, $h6.1d @ GHASH final-5 block - low + + pmull2 $rk4v.1q, $rk4v.2d, $h56k.2d @ GHASH final-5 block - mid + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-5 block - low + movi $t0.8b, #0 @ suppress further partial tag feed in + st1 { $res4b}, [$output_ptr], #16 @ AES final-5 block - store result + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-5 block - mid + eor3 $res4b, $res1b, $ctr3b, $t1.16b @ AES final-4 block - result +.L192_dec_blocks_more_than_4: @ blocks left > 4 + + rev64 $res0b, $res1b @ GHASH final-4 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + movi $t0.8b, #0 @ suppress further partial tag feed in + + ldr $res1q, [$input_ptr], #16 @ AES final-3 block - load ciphertext + ins $rk4v.d[0], $res0.d[1] @ GHASH final-4 block - mid + pmull $rk3q1, $res0.1d, $h5.1d @ GHASH final-4 block - low + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-4 block - mid + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-4 block - low + + pmull $rk4v.1q, $rk4v.1d, $h56k.1d @ GHASH final-4 block - mid + st1 { $res4b}, [$output_ptr], #16 @ AES final-4 block - store result + pmull2 $rk2q1, $res0.2d, $h5.2d @ GHASH final-4 block - high + + eor3 $res4b, $res1b, $ctr4b, $t1.16b @ AES final-3 block - result + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-4 block - mid + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-4 block - high +.L192_dec_blocks_more_than_3: @ blocks left > 3 + + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + rev64 $res0b, $res1b @ GHASH final-3 block + ldr $res1q, [$input_ptr], #16 @ AES final-2 block - load ciphertext + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-3 block - mid + pmull2 $rk2q1, $res0.2d, $h4.2d @ GHASH final-3 block - high + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-3 block - high + movi $t0.8b, #0 @ suppress further partial tag feed in + pmull $rk3q1, $res0.1d, $h4.1d @ GHASH final-3 block - low + + st1 { $res4b}, [$output_ptr], #16 @ AES final-3 block - store result + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-3 block - mid + eor3 $res4b, $res1b, $ctr5b, $t1.16b @ AES final-2 block - result + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-3 block - low + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-3 block - mid + + pmull2 $rk4v.1q, $rk4v.2d, $h34k.2d @ GHASH final-3 block - mid + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-3 block - mid +.L192_dec_blocks_more_than_2: @ blocks left > 2 + + rev64 $res0b, $res1b @ GHASH final-2 block + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-2 block - mid + ldr $res1q, [$input_ptr], #16 @ AES final-1 block - load ciphertext + + pmull2 $rk2q1, $res0.2d, $h3.2d @ GHASH final-2 block - high + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-2 block - mid + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-2 block - high + pmull $rk3q1, $res0.1d, $h3.1d @ GHASH final-2 block - low + + pmull $rk4v.1q, $rk4v.1d, $h34k.1d @ GHASH final-2 block - mid + movi $t0.8b, #0 @ suppress further partial tag feed in + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-2 block - low + st1 { $res4b}, [$output_ptr], #16 @ AES final-2 block - store result + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-2 block - mid + eor3 $res4b, $res1b, $ctr6b, $t1.16b @ AES final-1 block - result +.L192_dec_blocks_more_than_1: @ blocks left > 1 + + rev64 $res0b, $res1b @ GHASH final-1 block + ldr $res1q, [$input_ptr], #16 @ AES final block - load ciphertext + ldr $h2q, [$current_tag, #64] @ load h1l | h1h + ext $h2.16b, $h2.16b, $h2.16b, #8 + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + movi $t0.8b, #0 @ suppress further partial tag feed in + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + + pmull $rk3q1, $res0.1d, $h2.1d @ GHASH final-1 block - low + ins $rk4v.d[0], $res0.d[1] @ GHASH final-1 block - mid + st1 { $res4b}, [$output_ptr], #16 @ AES final-1 block - store result + + pmull2 $rk2q1, $res0.2d, $h2.2d @ GHASH final-1 block - high + + eor3 $res4b, $res1b, $ctr7b, $t1.16b @ AES final block - result + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-1 block - mid + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-1 block - mid + + pmull2 $rk4v.1q, $rk4v.2d, $h12k.2d @ GHASH final-1 block - mid + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-1 block - low + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-1 block - mid + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-1 block - high +.L192_dec_blocks_less_than_1: @ blocks left <= 1 + + rev32 $rtmp_ctr.16b, $rtmp_ctr.16b + and $bit_length, $bit_length, #127 @ bit_length %= 128 + + sub $bit_length, $bit_length, #128 @ bit_length -= 128 + str $rtmp_ctrq, [$counter] @ store the updated counter + + neg $bit_length, $bit_length @ bit_length = 128 - #bits in input (in range [1,128]) + mvn $temp0_x, xzr @ temp0_x = 0xffffffffffffffff + + and $bit_length, $bit_length, #127 @ bit_length %= 128 + + mvn $temp1_x, xzr @ temp1_x = 0xffffffffffffffff + lsr $temp0_x, $temp0_x, $bit_length @ temp0_x is mask for top 64b of last block + cmp $bit_length, #64 + + csel $temp2_x, $temp1_x, $temp0_x, lt + csel $temp3_x, $temp0_x, xzr, lt + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + + mov $ctr0.d[1], $temp3_x + ld1 { $rk0}, [$output_ptr] @ load existing bytes where the possibly partial last block is to be stored + + mov $ctr0.d[0], $temp2_x @ ctr0b is mask for last block + + and $res1b, $res1b, $ctr0b @ possibly partial last block has zeroes in highest bits + bif $res4b, $rk0, $ctr0b @ insert existing bytes in top end of result before storing + + rev64 $res0b, $res1b @ GHASH final block + + st1 { $res4b}, [$output_ptr] @ store all 16B + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $t0.d[0], $res0.d[1] @ GHASH final block - mid + pmull $rk3q1, $res0.1d, $h1.1d @ GHASH final block - low + + eor $t0.8b, $t0.8b, $res0.8b @ GHASH final block - mid + pmull2 $rk2q1, $res0.2d, $h1.2d @ GHASH final block - high + eor $acc_lb, $acc_lb, $rk3 @ GHASH final block - low + + pmull $t0.1q, $t0.1d, $h12k.1d @ GHASH final block - mid + eor $acc_hb, $acc_hb, $rk2 @ GHASH final block - high + + eor $t10.16b, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + eor $acc_mb, $acc_mb, $t0.16b @ GHASH final block - mid + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + + pmull $t11.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + ext $acc_hb, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + + eor $acc_mb, $acc_mb, $t10.16b @ MODULO - karatsuba tidy up + + eor3 $acc_mb, $acc_mb, $acc_hb, $t11.16b @ MODULO - fold into mid + + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + ext $acc_mb, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + + eor3 $acc_lb, $acc_lb, $acc_mb, $acc_hb @ MODULO - fold into low + ext $acc_lb, $acc_lb, $acc_lb, #8 + rev64 $acc_lb, $acc_lb + st1 { $acc_l.16b }, [$current_tag] + + mov x0, $byte_length + + ldp d10, d11, [sp, #16] + ldp d12, d13, [sp, #32] + ldp d14, d15, [sp, #48] + ldp d8, d9, [sp], #80 + ret + +.L192_dec_ret: + mov w0, #0x0 + ret +.size unroll8_eor3_aes_gcm_dec_192_kernel,.-unroll8_eor3_aes_gcm_dec_192_kernel +___ +} + +{ + +my ($end_input_ptr,$main_end_input_ptr,$temp0_x,$temp1_x)=map("x$_",(4..7)); +my ($temp2_x,$temp3_x)=map("x$_",(13..14)); +my ($ctr0b,$ctr1b,$ctr2b,$ctr3b,$ctr4b,$ctr5b,$ctr6b,$ctr7b,$res0b,$res1b,$res2b,$res3b,$res4b,$res5b,$res6b,$res7b)=map("v$_.16b",(0..15)); +my ($ctr0,$ctr1,$ctr2,$ctr3,$ctr4,$ctr5,$ctr6,$ctr7,$res0,$res1,$res2,$res3,$res4,$res5,$res6,$res7)=map("v$_",(0..15)); +my ($ctr0d,$ctr1d,$ctr2d,$ctr3d,$ctr4d,$ctr5d,$ctr6d,$ctr7d)=map("d$_",(0..7)); +my ($ctr0q,$ctr1q,$ctr2q,$ctr3q,$ctr4q,$ctr5q,$ctr6q,$ctr7q)=map("q$_",(0..7)); +my ($res0q,$res1q,$res2q,$res3q,$res4q,$res5q,$res6q,$res7q)=map("q$_",(8..15)); + +my ($ctr_t0,$ctr_t1,$ctr_t2,$ctr_t3,$ctr_t4,$ctr_t5,$ctr_t6,$ctr_t7)=map("v$_",(8..15)); +my ($ctr_t0b,$ctr_t1b,$ctr_t2b,$ctr_t3b,$ctr_t4b,$ctr_t5b,$ctr_t6b,$ctr_t7b)=map("v$_.16b",(8..15)); +my ($ctr_t0q,$ctr_t1q,$ctr_t2q,$ctr_t3q,$ctr_t4q,$ctr_t5q,$ctr_t6q,$ctr_t7q)=map("q$_",(8..15)); + +my ($acc_hb,$acc_mb,$acc_lb)=map("v$_.16b",(17..19)); +my ($acc_h,$acc_m,$acc_l)=map("v$_",(17..19)); + +my ($h1,$h12k,$h2,$h3,$h34k,$h4)=map("v$_",(20..25)); +my ($h5,$h56k,$h6,$h7,$h78k,$h8)=map("v$_",(20..25)); +my ($h1q,$h12kq,$h2q,$h3q,$h34kq,$h4q)=map("q$_",(20..25)); +my ($h5q,$h56kq,$h6q,$h7q,$h78kq,$h8q)=map("q$_",(20..25)); + +my $t0="v16"; +my $t0d="d16"; + +my $t1="v29"; +my $t2=$res1; +my $t3=$t1; + +my $t4=$res0; +my $t5=$res2; +my $t6=$t0; + +my $t7=$res3; +my $t8=$res4; +my $t9=$res5; + +my $t10=$res6; +my $t11="v21"; +my $t12=$t1; + +my $rtmp_ctr="v30"; +my $rtmp_ctrq="q30"; +my $rctr_inc="v31"; +my $rctr_incd="d31"; + +my $mod_constantd=$t0d; +my $mod_constant=$t0; + +my ($rk0,$rk1,$rk2)=map("v$_.16b",(26..28)); +my ($rk3,$rk4,$rk5)=map("v$_.16b",(26..28)); +my ($rk6,$rk7,$rk8)=map("v$_.16b",(26..28)); +my ($rk9,$rk10,$rk11)=map("v$_.16b",(26..28)); +my ($rk12,$rk13,$rk14)=map("v$_.16b",(26..28)); +my ($rk0q,$rk1q,$rk2q)=map("q$_",(26..28)); +my ($rk3q,$rk4q,$rk5q)=map("q$_",(26..28)); +my ($rk6q,$rk7q,$rk8q)=map("q$_",(26..28)); +my ($rk9q,$rk10q,$rk11q)=map("q$_",(26..28)); +my ($rk12q,$rk13q,$rk14q)=map("q$_",(26..28)); +my $rk2q1="v28.1q"; +my $rk3q1="v26.1q"; +my $rk4v="v27"; +######################################################################################### +# size_t unroll8_eor3_aes_gcm_enc_256_kernel(const uint8_t * plaintext, +# uint64_t plaintext_length, +# uint8_t * ciphertext, +# uint64_t *Xi, +# unsigned char ivec[16], +# const void *key); +# +$code.=<<___; +.global unroll8_eor3_aes_gcm_enc_256_kernel +.type unroll8_eor3_aes_gcm_enc_256_kernel,%function +.align 4 +unroll8_eor3_aes_gcm_enc_256_kernel: + AARCH64_VALID_CALL_TARGET + cbz x1, .L256_enc_ret + stp d8, d9, [sp, #-80]! + lsr $byte_length, $bit_length, #3 + mov $counter, x4 + mov $cc, x5 + stp d10, d11, [sp, #16] + stp d12, d13, [sp, #32] + stp d14, d15, [sp, #48] + mov x5, #0xc200000000000000 + stp x5, xzr, [sp, #64] + add $modulo_constant, sp, #64 + + ld1 { $ctr0b}, [$counter] @ CTR block 0 + + mov $main_end_input_ptr, $byte_length + + mov $constant_temp, #0x100000000 @ set up counter increment + movi $rctr_inc.16b, #0x0 + mov $rctr_inc.d[1], $constant_temp + sub $main_end_input_ptr, $main_end_input_ptr, #1 @ byte_len - 1 + + and $main_end_input_ptr, $main_end_input_ptr, #0xffffffffffffff80 @ number of bytes to be processed in main loop (at least 1 byte must be handled by tail) + + add $main_end_input_ptr, $main_end_input_ptr, $input_ptr + + rev32 $rtmp_ctr.16b, $ctr0.16b @ set up reversed counter + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 0 + + rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 1 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 1 + + rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 2 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 2 + + rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 3 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 3 + + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 4 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 4 + + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 5 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 5 + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 6 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 6 + + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 7 + + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 0 + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 0 + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 0 + + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 0 + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 0 + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 0 + + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 0 + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 0 + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 1 + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 1 + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 1 + + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 1 + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 1 + + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 1 + + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 1 + + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 2 + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 2 + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 1 + + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 2 + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 2 + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 2 + + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 2 + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 2 + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 2 + + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 3 + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 3 + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 3 + + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 3 + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 3 + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 3 + + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 3 + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 3 + + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 4 + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 4 + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 4 + + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 4 + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 4 + + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 4 + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 4 + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 4 + + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 5 + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 5 + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 5 + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 5 + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 5 + + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 5 + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 5 + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 5 + + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 6 + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 6 + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 6 + + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 6 + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 6 + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 6 + + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 6 + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 6 + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 7 + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 7 + + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 7 + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 7 + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 7 + + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 7 + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 7 + + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 7 + + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 8 + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 8 + + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 8 + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 8 + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 8 + + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 8 + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 8 + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 8 + + ld1 { $acc_lb}, [$current_tag] + ext $acc_lb, $acc_lb, $acc_lb, #8 + rev64 $acc_lb, $acc_lb + ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11 + + aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 9 + aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 9 + aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 9 + + aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 9 + aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 9 + aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 9 + + aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 9 + + aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 10 + aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 10 + aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 9 + + aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 10 + aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 10 + aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 10 + + aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 10 + aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 10 + aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 10 + + aese $ctr4b, $rk11 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 11 + ldp $rk12q, $rk13q, [$cc, #192] @ load rk12, rk13 + aese $ctr5b, $rk11 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 11 + + aese $ctr2b, $rk11 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 11 + aese $ctr6b, $rk11 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 11 + aese $ctr1b, $rk11 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 11 + + aese $ctr0b, $rk11 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 11 + aese $ctr3b, $rk11 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 11 + aese $ctr7b, $rk11 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 11 + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 7 + ldr $rk14q, [$cc, #224] @ load rk14 + + aese $ctr4b, $rk12 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 12 + aese $ctr2b, $rk12 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 12 + aese $ctr1b, $rk12 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 12 + + aese $ctr0b, $rk12 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 12 + aese $ctr5b, $rk12 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 12 + aese $ctr3b, $rk12 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 12 + + aese $ctr2b, $rk13 @ AES block 2 - round 13 + aese $ctr1b, $rk13 @ AES block 1 - round 13 + aese $ctr4b, $rk13 @ AES block 4 - round 13 + + aese $ctr6b, $rk12 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 12 + aese $ctr7b, $rk12 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 12 + + aese $ctr0b, $rk13 @ AES block 0 - round 13 + aese $ctr5b, $rk13 @ AES block 5 - round 13 + + aese $ctr6b, $rk13 @ AES block 6 - round 13 + aese $ctr7b, $rk13 @ AES block 7 - round 13 + aese $ctr3b, $rk13 @ AES block 3 - round 13 + + add $end_input_ptr, $input_ptr, $bit_length, lsr #3 @ end_input_ptr + cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks + b.ge .L256_enc_tail @ handle tail + + ldp $ctr_t0q, $ctr_t1q, [$input_ptr], #32 @ AES block 0, 1 - load plaintext + + ldp $ctr_t2q, $ctr_t3q, [$input_ptr], #32 @ AES block 2, 3 - load plaintext + + eor3 $res0b, $ctr_t0b, $ctr0b, $rk14 @ AES block 0 - result + rev32 $ctr0.16b, $rtmp_ctr.16b @ CTR block 8 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8 + + eor3 $res1b, $ctr_t1b, $ctr1b, $rk14 @ AES block 1 - result + eor3 $res3b, $ctr_t3b, $ctr3b, $rk14 @ AES block 3 - result + + rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 9 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 9 + ldp $ctr_t4q, $ctr_t5q, [$input_ptr], #32 @ AES block 4, 5 - load plaintext + + ldp $ctr_t6q, $ctr_t7q, [$input_ptr], #32 @ AES block 6, 7 - load plaintext + eor3 $res2b, $ctr_t2b, $ctr2b, $rk14 @ AES block 2 - result + cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks + + rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 10 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 10 + stp $res0q, $res1q, [$output_ptr], #32 @ AES block 0, 1 - store result + + stp $res2q, $res3q, [$output_ptr], #32 @ AES block 2, 3 - store result + + rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 11 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 11 + + eor3 $res4b, $ctr_t4b, $ctr4b, $rk14 @ AES block 4 - result + + eor3 $res7b, $ctr_t7b, $ctr7b, $rk14 @ AES block 7 - result + eor3 $res6b, $ctr_t6b, $ctr6b, $rk14 @ AES block 6 - result + eor3 $res5b, $ctr_t5b, $ctr5b, $rk14 @ AES block 5 - result + + stp $res4q, $res5q, [$output_ptr], #32 @ AES block 4, 5 - store result + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 12 + + stp $res6q, $res7q, [$output_ptr], #32 @ AES block 6, 7 - store result + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 12 + b.ge .L256_enc_prepretail @ do prepretail + +.L256_enc_main_loop: @ main loop start + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13 + ldr $h56kq, [$current_tag, #144] @ load h6k | h5k + ldr $h78kq, [$current_tag, #192] @ load h8k | h7k + + rev64 $res3b, $res3b @ GHASH block 8k+3 + ldr $h5q, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + ldr $h6q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + rev64 $res1b, $res1b @ GHASH block 8k+1 + + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14 + rev64 $res0b, $res0b @ GHASH block 8k + + rev64 $res4b, $res4b @ GHASH block 8k+4 + ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0 + ldr $h7q, [$current_tag, #176] @ load h7l | h7h + ext $h7.16b, $h7.16b, $h7.16b, #8 + ldr $h8q, [$current_tag, #208] @ load h8l | h8h + ext $h8.16b, $h8.16b, $h8.16b, #8 + + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0 + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0 + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15 + + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0 + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0 + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0 + + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0 + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0 + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0 + + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + eor $res0b, $res0b, $acc_lb @ PRE 1 + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1 + + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1 + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1 + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1 + + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1 + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1 + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1 + + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low + pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high + + trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1 + + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2 + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2 + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2 + + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2 + pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2 + + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3 + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3 + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2 + + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3 + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2 + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2 + + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3 + rev64 $res6b, $res6b @ GHASH block 8k+6 + pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high + + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3 + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + rev64 $res2b, $res2b @ GHASH block 8k+2 + + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3 + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3 + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3 + + eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high + pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high + rev64 $res5b, $res5b @ GHASH block 8k+5 + + pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low + eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + + trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high + pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low + + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4 + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4 + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4 + + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4 + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4 + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4 + + trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4 + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4 + + trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5 + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5 + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5 + + eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5 + rev64 $res7b, $res7b @ GHASH block 8k+7 + + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5 + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5 + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5 + + pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid + pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5 + + pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6 + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6 + + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6 + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6 + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6 + + eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid + pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6 + + eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6 + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6 + + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7 + + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + ldr $h2q, [$current_tag, #64] @ load h2l | h2h + ext $h2.16b, $h2.16b, $h2.16b, #8 + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7 + eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7 + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7 + + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7 + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7 + pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low + + trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7 + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7 + + pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8 + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8 + + pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low + trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8 + aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 9 + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8 + + pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid + pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8 + + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8 + pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high + pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low + + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8 + trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8 + + eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 9 + aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 9 + + eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 9 + aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 9 + + ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11 + aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 9 + aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 9 + + pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high + eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low + pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low + + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid + pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid + + aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 9 + + eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low + eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high + + aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 10 + aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 10 + aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 10 + + aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 10 + aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 10 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15 + + aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 10 + aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 10 + aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 10 + + eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high + + ldp $rk12q, $rk13q, [$cc, #192] @ load rk12, rk13 + rev32 $h1.16b, $rtmp_ctr.16b @ CTR block 8k+16 + + ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + ldp $ctr_t0q, $ctr_t1q, [$input_ptr], #32 @ AES block 8k+8, 8k+9 - load plaintext + aese $ctr2b, $rk11 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 11 + + aese $ctr6b, $rk11 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 11 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+16 + aese $ctr3b, $rk11 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 11 + + aese $ctr0b, $rk11 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 11 + aese $ctr7b, $rk11 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 11 + + pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + aese $ctr1b, $rk11 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 11 + + aese $ctr7b, $rk12 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 12 + aese $ctr5b, $rk11 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 11 + + aese $ctr3b, $rk12 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 12 + aese $ctr6b, $rk12 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 12 + rev32 $h2.16b, $rtmp_ctr.16b @ CTR block 8k+17 + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+17 + aese $ctr4b, $rk11 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 11 + eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + + aese $ctr5b, $rk12 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 12 + ldr $rk14q, [$cc, #224] @ load rk14 + aese $ctr7b, $rk13 @ AES block 8k+15 - round 13 + + ldp $ctr_t2q, $ctr_t3q, [$input_ptr], #32 @ AES block 8k+10, 8k+11 - load plaintext + aese $ctr2b, $rk12 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 12 + aese $ctr4b, $rk12 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 12 + + eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid + aese $ctr1b, $rk12 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 12 + ldp $ctr_t4q, $ctr_t5q, [$input_ptr], #32 @ AES block 4, 5 - load plaintext + + ldp $ctr_t6q, $ctr_t7q, [$input_ptr], #32 @ AES block 6, 7 - load plaintext + aese $ctr2b, $rk13 @ AES block 8k+10 - round 13 + aese $ctr4b, $rk13 @ AES block 8k+12 - round 13 + + rev32 $h3.16b, $rtmp_ctr.16b @ CTR block 8k+18 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+18 + aese $ctr5b, $rk13 @ AES block 8k+13 - round 13 + + aese $ctr0b, $rk12 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 12 + aese $ctr3b, $rk13 @ AES block 8k+11 - round 13 + cmp $input_ptr, $main_end_input_ptr @ LOOP CONTROL + + eor3 $res2b, $ctr_t2b, $ctr2b, $rk14 @ AES block 8k+10 - result + rev32 $h4.16b, $rtmp_ctr.16b @ CTR block 8k+19 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+19 + + aese $ctr0b, $rk13 @ AES block 8k+8 - round 13 + aese $ctr6b, $rk13 @ AES block 8k+14 - round 13 + eor3 $res5b, $ctr_t5b, $ctr5b, $rk14 @ AES block 5 - result + + ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + aese $ctr1b, $rk13 @ AES block 8k+9 - round 13 + + eor3 $res4b, $ctr_t4b, $ctr4b, $rk14 @ AES block 4 - result + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 8k+20 + eor3 $res3b, $ctr_t3b, $ctr3b, $rk14 @ AES block 8k+11 - result + + mov $ctr3.16b, $h4.16b @ CTR block 8k+19 + eor3 $res1b, $ctr_t1b, $ctr1b, $rk14 @ AES block 8k+9 - result + eor3 $res0b, $ctr_t0b, $ctr0b, $rk14 @ AES block 8k+8 - result + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+20 + stp $res0q, $res1q, [$output_ptr], #32 @ AES block 8k+8, 8k+9 - store result + mov $ctr2.16b, $h3.16b @ CTR block 8k+18 + + eor3 $res7b, $ctr_t7b, $ctr7b, $rk14 @ AES block 7 - result + eor3 $acc_lb, $acc_lb, $t11.16b, $acc_hb @ MODULO - fold into low + stp $res2q, $res3q, [$output_ptr], #32 @ AES block 8k+10, 8k+11 - store result + + eor3 $res6b, $ctr_t6b, $ctr6b, $rk14 @ AES block 6 - result + mov $ctr1.16b, $h2.16b @ CTR block 8k+17 + stp $res4q, $res5q, [$output_ptr], #32 @ AES block 4, 5 - store result + + stp $res6q, $res7q, [$output_ptr], #32 @ AES block 6, 7 - store result + mov $ctr0.16b, $h1.16b @ CTR block 8k+16 + b.lt .L256_enc_main_loop + +.L256_enc_prepretail: @ PREPRETAIL + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13 + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13 + + rev64 $res2b, $res2b @ GHASH block 8k+2 + + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14 + + rev64 $res5b, $res5b @ GHASH block 8k+5 + ldr $h56kq, [$current_tag, #144] @ load h6k | h5k + ldr $h78kq, [$current_tag, #192] @ load h8k | h7k + + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15 + + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0 + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0 + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0 + + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0 + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0 + + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0 + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0 + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0 + + ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0 + rev64 $res0b, $res0b @ GHASH block 8k + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1 + + rev64 $res1b, $res1b @ GHASH block 8k+1 + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1 + + ldr $h7q, [$current_tag, #176] @ load h7l | h7h + ext $h7.16b, $h7.16b, $h7.16b, #8 + ldr $h8q, [$current_tag, #208] @ load h8l | h8h + ext $h8.16b, $h8.16b, $h8.16b, #8 + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1 + + ldr $h5q, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + ldr $h6q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1 + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1 + + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1 + eor $res0b, $res0b, $acc_lb @ PRE 1 + + rev64 $res3b, $res3b @ GHASH block 8k+3 + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1 + + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2 + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2 + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1 + + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2 + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2 + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2 + + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2 + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2 + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2 + + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high + + rev64 $res6b, $res6b @ GHASH block 8k+6 + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3 + pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high + + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3 + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low + trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + + pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3 + + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3 + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3 + eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high + + pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low + pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3 + + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3 + eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3 + + pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4 + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4 + + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4 + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4 + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4 + + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5 + pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid + eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high + + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4 + trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4 + eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4 + + pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low + pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid + eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + + rev64 $res4b, $res4b @ GHASH block 8k+4 + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5 + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5 + + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5 + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5 + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid + pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid + + eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low + eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid + + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5 + rev64 $res7b, $res7b @ GHASH block 8k+7 + trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5 + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5 + eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6 + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6 + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6 + + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6 + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6 + + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6 + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6 + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6 + + pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high + pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + ldr $h2q, [$current_tag, #64] @ load h2l | h2h + ext $h2.16b, $h2.16b, $h2.16b, #8 + + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7 + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7 + + pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high + trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7 + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7 + pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low + + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7 + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7 + eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + + pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high + pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7 + + trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7 + + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8 + eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8 + + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8 + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8 + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8 + + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8 + eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8 + + pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid + pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8 + + pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high + pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid + pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid + + pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low + eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high + + ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11 + aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 9 + aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 9 + + eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high + eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + + eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low + + aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 9 + aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 9 + aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 9 + + aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 9 + aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 9 + + aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 10 + aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 10 + aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 9 + + aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 10 + aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 10 + aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 10 + + aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 10 + aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 10 + aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 10 + + pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + aese $ctr7b, $rk11 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 11 + + ldp $rk12q, $rk13q, [$cc, #192] @ load rk12, rk13 + ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + aese $ctr2b, $rk11 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 11 + + eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid + aese $ctr1b, $rk11 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 11 + aese $ctr6b, $rk11 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 11 + + aese $ctr0b, $rk11 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 11 + aese $ctr4b, $rk11 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 11 + aese $ctr5b, $rk11 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 11 + + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + aese $ctr3b, $rk11 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 11 + ldr $rk14q, [$cc, #224] @ load rk14 + + aese $ctr1b, $rk12 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 12 + aese $ctr2b, $rk12 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 12 + aese $ctr0b, $rk12 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 12 + + aese $ctr6b, $rk12 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 12 + aese $ctr5b, $rk12 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 12 + ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + + aese $ctr4b, $rk12 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 12 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15 + + aese $ctr3b, $rk12 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 12 + aese $ctr7b, $rk12 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 12 + aese $ctr0b, $rk13 @ AES block 8k+8 - round 13 + + eor3 $acc_lb, $acc_lb, $t11.16b, $acc_hb @ MODULO - fold into low + aese $ctr5b, $rk13 @ AES block 8k+13 - round 13 + aese $ctr1b, $rk13 @ AES block 8k+9 - round 13 + + aese $ctr3b, $rk13 @ AES block 8k+11 - round 13 + aese $ctr4b, $rk13 @ AES block 8k+12 - round 13 + aese $ctr7b, $rk13 @ AES block 8k+15 - round 13 + + aese $ctr2b, $rk13 @ AES block 8k+10 - round 13 + aese $ctr6b, $rk13 @ AES block 8k+14 - round 13 +.L256_enc_tail: @ TAIL + + ldp $h78kq, $h8q, [$current_tag, #192] @ load h8l | h8h + ext $h8.16b, $h8.16b, $h8.16b, #8 + sub $main_end_input_ptr, $end_input_ptr, $input_ptr @ main_end_input_ptr is number of bytes left to process + + ldr $ctr_t0q, [$input_ptr], #16 @ AES block 8k+8 - load plaintext + + ldp $h5q, $h56kq, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + + ext $t0.16b, $acc_lb, $acc_lb, #8 @ prepare final partial tag + ldp $h6q, $h7q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + ext $h7.16b, $h7.16b, $h7.16b, #8 + mov $t1.16b, $rk14 + + cmp $main_end_input_ptr, #112 + eor3 $res1b, $ctr_t0b, $ctr0b, $t1.16b @ AES block 8k+8 - result + b.gt .L256_enc_blocks_more_than_7 + + movi $acc_l.8b, #0 + mov $ctr7b, $ctr6b + movi $acc_h.8b, #0 + + mov $ctr6b, $ctr5b + mov $ctr5b, $ctr4b + mov $ctr4b, $ctr3b + + mov $ctr3b, $ctr2b + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + mov $ctr2b, $ctr1b + + movi $acc_m.8b, #0 + cmp $main_end_input_ptr, #96 + b.gt .L256_enc_blocks_more_than_6 + + mov $ctr7b, $ctr6b + mov $ctr6b, $ctr5b + cmp $main_end_input_ptr, #80 + + mov $ctr5b, $ctr4b + mov $ctr4b, $ctr3b + mov $ctr3b, $ctr1b + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + b.gt .L256_enc_blocks_more_than_5 + + mov $ctr7b, $ctr6b + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + + mov $ctr6b, $ctr5b + mov $ctr5b, $ctr4b + + cmp $main_end_input_ptr, #64 + mov $ctr4b, $ctr1b + b.gt .L256_enc_blocks_more_than_4 + + cmp $main_end_input_ptr, #48 + mov $ctr7b, $ctr6b + mov $ctr6b, $ctr5b + + mov $ctr5b, $ctr1b + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + b.gt .L256_enc_blocks_more_than_3 + + cmp $main_end_input_ptr, #32 + mov $ctr7b, $ctr6b + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + + mov $ctr6b, $ctr1b + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + b.gt .L256_enc_blocks_more_than_2 + + mov $ctr7b, $ctr1b + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + cmp $main_end_input_ptr, #16 + b.gt .L256_enc_blocks_more_than_1 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + b .L256_enc_blocks_less_than_1 +.L256_enc_blocks_more_than_7: @ blocks left > 7 + st1 { $res1b}, [$output_ptr], #16 @ AES final-7 block - store result + + rev64 $res0b, $res1b @ GHASH final-7 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-6 block - load plaintext + + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH final-7 block - high + ins $rk4v.d[0], $res0.d[1] @ GHASH final-7 block - mid + ins $acc_m.d[0], $h78k.d[1] @ GHASH final-7 block - mid + + movi $t0.8b, #0 @ suppress further partial tag feed in + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-7 block - mid + eor3 $res1b, $ctr_t1b, $ctr1b, $t1.16b @ AES final-6 block - result + + pmull $acc_m.1q, $rk4v.1d, $acc_m.1d @ GHASH final-7 block - mid + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH final-7 block - low +.L256_enc_blocks_more_than_6: @ blocks left > 6 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-6 block - store result + + rev64 $res0b, $res1b @ GHASH final-6 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + pmull $rk3q1, $res0.1d, $h7.1d @ GHASH final-6 block - low + ins $rk4v.d[0], $res0.d[1] @ GHASH final-6 block - mid + pmull2 $rk2q1, $res0.2d, $h7.2d @ GHASH final-6 block - high + + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-5 block - load plaintext + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-6 block - low + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-6 block - mid + + pmull $rk4v.1q, $rk4v.1d, $h78k.1d @ GHASH final-6 block - mid + eor3 $res1b, $ctr_t1b, $ctr2b, $t1.16b @ AES final-5 block - result + + movi $t0.8b, #0 @ suppress further partial tag feed in + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-6 block - mid + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-6 block - high +.L256_enc_blocks_more_than_5: @ blocks left > 5 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-5 block - store result + + rev64 $res0b, $res1b @ GHASH final-5 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-5 block - mid + + pmull2 $rk2q1, $res0.2d, $h6.2d @ GHASH final-5 block - high + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-5 block - high + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-5 block - mid + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-5 block - mid + + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-4 block - load plaintext + pmull $rk3q1, $res0.1d, $h6.1d @ GHASH final-5 block - low + + pmull2 $rk4v.1q, $rk4v.2d, $h56k.2d @ GHASH final-5 block - mid + movi $t0.8b, #0 @ suppress further partial tag feed in + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-5 block - low + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-5 block - mid + eor3 $res1b, $ctr_t1b, $ctr3b, $t1.16b @ AES final-4 block - result +.L256_enc_blocks_more_than_4: @ blocks left > 4 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-4 block - store result + + rev64 $res0b, $res1b @ GHASH final-4 block + + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-3 block - load plaintext + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-4 block - mid + pmull2 $rk2q1, $res0.2d, $h5.2d @ GHASH final-4 block - high + + eor3 $res1b, $ctr_t1b, $ctr4b, $t1.16b @ AES final-3 block - result + pmull $rk3q1, $res0.1d, $h5.1d @ GHASH final-4 block - low + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-4 block - mid + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-4 block - low + + pmull $rk4v.1q, $rk4v.1d, $h56k.1d @ GHASH final-4 block - mid + + movi $t0.8b, #0 @ suppress further partial tag feed in + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-4 block - mid + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-4 block - high +.L256_enc_blocks_more_than_3: @ blocks left > 3 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-3 block - store result + + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + rev64 $res0b, $res1b @ GHASH final-3 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-3 block - mid + pmull2 $rk2q1, $res0.2d, $h4.2d @ GHASH final-3 block - high + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-3 block - high + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-3 block - mid + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-3 block - mid + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-2 block - load plaintext + + pmull2 $rk4v.1q, $rk4v.2d, $h34k.2d @ GHASH final-3 block - mid + pmull $rk3q1, $res0.1d, $h4.1d @ GHASH final-3 block - low + + eor3 $res1b, $ctr_t1b, $ctr5b, $t1.16b @ AES final-2 block - result + movi $t0.8b, #0 @ suppress further partial tag feed in + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-3 block - mid + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-3 block - low +.L256_enc_blocks_more_than_2: @ blocks left > 2 + + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-2 block - store result + + rev64 $res0b, $res1b @ GHASH final-2 block + ldr $ctr_t1q, [$input_ptr], #16 @ AES final-1 block - load plaintext + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-2 block - mid + + movi $t0.8b, #0 @ suppress further partial tag feed in + + pmull2 $rk2q1, $res0.2d, $h3.2d @ GHASH final-2 block - high + eor3 $res1b, $ctr_t1b, $ctr6b, $t1.16b @ AES final-1 block - result + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-2 block - mid + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-2 block - high + + pmull $rk4v.1q, $rk4v.1d, $h34k.1d @ GHASH final-2 block - mid + pmull $rk3q1, $res0.1d, $h3.1d @ GHASH final-2 block - low + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-2 block - mid + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-2 block - low +.L256_enc_blocks_more_than_1: @ blocks left > 1 + + st1 { $res1b}, [$output_ptr], #16 @ AES final-1 block - store result + + ldr $h2q, [$current_tag, #64] @ load h2l | h2h + ext $h2.16b, $h2.16b, $h2.16b, #8 + rev64 $res0b, $res1b @ GHASH final-1 block + ldr $ctr_t1q, [$input_ptr], #16 @ AES final block - load plaintext + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + movi $t0.8b, #0 @ suppress further partial tag feed in + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-1 block - mid + pmull2 $rk2q1, $res0.2d, $h2.2d @ GHASH final-1 block - high + + eor3 $res1b, $ctr_t1b, $ctr7b, $t1.16b @ AES final block - result + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-1 block - high + + pmull $rk3q1, $res0.1d, $h2.1d @ GHASH final-1 block - low + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-1 block - mid + + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-1 block - low + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-1 block - mid + + pmull2 $rk4v.1q, $rk4v.2d, $h12k.2d @ GHASH final-1 block - mid + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-1 block - mid +.L256_enc_blocks_less_than_1: @ blocks left <= 1 + + and $bit_length, $bit_length, #127 @ bit_length %= 128 + + sub $bit_length, $bit_length, #128 @ bit_length -= 128 + + neg $bit_length, $bit_length @ bit_length = 128 - #bits in input (in range [1,128]) + + mvn $temp0_x, xzr @ temp0_x = 0xffffffffffffffff + and $bit_length, $bit_length, #127 @ bit_length %= 128 + + lsr $temp0_x, $temp0_x, $bit_length @ temp0_x is mask for top 64b of last block + cmp $bit_length, #64 + mvn $temp1_x, xzr @ temp1_x = 0xffffffffffffffff + + csel $temp3_x, $temp0_x, xzr, lt + csel $temp2_x, $temp1_x, $temp0_x, lt + + mov $ctr0.d[0], $temp2_x @ ctr0b is mask for last block + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + + ld1 { $rk0}, [$output_ptr] @ load existing bytes where the possibly partial last block is to be stored + mov $ctr0.d[1], $temp3_x + + and $res1b, $res1b, $ctr0b @ possibly partial last block has zeroes in highest bits + + rev64 $res0b, $res1b @ GHASH final block + + rev32 $rtmp_ctr.16b, $rtmp_ctr.16b + bif $res1b, $rk0, $ctr0b @ insert existing bytes in top end of result before storing + str $rtmp_ctrq, [$counter] @ store the updated counter + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + st1 { $res1b}, [$output_ptr] @ store all 16B + + ins $t0.d[0], $res0.d[1] @ GHASH final block - mid + pmull2 $rk2q1, $res0.2d, $h1.2d @ GHASH final block - high + pmull $rk3q1, $res0.1d, $h1.1d @ GHASH final block - low + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final block - high + eor $acc_lb, $acc_lb, $rk3 @ GHASH final block - low + + eor $t0.8b, $t0.8b, $res0.8b @ GHASH final block - mid + + pmull $t0.1q, $t0.1d, $h12k.1d @ GHASH final block - mid + + eor $acc_mb, $acc_mb, $t0.16b @ GHASH final block - mid + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + + ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + + eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + + eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid + + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + + eor3 $acc_lb, $acc_lb, $acc_hb, $t11.16b @ MODULO - fold into low + ext $acc_lb, $acc_lb, $acc_lb, #8 + rev64 $acc_lb, $acc_lb + st1 { $acc_l.16b }, [$current_tag] + mov x0, $byte_length @ return sizes + + ldp d10, d11, [sp, #16] + ldp d12, d13, [sp, #32] + ldp d14, d15, [sp, #48] + ldp d8, d9, [sp], #80 + ret + +.L256_enc_ret: + mov w0, #0x0 + ret +.size unroll8_eor3_aes_gcm_enc_256_kernel,.-unroll8_eor3_aes_gcm_enc_256_kernel +___ + +{ +######################################################################################### +# size_t unroll8_eor3_aes_gcm_dec_256_kernel(const uint8_t * ciphertext, +# uint64_t plaintext_length, +# uint8_t * plaintext, +# uint64_t *Xi, +# unsigned char ivec[16], +# const void *key); +# +$code.=<<___; +.global unroll8_eor3_aes_gcm_dec_256_kernel +.type unroll8_eor3_aes_gcm_dec_256_kernel,%function +.align 4 +unroll8_eor3_aes_gcm_dec_256_kernel: + AARCH64_VALID_CALL_TARGET + cbz x1, .L256_dec_ret + stp d8, d9, [sp, #-80]! + lsr $byte_length, $bit_length, #3 + mov $counter, x4 + mov $cc, x5 + stp d10, d11, [sp, #16] + stp d12, d13, [sp, #32] + stp d14, d15, [sp, #48] + mov x5, #0xc200000000000000 + stp x5, xzr, [sp, #64] + add $modulo_constant, sp, #64 + + ld1 { $ctr0b}, [$counter] @ CTR block 0 + + mov $constant_temp, #0x100000000 @ set up counter increment + movi $rctr_inc.16b, #0x0 + mov $rctr_inc.d[1], $constant_temp + mov $main_end_input_ptr, $byte_length + + sub $main_end_input_ptr, $main_end_input_ptr, #1 @ byte_len - 1 + + rev32 $rtmp_ctr.16b, $ctr0.16b @ set up reversed counter + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 0 + + rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 1 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 1 + + rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 2 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 2 + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + + rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 3 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 3 + + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 4 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 4 + + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 0 + + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 5 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 5 + + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 0 + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 0 + + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 6 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 6 + + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 7 + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 0 + + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 0 + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 0 + + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 0 + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 0 + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 1 + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 1 + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 1 + + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 1 + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 1 + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 1 + + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 1 + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 1 + + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 2 + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 2 + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 2 + + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 2 + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 2 + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 2 + + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 2 + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 2 + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 3 + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 3 + + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 3 + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 3 + + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 3 + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 3 + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 3 + + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 3 + + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 4 + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 4 + + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 4 + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 4 + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 4 + + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 4 + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 4 + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 4 + + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 5 + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 5 + + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 5 + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 5 + + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 5 + + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 5 + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 5 + + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 5 + + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 6 + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 6 + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 6 + + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 6 + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 6 + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 6 + + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 6 + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 6 + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 7 + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 7 + + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 7 + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 7 + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 7 + + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 7 + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 7 + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 7 + + and $main_end_input_ptr, $main_end_input_ptr, #0xffffffffffffff80 @ number of bytes to be processed in main loop (at least 1 byte must be handled by tail) + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 8 + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 8 + + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 8 + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 8 + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 8 + + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 8 + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 8 + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 8 + + aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 9 + + ld1 { $acc_lb}, [$current_tag] + ext $acc_lb, $acc_lb, $acc_lb, #8 + rev64 $acc_lb, $acc_lb + ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11 + add $end_input_ptr, $input_ptr, $bit_length, lsr #3 @ end_input_ptr + add $main_end_input_ptr, $main_end_input_ptr, $input_ptr + + aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 9 + aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 9 + + aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 9 + aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 9 + + aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 9 + + aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 9 + aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 9 + + aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 10 + aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 10 + aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 10 + + aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 10 + aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 10 + aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 10 + + aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 10 + aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 10 + ldp $rk12q, $rk13q, [$cc, #192] @ load rk12, rk13 + + aese $ctr0b, $rk11 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 11 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 7 + + aese $ctr7b, $rk11 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 11 + aese $ctr3b, $rk11 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 11 + aese $ctr1b, $rk11 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 11 + + aese $ctr5b, $rk11 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 11 + aese $ctr4b, $rk11 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 11 + aese $ctr2b, $rk11 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 11 + + aese $ctr6b, $rk11 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 11 + ldr $rk14q, [$cc, #224] @ load rk14 + + aese $ctr1b, $rk12 \n aesmc $ctr1b, $ctr1b @ AES block 1 - round 12 + aese $ctr4b, $rk12 \n aesmc $ctr4b, $ctr4b @ AES block 4 - round 12 + aese $ctr5b, $rk12 \n aesmc $ctr5b, $ctr5b @ AES block 5 - round 12 + + cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks + aese $ctr3b, $rk12 \n aesmc $ctr3b, $ctr3b @ AES block 3 - round 12 + aese $ctr2b, $rk12 \n aesmc $ctr2b, $ctr2b @ AES block 2 - round 12 + + aese $ctr6b, $rk12 \n aesmc $ctr6b, $ctr6b @ AES block 6 - round 12 + aese $ctr0b, $rk12 \n aesmc $ctr0b, $ctr0b @ AES block 0 - round 12 + aese $ctr7b, $rk12 \n aesmc $ctr7b, $ctr7b @ AES block 7 - round 12 + + aese $ctr5b, $rk13 @ AES block 5 - round 13 + aese $ctr1b, $rk13 @ AES block 1 - round 13 + aese $ctr2b, $rk13 @ AES block 2 - round 13 + + aese $ctr0b, $rk13 @ AES block 0 - round 13 + aese $ctr4b, $rk13 @ AES block 4 - round 13 + aese $ctr6b, $rk13 @ AES block 6 - round 13 + + aese $ctr3b, $rk13 @ AES block 3 - round 13 + aese $ctr7b, $rk13 @ AES block 7 - round 13 + b.ge .L256_dec_tail @ handle tail + + ldp $res0q, $res1q, [$input_ptr], #32 @ AES block 0, 1 - load ciphertext + + ldp $res2q, $res3q, [$input_ptr], #32 @ AES block 2, 3 - load ciphertext + + ldp $res4q, $res5q, [$input_ptr], #32 @ AES block 4, 5 - load ciphertext + + ldp $res6q, $res7q, [$input_ptr], #32 @ AES block 6, 7 - load ciphertext + cmp $input_ptr, $main_end_input_ptr @ check if we have <= 8 blocks + + eor3 $ctr1b, $res1b, $ctr1b, $rk14 @ AES block 1 - result + eor3 $ctr0b, $res0b, $ctr0b, $rk14 @ AES block 0 - result + stp $ctr0q, $ctr1q, [$output_ptr], #32 @ AES block 0, 1 - store result + + rev32 $ctr0.16b, $rtmp_ctr.16b @ CTR block 8 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8 + eor3 $ctr3b, $res3b, $ctr3b, $rk14 @ AES block 3 - result + + eor3 $ctr5b, $res5b, $ctr5b, $rk14 @ AES block 5 - result + + eor3 $ctr4b, $res4b, $ctr4b, $rk14 @ AES block 4 - result + rev32 $ctr1.16b, $rtmp_ctr.16b @ CTR block 9 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 9 + + eor3 $ctr2b, $res2b, $ctr2b, $rk14 @ AES block 2 - result + stp $ctr2q, $ctr3q, [$output_ptr], #32 @ AES block 2, 3 - store result + + rev32 $ctr2.16b, $rtmp_ctr.16b @ CTR block 10 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 10 + + eor3 $ctr6b, $res6b, $ctr6b, $rk14 @ AES block 6 - result + + rev32 $ctr3.16b, $rtmp_ctr.16b @ CTR block 11 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 11 + stp $ctr4q, $ctr5q, [$output_ptr], #32 @ AES block 4, 5 - store result + + eor3 $ctr7b, $res7b, $ctr7b, $rk14 @ AES block 7 - result + stp $ctr6q, $ctr7q, [$output_ptr], #32 @ AES block 6, 7 - store result + + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 12 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 12 + b.ge .L256_dec_prepretail @ do prepretail + +.L256_dec_main_loop: @ main loop start + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13 + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13 + + rev64 $res1b, $res1b @ GHASH block 8k+1 + ldr $h7q, [$current_tag, #176] @ load h7l | h7h + ext $h7.16b, $h7.16b, $h7.16b, #8 + ldr $h8q, [$current_tag, #208] @ load h8l | h8h + ext $h8.16b, $h8.16b, $h8.16b, #8 + + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14 + rev64 $res0b, $res0b @ GHASH block 8k + + ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0 + rev64 $res4b, $res4b @ GHASH block 8k+4 + rev64 $res3b, $res3b @ GHASH block 8k+3 + + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15 + rev64 $res7b, $res7b @ GHASH block 8k+7 + + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0 + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0 + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0 + + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0 + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0 + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0 + + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0 + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0 + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + + eor $res0b, $res0b, $acc_lb @ PRE 1 + ldr $h5q, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + ldr $h6q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1 + + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1 + rev64 $res2b, $res2b @ GHASH block 8k+2 + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1 + + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1 + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1 + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1 + + trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1 + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1 + + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2 + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2 + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2 + + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2 + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2 + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low + + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2 + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2 + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2 + + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3 + + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3 + pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high + pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low + + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3 + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3 + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high + + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3 + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3 + trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + + pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3 + eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high + + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4 + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3 + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4 + + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4 + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4 + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4 + + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4 + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4 + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4 + + ldr $h56kq, [$current_tag, #144] @ load h6k | h5k + ldr $h78kq, [$current_tag, #192] @ load h8k | h7k + eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid + pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low + + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5 + eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low + + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5 + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5 + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5 + + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5 + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5 + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5 + + eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high + trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + rev64 $res5b, $res5b @ GHASH block 8k+5 + + pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid + pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid + trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6 + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6 + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5 + + trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6 + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6 + + eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6 + + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6 + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6 + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6 + + pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid + pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid + eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low + + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + rev64 $res6b, $res6b @ GHASH block 8k+6 + eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid + + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7 + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7 + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + ldr $h2q, [$current_tag, #64] @ load h2l | h2h + ext $h2.16b, $h2.16b, $h2.16b, #8 + eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7 + + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7 + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7 + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7 + + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7 + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7 + + pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high + pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low + trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8 + pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8 + + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8 + pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8 + + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8 + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8 + pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high + + trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8 + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8 + + ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11 + pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low + trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15 + eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high + aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 9 + + aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 9 + eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 9 + + ldp $res0q, $res1q, [$input_ptr], #32 @ AES block 8k+8, 8k+9 - load ciphertext + eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 9 + + pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid + aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 9 + aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 9 + + pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid + pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid + pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high + + pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low + aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 10 + aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 10 + + pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid + aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 9 + eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low + + aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 9 + eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high + + aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 10 + aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 10 + aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 10 + + aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 10 + aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 10 + aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 10 + + eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low + rev32 $h1.16b, $rtmp_ctr.16b @ CTR block 8k+16 + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+16 + aese $ctr1b, $rk11 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 11 + ldp $rk12q, $rk13q, [$cc, #192] @ load rk12, rk13 + + aese $ctr0b, $rk11 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 11 + aese $ctr6b, $rk11 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 11 + + eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + rev32 $h2.16b, $rtmp_ctr.16b @ CTR block 8k+17 + aese $ctr2b, $rk11 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 11 + + ldp $res2q, $res3q, [$input_ptr], #32 @ AES block 8k+10, 8k+11 - load ciphertext + aese $ctr7b, $rk11 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 11 + ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + + aese $ctr5b, $rk11 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 11 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+17 + aese $ctr3b, $rk11 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 11 + + aese $ctr2b, $rk12 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 12 + aese $ctr7b, $rk12 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 12 + aese $ctr6b, $rk12 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 12 + + rev32 $h3.16b, $rtmp_ctr.16b @ CTR block 8k+18 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+18 + pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + + eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + aese $ctr1b, $rk12 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 12 + aese $ctr4b, $rk11 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 11 + + ldr $rk14q, [$cc, #224] @ load rk14 + aese $ctr5b, $rk12 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 12 + aese $ctr3b, $rk12 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 12 + + eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid + aese $ctr0b, $rk12 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 12 + aese $ctr4b, $rk12 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 12 + + ldp $res4q, $res5q, [$input_ptr], #32 @ AES block 8k+12, 8k+13 - load ciphertext + aese $ctr1b, $rk13 @ AES block 8k+9 - round 13 + aese $ctr2b, $rk13 @ AES block 8k+10 - round 13 + + ldp $res6q, $res7q, [$input_ptr], #32 @ AES block 8k+14, 8k+15 - load ciphertext + aese $ctr0b, $rk13 @ AES block 8k+8 - round 13 + aese $ctr5b, $rk13 @ AES block 8k+13 - round 13 + + rev32 $h4.16b, $rtmp_ctr.16b @ CTR block 8k+19 + eor3 $ctr2b, $res2b, $ctr2b, $rk14 @ AES block 8k+10 - result + eor3 $ctr1b, $res1b, $ctr1b, $rk14 @ AES block 8k+9 - result + + ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + aese $ctr7b, $rk13 @ AES block 8k+15 - round 13 + + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+19 + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + aese $ctr4b, $rk13 @ AES block 8k+12 - round 13 + + eor3 $ctr5b, $res5b, $ctr5b, $rk14 @ AES block 8k+13 - result + eor3 $ctr0b, $res0b, $ctr0b, $rk14 @ AES block 8k+8 - result + aese $ctr3b, $rk13 @ AES block 8k+11 - round 13 + + stp $ctr0q, $ctr1q, [$output_ptr], #32 @ AES block 8k+8, 8k+9 - store result + mov $ctr0.16b, $h1.16b @ CTR block 8k+16 + eor3 $ctr4b, $res4b, $ctr4b, $rk14 @ AES block 8k+12 - result + + eor3 $acc_lb, $acc_lb, $t11.16b, $acc_hb @ MODULO - fold into low + eor3 $ctr3b, $res3b, $ctr3b, $rk14 @ AES block 8k+11 - result + stp $ctr2q, $ctr3q, [$output_ptr], #32 @ AES block 8k+10, 8k+11 - store result + + mov $ctr3.16b, $h4.16b @ CTR block 8k+19 + mov $ctr2.16b, $h3.16b @ CTR block 8k+18 + aese $ctr6b, $rk13 @ AES block 8k+14 - round 13 + + mov $ctr1.16b, $h2.16b @ CTR block 8k+17 + stp $ctr4q, $ctr5q, [$output_ptr], #32 @ AES block 8k+12, 8k+13 - store result + eor3 $ctr7b, $res7b, $ctr7b, $rk14 @ AES block 8k+15 - result + + eor3 $ctr6b, $res6b, $ctr6b, $rk14 @ AES block 8k+14 - result + rev32 $ctr4.16b, $rtmp_ctr.16b @ CTR block 8k+20 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+20 + + cmp $input_ptr, $main_end_input_ptr @ LOOP CONTROL + stp $ctr6q, $ctr7q, [$output_ptr], #32 @ AES block 8k+14, 8k+15 - store result + b.lt .L256_dec_main_loop + +.L256_dec_prepretail: @ PREPRETAIL + ldp $rk0q, $rk1q, [$cc, #0] @ load rk0, rk1 + rev32 $ctr5.16b, $rtmp_ctr.16b @ CTR block 8k+13 + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+13 + + rev64 $res4b, $res4b @ GHASH block 8k+4 + ldr $h56kq, [$current_tag, #144] @ load h6k | h5k + ldr $h78kq, [$current_tag, #192] @ load h8k | h7k + + rev32 $ctr6.16b, $rtmp_ctr.16b @ CTR block 8k+14 + rev64 $res0b, $res0b @ GHASH block 8k + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+14 + + ext $acc_lb, $acc_lb, $acc_lb, #8 @ PRE 0 + ldr $h7q, [$current_tag, #176] @ load h7l | h7h + ext $h7.16b, $h7.16b, $h7.16b, #8 + ldr $h8q, [$current_tag, #208] @ load h8l | h8h + ext $h8.16b, $h8.16b, $h8.16b, #8 + rev64 $res1b, $res1b @ GHASH block 8k+1 + + rev32 $ctr7.16b, $rtmp_ctr.16b @ CTR block 8k+15 + rev64 $res2b, $res2b @ GHASH block 8k+2 + ldr $h5q, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + ldr $h6q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + + aese $ctr0b, $rk0 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 0 + aese $ctr1b, $rk0 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 0 + aese $ctr4b, $rk0 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 0 + + aese $ctr3b, $rk0 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 0 + aese $ctr5b, $rk0 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 0 + aese $ctr6b, $rk0 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 0 + + aese $ctr4b, $rk1 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 1 + aese $ctr7b, $rk0 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 0 + aese $ctr2b, $rk0 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 0 + + ldp $rk2q, $rk3q, [$cc, #32] @ load rk2, rk3 + aese $ctr0b, $rk1 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 1 + eor $res0b, $res0b, $acc_lb @ PRE 1 + + aese $ctr7b, $rk1 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 1 + aese $ctr6b, $rk1 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 1 + aese $ctr2b, $rk1 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 1 + + aese $ctr3b, $rk1 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 1 + aese $ctr1b, $rk1 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 1 + aese $ctr5b, $rk1 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 1 + + pmull2 $t0.1q, $res1.2d, $h7.2d @ GHASH block 8k+1 - high + trn1 $acc_m.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH block 8k - low + + rev64 $res3b, $res3b @ GHASH block 8k+3 + pmull $h7.1q, $res1.1d, $h7.1d @ GHASH block 8k+1 - low + + aese $ctr5b, $rk2 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 2 + aese $ctr7b, $rk2 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 2 + aese $ctr1b, $rk2 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 2 + + aese $ctr3b, $rk2 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 2 + aese $ctr6b, $rk2 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 2 + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH block 8k - high + + aese $ctr0b, $rk2 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 2 + aese $ctr7b, $rk3 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 3 + + aese $ctr5b, $rk3 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 3 + rev64 $res6b, $res6b @ GHASH block 8k+6 + + aese $ctr0b, $rk3 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 3 + aese $ctr2b, $rk2 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 2 + aese $ctr6b, $rk3 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 3 + + pmull2 $t1.1q, $res2.2d, $h6.2d @ GHASH block 8k+2 - high + trn2 $res0.2d, $res1.2d, $res0.2d @ GHASH block 8k, 8k+1 - mid + aese $ctr4b, $rk2 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 2 + + ldp $rk4q, $rk5q, [$cc, #64] @ load rk4, rk5 + aese $ctr1b, $rk3 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 3 + pmull2 $t2.1q, $res3.2d, $h5.2d @ GHASH block 8k+3 - high + + aese $ctr2b, $rk3 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 3 + eor $acc_hb, $acc_hb, $t0.16b @ GHASH block 8k+1 - high + eor $res0.16b, $res0.16b, $acc_m.16b @ GHASH block 8k, 8k+1 - mid + + aese $ctr4b, $rk3 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 3 + pmull $h6.1q, $res2.1d, $h6.1d @ GHASH block 8k+2 - low + aese $ctr3b, $rk3 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 3 + + eor3 $acc_hb, $acc_hb, $t1.16b, $t2.16b @ GHASH block 8k+2, 8k+3 - high + trn1 $t3.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + trn2 $res2.2d, $res3.2d, $res2.2d @ GHASH block 8k+2, 8k+3 - mid + + pmull2 $acc_m.1q, $res0.2d, $h78k.2d @ GHASH block 8k - mid + pmull $h5.1q, $res3.1d, $h5.1d @ GHASH block 8k+3 - low + eor $acc_lb, $acc_lb, $h7.16b @ GHASH block 8k+1 - low + + pmull $h78k.1q, $res0.1d, $h78k.1d @ GHASH block 8k+1 - mid + aese $ctr5b, $rk4 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 4 + aese $ctr0b, $rk4 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 4 + + eor3 $acc_lb, $acc_lb, $h6.16b, $h5.16b @ GHASH block 8k+2, 8k+3 - low + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + ldr $h2q, [$current_tag, #64] @ load h2l | h2h + ext $h2.16b, $h2.16b, $h2.16b, #8 + aese $ctr7b, $rk4 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 4 + + aese $ctr2b, $rk4 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 4 + aese $ctr6b, $rk4 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 4 + eor $acc_mb, $acc_mb, $h78k.16b @ GHASH block 8k+1 - mid + + eor $res2.16b, $res2.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + aese $ctr7b, $rk5 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 5 + aese $ctr1b, $rk4 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 4 + + aese $ctr2b, $rk5 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 5 + aese $ctr3b, $rk4 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 4 + aese $ctr4b, $rk4 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 4 + + aese $ctr1b, $rk5 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 5 + pmull2 $t3.1q, $res2.2d, $h56k.2d @ GHASH block 8k+2 - mid + aese $ctr6b, $rk5 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 5 + + aese $ctr4b, $rk5 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 5 + aese $ctr3b, $rk5 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 5 + pmull $h56k.1q, $res2.1d, $h56k.1d @ GHASH block 8k+3 - mid + + aese $ctr0b, $rk5 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 5 + aese $ctr5b, $rk5 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 5 + ldp $rk6q, $rk7q, [$cc, #96] @ load rk6, rk7 + + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + rev64 $res7b, $res7b @ GHASH block 8k+7 + rev64 $res5b, $res5b @ GHASH block 8k+5 + + eor3 $acc_mb, $acc_mb, $h56k.16b, $t3.16b @ GHASH block 8k+2, 8k+3 - mid + + trn1 $t6.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + + aese $ctr0b, $rk6 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 6 + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + aese $ctr6b, $rk6 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 6 + + aese $ctr5b, $rk6 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 6 + aese $ctr7b, $rk6 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 6 + + pmull2 $t4.1q, $res4.2d, $h4.2d @ GHASH block 8k+4 - high + pmull2 $t5.1q, $res5.2d, $h3.2d @ GHASH block 8k+5 - high + pmull $h4.1q, $res4.1d, $h4.1d @ GHASH block 8k+4 - low + + trn2 $res4.2d, $res5.2d, $res4.2d @ GHASH block 8k+4, 8k+5 - mid + pmull $h3.1q, $res5.1d, $h3.1d @ GHASH block 8k+5 - low + trn1 $t9.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + + aese $ctr7b, $rk7 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 7 + pmull2 $t7.1q, $res6.2d, $h2.2d @ GHASH block 8k+6 - high + aese $ctr1b, $rk6 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 6 + + aese $ctr2b, $rk6 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 6 + aese $ctr3b, $rk6 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 6 + aese $ctr4b, $rk6 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 6 + + ldp $rk8q, $rk9q, [$cc, #128] @ load rk8, rk9 + pmull $h2.1q, $res6.1d, $h2.1d @ GHASH block 8k+6 - low + aese $ctr5b, $rk7 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 7 + + aese $ctr1b, $rk7 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 7 + aese $ctr4b, $rk7 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 7 + + aese $ctr6b, $rk7 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 7 + aese $ctr2b, $rk7 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 7 + eor3 $acc_hb, $acc_hb, $t4.16b, $t5.16b @ GHASH block 8k+4, 8k+5 - high + + aese $ctr0b, $rk7 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 7 + trn2 $res6.2d, $res7.2d, $res6.2d @ GHASH block 8k+6, 8k+7 - mid + aese $ctr3b, $rk7 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 7 + + aese $ctr0b, $rk8 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 8 + aese $ctr7b, $rk8 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 8 + aese $ctr4b, $rk8 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 8 + + aese $ctr1b, $rk8 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 8 + aese $ctr5b, $rk8 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 8 + aese $ctr6b, $rk8 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 8 + + aese $ctr3b, $rk8 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 8 + aese $ctr4b, $rk9 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 9 + eor $res4.16b, $res4.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + + aese $ctr0b, $rk9 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 9 + aese $ctr1b, $rk9 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 9 + eor $res6.16b, $res6.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + + aese $ctr6b, $rk9 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 9 + aese $ctr7b, $rk9 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 9 + pmull2 $t6.1q, $res4.2d, $h34k.2d @ GHASH block 8k+4 - mid + + aese $ctr2b, $rk8 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 8 + pmull $h34k.1q, $res4.1d, $h34k.1d @ GHASH block 8k+5 - mid + pmull2 $t8.1q, $res7.2d, $h1.2d @ GHASH block 8k+7 - high + + pmull2 $t9.1q, $res6.2d, $h12k.2d @ GHASH block 8k+6 - mid + pmull $h12k.1q, $res6.1d, $h12k.1d @ GHASH block 8k+7 - mid + pmull $h1.1q, $res7.1d, $h1.1d @ GHASH block 8k+7 - low + + ldp $rk10q, $rk11q, [$cc, #160] @ load rk10, rk11 + eor3 $acc_lb, $acc_lb, $h4.16b, $h3.16b @ GHASH block 8k+4, 8k+5 - low + eor3 $acc_mb, $acc_mb, $h34k.16b, $t6.16b @ GHASH block 8k+4, 8k+5 - mid + + aese $ctr2b, $rk9 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 9 + aese $ctr3b, $rk9 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 9 + aese $ctr5b, $rk9 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 9 + + eor3 $acc_hb, $acc_hb, $t7.16b, $t8.16b @ GHASH block 8k+6, 8k+7 - high + eor3 $acc_lb, $acc_lb, $h2.16b, $h1.16b @ GHASH block 8k+6, 8k+7 - low + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + + eor3 $acc_mb, $acc_mb, $h12k.16b, $t9.16b @ GHASH block 8k+6, 8k+7 - mid + + aese $ctr4b, $rk10 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 10 + aese $ctr6b, $rk10 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 10 + aese $ctr5b, $rk10 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 10 + + aese $ctr0b, $rk10 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 10 + aese $ctr2b, $rk10 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 10 + aese $ctr3b, $rk10 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 10 + + eor3 $acc_mb, $acc_mb, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + + aese $ctr7b, $rk10 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 10 + aese $ctr1b, $rk10 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 10 + ldp $rk12q, $rk13q, [$cc, #192] @ load rk12, rk13 + + ext $t11.16b, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + + aese $ctr2b, $rk11 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 11 + aese $ctr1b, $rk11 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 11 + aese $ctr0b, $rk11 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 11 + + pmull $t12.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + aese $ctr3b, $rk11 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 11 + + aese $ctr7b, $rk11 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 11 + aese $ctr6b, $rk11 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 11 + aese $ctr4b, $rk11 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 11 + + aese $ctr5b, $rk11 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 11 + aese $ctr3b, $rk12 \n aesmc $ctr3b, $ctr3b @ AES block 8k+11 - round 12 + + eor3 $acc_mb, $acc_mb, $t12.16b, $t11.16b @ MODULO - fold into mid + + aese $ctr3b, $rk13 @ AES block 8k+11 - round 13 + aese $ctr2b, $rk12 \n aesmc $ctr2b, $ctr2b @ AES block 8k+10 - round 12 + aese $ctr6b, $rk12 \n aesmc $ctr6b, $ctr6b @ AES block 8k+14 - round 12 + + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + aese $ctr4b, $rk12 \n aesmc $ctr4b, $ctr4b @ AES block 8k+12 - round 12 + aese $ctr7b, $rk12 \n aesmc $ctr7b, $ctr7b @ AES block 8k+15 - round 12 + + aese $ctr0b, $rk12 \n aesmc $ctr0b, $ctr0b @ AES block 8k+8 - round 12 + ldr $rk14q, [$cc, #224] @ load rk14 + aese $ctr1b, $rk12 \n aesmc $ctr1b, $ctr1b @ AES block 8k+9 - round 12 + + aese $ctr4b, $rk13 @ AES block 8k+12 - round 13 + ext $t11.16b, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + aese $ctr5b, $rk12 \n aesmc $ctr5b, $ctr5b @ AES block 8k+13 - round 12 + + aese $ctr6b, $rk13 @ AES block 8k+14 - round 13 + aese $ctr2b, $rk13 @ AES block 8k+10 - round 13 + aese $ctr1b, $rk13 @ AES block 8k+9 - round 13 + + aese $ctr5b, $rk13 @ AES block 8k+13 - round 13 + eor3 $acc_lb, $acc_lb, $t11.16b, $acc_hb @ MODULO - fold into low + add $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s @ CTR block 8k+15 + + aese $ctr7b, $rk13 @ AES block 8k+15 - round 13 + aese $ctr0b, $rk13 @ AES block 8k+8 - round 13 +.L256_dec_tail: @ TAIL + + ext $t0.16b, $acc_lb, $acc_lb, #8 @ prepare final partial tag + sub $main_end_input_ptr, $end_input_ptr, $input_ptr @ main_end_input_ptr is number of bytes left to process + cmp $main_end_input_ptr, #112 + + ldr $res1q, [$input_ptr], #16 @ AES block 8k+8 - load ciphertext + + ldp $h78kq, $h8q, [$current_tag, #192] @ load h8k | h7k + ext $h8.16b, $h8.16b, $h8.16b, #8 + mov $t1.16b, $rk14 + + ldp $h5q, $h56kq, [$current_tag, #128] @ load h5l | h5h + ext $h5.16b, $h5.16b, $h5.16b, #8 + + eor3 $res4b, $res1b, $ctr0b, $t1.16b @ AES block 8k+8 - result + ldp $h6q, $h7q, [$current_tag, #160] @ load h6l | h6h + ext $h6.16b, $h6.16b, $h6.16b, #8 + ext $h7.16b, $h7.16b, $h7.16b, #8 + b.gt .L256_dec_blocks_more_than_7 + + mov $ctr7b, $ctr6b + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + mov $ctr6b, $ctr5b + + mov $ctr5b, $ctr4b + mov $ctr4b, $ctr3b + movi $acc_l.8b, #0 + + movi $acc_h.8b, #0 + movi $acc_m.8b, #0 + mov $ctr3b, $ctr2b + + cmp $main_end_input_ptr, #96 + mov $ctr2b, $ctr1b + b.gt .L256_dec_blocks_more_than_6 + + mov $ctr7b, $ctr6b + mov $ctr6b, $ctr5b + + mov $ctr5b, $ctr4b + cmp $main_end_input_ptr, #80 + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + + mov $ctr4b, $ctr3b + mov $ctr3b, $ctr1b + b.gt .L256_dec_blocks_more_than_5 + + cmp $main_end_input_ptr, #64 + mov $ctr7b, $ctr6b + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + + mov $ctr6b, $ctr5b + + mov $ctr5b, $ctr4b + mov $ctr4b, $ctr1b + b.gt .L256_dec_blocks_more_than_4 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + mov $ctr7b, $ctr6b + cmp $main_end_input_ptr, #48 + + mov $ctr6b, $ctr5b + mov $ctr5b, $ctr1b + b.gt .L256_dec_blocks_more_than_3 + + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + mov $ctr7b, $ctr6b + + cmp $main_end_input_ptr, #32 + mov $ctr6b, $ctr1b + b.gt .L256_dec_blocks_more_than_2 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + + mov $ctr7b, $ctr1b + cmp $main_end_input_ptr, #16 + b.gt .L256_dec_blocks_more_than_1 + + sub $rtmp_ctr.4s, $rtmp_ctr.4s, $rctr_inc.4s + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + b .L256_dec_blocks_less_than_1 +.L256_dec_blocks_more_than_7: @ blocks left > 7 + rev64 $res0b, $res1b @ GHASH final-7 block + ldr $res1q, [$input_ptr], #16 @ AES final-6 block - load ciphertext + st1 { $res4b}, [$output_ptr], #16 @ AES final-7 block - store result + + ins $acc_m.d[0], $h78k.d[1] @ GHASH final-7 block - mid + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-7 block - mid + eor3 $res4b, $res1b, $ctr1b, $t1.16b @ AES final-6 block - result + + pmull2 $acc_h.1q, $res0.2d, $h8.2d @ GHASH final-7 block - high + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-7 block - mid + movi $t0.8b, #0 @ suppress further partial tag feed in + + pmull $acc_l.1q, $res0.1d, $h8.1d @ GHASH final-7 block - low + pmull $acc_m.1q, $rk4v.1d, $acc_m.1d @ GHASH final-7 block - mid +.L256_dec_blocks_more_than_6: @ blocks left > 6 + + rev64 $res0b, $res1b @ GHASH final-6 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + ldr $res1q, [$input_ptr], #16 @ AES final-5 block - load ciphertext + movi $t0.8b, #0 @ suppress further partial tag feed in + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-6 block - mid + st1 { $res4b}, [$output_ptr], #16 @ AES final-6 block - store result + pmull2 $rk2q1, $res0.2d, $h7.2d @ GHASH final-6 block - high + + pmull $rk3q1, $res0.1d, $h7.1d @ GHASH final-6 block - low + + eor3 $res4b, $res1b, $ctr2b, $t1.16b @ AES final-5 block - result + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-6 block - low + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-6 block - mid + + pmull $rk4v.1q, $rk4v.1d, $h78k.1d @ GHASH final-6 block - mid + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-6 block - mid + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-6 block - high +.L256_dec_blocks_more_than_5: @ blocks left > 5 + + rev64 $res0b, $res1b @ GHASH final-5 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + pmull2 $rk2q1, $res0.2d, $h6.2d @ GHASH final-5 block - high + ins $rk4v.d[0], $res0.d[1] @ GHASH final-5 block - mid + + ldr $res1q, [$input_ptr], #16 @ AES final-4 block - load ciphertext + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-5 block - mid + st1 { $res4b}, [$output_ptr], #16 @ AES final-5 block - store result + + pmull $rk3q1, $res0.1d, $h6.1d @ GHASH final-5 block - low + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-5 block - mid + + pmull2 $rk4v.1q, $rk4v.2d, $h56k.2d @ GHASH final-5 block - mid + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-5 block - high + eor3 $res4b, $res1b, $ctr3b, $t1.16b @ AES final-4 block - result + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-5 block - low + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-5 block - mid + movi $t0.8b, #0 @ suppress further partial tag feed in +.L256_dec_blocks_more_than_4: @ blocks left > 4 + + rev64 $res0b, $res1b @ GHASH final-4 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-4 block - mid + ldr $res1q, [$input_ptr], #16 @ AES final-3 block - load ciphertext + + movi $t0.8b, #0 @ suppress further partial tag feed in + + pmull $rk3q1, $res0.1d, $h5.1d @ GHASH final-4 block - low + pmull2 $rk2q1, $res0.2d, $h5.2d @ GHASH final-4 block - high + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-4 block - mid + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-4 block - high + + pmull $rk4v.1q, $rk4v.1d, $h56k.1d @ GHASH final-4 block - mid + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-4 block - low + st1 { $res4b}, [$output_ptr], #16 @ AES final-4 block - store result + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-4 block - mid + eor3 $res4b, $res1b, $ctr4b, $t1.16b @ AES final-3 block - result +.L256_dec_blocks_more_than_3: @ blocks left > 3 + + ldr $h4q, [$current_tag, #112] @ load h4l | h4h + ext $h4.16b, $h4.16b, $h4.16b, #8 + rev64 $res0b, $res1b @ GHASH final-3 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + ldr $res1q, [$input_ptr], #16 @ AES final-2 block - load ciphertext + ldr $h34kq, [$current_tag, #96] @ load h4k | h3k + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-3 block - mid + st1 { $res4b}, [$output_ptr], #16 @ AES final-3 block - store result + + eor3 $res4b, $res1b, $ctr5b, $t1.16b @ AES final-2 block - result + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-3 block - mid + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-3 block - mid + pmull $rk3q1, $res0.1d, $h4.1d @ GHASH final-3 block - low + pmull2 $rk2q1, $res0.2d, $h4.2d @ GHASH final-3 block - high + + movi $t0.8b, #0 @ suppress further partial tag feed in + pmull2 $rk4v.1q, $rk4v.2d, $h34k.2d @ GHASH final-3 block - mid + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-3 block - low + + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-3 block - high + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-3 block - mid +.L256_dec_blocks_more_than_2: @ blocks left > 2 + + rev64 $res0b, $res1b @ GHASH final-2 block + + ldr $h3q, [$current_tag, #80] @ load h3l | h3h + ext $h3.16b, $h3.16b, $h3.16b, #8 + ldr $res1q, [$input_ptr], #16 @ AES final-1 block - load ciphertext + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-2 block - mid + + pmull $rk3q1, $res0.1d, $h3.1d @ GHASH final-2 block - low + st1 { $res4b}, [$output_ptr], #16 @ AES final-2 block - store result + eor3 $res4b, $res1b, $ctr6b, $t1.16b @ AES final-1 block - result + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-2 block - mid + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-2 block - low + movi $t0.8b, #0 @ suppress further partial tag feed in + + pmull $rk4v.1q, $rk4v.1d, $h34k.1d @ GHASH final-2 block - mid + pmull2 $rk2q1, $res0.2d, $h3.2d @ GHASH final-2 block - high + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-2 block - mid + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-2 block - high +.L256_dec_blocks_more_than_1: @ blocks left > 1 + + rev64 $res0b, $res1b @ GHASH final-1 block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $rk4v.d[0], $res0.d[1] @ GHASH final-1 block - mid + ldr $h2q, [$current_tag, #64] @ load h2l | h2h + ext $h2.16b, $h2.16b, $h2.16b, #8 + + eor $rk4v.8b, $rk4v.8b, $res0.8b @ GHASH final-1 block - mid + ldr $res1q, [$input_ptr], #16 @ AES final block - load ciphertext + st1 { $res4b}, [$output_ptr], #16 @ AES final-1 block - store result + + ldr $h12kq, [$current_tag, #48] @ load h2k | h1k + pmull $rk3q1, $res0.1d, $h2.1d @ GHASH final-1 block - low + + ins $rk4v.d[1], $rk4v.d[0] @ GHASH final-1 block - mid + + eor $acc_lb, $acc_lb, $rk3 @ GHASH final-1 block - low + + eor3 $res4b, $res1b, $ctr7b, $t1.16b @ AES final block - result + pmull2 $rk2q1, $res0.2d, $h2.2d @ GHASH final-1 block - high + + pmull2 $rk4v.1q, $rk4v.2d, $h12k.2d @ GHASH final-1 block - mid + + movi $t0.8b, #0 @ suppress further partial tag feed in + eor $acc_hb, $acc_hb, $rk2 @ GHASH final-1 block - high + + eor $acc_mb, $acc_mb, $rk4v.16b @ GHASH final-1 block - mid +.L256_dec_blocks_less_than_1: @ blocks left <= 1 + + ld1 { $rk0}, [$output_ptr] @ load existing bytes where the possibly partial last block is to be stored + mvn $temp0_x, xzr @ temp0_x = 0xffffffffffffffff + and $bit_length, $bit_length, #127 @ bit_length %= 128 + + sub $bit_length, $bit_length, #128 @ bit_length -= 128 + rev32 $rtmp_ctr.16b, $rtmp_ctr.16b + str $rtmp_ctrq, [$counter] @ store the updated counter + + neg $bit_length, $bit_length @ bit_length = 128 - #bits in input (in range [1,128]) + + and $bit_length, $bit_length, #127 @ bit_length %= 128 + + lsr $temp0_x, $temp0_x, $bit_length @ temp0_x is mask for top 64b of last block + cmp $bit_length, #64 + mvn $temp1_x, xzr @ temp1_x = 0xffffffffffffffff + + csel $temp3_x, $temp0_x, xzr, lt + csel $temp2_x, $temp1_x, $temp0_x, lt + + mov $ctr0.d[0], $temp2_x @ ctr0b is mask for last block + mov $ctr0.d[1], $temp3_x + + and $res1b, $res1b, $ctr0b @ possibly partial last block has zeroes in highest bits + ldr $h1q, [$current_tag, #32] @ load h1l | h1h + ext $h1.16b, $h1.16b, $h1.16b, #8 + bif $res4b, $rk0, $ctr0b @ insert existing bytes in top end of result before storing + + rev64 $res0b, $res1b @ GHASH final block + + eor $res0b, $res0b, $t0.16b @ feed in partial tag + + ins $t0.d[0], $res0.d[1] @ GHASH final block - mid + pmull2 $rk2q1, $res0.2d, $h1.2d @ GHASH final block - high + + eor $t0.8b, $t0.8b, $res0.8b @ GHASH final block - mid + + pmull $rk3q1, $res0.1d, $h1.1d @ GHASH final block - low + eor $acc_hb, $acc_hb, $rk2 @ GHASH final block - high + + pmull $t0.1q, $t0.1d, $h12k.1d @ GHASH final block - mid + + eor $acc_mb, $acc_mb, $t0.16b @ GHASH final block - mid + ldr $mod_constantd, [$modulo_constant] @ MODULO - load modulo constant + eor $acc_lb, $acc_lb, $rk3 @ GHASH final block - low + + pmull $t11.1q, $acc_h.1d, $mod_constant.1d @ MODULO - top 64b align with mid + eor $t10.16b, $acc_hb, $acc_lb @ MODULO - karatsuba tidy up + + ext $acc_hb, $acc_hb, $acc_hb, #8 @ MODULO - other top alignment + st1 { $res4b}, [$output_ptr] @ store all 16B + + eor $acc_mb, $acc_mb, $t10.16b @ MODULO - karatsuba tidy up + + eor $t11.16b, $acc_hb, $t11.16b @ MODULO - fold into mid + eor $acc_mb, $acc_mb, $t11.16b @ MODULO - fold into mid + + pmull $acc_h.1q, $acc_m.1d, $mod_constant.1d @ MODULO - mid 64b align with low + + ext $acc_mb, $acc_mb, $acc_mb, #8 @ MODULO - other mid alignment + eor $acc_lb, $acc_lb, $acc_hb @ MODULO - fold into low + + eor $acc_lb, $acc_lb, $acc_mb @ MODULO - fold into low + ext $acc_lb, $acc_lb, $acc_lb, #8 + rev64 $acc_lb, $acc_lb + st1 { $acc_l.16b }, [$current_tag] + mov x0, $byte_length + + ldp d10, d11, [sp, #16] + ldp d12, d13, [sp, #32] + ldp d14, d15, [sp, #48] + ldp d8, d9, [sp], #80 + ret + +.L256_dec_ret: + mov w0, #0x0 + ret +.size unroll8_eor3_aes_gcm_dec_256_kernel,.-unroll8_eor3_aes_gcm_dec_256_kernel +___ +} +} + +$code.=<<___; +.asciz "AES GCM module for ARMv8, SPDX BSD-3-Clause by " +.align 2 +#endif +___ + +{ + my %opcode = ( + "rax1" => 0xce608c00, "eor3" => 0xce000000, + "bcax" => 0xce200000, "xar" => 0xce800000 ); + + sub unsha3 { + my ($mnemonic,$arg)=@_; + + $arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv#]([0-9\-]+))?)?/ + && + sprintf ".inst\t0x%08x\t//%s %s", + $opcode{$mnemonic}|$1|($2<<5)|($3<<16)|(eval($4)<<10), + $mnemonic,$arg; + } + sub unvmov { + my $arg=shift; + + $arg =~ m/q([0-9]+)#(lo|hi),\s*q([0-9]+)#(lo|hi)/o && + sprintf "ins v%d.d[%d],v%d.d[%d]",$1<8?$1:$1+8,($2 eq "lo")?0:1, + $3<8?$3:$3+8,($4 eq "lo")?0:1; + } + + foreach(split("\n",$code)) { + s/@\s/\/\//o; # old->new style commentary + s/\`([^\`]*)\`/eval($1)/ge; + + m/\bld1r\b/ and s/\.16b/.2d/g or + s/\b(eor3|rax1|xar|bcax)\s+(v.*)/unsha3($1,$2)/ge; + print $_,"\n"; + } +} + +close STDOUT or die "error closing STDOUT: $!"; # enforce flush diff --git a/crypto/openssl/crypto/modes/asm/aes-gcm-armv8_64.pl b/crypto/openssl/crypto/modes/asm/aes-gcm-armv8_64.pl index bcbd52d18f91..2a97311b1a7c 100755 --- a/crypto/openssl/crypto/modes/asm/aes-gcm-armv8_64.pl +++ b/crypto/openssl/crypto/modes/asm/aes-gcm-armv8_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2019-2023 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2019-2025 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -250,12 +250,12 @@ $code.=<<___ if ($flavour !~ /64/); ___ ######################################################################################### -# size_t aes_gcm_enc_128_kernel(const unsigned char *in, -# size_t len, -# unsigned char *out, -# const void *key, +# size_t aes_gcm_enc_128_kernel(const uint8_t * plaintext, +# uint64_t plaintext_length, +# uint8_t * ciphertext, +# uint64_t *Xi, # unsigned char ivec[16], -# u64 *Xi); +# const void *key); # $code.=<<___; .global aes_gcm_enc_128_kernel @@ -1141,12 +1141,12 @@ aes_gcm_enc_128_kernel: ___ ######################################################################################### -# size_t aes_gcm_dec_128_kernel(const unsigned char *in, -# size_t len, -# unsigned char *out, -# const void *key, +# size_t aes_gcm_dec_128_kernel(const uint8_t * ciphertext, +# uint64_t plaintext_length, +# uint8_t * plaintext, +# uint64_t *Xi, # unsigned char ivec[16], -# u64 *Xi); +# const void *key); # $code.=<<___; .global aes_gcm_dec_128_kernel @@ -2105,12 +2105,12 @@ my $rk4v="v22"; my $rk4d="d22"; ######################################################################################### -# size_t aes_gcm_enc_192_kernel(const unsigned char *in, -# size_t len, -# unsigned char *out, -# const void *key, +# size_t aes_gcm_enc_192_kernel(const uint8_t * plaintext, +# uint64_t plaintext_length, +# uint8_t * ciphertext, +# uint64_t *Xi, # unsigned char ivec[16], -# u64 *Xi); +# const void *key); # $code.=<<___; .global aes_gcm_enc_192_kernel @@ -3047,12 +3047,12 @@ aes_gcm_enc_192_kernel: ___ ######################################################################################### -# size_t aes_gcm_dec_192_kernel(const unsigned char *in, -# size_t len, -# unsigned char *out, -# const void *key, +# size_t aes_gcm_dec_192_kernel(const uint8_t * ciphertext, +# uint64_t plaintext_length, +# uint8_t * plaintext, +# uint64_t *Xi, # unsigned char ivec[16], -# u64 *Xi); +# const void *key); # $code.=<<___; .global aes_gcm_dec_192_kernel @@ -4061,12 +4061,12 @@ my $rk4v="v22"; my $rk4d="d22"; ######################################################################################### -# size_t aes_gcm_enc_256_kernel(const unsigned char *in, -# size_t len, -# unsigned char *out, -# const void *key, +# size_t aes_gcm_enc_256_kernel(const uint8_t * plaintext, +# uint64_t plaintext_length, +# uint8_t * ciphertext, +# uint64_t *Xi, # unsigned char ivec[16], -# u64 *Xi); +# const void *key); # $code.=<<___; .global aes_gcm_enc_256_kernel @@ -5051,12 +5051,12 @@ my $t8d="d4"; my $t9="v6"; my $t9d="d6"; ######################################################################################### -# size_t aes_gcm_dec_256_kernel(const unsigned char *in, -# size_t len, -# unsigned char *out, -# const void *key, +# size_t aes_gcm_dec_256_kernel(const uint8_t * ciphertext, +# uint64_t plaintext_length, +# uint8_t * plaintext, +# uint64_t *Xi, # unsigned char ivec[16], -# u64 *Xi); +# const void *key); # $code.=<<___; .global aes_gcm_dec_256_kernel @@ -6072,6 +6072,7 @@ ___ } $code.=<<___; +.rodata .asciz "GHASH for ARMv8, CRYPTOGAMS by " .align 2 #endif diff --git a/crypto/openssl/crypto/modes/asm/aes-gcm-avx512.pl b/crypto/openssl/crypto/modes/asm/aes-gcm-avx512.pl new file mode 100644 index 000000000000..79ee59f9e07d --- /dev/null +++ b/crypto/openssl/crypto/modes/asm/aes-gcm-avx512.pl @@ -0,0 +1,4978 @@ +# Copyright 2021-2024 The OpenSSL Project Authors. All Rights Reserved. +# Copyright (c) 2021, Intel Corporation. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html +# +# +# This implementation is based on the AES-GCM code (AVX512VAES + VPCLMULQDQ) +# from Intel(R) Multi-Buffer Crypto for IPsec Library v1.1 +# (https://github.com/intel/intel-ipsec-mb). +# Original author is Tomasz Kantecki . +# +# References: +# [1] Vinodh Gopal et. al. Optimized Galois-Counter-Mode Implementation on +# Intel Architecture Processors. August, 2010. +# [2] Erdinc Ozturk et. al. Enabling High-Performance Galois-Counter-Mode on +# Intel Architecture Processors. October, 2012. +# [3] Shay Gueron et. al. Intel Carry-Less Multiplication Instruction and its +# Usage for Computing the GCM Mode. May, 2010. +# +# +# December 2021 +# +# Initial release. +# +# GCM128_CONTEXT structure has storage for 16 hkeys only, but this +# implementation can use up to 48. To avoid extending the context size, +# precompute and store in the context first 16 hkeys only, and compute the rest +# on demand keeping them in the local frame. +# +#====================================================================== +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$win64 = 0; +$win64 = 1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); + +$avx512vaes = 0; + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; +$dir = $1; +($xlate = "${dir}x86_64-xlate.pl" and -f $xlate) + or ($xlate = "${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) + or die "can't locate x86_64-xlate.pl"; + +if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` =~ /GNU assembler version ([2-9]\.[0-9]+)/) { + $avx512vaes = ($1 >= 2.30); +} + +if (!$avx512vaes + && $win64 + && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) + && `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?/) +{ + $avx512vaes = ($1 == 2.13 && $2 >= 3) + ($1 >= 2.14); +} + +if (!$avx512vaes && `$ENV{CC} -v 2>&1` + =~ /(Apple)?\s*((?:clang|LLVM) version|.*based on LLVM) ([0-9]+)\.([0-9]+)\.([0-9]+)?/) { + my $ver = $3 + $4/100.0 + $5/10000.0; # 3.1.0->3.01, 3.10.1->3.1001 + if ($1) { + # Apple conditions, they use a different version series, see + # https://en.wikipedia.org/wiki/Xcode#Xcode_7.0_-_10.x_(since_Free_On-Device_Development)_2 + # clang 7.0.0 is Apple clang 10.0.1 + $avx512vaes = ($ver>=10.0001) + } else { + $avx512vaes = ($ver>=7.0); + } +} + +open OUT, "| \"$^X\" \"$xlate\" $flavour \"$output\"" + or die "can't call $xlate: $!"; +*STDOUT = *OUT; + +#====================================================================== +if ($avx512vaes>0) { #<<< + +$code .= <<___; +.extern OPENSSL_ia32cap_P +.globl ossl_vaes_vpclmulqdq_capable +.type ossl_vaes_vpclmulqdq_capable,\@abi-omnipotent +.align 32 +ossl_vaes_vpclmulqdq_capable: + mov OPENSSL_ia32cap_P+8(%rip), %rcx + # avx512vpclmulqdq + avx512vaes + avx512vl + avx512bw + avx512dq + avx512f + mov \$`1<<42|1<<41|1<<31|1<<30|1<<17|1<<16`,%rdx + xor %eax,%eax + and %rdx,%rcx + cmp %rdx,%rcx + cmove %rcx,%rax + ret +.size ossl_vaes_vpclmulqdq_capable, .-ossl_vaes_vpclmulqdq_capable +___ + +# ; Mapping key length -> AES rounds count +my %aes_rounds = ( + 128 => 9, + 192 => 11, + 256 => 13); + +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +# ;;; Code generation control switches +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +# ; ABI-aware zeroing of volatile registers in EPILOG(). +# ; Disabled due to performance reasons. +my $CLEAR_SCRATCH_REGISTERS = 0; + +# ; Zero HKeys storage from the stack if they are stored there +my $CLEAR_HKEYS_STORAGE_ON_EXIT = 1; + +# ; Enable / disable check of function arguments for null pointer +# ; Currently disabled, as this check is handled outside. +my $CHECK_FUNCTION_ARGUMENTS = 0; + +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +# ;;; Global constants +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +# AES block size in bytes +my $AES_BLOCK_SIZE = 16; + +# Storage capacity in elements +my $HKEYS_STORAGE_CAPACITY = 48; +my $LOCAL_STORAGE_CAPACITY = 48; +my $HKEYS_CONTEXT_CAPACITY = 16; + +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +# ;;; Stack frame definition +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +# (1) -> +64(Win)/+48(Lin)-byte space for pushed GPRs +# (2) -> +8-byte space for 16-byte alignment of XMM storage +# (3) -> Frame pointer (%RBP) +# (4) -> +160-byte XMM storage (Windows only, zero on Linux) +# (5) -> +48-byte space for 64-byte alignment of %RSP from p.8 +# (6) -> +768-byte LOCAL storage (optional, can be omitted in some functions) +# (7) -> +768-byte HKEYS storage +# (8) -> Stack pointer (%RSP) aligned on 64-byte boundary + +my $GP_STORAGE = $win64 ? 8 * 8 : 8 * 6; # ; space for saved non-volatile GP registers (pushed on stack) +my $XMM_STORAGE = $win64 ? (10 * 16) : 0; # ; space for saved XMM registers +my $HKEYS_STORAGE = ($HKEYS_STORAGE_CAPACITY * $AES_BLOCK_SIZE); # ; space for HKeys^i, i=1..48 +my $LOCAL_STORAGE = ($LOCAL_STORAGE_CAPACITY * $AES_BLOCK_SIZE); # ; space for up to 48 AES blocks + +my $STACK_HKEYS_OFFSET = 0; +my $STACK_LOCAL_OFFSET = ($STACK_HKEYS_OFFSET + $HKEYS_STORAGE); + +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +# ;;; Function arguments abstraction +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +my ($arg1, $arg2, $arg3, $arg4, $arg5, $arg6, $arg7, $arg8, $arg9, $arg10, $arg11); + +# ; Counter used for assembly label generation +my $label_count = 0; + +# ; This implementation follows the convention: for non-leaf functions (they +# ; must call PROLOG) %rbp is used as a frame pointer, and has fixed offset from +# ; the function entry: $GP_STORAGE + [8 bytes alignment (Windows only)]. This +# ; helps to facilitate SEH handlers writing. +# +# ; Leaf functions here do not use more than 4 input arguments. +if ($win64) { + $arg1 = "%rcx"; + $arg2 = "%rdx"; + $arg3 = "%r8"; + $arg4 = "%r9"; + $arg5 = "`$GP_STORAGE + 8 + 8*5`(%rbp)"; # +8 - alignment bytes + $arg6 = "`$GP_STORAGE + 8 + 8*6`(%rbp)"; + $arg7 = "`$GP_STORAGE + 8 + 8*7`(%rbp)"; + $arg8 = "`$GP_STORAGE + 8 + 8*8`(%rbp)"; + $arg9 = "`$GP_STORAGE + 8 + 8*9`(%rbp)"; + $arg10 = "`$GP_STORAGE + 8 + 8*10`(%rbp)"; + $arg11 = "`$GP_STORAGE + 8 + 8*11`(%rbp)"; +} else { + $arg1 = "%rdi"; + $arg2 = "%rsi"; + $arg3 = "%rdx"; + $arg4 = "%rcx"; + $arg5 = "%r8"; + $arg6 = "%r9"; + $arg7 = "`$GP_STORAGE + 8*1`(%rbp)"; + $arg8 = "`$GP_STORAGE + 8*2`(%rbp)"; + $arg9 = "`$GP_STORAGE + 8*3`(%rbp)"; + $arg10 = "`$GP_STORAGE + 8*4`(%rbp)"; + $arg11 = "`$GP_STORAGE + 8*5`(%rbp)"; +} + +# ; Offsets in gcm128_context structure (see include/crypto/modes.h) +my $CTX_OFFSET_CurCount = (16 * 0); # ; (Yi) Current counter for generation of encryption key +my $CTX_OFFSET_PEncBlock = (16 * 1); # ; (repurposed EKi field) Partial block buffer +my $CTX_OFFSET_EK0 = (16 * 2); # ; (EK0) Encrypted Y0 counter (see gcm spec notation) +my $CTX_OFFSET_AadLen = (16 * 3); # ; (len.u[0]) Length of Hash which has been input +my $CTX_OFFSET_InLen = ((16 * 3) + 8); # ; (len.u[1]) Length of input data which will be encrypted or decrypted +my $CTX_OFFSET_AadHash = (16 * 4); # ; (Xi) Current hash +my $CTX_OFFSET_HTable = (16 * 6); # ; (Htable) Precomputed table (allows 16 values) + +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +# ;;; Helper functions +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +sub BYTE { + my ($reg) = @_; + if ($reg =~ /%r[abcd]x/i) { + $reg =~ s/%r([abcd])x/%${1}l/i; + } elsif ($reg =~ /%r[sdb][ip]/i) { + $reg =~ s/%r([sdb][ip])/%${1}l/i; + } elsif ($reg =~ /%r[0-9]{1,2}/i) { + $reg =~ s/%(r[0-9]{1,2})/%${1}b/i; + } else { + die "BYTE: unknown register: $reg\n"; + } + return $reg; +} + +sub WORD { + my ($reg) = @_; + if ($reg =~ /%r[abcdsdb][xip]/i) { + $reg =~ s/%r([abcdsdb])([xip])/%${1}${2}/i; + } elsif ($reg =~ /%r[0-9]{1,2}/) { + $reg =~ s/%(r[0-9]{1,2})/%${1}w/i; + } else { + die "WORD: unknown register: $reg\n"; + } + return $reg; +} + +sub DWORD { + my ($reg) = @_; + if ($reg =~ /%r[abcdsdb][xip]/i) { + $reg =~ s/%r([abcdsdb])([xip])/%e${1}${2}/i; + } elsif ($reg =~ /%r[0-9]{1,2}/i) { + $reg =~ s/%(r[0-9]{1,2})/%${1}d/i; + } else { + die "DWORD: unknown register: $reg\n"; + } + return $reg; +} + +sub XWORD { + my ($reg) = @_; + if ($reg =~ /%[xyz]mm/i) { + $reg =~ s/%[xyz]mm/%xmm/i; + } else { + die "XWORD: unknown register: $reg\n"; + } + return $reg; +} + +sub YWORD { + my ($reg) = @_; + if ($reg =~ /%[xyz]mm/i) { + $reg =~ s/%[xyz]mm/%ymm/i; + } else { + die "YWORD: unknown register: $reg\n"; + } + return $reg; +} + +sub ZWORD { + my ($reg) = @_; + if ($reg =~ /%[xyz]mm/i) { + $reg =~ s/%[xyz]mm/%zmm/i; + } else { + die "ZWORD: unknown register: $reg\n"; + } + return $reg; +} + +# ; Helper function to construct effective address based on two kinds of +# ; offsets: numerical or located in the register +sub EffectiveAddress { + my ($base, $offset, $displacement) = @_; + $displacement = 0 if (!$displacement); + + if ($offset =~ /^\d+\z/) { # numerical offset + return "`$offset + $displacement`($base)"; + } else { # offset resides in register + return "$displacement($base,$offset,1)"; + } +} + +# ; Provides memory location of corresponding HashKey power +sub HashKeyByIdx { + my ($idx, $base) = @_; + my $base_str = ($base eq "%rsp") ? "frame" : "context"; + + my $offset = &HashKeyOffsetByIdx($idx, $base_str); + return "$offset($base)"; +} + +# ; Provides offset (in bytes) of corresponding HashKey power from the highest key in the storage +sub HashKeyOffsetByIdx { + my ($idx, $base) = @_; + die "HashKeyOffsetByIdx: base should be either 'frame' or 'context'; base = $base" + if (($base ne "frame") && ($base ne "context")); + + my $offset_base; + my $offset_idx; + if ($base eq "frame") { # frame storage + die "HashKeyOffsetByIdx: idx out of bounds (1..48)! idx = $idx\n" if ($idx > $HKEYS_STORAGE_CAPACITY || $idx < 1); + $offset_base = $STACK_HKEYS_OFFSET; + $offset_idx = ($AES_BLOCK_SIZE * ($HKEYS_STORAGE_CAPACITY - $idx)); + } else { # context storage + die "HashKeyOffsetByIdx: idx out of bounds (1..16)! idx = $idx\n" if ($idx > $HKEYS_CONTEXT_CAPACITY || $idx < 1); + $offset_base = $CTX_OFFSET_HTable; + $offset_idx = ($AES_BLOCK_SIZE * ($HKEYS_CONTEXT_CAPACITY - $idx)); + } + return $offset_base + $offset_idx; +} + +# ; Creates local frame and does back up of non-volatile registers. +# ; Holds stack unwinding directives. +sub PROLOG { + my ($need_hkeys_stack_storage, $need_aes_stack_storage, $func_name) = @_; + + my $DYNAMIC_STACK_ALLOC_SIZE = 0; + my $DYNAMIC_STACK_ALLOC_ALIGNMENT_SPACE = $win64 ? 48 : 52; + + if ($need_hkeys_stack_storage) { + $DYNAMIC_STACK_ALLOC_SIZE += $HKEYS_STORAGE; + } + + if ($need_aes_stack_storage) { + if (!$need_hkeys_stack_storage) { + die "PROLOG: unsupported case - aes storage without hkeys one"; + } + $DYNAMIC_STACK_ALLOC_SIZE += $LOCAL_STORAGE; + } + + $code .= <<___; + push %rbx +.cfi_push %rbx +.L${func_name}_seh_push_rbx: + push %rbp +.cfi_push %rbp +.L${func_name}_seh_push_rbp: + push %r12 +.cfi_push %r12 +.L${func_name}_seh_push_r12: + push %r13 +.cfi_push %r13 +.L${func_name}_seh_push_r13: + push %r14 +.cfi_push %r14 +.L${func_name}_seh_push_r14: + push %r15 +.cfi_push %r15 +.L${func_name}_seh_push_r15: +___ + + if ($win64) { + $code .= <<___; + push %rdi +.L${func_name}_seh_push_rdi: + push %rsi +.L${func_name}_seh_push_rsi: + + sub \$`$XMM_STORAGE+8`,%rsp # +8 alignment +.L${func_name}_seh_allocstack_xmm: +___ + } + $code .= <<___; + # ; %rbp contains stack pointer right after GP regs pushed at stack + [8 + # ; bytes of alignment (Windows only)]. It serves as a frame pointer in SEH + # ; handlers. The requirement for a frame pointer is that its offset from + # ; RSP shall be multiple of 16, and not exceed 240 bytes. The frame pointer + # ; itself seems to be reasonable to use here, because later we do 64-byte stack + # ; alignment which gives us non-determinate offsets and complicates writing + # ; SEH handlers. + # + # ; It also serves as an anchor for retrieving stack arguments on both Linux + # ; and Windows. + lea `$XMM_STORAGE`(%rsp),%rbp +.cfi_def_cfa_register %rbp +.L${func_name}_seh_setfp: +___ + if ($win64) { + + # ; xmm6:xmm15 need to be preserved on Windows + foreach my $reg_idx (6 .. 15) { + my $xmm_reg_offset = ($reg_idx - 6) * 16; + $code .= <<___; + vmovdqu %xmm${reg_idx},$xmm_reg_offset(%rsp) +.L${func_name}_seh_save_xmm${reg_idx}: +___ + } + } + + $code .= <<___; +# Prolog ends here. Next stack allocation is treated as "dynamic". +.L${func_name}_seh_prolog_end: +___ + + if ($DYNAMIC_STACK_ALLOC_SIZE) { + $code .= <<___; + sub \$`$DYNAMIC_STACK_ALLOC_SIZE + $DYNAMIC_STACK_ALLOC_ALIGNMENT_SPACE`,%rsp + and \$(-64),%rsp +___ + } +} + +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +# ;;; Restore register content for the caller. +# ;;; And cleanup stack. +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +sub EPILOG { + my ($hkeys_storage_on_stack, $payload_len) = @_; + + my $label_suffix = $label_count++; + + if ($hkeys_storage_on_stack && $CLEAR_HKEYS_STORAGE_ON_EXIT) { + + # ; There is no need in hkeys cleanup if payload len was small, i.e. no hkeys + # ; were stored in the local frame storage + $code .= <<___; + cmpq \$`16*16`,$payload_len + jbe .Lskip_hkeys_cleanup_${label_suffix} + vpxor %xmm0,%xmm0,%xmm0 +___ + for (my $i = 0; $i < int($HKEYS_STORAGE / 64); $i++) { + $code .= "vmovdqa64 %zmm0,`$STACK_HKEYS_OFFSET + 64*$i`(%rsp)\n"; + } + $code .= ".Lskip_hkeys_cleanup_${label_suffix}:\n"; + } + + if ($CLEAR_SCRATCH_REGISTERS) { + &clear_scratch_gps_asm(); + &clear_scratch_zmms_asm(); + } else { + $code .= "vzeroupper\n"; + } + + if ($win64) { + + # ; restore xmm15:xmm6 + for (my $reg_idx = 15; $reg_idx >= 6; $reg_idx--) { + my $xmm_reg_offset = -$XMM_STORAGE + ($reg_idx - 6) * 16; + $code .= <<___; + vmovdqu $xmm_reg_offset(%rbp),%xmm${reg_idx}, +___ + } + } + + if ($win64) { + + # Forming valid epilog for SEH with use of frame pointer. + # https://docs.microsoft.com/en-us/cpp/build/prolog-and-epilog?view=msvc-160#epilog-code + $code .= "lea 8(%rbp),%rsp\n"; + } else { + $code .= "lea (%rbp),%rsp\n"; + $code .= ".cfi_def_cfa_register %rsp\n"; + } + + if ($win64) { + $code .= <<___; + pop %rsi +.cfi_pop %rsi + pop %rdi +.cfi_pop %rdi +___ + } + $code .= <<___; + pop %r15 +.cfi_pop %r15 + pop %r14 +.cfi_pop %r14 + pop %r13 +.cfi_pop %r13 + pop %r12 +.cfi_pop %r12 + pop %rbp +.cfi_pop %rbp + pop %rbx +.cfi_pop %rbx +___ +} + +# ; Clears all scratch ZMM registers +# ; +# ; It should be called before restoring the XMM registers +# ; for Windows (XMM6-XMM15). +# ; +sub clear_scratch_zmms_asm { + + # ; On Linux, all ZMM registers are scratch registers + if (!$win64) { + $code .= "vzeroall\n"; + } else { + foreach my $i (0 .. 5) { + $code .= "vpxorq %xmm${i},%xmm${i},%xmm${i}\n"; + } + } + foreach my $i (16 .. 31) { + $code .= "vpxorq %xmm${i},%xmm${i},%xmm${i}\n"; + } +} + +# Clears all scratch GP registers +sub clear_scratch_gps_asm { + foreach my $reg ("%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11") { + $code .= "xor $reg,$reg\n"; + } + if (!$win64) { + foreach my $reg ("%rsi", "%rdi") { + $code .= "xor $reg,$reg\n"; + } + } +} + +sub precompute_hkeys_on_stack { + my $GCM128_CTX = $_[0]; + my $HKEYS_READY = $_[1]; + my $ZTMP0 = $_[2]; + my $ZTMP1 = $_[3]; + my $ZTMP2 = $_[4]; + my $ZTMP3 = $_[5]; + my $ZTMP4 = $_[6]; + my $ZTMP5 = $_[7]; + my $ZTMP6 = $_[8]; + my $HKEYS_RANGE = $_[9]; # ; "first16", "mid16", "all", "first32", "last32" + + die "precompute_hkeys_on_stack: Unexpected value of HKEYS_RANGE: $HKEYS_RANGE" + if ($HKEYS_RANGE ne "first16" + && $HKEYS_RANGE ne "mid16" + && $HKEYS_RANGE ne "all" + && $HKEYS_RANGE ne "first32" + && $HKEYS_RANGE ne "last32"); + + my $label_suffix = $label_count++; + + $code .= <<___; + test $HKEYS_READY,$HKEYS_READY + jnz .L_skip_hkeys_precomputation_${label_suffix} +___ + + if ($HKEYS_RANGE eq "first16" || $HKEYS_RANGE eq "first32" || $HKEYS_RANGE eq "all") { + + # ; Fill the stack with the first 16 hkeys from the context + $code .= <<___; + # ; Move 16 hkeys from the context to stack + vmovdqu64 @{[HashKeyByIdx(4,$GCM128_CTX)]},$ZTMP0 + vmovdqu64 $ZTMP0,@{[HashKeyByIdx(4,"%rsp")]} + + vmovdqu64 @{[HashKeyByIdx(8,$GCM128_CTX)]},$ZTMP1 + vmovdqu64 $ZTMP1,@{[HashKeyByIdx(8,"%rsp")]} + + # ; broadcast HashKey^8 + vshufi64x2 \$0x00,$ZTMP1,$ZTMP1,$ZTMP1 + + vmovdqu64 @{[HashKeyByIdx(12,$GCM128_CTX)]},$ZTMP2 + vmovdqu64 $ZTMP2,@{[HashKeyByIdx(12,"%rsp")]} + + vmovdqu64 @{[HashKeyByIdx(16,$GCM128_CTX)]},$ZTMP3 + vmovdqu64 $ZTMP3,@{[HashKeyByIdx(16,"%rsp")]} +___ + } + + if ($HKEYS_RANGE eq "mid16" || $HKEYS_RANGE eq "last32") { + $code .= <<___; + vmovdqu64 @{[HashKeyByIdx(8,"%rsp")]},$ZTMP1 + + # ; broadcast HashKey^8 + vshufi64x2 \$0x00,$ZTMP1,$ZTMP1,$ZTMP1 + + vmovdqu64 @{[HashKeyByIdx(12,"%rsp")]},$ZTMP2 + vmovdqu64 @{[HashKeyByIdx(16,"%rsp")]},$ZTMP3 +___ + + } + + if ($HKEYS_RANGE eq "mid16" || $HKEYS_RANGE eq "first32" || $HKEYS_RANGE eq "last32" || $HKEYS_RANGE eq "all") { + + # ; Precompute hkeys^i, i=17..32 + my $i = 20; + foreach (1 .. int((32 - 16) / 8)) { + + # ;; compute HashKey^(4 + n), HashKey^(3 + n), ... HashKey^(1 + n) + &GHASH_MUL($ZTMP2, $ZTMP1, $ZTMP4, $ZTMP5, $ZTMP6); + $code .= "vmovdqu64 $ZTMP2,@{[HashKeyByIdx($i,\"%rsp\")]}\n"; + $i += 4; + + # ;; compute HashKey^(8 + n), HashKey^(7 + n), ... HashKey^(5 + n) + &GHASH_MUL($ZTMP3, $ZTMP1, $ZTMP4, $ZTMP5, $ZTMP6); + $code .= "vmovdqu64 $ZTMP3,@{[HashKeyByIdx($i,\"%rsp\")]}\n"; + $i += 4; + } + } + + if ($HKEYS_RANGE eq "last32" || $HKEYS_RANGE eq "all") { + + # ; Precompute hkeys^i, i=33..48 (HKEYS_STORAGE_CAPACITY = 48) + my $i = 36; + foreach (1 .. int((48 - 32) / 8)) { + + # ;; compute HashKey^(4 + n), HashKey^(3 + n), ... HashKey^(1 + n) + &GHASH_MUL($ZTMP2, $ZTMP1, $ZTMP4, $ZTMP5, $ZTMP6); + $code .= "vmovdqu64 $ZTMP2,@{[HashKeyByIdx($i,\"%rsp\")]}\n"; + $i += 4; + + # ;; compute HashKey^(8 + n), HashKey^(7 + n), ... HashKey^(5 + n) + &GHASH_MUL($ZTMP3, $ZTMP1, $ZTMP4, $ZTMP5, $ZTMP6); + $code .= "vmovdqu64 $ZTMP3,@{[HashKeyByIdx($i,\"%rsp\")]}\n"; + $i += 4; + } + } + + $code .= ".L_skip_hkeys_precomputation_${label_suffix}:\n"; +} + +# ;; ============================================================================= +# ;; Generic macro to produce code that executes $OPCODE instruction +# ;; on selected number of AES blocks (16 bytes long ) between 0 and 16. +# ;; All three operands of the instruction come from registers. +# ;; Note: if 3 blocks are left at the end instruction is produced to operate all +# ;; 4 blocks (full width of ZMM) +sub ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 { + my $NUM_BLOCKS = $_[0]; # [in] numerical value, number of AES blocks (0 to 16) + my $OPCODE = $_[1]; # [in] instruction name + my @DST; + $DST[0] = $_[2]; # [out] destination ZMM register + $DST[1] = $_[3]; # [out] destination ZMM register + $DST[2] = $_[4]; # [out] destination ZMM register + $DST[3] = $_[5]; # [out] destination ZMM register + my @SRC1; + $SRC1[0] = $_[6]; # [in] source 1 ZMM register + $SRC1[1] = $_[7]; # [in] source 1 ZMM register + $SRC1[2] = $_[8]; # [in] source 1 ZMM register + $SRC1[3] = $_[9]; # [in] source 1 ZMM register + my @SRC2; + $SRC2[0] = $_[10]; # [in] source 2 ZMM register + $SRC2[1] = $_[11]; # [in] source 2 ZMM register + $SRC2[2] = $_[12]; # [in] source 2 ZMM register + $SRC2[3] = $_[13]; # [in] source 2 ZMM register + + die "ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16: num_blocks is out of bounds = $NUM_BLOCKS\n" + if ($NUM_BLOCKS > 16 || $NUM_BLOCKS < 0); + + my $reg_idx = 0; + my $blocks_left = $NUM_BLOCKS; + + foreach (1 .. ($NUM_BLOCKS / 4)) { + $code .= "$OPCODE $SRC2[$reg_idx],$SRC1[$reg_idx],$DST[$reg_idx]\n"; + $reg_idx++; + $blocks_left -= 4; + } + + my $DSTREG = $DST[$reg_idx]; + my $SRC1REG = $SRC1[$reg_idx]; + my $SRC2REG = $SRC2[$reg_idx]; + + if ($blocks_left == 1) { + $code .= "$OPCODE @{[XWORD($SRC2REG)]},@{[XWORD($SRC1REG)]},@{[XWORD($DSTREG)]}\n"; + } elsif ($blocks_left == 2) { + $code .= "$OPCODE @{[YWORD($SRC2REG)]},@{[YWORD($SRC1REG)]},@{[YWORD($DSTREG)]}\n"; + } elsif ($blocks_left == 3) { + $code .= "$OPCODE $SRC2REG,$SRC1REG,$DSTREG\n"; + } +} + +# ;; ============================================================================= +# ;; Loads specified number of AES blocks into ZMM registers using mask register +# ;; for the last loaded register (xmm, ymm or zmm). +# ;; Loads take place at 1 byte granularity. +sub ZMM_LOAD_MASKED_BLOCKS_0_16 { + my $NUM_BLOCKS = $_[0]; # [in] numerical value, number of AES blocks (0 to 16) + my $INP = $_[1]; # [in] input data pointer to read from + my $DATA_OFFSET = $_[2]; # [in] offset to the output pointer (GP or numerical) + my @DST; + $DST[0] = $_[3]; # [out] ZMM register with loaded data + $DST[1] = $_[4]; # [out] ZMM register with loaded data + $DST[2] = $_[5]; # [out] ZMM register with loaded data + $DST[3] = $_[6]; # [out] ZMM register with loaded data + my $MASK = $_[7]; # [in] mask register + + die "ZMM_LOAD_MASKED_BLOCKS_0_16: num_blocks is out of bounds = $NUM_BLOCKS\n" + if ($NUM_BLOCKS > 16 || $NUM_BLOCKS < 0); + + my $src_offset = 0; + my $dst_idx = 0; + my $blocks_left = $NUM_BLOCKS; + + if ($NUM_BLOCKS > 0) { + foreach (1 .. (int(($NUM_BLOCKS + 3) / 4) - 1)) { + $code .= "vmovdqu8 @{[EffectiveAddress($INP,$DATA_OFFSET,$src_offset)]},$DST[$dst_idx]\n"; + $src_offset += 64; + $dst_idx++; + $blocks_left -= 4; + } + } + + my $DSTREG = $DST[$dst_idx]; + + if ($blocks_left == 1) { + $code .= "vmovdqu8 @{[EffectiveAddress($INP,$DATA_OFFSET,$src_offset)]},@{[XWORD($DSTREG)]}\{$MASK\}{z}\n"; + } elsif ($blocks_left == 2) { + $code .= "vmovdqu8 @{[EffectiveAddress($INP,$DATA_OFFSET,$src_offset)]},@{[YWORD($DSTREG)]}\{$MASK\}{z}\n"; + } elsif (($blocks_left == 3 || $blocks_left == 4)) { + $code .= "vmovdqu8 @{[EffectiveAddress($INP,$DATA_OFFSET,$src_offset)]},$DSTREG\{$MASK\}{z}\n"; + } +} + +# ;; ============================================================================= +# ;; Stores specified number of AES blocks from ZMM registers with mask register +# ;; for the last loaded register (xmm, ymm or zmm). +# ;; Stores take place at 1 byte granularity. +sub ZMM_STORE_MASKED_BLOCKS_0_16 { + my $NUM_BLOCKS = $_[0]; # [in] numerical value, number of AES blocks (0 to 16) + my $OUTP = $_[1]; # [in] output data pointer to write to + my $DATA_OFFSET = $_[2]; # [in] offset to the output pointer (GP or numerical) + my @SRC; + $SRC[0] = $_[3]; # [in] ZMM register with data to store + $SRC[1] = $_[4]; # [in] ZMM register with data to store + $SRC[2] = $_[5]; # [in] ZMM register with data to store + $SRC[3] = $_[6]; # [in] ZMM register with data to store + my $MASK = $_[7]; # [in] mask register + + die "ZMM_STORE_MASKED_BLOCKS_0_16: num_blocks is out of bounds = $NUM_BLOCKS\n" + if ($NUM_BLOCKS > 16 || $NUM_BLOCKS < 0); + + my $dst_offset = 0; + my $src_idx = 0; + my $blocks_left = $NUM_BLOCKS; + + if ($NUM_BLOCKS > 0) { + foreach (1 .. (int(($NUM_BLOCKS + 3) / 4) - 1)) { + $code .= "vmovdqu8 $SRC[$src_idx],`$dst_offset`($OUTP,$DATA_OFFSET,1)\n"; + $dst_offset += 64; + $src_idx++; + $blocks_left -= 4; + } + } + + my $SRCREG = $SRC[$src_idx]; + + if ($blocks_left == 1) { + $code .= "vmovdqu8 @{[XWORD($SRCREG)]},`$dst_offset`($OUTP,$DATA_OFFSET,1){$MASK}\n"; + } elsif ($blocks_left == 2) { + $code .= "vmovdqu8 @{[YWORD($SRCREG)]},`$dst_offset`($OUTP,$DATA_OFFSET,1){$MASK}\n"; + } elsif ($blocks_left == 3 || $blocks_left == 4) { + $code .= "vmovdqu8 $SRCREG,`$dst_offset`($OUTP,$DATA_OFFSET,1){$MASK}\n"; + } +} + +# ;;; =========================================================================== +# ;;; Handles AES encryption rounds +# ;;; It handles special cases: the last and first rounds +# ;;; Optionally, it performs XOR with data after the last AES round. +# ;;; Uses NROUNDS parameter to check what needs to be done for the current round. +# ;;; If 3 blocks are trailing then operation on whole ZMM is performed (4 blocks). +sub ZMM_AESENC_ROUND_BLOCKS_0_16 { + my $L0B0_3 = $_[0]; # [in/out] zmm; blocks 0 to 3 + my $L0B4_7 = $_[1]; # [in/out] zmm; blocks 4 to 7 + my $L0B8_11 = $_[2]; # [in/out] zmm; blocks 8 to 11 + my $L0B12_15 = $_[3]; # [in/out] zmm; blocks 12 to 15 + my $KEY = $_[4]; # [in] zmm containing round key + my $ROUND = $_[5]; # [in] round number + my $D0_3 = $_[6]; # [in] zmm or no_data; plain/cipher text blocks 0-3 + my $D4_7 = $_[7]; # [in] zmm or no_data; plain/cipher text blocks 4-7 + my $D8_11 = $_[8]; # [in] zmm or no_data; plain/cipher text blocks 8-11 + my $D12_15 = $_[9]; # [in] zmm or no_data; plain/cipher text blocks 12-15 + my $NUMBL = $_[10]; # [in] number of blocks; numerical value + my $NROUNDS = $_[11]; # [in] number of rounds; numerical value + + # ;;; === first AES round + if ($ROUND < 1) { + + # ;; round 0 + &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16( + $NUMBL, "vpxorq", $L0B0_3, $L0B4_7, $L0B8_11, $L0B12_15, $L0B0_3, + $L0B4_7, $L0B8_11, $L0B12_15, $KEY, $KEY, $KEY, $KEY); + } + + # ;;; === middle AES rounds + if ($ROUND >= 1 && $ROUND <= $NROUNDS) { + + # ;; rounds 1 to 9/11/13 + &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16( + $NUMBL, "vaesenc", $L0B0_3, $L0B4_7, $L0B8_11, $L0B12_15, $L0B0_3, + $L0B4_7, $L0B8_11, $L0B12_15, $KEY, $KEY, $KEY, $KEY); + } + + # ;;; === last AES round + if ($ROUND > $NROUNDS) { + + # ;; the last round - mix enclast with text xor's + &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16( + $NUMBL, "vaesenclast", $L0B0_3, $L0B4_7, $L0B8_11, $L0B12_15, $L0B0_3, + $L0B4_7, $L0B8_11, $L0B12_15, $KEY, $KEY, $KEY, $KEY); + + # ;;; === XOR with data + if ( ($D0_3 ne "no_data") + && ($D4_7 ne "no_data") + && ($D8_11 ne "no_data") + && ($D12_15 ne "no_data")) + { + &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16( + $NUMBL, "vpxorq", $L0B0_3, $L0B4_7, $L0B8_11, $L0B12_15, $L0B0_3, + $L0B4_7, $L0B8_11, $L0B12_15, $D0_3, $D4_7, $D8_11, $D12_15); + } + } +} + +# ;;; Horizontal XOR - 4 x 128bits xored together +sub VHPXORI4x128 { + my $REG = $_[0]; # [in/out] ZMM with 4x128bits to xor; 128bit output + my $TMP = $_[1]; # [clobbered] ZMM temporary register + $code .= <<___; + vextracti64x4 \$1,$REG,@{[YWORD($TMP)]} + vpxorq @{[YWORD($TMP)]},@{[YWORD($REG)]},@{[YWORD($REG)]} + vextracti32x4 \$1,@{[YWORD($REG)]},@{[XWORD($TMP)]} + vpxorq @{[XWORD($TMP)]},@{[XWORD($REG)]},@{[XWORD($REG)]} +___ +} + +# ;;; AVX512 reduction macro +sub VCLMUL_REDUCE { + my $OUT = $_[0]; # [out] zmm/ymm/xmm: result (must not be $TMP1 or $HI128) + my $POLY = $_[1]; # [in] zmm/ymm/xmm: polynomial + my $HI128 = $_[2]; # [in] zmm/ymm/xmm: high 128b of hash to reduce + my $LO128 = $_[3]; # [in] zmm/ymm/xmm: low 128b of hash to reduce + my $TMP0 = $_[4]; # [in] zmm/ymm/xmm: temporary register + my $TMP1 = $_[5]; # [in] zmm/ymm/xmm: temporary register + + $code .= <<___; + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; first phase of the reduction + vpclmulqdq \$0x01,$LO128,$POLY,$TMP0 + vpslldq \$8,$TMP0,$TMP0 # ; shift-L 2 DWs + vpxorq $TMP0,$LO128,$TMP0 # ; first phase of the reduction complete + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; second phase of the reduction + vpclmulqdq \$0x00,$TMP0,$POLY,$TMP1 + vpsrldq \$4,$TMP1,$TMP1 # ; shift-R only 1-DW to obtain 2-DWs shift-R + vpclmulqdq \$0x10,$TMP0,$POLY,$OUT + vpslldq \$4,$OUT,$OUT # ; shift-L 1-DW to obtain result with no shifts + vpternlogq \$0x96,$HI128,$TMP1,$OUT # ; OUT/GHASH = OUT xor TMP1 xor HI128 + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +___ +} + +# ;; =========================================================================== +# ;; schoolbook multiply of 16 blocks (16 x 16 bytes) +# ;; - it is assumed that data read from $INPTR is already shuffled and +# ;; $INPTR address is 64 byte aligned +# ;; - there is an option to pass ready blocks through ZMM registers too. +# ;; 4 extra parameters need to be passed in such case and 21st ($ZTMP9) argument can be empty +sub GHASH_16 { + my $TYPE = $_[0]; # [in] ghash type: start (xor hash), mid, end (same as mid; no reduction), + # end_reduce (end with reduction), start_reduce + my $GH = $_[1]; # [in/out] ZMM ghash sum: high 128-bits + my $GM = $_[2]; # [in/out] ZMM ghash sum: middle 128-bits + my $GL = $_[3]; # [in/out] ZMM ghash sum: low 128-bits + my $INPTR = $_[4]; # [in] data input pointer + my $INOFF = $_[5]; # [in] data input offset + my $INDIS = $_[6]; # [in] data input displacement + my $HKPTR = $_[7]; # [in] hash key pointer + my $HKOFF = $_[8]; # [in] hash key offset (can be either numerical offset, or register containing offset) + my $HKDIS = $_[9]; # [in] hash key displacement + my $HASH = $_[10]; # [in/out] ZMM hash value in/out + my $ZTMP0 = $_[11]; # [clobbered] temporary ZMM + my $ZTMP1 = $_[12]; # [clobbered] temporary ZMM + my $ZTMP2 = $_[13]; # [clobbered] temporary ZMM + my $ZTMP3 = $_[14]; # [clobbered] temporary ZMM + my $ZTMP4 = $_[15]; # [clobbered] temporary ZMM + my $ZTMP5 = $_[16]; # [clobbered] temporary ZMM + my $ZTMP6 = $_[17]; # [clobbered] temporary ZMM + my $ZTMP7 = $_[18]; # [clobbered] temporary ZMM + my $ZTMP8 = $_[19]; # [clobbered] temporary ZMM + my $ZTMP9 = $_[20]; # [clobbered] temporary ZMM, can be empty if 4 extra parameters below are provided + my $DAT0 = $_[21]; # [in] ZMM with 4 blocks of input data (INPTR, INOFF, INDIS unused) + my $DAT1 = $_[22]; # [in] ZMM with 4 blocks of input data (INPTR, INOFF, INDIS unused) + my $DAT2 = $_[23]; # [in] ZMM with 4 blocks of input data (INPTR, INOFF, INDIS unused) + my $DAT3 = $_[24]; # [in] ZMM with 4 blocks of input data (INPTR, INOFF, INDIS unused) + + my $start_ghash = 0; + my $do_reduction = 0; + if ($TYPE eq "start") { + $start_ghash = 1; + } + + if ($TYPE eq "start_reduce") { + $start_ghash = 1; + $do_reduction = 1; + } + + if ($TYPE eq "end_reduce") { + $do_reduction = 1; + } + + # ;; ghash blocks 0-3 + if (scalar(@_) == 21) { + $code .= "vmovdqa64 @{[EffectiveAddress($INPTR,$INOFF,($INDIS+0*64))]},$ZTMP9\n"; + } else { + $ZTMP9 = $DAT0; + } + + if ($start_ghash != 0) { + $code .= "vpxorq $HASH,$ZTMP9,$ZTMP9\n"; + } + $code .= <<___; + vmovdqu64 @{[EffectiveAddress($HKPTR,$HKOFF,($HKDIS+0*64))]},$ZTMP8 + vpclmulqdq \$0x11,$ZTMP8,$ZTMP9,$ZTMP0 # ; T0H = a1*b1 + vpclmulqdq \$0x00,$ZTMP8,$ZTMP9,$ZTMP1 # ; T0L = a0*b0 + vpclmulqdq \$0x01,$ZTMP8,$ZTMP9,$ZTMP2 # ; T0M1 = a1*b0 + vpclmulqdq \$0x10,$ZTMP8,$ZTMP9,$ZTMP3 # ; T0M2 = a0*b1 +___ + + # ;; ghash blocks 4-7 + if (scalar(@_) == 21) { + $code .= "vmovdqa64 @{[EffectiveAddress($INPTR,$INOFF,($INDIS+1*64))]},$ZTMP9\n"; + } else { + $ZTMP9 = $DAT1; + } + $code .= <<___; + vmovdqu64 @{[EffectiveAddress($HKPTR,$HKOFF,($HKDIS+1*64))]},$ZTMP8 + vpclmulqdq \$0x11,$ZTMP8,$ZTMP9,$ZTMP4 # ; T1H = a1*b1 + vpclmulqdq \$0x00,$ZTMP8,$ZTMP9,$ZTMP5 # ; T1L = a0*b0 + vpclmulqdq \$0x01,$ZTMP8,$ZTMP9,$ZTMP6 # ; T1M1 = a1*b0 + vpclmulqdq \$0x10,$ZTMP8,$ZTMP9,$ZTMP7 # ; T1M2 = a0*b1 +___ + + # ;; update sums + if ($start_ghash != 0) { + $code .= <<___; + vpxorq $ZTMP6,$ZTMP2,$GM # ; GM = T0M1 + T1M1 + vpxorq $ZTMP4,$ZTMP0,$GH # ; GH = T0H + T1H + vpxorq $ZTMP5,$ZTMP1,$GL # ; GL = T0L + T1L + vpternlogq \$0x96,$ZTMP7,$ZTMP3,$GM # ; GM = T0M2 + T1M1 +___ + } else { # ;; mid, end, end_reduce + $code .= <<___; + vpternlogq \$0x96,$ZTMP6,$ZTMP2,$GM # ; GM += T0M1 + T1M1 + vpternlogq \$0x96,$ZTMP4,$ZTMP0,$GH # ; GH += T0H + T1H + vpternlogq \$0x96,$ZTMP5,$ZTMP1,$GL # ; GL += T0L + T1L + vpternlogq \$0x96,$ZTMP7,$ZTMP3,$GM # ; GM += T0M2 + T1M1 +___ + } + + # ;; ghash blocks 8-11 + if (scalar(@_) == 21) { + $code .= "vmovdqa64 @{[EffectiveAddress($INPTR,$INOFF,($INDIS+2*64))]},$ZTMP9\n"; + } else { + $ZTMP9 = $DAT2; + } + $code .= <<___; + vmovdqu64 @{[EffectiveAddress($HKPTR,$HKOFF,($HKDIS+2*64))]},$ZTMP8 + vpclmulqdq \$0x11,$ZTMP8,$ZTMP9,$ZTMP0 # ; T0H = a1*b1 + vpclmulqdq \$0x00,$ZTMP8,$ZTMP9,$ZTMP1 # ; T0L = a0*b0 + vpclmulqdq \$0x01,$ZTMP8,$ZTMP9,$ZTMP2 # ; T0M1 = a1*b0 + vpclmulqdq \$0x10,$ZTMP8,$ZTMP9,$ZTMP3 # ; T0M2 = a0*b1 +___ + + # ;; ghash blocks 12-15 + if (scalar(@_) == 21) { + $code .= "vmovdqa64 @{[EffectiveAddress($INPTR,$INOFF,($INDIS+3*64))]},$ZTMP9\n"; + } else { + $ZTMP9 = $DAT3; + } + $code .= <<___; + vmovdqu64 @{[EffectiveAddress($HKPTR,$HKOFF,($HKDIS+3*64))]},$ZTMP8 + vpclmulqdq \$0x11,$ZTMP8,$ZTMP9,$ZTMP4 # ; T1H = a1*b1 + vpclmulqdq \$0x00,$ZTMP8,$ZTMP9,$ZTMP5 # ; T1L = a0*b0 + vpclmulqdq \$0x01,$ZTMP8,$ZTMP9,$ZTMP6 # ; T1M1 = a1*b0 + vpclmulqdq \$0x10,$ZTMP8,$ZTMP9,$ZTMP7 # ; T1M2 = a0*b1 + # ;; update sums + vpternlogq \$0x96,$ZTMP6,$ZTMP2,$GM # ; GM += T0M1 + T1M1 + vpternlogq \$0x96,$ZTMP4,$ZTMP0,$GH # ; GH += T0H + T1H + vpternlogq \$0x96,$ZTMP5,$ZTMP1,$GL # ; GL += T0L + T1L + vpternlogq \$0x96,$ZTMP7,$ZTMP3,$GM # ; GM += T0M2 + T1M1 +___ + if ($do_reduction != 0) { + $code .= <<___; + # ;; integrate GM into GH and GL + vpsrldq \$8,$GM,$ZTMP0 + vpslldq \$8,$GM,$ZTMP1 + vpxorq $ZTMP0,$GH,$GH + vpxorq $ZTMP1,$GL,$GL +___ + + # ;; add GH and GL 128-bit words horizontally + &VHPXORI4x128($GH, $ZTMP0); + &VHPXORI4x128($GL, $ZTMP1); + + # ;; reduction + $code .= "vmovdqa64 POLY2(%rip),@{[XWORD($ZTMP2)]}\n"; + &VCLMUL_REDUCE(&XWORD($HASH), &XWORD($ZTMP2), &XWORD($GH), &XWORD($GL), &XWORD($ZTMP0), &XWORD($ZTMP1)); + } +} + +# ;; =========================================================================== +# ;; GHASH 1 to 16 blocks of cipher text +# ;; - performs reduction at the end +# ;; - it doesn't load the data and it assumed it is already loaded and shuffled +sub GHASH_1_TO_16 { + my $GCM128_CTX = $_[0]; # [in] pointer to expanded keys + my $GHASH = $_[1]; # [out] ghash output + my $T0H = $_[2]; # [clobbered] temporary ZMM + my $T0L = $_[3]; # [clobbered] temporary ZMM + my $T0M1 = $_[4]; # [clobbered] temporary ZMM + my $T0M2 = $_[5]; # [clobbered] temporary ZMM + my $T1H = $_[6]; # [clobbered] temporary ZMM + my $T1L = $_[7]; # [clobbered] temporary ZMM + my $T1M1 = $_[8]; # [clobbered] temporary ZMM + my $T1M2 = $_[9]; # [clobbered] temporary ZMM + my $HK = $_[10]; # [clobbered] temporary ZMM + my $AAD_HASH_IN = $_[11]; # [in] input hash value + my @CIPHER_IN; + $CIPHER_IN[0] = $_[12]; # [in] ZMM with cipher text blocks 0-3 + $CIPHER_IN[1] = $_[13]; # [in] ZMM with cipher text blocks 4-7 + $CIPHER_IN[2] = $_[14]; # [in] ZMM with cipher text blocks 8-11 + $CIPHER_IN[3] = $_[15]; # [in] ZMM with cipher text blocks 12-15 + my $NUM_BLOCKS = $_[16]; # [in] numerical value, number of blocks + my $GH = $_[17]; # [in] ZMM with hi product part + my $GM = $_[18]; # [in] ZMM with mid product part + my $GL = $_[19]; # [in] ZMM with lo product part + + die "GHASH_1_TO_16: num_blocks is out of bounds = $NUM_BLOCKS\n" if ($NUM_BLOCKS > 16 || $NUM_BLOCKS < 0); + + if (scalar(@_) == 17) { + $code .= "vpxorq $AAD_HASH_IN,$CIPHER_IN[0],$CIPHER_IN[0]\n"; + } + + if ($NUM_BLOCKS == 16) { + $code .= <<___; + vmovdqu64 @{[HashKeyByIdx($NUM_BLOCKS, $GCM128_CTX)]},$HK + vpclmulqdq \$0x11,$HK,$CIPHER_IN[0],$T0H # ; H = a1*b1 + vpclmulqdq \$0x00,$HK,$CIPHER_IN[0],$T0L # ; L = a0*b0 + vpclmulqdq \$0x01,$HK,$CIPHER_IN[0],$T0M1 # ; M1 = a1*b0 + vpclmulqdq \$0x10,$HK,$CIPHER_IN[0],$T0M2 # ; M2 = a0*b1 + vmovdqu64 @{[HashKeyByIdx($NUM_BLOCKS-1*4, $GCM128_CTX)]},$HK + vpclmulqdq \$0x11,$HK,$CIPHER_IN[1],$T1H # ; H = a1*b1 + vpclmulqdq \$0x00,$HK,$CIPHER_IN[1],$T1L # ; L = a0*b0 + vpclmulqdq \$0x01,$HK,$CIPHER_IN[1],$T1M1 # ; M1 = a1*b0 + vpclmulqdq \$0x10,$HK,$CIPHER_IN[1],$T1M2 # ; M2 = a0*b1 + vmovdqu64 @{[HashKeyByIdx($NUM_BLOCKS-2*4, $GCM128_CTX)]},$HK + vpclmulqdq \$0x11,$HK,$CIPHER_IN[2],$CIPHER_IN[0] # ; H = a1*b1 + vpclmulqdq \$0x00,$HK,$CIPHER_IN[2],$CIPHER_IN[1] # ; L = a0*b0 + vpternlogq \$0x96,$T1H,$CIPHER_IN[0],$T0H + vpternlogq \$0x96,$T1L,$CIPHER_IN[1],$T0L + vpclmulqdq \$0x01,$HK,$CIPHER_IN[2],$CIPHER_IN[0] # ; M1 = a1*b0 + vpclmulqdq \$0x10,$HK,$CIPHER_IN[2],$CIPHER_IN[1] # ; M2 = a0*b1 + vpternlogq \$0x96,$T1M1,$CIPHER_IN[0],$T0M1 + vpternlogq \$0x96,$T1M2,$CIPHER_IN[1],$T0M2 + vmovdqu64 @{[HashKeyByIdx($NUM_BLOCKS-3*4, $GCM128_CTX)]},$HK + vpclmulqdq \$0x11,$HK,$CIPHER_IN[3],$T1H # ; H = a1*b1 + vpclmulqdq \$0x00,$HK,$CIPHER_IN[3],$T1L # ; L = a0*b0 + vpclmulqdq \$0x01,$HK,$CIPHER_IN[3],$T1M1 # ; M1 = a1*b0 + vpclmulqdq \$0x10,$HK,$CIPHER_IN[3],$T1M2 # ; M2 = a0*b1 + vpxorq $T1H,$T0H,$T1H + vpxorq $T1L,$T0L,$T1L + vpxorq $T1M1,$T0M1,$T1M1 + vpxorq $T1M2,$T0M2,$T1M2 +___ + } elsif ($NUM_BLOCKS >= 12) { + $code .= <<___; + vmovdqu64 @{[HashKeyByIdx($NUM_BLOCKS, $GCM128_CTX)]},$HK + vpclmulqdq \$0x11,$HK,$CIPHER_IN[0],$T0H # ; H = a1*b1 + vpclmulqdq \$0x00,$HK,$CIPHER_IN[0],$T0L # ; L = a0*b0 + vpclmulqdq \$0x01,$HK,$CIPHER_IN[0],$T0M1 # ; M1 = a1*b0 + vpclmulqdq \$0x10,$HK,$CIPHER_IN[0],$T0M2 # ; M2 = a0*b1 + vmovdqu64 @{[HashKeyByIdx($NUM_BLOCKS-1*4, $GCM128_CTX)]},$HK + vpclmulqdq \$0x11,$HK,$CIPHER_IN[1],$T1H # ; H = a1*b1 + vpclmulqdq \$0x00,$HK,$CIPHER_IN[1],$T1L # ; L = a0*b0 + vpclmulqdq \$0x01,$HK,$CIPHER_IN[1],$T1M1 # ; M1 = a1*b0 + vpclmulqdq \$0x10,$HK,$CIPHER_IN[1],$T1M2 # ; M2 = a0*b1 + vmovdqu64 @{[HashKeyByIdx($NUM_BLOCKS-2*4, $GCM128_CTX)]},$HK + vpclmulqdq \$0x11,$HK,$CIPHER_IN[2],$CIPHER_IN[0] # ; H = a1*b1 + vpclmulqdq \$0x00,$HK,$CIPHER_IN[2],$CIPHER_IN[1] # ; L = a0*b0 + vpternlogq \$0x96,$T0H,$CIPHER_IN[0],$T1H + vpternlogq \$0x96,$T0L,$CIPHER_IN[1],$T1L + vpclmulqdq \$0x01,$HK,$CIPHER_IN[2],$CIPHER_IN[0] # ; M1 = a1*b0 + vpclmulqdq \$0x10,$HK,$CIPHER_IN[2],$CIPHER_IN[1] # ; M2 = a0*b1 + vpternlogq \$0x96,$T0M1,$CIPHER_IN[0],$T1M1 + vpternlogq \$0x96,$T0M2,$CIPHER_IN[1],$T1M2 +___ + } elsif ($NUM_BLOCKS >= 8) { + $code .= <<___; + vmovdqu64 @{[HashKeyByIdx($NUM_BLOCKS, $GCM128_CTX)]},$HK + vpclmulqdq \$0x11,$HK,$CIPHER_IN[0],$T0H # ; H = a1*b1 + vpclmulqdq \$0x00,$HK,$CIPHER_IN[0],$T0L # ; L = a0*b0 + vpclmulqdq \$0x01,$HK,$CIPHER_IN[0],$T0M1 # ; M1 = a1*b0 + vpclmulqdq \$0x10,$HK,$CIPHER_IN[0],$T0M2 # ; M2 = a0*b1 + vmovdqu64 @{[HashKeyByIdx($NUM_BLOCKS-1*4, $GCM128_CTX)]},$HK + vpclmulqdq \$0x11,$HK,$CIPHER_IN[1],$T1H # ; H = a1*b1 + vpclmulqdq \$0x00,$HK,$CIPHER_IN[1],$T1L # ; L = a0*b0 + vpclmulqdq \$0x01,$HK,$CIPHER_IN[1],$T1M1 # ; M1 = a1*b0 + vpclmulqdq \$0x10,$HK,$CIPHER_IN[1],$T1M2 # ; M2 = a0*b1 + vpxorq $T1H,$T0H,$T1H + vpxorq $T1L,$T0L,$T1L + vpxorq $T1M1,$T0M1,$T1M1 + vpxorq $T1M2,$T0M2,$T1M2 +___ + } elsif ($NUM_BLOCKS >= 4) { + $code .= <<___; + vmovdqu64 @{[HashKeyByIdx($NUM_BLOCKS, $GCM128_CTX)]},$HK + vpclmulqdq \$0x11,$HK,$CIPHER_IN[0],$T1H # ; H = a1*b1 + vpclmulqdq \$0x00,$HK,$CIPHER_IN[0],$T1L # ; L = a0*b0 + vpclmulqdq \$0x01,$HK,$CIPHER_IN[0],$T1M1 # ; M1 = a1*b0 + vpclmulqdq \$0x10,$HK,$CIPHER_IN[0],$T1M2 # ; M2 = a0*b1 +___ + } + + # ;; T1H/L/M1/M2 - hold current product sums (provided $NUM_BLOCKS >= 4) + my $blocks_left = ($NUM_BLOCKS % 4); + if ($blocks_left > 0) { + + # ;; ===================================================== + # ;; There are 1, 2 or 3 blocks left to process. + # ;; It may also be that they are the only blocks to process. + + # ;; Set hash key and register index position for the remaining 1 to 3 blocks + my $reg_idx = ($NUM_BLOCKS / 4); + my $REG_IN = $CIPHER_IN[$reg_idx]; + + if ($blocks_left == 1) { + $code .= <<___; + vmovdqu64 @{[HashKeyByIdx($blocks_left, $GCM128_CTX)]},@{[XWORD($HK)]} + vpclmulqdq \$0x01,@{[XWORD($HK)]},@{[XWORD($REG_IN)]},@{[XWORD($T0M1)]} # ; M1 = a1*b0 + vpclmulqdq \$0x10,@{[XWORD($HK)]},@{[XWORD($REG_IN)]},@{[XWORD($T0M2)]} # ; M2 = a0*b1 + vpclmulqdq \$0x11,@{[XWORD($HK)]},@{[XWORD($REG_IN)]},@{[XWORD($T0H)]} # ; H = a1*b1 + vpclmulqdq \$0x00,@{[XWORD($HK)]},@{[XWORD($REG_IN)]},@{[XWORD($T0L)]} # ; L = a0*b0 +___ + } elsif ($blocks_left == 2) { + $code .= <<___; + vmovdqu64 @{[HashKeyByIdx($blocks_left, $GCM128_CTX)]},@{[YWORD($HK)]} + vpclmulqdq \$0x01,@{[YWORD($HK)]},@{[YWORD($REG_IN)]},@{[YWORD($T0M1)]} # ; M1 = a1*b0 + vpclmulqdq \$0x10,@{[YWORD($HK)]},@{[YWORD($REG_IN)]},@{[YWORD($T0M2)]} # ; M2 = a0*b1 + vpclmulqdq \$0x11,@{[YWORD($HK)]},@{[YWORD($REG_IN)]},@{[YWORD($T0H)]} # ; H = a1*b1 + vpclmulqdq \$0x00,@{[YWORD($HK)]},@{[YWORD($REG_IN)]},@{[YWORD($T0L)]} # ; L = a0*b0 +___ + } else { # ; blocks_left == 3 + $code .= <<___; + vmovdqu64 @{[HashKeyByIdx($blocks_left, $GCM128_CTX)]},@{[YWORD($HK)]} + vinserti64x2 \$2,@{[HashKeyByIdx($blocks_left-2, $GCM128_CTX)]},$HK,$HK + vpclmulqdq \$0x01,$HK,$REG_IN,$T0M1 # ; M1 = a1*b0 + vpclmulqdq \$0x10,$HK,$REG_IN,$T0M2 # ; M2 = a0*b1 + vpclmulqdq \$0x11,$HK,$REG_IN,$T0H # ; H = a1*b1 + vpclmulqdq \$0x00,$HK,$REG_IN,$T0L # ; L = a0*b0 +___ + } + + if (scalar(@_) == 20) { + + # ;; *** GH/GM/GL passed as arguments + if ($NUM_BLOCKS >= 4) { + $code .= <<___; + # ;; add ghash product sums from the first 4, 8 or 12 blocks + vpxorq $T1M1,$T0M1,$T0M1 + vpternlogq \$0x96,$T1M2,$GM,$T0M2 + vpternlogq \$0x96,$T1H,$GH,$T0H + vpternlogq \$0x96,$T1L,$GL,$T0L +___ + } else { + $code .= <<___; + vpxorq $GM,$T0M1,$T0M1 + vpxorq $GH,$T0H,$T0H + vpxorq $GL,$T0L,$T0L +___ + } + } else { + + # ;; *** GH/GM/GL NOT passed as arguments + if ($NUM_BLOCKS >= 4) { + $code .= <<___; + # ;; add ghash product sums from the first 4, 8 or 12 blocks + vpxorq $T1M1,$T0M1,$T0M1 + vpxorq $T1M2,$T0M2,$T0M2 + vpxorq $T1H,$T0H,$T0H + vpxorq $T1L,$T0L,$T0L +___ + } + } + $code .= <<___; + # ;; integrate TM into TH and TL + vpxorq $T0M2,$T0M1,$T0M1 + vpsrldq \$8,$T0M1,$T1M1 + vpslldq \$8,$T0M1,$T1M2 + vpxorq $T1M1,$T0H,$T0H + vpxorq $T1M2,$T0L,$T0L +___ + } else { + + # ;; ===================================================== + # ;; number of blocks is 4, 8, 12 or 16 + # ;; T1H/L/M1/M2 include product sums not T0H/L/M1/M2 + if (scalar(@_) == 20) { + $code .= <<___; + # ;; *** GH/GM/GL passed as arguments + vpxorq $GM,$T1M1,$T1M1 + vpxorq $GH,$T1H,$T1H + vpxorq $GL,$T1L,$T1L +___ + } + $code .= <<___; + # ;; integrate TM into TH and TL + vpxorq $T1M2,$T1M1,$T1M1 + vpsrldq \$8,$T1M1,$T0M1 + vpslldq \$8,$T1M1,$T0M2 + vpxorq $T0M1,$T1H,$T0H + vpxorq $T0M2,$T1L,$T0L +___ + } + + # ;; add TH and TL 128-bit words horizontally + &VHPXORI4x128($T0H, $T1M1); + &VHPXORI4x128($T0L, $T1M2); + + # ;; reduction + $code .= "vmovdqa64 POLY2(%rip),@{[XWORD($HK)]}\n"; + &VCLMUL_REDUCE( + @{[XWORD($GHASH)]}, + @{[XWORD($HK)]}, + @{[XWORD($T0H)]}, + @{[XWORD($T0L)]}, + @{[XWORD($T0M1)]}, + @{[XWORD($T0M2)]}); +} + +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +# ;; GHASH_MUL MACRO to implement: Data*HashKey mod (x^128 + x^127 + x^126 +x^121 + 1) +# ;; Input: A and B (128-bits each, bit-reflected) +# ;; Output: C = A*B*x mod poly, (i.e. >>1 ) +# ;; To compute GH = GH*HashKey mod poly, give HK = HashKey<<1 mod poly as input +# ;; GH = GH * HK * x mod poly which is equivalent to GH*HashKey mod poly. +# ;; +# ;; Refer to [3] for more details. +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +sub GHASH_MUL { + my $GH = $_[0]; #; [in/out] xmm/ymm/zmm with multiply operand(s) (128-bits) + my $HK = $_[1]; #; [in] xmm/ymm/zmm with hash key value(s) (128-bits) + my $T1 = $_[2]; #; [clobbered] xmm/ymm/zmm + my $T2 = $_[3]; #; [clobbered] xmm/ymm/zmm + my $T3 = $_[4]; #; [clobbered] xmm/ymm/zmm + + $code .= <<___; + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + vpclmulqdq \$0x11,$HK,$GH,$T1 # ; $T1 = a1*b1 + vpclmulqdq \$0x00,$HK,$GH,$T2 # ; $T2 = a0*b0 + vpclmulqdq \$0x01,$HK,$GH,$T3 # ; $T3 = a1*b0 + vpclmulqdq \$0x10,$HK,$GH,$GH # ; $GH = a0*b1 + vpxorq $T3,$GH,$GH + + vpsrldq \$8,$GH,$T3 # ; shift-R $GH 2 DWs + vpslldq \$8,$GH,$GH # ; shift-L $GH 2 DWs + vpxorq $T3,$T1,$T1 + vpxorq $T2,$GH,$GH + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;first phase of the reduction + vmovdqu64 POLY2(%rip),$T3 + + vpclmulqdq \$0x01,$GH,$T3,$T2 + vpslldq \$8,$T2,$T2 # ; shift-L $T2 2 DWs + vpxorq $T2,$GH,$GH # ; first phase of the reduction complete + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;second phase of the reduction + vpclmulqdq \$0x00,$GH,$T3,$T2 + vpsrldq \$4,$T2,$T2 # ; shift-R only 1-DW to obtain 2-DWs shift-R + vpclmulqdq \$0x10,$GH,$T3,$GH + vpslldq \$4,$GH,$GH # ; Shift-L 1-DW to obtain result with no shifts + # ; second phase of the reduction complete, the result is in $GH + vpternlogq \$0x96,$T2,$T1,$GH # ; GH = GH xor T1 xor T2 + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +___ +} + +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +# ;;; PRECOMPUTE computes HashKey_i +sub PRECOMPUTE { + my $GCM128_CTX = $_[0]; #; [in/out] context pointer, hkeys content updated + my $HK = $_[1]; #; [in] xmm, hash key + my $T1 = $_[2]; #; [clobbered] xmm + my $T2 = $_[3]; #; [clobbered] xmm + my $T3 = $_[4]; #; [clobbered] xmm + my $T4 = $_[5]; #; [clobbered] xmm + my $T5 = $_[6]; #; [clobbered] xmm + my $T6 = $_[7]; #; [clobbered] xmm + + my $ZT1 = &ZWORD($T1); + my $ZT2 = &ZWORD($T2); + my $ZT3 = &ZWORD($T3); + my $ZT4 = &ZWORD($T4); + my $ZT5 = &ZWORD($T5); + my $ZT6 = &ZWORD($T6); + + my $YT1 = &YWORD($T1); + my $YT2 = &YWORD($T2); + my $YT3 = &YWORD($T3); + my $YT4 = &YWORD($T4); + my $YT5 = &YWORD($T5); + my $YT6 = &YWORD($T6); + + $code .= <<___; + vshufi32x4 \$0x00,@{[YWORD($HK)]},@{[YWORD($HK)]},$YT5 + vmovdqa $YT5,$YT4 +___ + + # ;; calculate HashKey^2<<1 mod poly + &GHASH_MUL($YT4, $YT5, $YT1, $YT2, $YT3); + + $code .= <<___; + vmovdqu64 $T4,@{[HashKeyByIdx(2,$GCM128_CTX)]} + vinserti64x2 \$1,$HK,$YT4,$YT5 + vmovdqa64 $YT5,$YT6 # ;; YT6 = HashKey | HashKey^2 +___ + + # ;; use 2x128-bit computation + # ;; calculate HashKey^4<<1 mod poly, HashKey^3<<1 mod poly + &GHASH_MUL($YT5, $YT4, $YT1, $YT2, $YT3); # ;; YT5 = HashKey^3 | HashKey^4 + + $code .= <<___; + vmovdqu64 $YT5,@{[HashKeyByIdx(4,$GCM128_CTX)]} + + vinserti64x4 \$1,$YT6,$ZT5,$ZT5 # ;; ZT5 = YT6 | YT5 + + # ;; switch to 4x128-bit computations now + vshufi64x2 \$0x00,$ZT5,$ZT5,$ZT4 # ;; broadcast HashKey^4 across all ZT4 + vmovdqa64 $ZT5,$ZT6 # ;; save HashKey^4 to HashKey^1 in ZT6 +___ + + # ;; calculate HashKey^5<<1 mod poly, HashKey^6<<1 mod poly, ... HashKey^8<<1 mod poly + &GHASH_MUL($ZT5, $ZT4, $ZT1, $ZT2, $ZT3); + $code .= <<___; + vmovdqu64 $ZT5,@{[HashKeyByIdx(8,$GCM128_CTX)]} # ;; HashKey^8 to HashKey^5 in ZT5 now + vshufi64x2 \$0x00,$ZT5,$ZT5,$ZT4 # ;; broadcast HashKey^8 across all ZT4 +___ + + # ;; calculate HashKey^9<<1 mod poly, HashKey^10<<1 mod poly, ... HashKey^16<<1 mod poly + # ;; use HashKey^8 as multiplier against ZT6 and ZT5 - this allows deeper ooo execution + + # ;; compute HashKey^(12), HashKey^(11), ... HashKey^(9) + &GHASH_MUL($ZT6, $ZT4, $ZT1, $ZT2, $ZT3); + $code .= "vmovdqu64 $ZT6,@{[HashKeyByIdx(12,$GCM128_CTX)]}\n"; + + # ;; compute HashKey^(16), HashKey^(15), ... HashKey^(13) + &GHASH_MUL($ZT5, $ZT4, $ZT1, $ZT2, $ZT3); + $code .= "vmovdqu64 $ZT5,@{[HashKeyByIdx(16,$GCM128_CTX)]}\n"; + + # ; Hkeys 17..48 will be precomputed somewhere else as context can hold only 16 hkeys +} + +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +# ;; READ_SMALL_DATA_INPUT +# ;; Packs xmm register with data when data input is less or equal to 16 bytes +# ;; Returns 0 if data has length 0 +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +sub READ_SMALL_DATA_INPUT { + my $OUTPUT = $_[0]; # [out] xmm register + my $INPUT = $_[1]; # [in] buffer pointer to read from + my $LENGTH = $_[2]; # [in] number of bytes to read + my $TMP1 = $_[3]; # [clobbered] + my $TMP2 = $_[4]; # [clobbered] + my $MASK = $_[5]; # [out] k1 to k7 register to store the partial block mask + + $code .= <<___; + mov \$16,@{[DWORD($TMP2)]} + lea byte_len_to_mask_table(%rip),$TMP1 + cmp $TMP2,$LENGTH + cmovc $LENGTH,$TMP2 +___ + if ($win64) { + $code .= <<___; + add $TMP2,$TMP1 + add $TMP2,$TMP1 + kmovw ($TMP1),$MASK +___ + } else { + $code .= "kmovw ($TMP1,$TMP2,2),$MASK\n"; + } + $code .= "vmovdqu8 ($INPUT),${OUTPUT}{$MASK}{z}\n"; +} + +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +# CALC_AAD_HASH: Calculates the hash of the data which will not be encrypted. +# Input: The input data (A_IN), that data's length (A_LEN), and the hash key (HASH_KEY). +# Output: The hash of the data (AAD_HASH). +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +sub CALC_AAD_HASH { + my $A_IN = $_[0]; # [in] AAD text pointer + my $A_LEN = $_[1]; # [in] AAD length + my $AAD_HASH = $_[2]; # [in/out] xmm ghash value + my $GCM128_CTX = $_[3]; # [in] pointer to context + my $ZT0 = $_[4]; # [clobbered] ZMM register + my $ZT1 = $_[5]; # [clobbered] ZMM register + my $ZT2 = $_[6]; # [clobbered] ZMM register + my $ZT3 = $_[7]; # [clobbered] ZMM register + my $ZT4 = $_[8]; # [clobbered] ZMM register + my $ZT5 = $_[9]; # [clobbered] ZMM register + my $ZT6 = $_[10]; # [clobbered] ZMM register + my $ZT7 = $_[11]; # [clobbered] ZMM register + my $ZT8 = $_[12]; # [clobbered] ZMM register + my $ZT9 = $_[13]; # [clobbered] ZMM register + my $ZT10 = $_[14]; # [clobbered] ZMM register + my $ZT11 = $_[15]; # [clobbered] ZMM register + my $ZT12 = $_[16]; # [clobbered] ZMM register + my $ZT13 = $_[17]; # [clobbered] ZMM register + my $ZT14 = $_[18]; # [clobbered] ZMM register + my $ZT15 = $_[19]; # [clobbered] ZMM register + my $ZT16 = $_[20]; # [clobbered] ZMM register + my $T1 = $_[21]; # [clobbered] GP register + my $T2 = $_[22]; # [clobbered] GP register + my $T3 = $_[23]; # [clobbered] GP register + my $MASKREG = $_[24]; # [clobbered] mask register + + my $HKEYS_READY = "%rbx"; + + my $SHFMSK = $ZT13; + + my $label_suffix = $label_count++; + + $code .= <<___; + mov $A_IN,$T1 # ; T1 = AAD + mov $A_LEN,$T2 # ; T2 = aadLen + or $T2,$T2 + jz .L_CALC_AAD_done_${label_suffix} + + xor $HKEYS_READY,$HKEYS_READY + vmovdqa64 SHUF_MASK(%rip),$SHFMSK + +.L_get_AAD_loop48x16_${label_suffix}: + cmp \$`(48*16)`,$T2 + jl .L_exit_AAD_loop48x16_${label_suffix} +___ + + $code .= <<___; + vmovdqu64 `64*0`($T1),$ZT1 # ; Blocks 0-3 + vmovdqu64 `64*1`($T1),$ZT2 # ; Blocks 4-7 + vmovdqu64 `64*2`($T1),$ZT3 # ; Blocks 8-11 + vmovdqu64 `64*3`($T1),$ZT4 # ; Blocks 12-15 + vpshufb $SHFMSK,$ZT1,$ZT1 + vpshufb $SHFMSK,$ZT2,$ZT2 + vpshufb $SHFMSK,$ZT3,$ZT3 + vpshufb $SHFMSK,$ZT4,$ZT4 +___ + + &precompute_hkeys_on_stack($GCM128_CTX, $HKEYS_READY, $ZT0, $ZT8, $ZT9, $ZT10, $ZT11, $ZT12, $ZT14, "all"); + $code .= "mov \$1,$HKEYS_READY\n"; + + &GHASH_16( + "start", $ZT5, $ZT6, $ZT7, + "NO_INPUT_PTR", "NO_INPUT_PTR", "NO_INPUT_PTR", "%rsp", + &HashKeyOffsetByIdx(48, "frame"), 0, "@{[ZWORD($AAD_HASH)]}", $ZT0, + $ZT8, $ZT9, $ZT10, $ZT11, + $ZT12, $ZT14, $ZT15, $ZT16, + "NO_ZMM", $ZT1, $ZT2, $ZT3, + $ZT4); + + $code .= <<___; + vmovdqu64 `16*16 + 64*0`($T1),$ZT1 # ; Blocks 16-19 + vmovdqu64 `16*16 + 64*1`($T1),$ZT2 # ; Blocks 20-23 + vmovdqu64 `16*16 + 64*2`($T1),$ZT3 # ; Blocks 24-27 + vmovdqu64 `16*16 + 64*3`($T1),$ZT4 # ; Blocks 28-31 + vpshufb $SHFMSK,$ZT1,$ZT1 + vpshufb $SHFMSK,$ZT2,$ZT2 + vpshufb $SHFMSK,$ZT3,$ZT3 + vpshufb $SHFMSK,$ZT4,$ZT4 +___ + + &GHASH_16( + "mid", $ZT5, $ZT6, $ZT7, + "NO_INPUT_PTR", "NO_INPUT_PTR", "NO_INPUT_PTR", "%rsp", + &HashKeyOffsetByIdx(32, "frame"), 0, "NO_HASH_IN_OUT", $ZT0, + $ZT8, $ZT9, $ZT10, $ZT11, + $ZT12, $ZT14, $ZT15, $ZT16, + "NO_ZMM", $ZT1, $ZT2, $ZT3, + $ZT4); + + $code .= <<___; + vmovdqu64 `32*16 + 64*0`($T1),$ZT1 # ; Blocks 32-35 + vmovdqu64 `32*16 + 64*1`($T1),$ZT2 # ; Blocks 36-39 + vmovdqu64 `32*16 + 64*2`($T1),$ZT3 # ; Blocks 40-43 + vmovdqu64 `32*16 + 64*3`($T1),$ZT4 # ; Blocks 44-47 + vpshufb $SHFMSK,$ZT1,$ZT1 + vpshufb $SHFMSK,$ZT2,$ZT2 + vpshufb $SHFMSK,$ZT3,$ZT3 + vpshufb $SHFMSK,$ZT4,$ZT4 +___ + + &GHASH_16( + "end_reduce", $ZT5, $ZT6, $ZT7, + "NO_INPUT_PTR", "NO_INPUT_PTR", "NO_INPUT_PTR", "%rsp", + &HashKeyOffsetByIdx(16, "frame"), 0, &ZWORD($AAD_HASH), $ZT0, + $ZT8, $ZT9, $ZT10, $ZT11, + $ZT12, $ZT14, $ZT15, $ZT16, + "NO_ZMM", $ZT1, $ZT2, $ZT3, + $ZT4); + + $code .= <<___; + sub \$`(48*16)`,$T2 + je .L_CALC_AAD_done_${label_suffix} + + add \$`(48*16)`,$T1 + jmp .L_get_AAD_loop48x16_${label_suffix} + +.L_exit_AAD_loop48x16_${label_suffix}: + # ; Less than 48x16 bytes remaining + cmp \$`(32*16)`,$T2 + jl .L_less_than_32x16_${label_suffix} +___ + + $code .= <<___; + # ; Get next 16 blocks + vmovdqu64 `64*0`($T1),$ZT1 + vmovdqu64 `64*1`($T1),$ZT2 + vmovdqu64 `64*2`($T1),$ZT3 + vmovdqu64 `64*3`($T1),$ZT4 + vpshufb $SHFMSK,$ZT1,$ZT1 + vpshufb $SHFMSK,$ZT2,$ZT2 + vpshufb $SHFMSK,$ZT3,$ZT3 + vpshufb $SHFMSK,$ZT4,$ZT4 +___ + + &precompute_hkeys_on_stack($GCM128_CTX, $HKEYS_READY, $ZT0, $ZT8, $ZT9, $ZT10, $ZT11, $ZT12, $ZT14, "first32"); + $code .= "mov \$1,$HKEYS_READY\n"; + + &GHASH_16( + "start", $ZT5, $ZT6, $ZT7, + "NO_INPUT_PTR", "NO_INPUT_PTR", "NO_INPUT_PTR", "%rsp", + &HashKeyOffsetByIdx(32, "frame"), 0, &ZWORD($AAD_HASH), $ZT0, + $ZT8, $ZT9, $ZT10, $ZT11, + $ZT12, $ZT14, $ZT15, $ZT16, + "NO_ZMM", $ZT1, $ZT2, $ZT3, + $ZT4); + + $code .= <<___; + vmovdqu64 `16*16 + 64*0`($T1),$ZT1 + vmovdqu64 `16*16 + 64*1`($T1),$ZT2 + vmovdqu64 `16*16 + 64*2`($T1),$ZT3 + vmovdqu64 `16*16 + 64*3`($T1),$ZT4 + vpshufb $SHFMSK,$ZT1,$ZT1 + vpshufb $SHFMSK,$ZT2,$ZT2 + vpshufb $SHFMSK,$ZT3,$ZT3 + vpshufb $SHFMSK,$ZT4,$ZT4 +___ + + &GHASH_16( + "end_reduce", $ZT5, $ZT6, $ZT7, + "NO_INPUT_PTR", "NO_INPUT_PTR", "NO_INPUT_PTR", "%rsp", + &HashKeyOffsetByIdx(16, "frame"), 0, &ZWORD($AAD_HASH), $ZT0, + $ZT8, $ZT9, $ZT10, $ZT11, + $ZT12, $ZT14, $ZT15, $ZT16, + "NO_ZMM", $ZT1, $ZT2, $ZT3, + $ZT4); + + $code .= <<___; + sub \$`(32*16)`,$T2 + je .L_CALC_AAD_done_${label_suffix} + + add \$`(32*16)`,$T1 + jmp .L_less_than_16x16_${label_suffix} + +.L_less_than_32x16_${label_suffix}: + cmp \$`(16*16)`,$T2 + jl .L_less_than_16x16_${label_suffix} + # ; Get next 16 blocks + vmovdqu64 `64*0`($T1),$ZT1 + vmovdqu64 `64*1`($T1),$ZT2 + vmovdqu64 `64*2`($T1),$ZT3 + vmovdqu64 `64*3`($T1),$ZT4 + vpshufb $SHFMSK,$ZT1,$ZT1 + vpshufb $SHFMSK,$ZT2,$ZT2 + vpshufb $SHFMSK,$ZT3,$ZT3 + vpshufb $SHFMSK,$ZT4,$ZT4 +___ + + # ; This code path does not use more than 16 hkeys, so they can be taken from the context + # ; (not from the stack storage) + &GHASH_16( + "start_reduce", $ZT5, $ZT6, $ZT7, + "NO_INPUT_PTR", "NO_INPUT_PTR", "NO_INPUT_PTR", $GCM128_CTX, + &HashKeyOffsetByIdx(16, "context"), 0, &ZWORD($AAD_HASH), $ZT0, + $ZT8, $ZT9, $ZT10, $ZT11, + $ZT12, $ZT14, $ZT15, $ZT16, + "NO_ZMM", $ZT1, $ZT2, $ZT3, + $ZT4); + + $code .= <<___; + sub \$`(16*16)`,$T2 + je .L_CALC_AAD_done_${label_suffix} + + add \$`(16*16)`,$T1 + # ; Less than 16x16 bytes remaining +.L_less_than_16x16_${label_suffix}: + # ;; prep mask source address + lea byte64_len_to_mask_table(%rip),$T3 + lea ($T3,$T2,8),$T3 + + # ;; calculate number of blocks to ghash (including partial bytes) + add \$15,@{[DWORD($T2)]} + shr \$4,@{[DWORD($T2)]} + cmp \$2,@{[DWORD($T2)]} + jb .L_AAD_blocks_1_${label_suffix} + je .L_AAD_blocks_2_${label_suffix} + cmp \$4,@{[DWORD($T2)]} + jb .L_AAD_blocks_3_${label_suffix} + je .L_AAD_blocks_4_${label_suffix} + cmp \$6,@{[DWORD($T2)]} + jb .L_AAD_blocks_5_${label_suffix} + je .L_AAD_blocks_6_${label_suffix} + cmp \$8,@{[DWORD($T2)]} + jb .L_AAD_blocks_7_${label_suffix} + je .L_AAD_blocks_8_${label_suffix} + cmp \$10,@{[DWORD($T2)]} + jb .L_AAD_blocks_9_${label_suffix} + je .L_AAD_blocks_10_${label_suffix} + cmp \$12,@{[DWORD($T2)]} + jb .L_AAD_blocks_11_${label_suffix} + je .L_AAD_blocks_12_${label_suffix} + cmp \$14,@{[DWORD($T2)]} + jb .L_AAD_blocks_13_${label_suffix} + je .L_AAD_blocks_14_${label_suffix} + cmp \$15,@{[DWORD($T2)]} + je .L_AAD_blocks_15_${label_suffix} +___ + + # ;; fall through for 16 blocks + + # ;; The flow of each of these cases is identical: + # ;; - load blocks plain text + # ;; - shuffle loaded blocks + # ;; - xor in current hash value into block 0 + # ;; - perform up multiplications with ghash keys + # ;; - jump to reduction code + + for (my $aad_blocks = 16; $aad_blocks > 0; $aad_blocks--) { + $code .= ".L_AAD_blocks_${aad_blocks}_${label_suffix}:\n"; + if ($aad_blocks > 12) { + $code .= "sub \$`12*16*8`, $T3\n"; + } elsif ($aad_blocks > 8) { + $code .= "sub \$`8*16*8`, $T3\n"; + } elsif ($aad_blocks > 4) { + $code .= "sub \$`4*16*8`, $T3\n"; + } + $code .= "kmovq ($T3),$MASKREG\n"; + + &ZMM_LOAD_MASKED_BLOCKS_0_16($aad_blocks, $T1, 0, $ZT1, $ZT2, $ZT3, $ZT4, $MASKREG); + + &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16($aad_blocks, "vpshufb", $ZT1, $ZT2, $ZT3, $ZT4, + $ZT1, $ZT2, $ZT3, $ZT4, $SHFMSK, $SHFMSK, $SHFMSK, $SHFMSK); + + &GHASH_1_TO_16($GCM128_CTX, &ZWORD($AAD_HASH), + $ZT0, $ZT5, $ZT6, $ZT7, $ZT8, $ZT9, $ZT10, $ZT11, $ZT12, &ZWORD($AAD_HASH), $ZT1, $ZT2, $ZT3, $ZT4, $aad_blocks); + + if ($aad_blocks > 1) { + + # ;; fall through to CALC_AAD_done in 1 block case + $code .= "jmp .L_CALC_AAD_done_${label_suffix}\n"; + } + + } + $code .= ".L_CALC_AAD_done_${label_suffix}:\n"; + + # ;; result in AAD_HASH +} + +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +# ;; PARTIAL_BLOCK +# ;; Handles encryption/decryption and the tag partial blocks between +# ;; update calls. +# ;; Requires the input data be at least 1 byte long. +# ;; Output: +# ;; A cipher/plain of the first partial block (CIPH_PLAIN_OUT), +# ;; AAD_HASH and updated GCM128_CTX +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +sub PARTIAL_BLOCK { + my $GCM128_CTX = $_[0]; # [in] key pointer + my $PBLOCK_LEN = $_[1]; # [in] partial block length + my $CIPH_PLAIN_OUT = $_[2]; # [in] output buffer + my $PLAIN_CIPH_IN = $_[3]; # [in] input buffer + my $PLAIN_CIPH_LEN = $_[4]; # [in] buffer length + my $DATA_OFFSET = $_[5]; # [out] data offset (gets set) + my $AAD_HASH = $_[6]; # [out] updated GHASH value + my $ENC_DEC = $_[7]; # [in] cipher direction + my $GPTMP0 = $_[8]; # [clobbered] GP temporary register + my $GPTMP1 = $_[9]; # [clobbered] GP temporary register + my $GPTMP2 = $_[10]; # [clobbered] GP temporary register + my $ZTMP0 = $_[11]; # [clobbered] ZMM temporary register + my $ZTMP1 = $_[12]; # [clobbered] ZMM temporary register + my $ZTMP2 = $_[13]; # [clobbered] ZMM temporary register + my $ZTMP3 = $_[14]; # [clobbered] ZMM temporary register + my $ZTMP4 = $_[15]; # [clobbered] ZMM temporary register + my $ZTMP5 = $_[16]; # [clobbered] ZMM temporary register + my $ZTMP6 = $_[17]; # [clobbered] ZMM temporary register + my $ZTMP7 = $_[18]; # [clobbered] ZMM temporary register + my $MASKREG = $_[19]; # [clobbered] mask temporary register + + my $XTMP0 = &XWORD($ZTMP0); + my $XTMP1 = &XWORD($ZTMP1); + my $XTMP2 = &XWORD($ZTMP2); + my $XTMP3 = &XWORD($ZTMP3); + my $XTMP4 = &XWORD($ZTMP4); + my $XTMP5 = &XWORD($ZTMP5); + my $XTMP6 = &XWORD($ZTMP6); + my $XTMP7 = &XWORD($ZTMP7); + + my $LENGTH = $DATA_OFFSET; + my $IA0 = $GPTMP1; + my $IA1 = $GPTMP2; + my $IA2 = $GPTMP0; + + my $label_suffix = $label_count++; + + $code .= <<___; + # ;; if no partial block present then LENGTH/DATA_OFFSET will be set to zero + mov ($PBLOCK_LEN),$LENGTH + or $LENGTH,$LENGTH + je .L_partial_block_done_${label_suffix} # ;Leave Macro if no partial blocks +___ + + &READ_SMALL_DATA_INPUT($XTMP0, $PLAIN_CIPH_IN, $PLAIN_CIPH_LEN, $IA0, $IA2, $MASKREG); + + $code .= <<___; + # ;; XTMP1 = my_ctx_data.partial_block_enc_key + vmovdqu64 $CTX_OFFSET_PEncBlock($GCM128_CTX),$XTMP1 + vmovdqu64 @{[HashKeyByIdx(1,$GCM128_CTX)]},$XTMP2 + + # ;; adjust the shuffle mask pointer to be able to shift right $LENGTH bytes + # ;; (16 - $LENGTH) is the number of bytes in plaintext mod 16) + lea SHIFT_MASK(%rip),$IA0 + add $LENGTH,$IA0 + vmovdqu64 ($IA0),$XTMP3 # ; shift right shuffle mask + vpshufb $XTMP3,$XTMP1,$XTMP1 +___ + + if ($ENC_DEC eq "DEC") { + $code .= <<___; + # ;; keep copy of cipher text in $XTMP4 + vmovdqa64 $XTMP0,$XTMP4 +___ + } + $code .= <<___; + vpxorq $XTMP0,$XTMP1,$XTMP1 # ; Ciphertext XOR E(K, Yn) + # ;; Set $IA1 to be the amount of data left in CIPH_PLAIN_IN after filling the block + # ;; Determine if partial block is not being filled and shift mask accordingly +___ + if ($win64) { + $code .= <<___; + mov $PLAIN_CIPH_LEN,$IA1 + add $LENGTH,$IA1 +___ + } else { + $code .= "lea ($PLAIN_CIPH_LEN, $LENGTH, 1),$IA1\n"; + } + $code .= <<___; + sub \$16,$IA1 + jge .L_no_extra_mask_${label_suffix} + sub $IA1,$IA0 +.L_no_extra_mask_${label_suffix}: + # ;; get the appropriate mask to mask out bottom $LENGTH bytes of $XTMP1 + # ;; - mask out bottom $LENGTH bytes of $XTMP1 + # ;; sizeof(SHIFT_MASK) == 16 bytes + vmovdqu64 16($IA0),$XTMP0 + vpand $XTMP0,$XTMP1,$XTMP1 +___ + + if ($ENC_DEC eq "DEC") { + $code .= <<___; + vpand $XTMP0,$XTMP4,$XTMP4 + vpshufb SHUF_MASK(%rip),$XTMP4,$XTMP4 + vpshufb $XTMP3,$XTMP4,$XTMP4 + vpxorq $XTMP4,$AAD_HASH,$AAD_HASH +___ + } else { + $code .= <<___; + vpshufb SHUF_MASK(%rip),$XTMP1,$XTMP1 + vpshufb $XTMP3,$XTMP1,$XTMP1 + vpxorq $XTMP1,$AAD_HASH,$AAD_HASH +___ + } + $code .= <<___; + cmp \$0,$IA1 + jl .L_partial_incomplete_${label_suffix} +___ + + # ;; GHASH computation for the last <16 Byte block + &GHASH_MUL($AAD_HASH, $XTMP2, $XTMP5, $XTMP6, $XTMP7); + + $code .= <<___; + movq \$0, ($PBLOCK_LEN) + # ;; Set $LENGTH to be the number of bytes to write out + mov $LENGTH,$IA0 + mov \$16,$LENGTH + sub $IA0,$LENGTH + jmp .L_enc_dec_done_${label_suffix} + +.L_partial_incomplete_${label_suffix}: +___ + if ($win64) { + $code .= <<___; + mov $PLAIN_CIPH_LEN,$IA0 + add $IA0,($PBLOCK_LEN) +___ + } else { + $code .= "add $PLAIN_CIPH_LEN,($PBLOCK_LEN)\n"; + } + $code .= <<___; + mov $PLAIN_CIPH_LEN,$LENGTH + +.L_enc_dec_done_${label_suffix}: + # ;; output encrypted Bytes + + lea byte_len_to_mask_table(%rip),$IA0 + kmovw ($IA0,$LENGTH,2),$MASKREG + vmovdqu64 $AAD_HASH,$CTX_OFFSET_AadHash($GCM128_CTX) +___ + + if ($ENC_DEC eq "ENC") { + $code .= <<___; + # ;; shuffle XTMP1 back to output as ciphertext + vpshufb SHUF_MASK(%rip),$XTMP1,$XTMP1 + vpshufb $XTMP3,$XTMP1,$XTMP1 +___ + } + $code .= <<___; + mov $CIPH_PLAIN_OUT,$IA0 + vmovdqu8 $XTMP1,($IA0){$MASKREG} +.L_partial_block_done_${label_suffix}: +___ +} + +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +# ;; Ciphers 1 to 16 blocks and prepares them for later GHASH compute operation +sub INITIAL_BLOCKS_PARTIAL_CIPHER { + my $AES_KEYS = $_[0]; # [in] key pointer + my $GCM128_CTX = $_[1]; # [in] context pointer + my $CIPH_PLAIN_OUT = $_[2]; # [in] text output pointer + my $PLAIN_CIPH_IN = $_[3]; # [in] text input pointer + my $LENGTH = $_[4]; # [in/clobbered] length in bytes + my $DATA_OFFSET = $_[5]; # [in/out] current data offset (updated) + my $NUM_BLOCKS = $_[6]; # [in] can only be 1, 2, 3, 4, 5, ..., 15 or 16 (not 0) + my $CTR = $_[7]; # [in/out] current counter value + my $ENC_DEC = $_[8]; # [in] cipher direction (ENC/DEC) + my $DAT0 = $_[9]; # [out] ZMM with cipher text shuffled for GHASH + my $DAT1 = $_[10]; # [out] ZMM with cipher text shuffled for GHASH + my $DAT2 = $_[11]; # [out] ZMM with cipher text shuffled for GHASH + my $DAT3 = $_[12]; # [out] ZMM with cipher text shuffled for GHASH + my $LAST_CIPHER_BLK = $_[13]; # [out] XMM to put ciphered counter block partially xor'ed with text + my $LAST_GHASH_BLK = $_[14]; # [out] XMM to put last cipher text block shuffled for GHASH + my $CTR0 = $_[15]; # [clobbered] ZMM temporary + my $CTR1 = $_[16]; # [clobbered] ZMM temporary + my $CTR2 = $_[17]; # [clobbered] ZMM temporary + my $CTR3 = $_[18]; # [clobbered] ZMM temporary + my $ZT1 = $_[19]; # [clobbered] ZMM temporary + my $IA0 = $_[20]; # [clobbered] GP temporary + my $IA1 = $_[21]; # [clobbered] GP temporary + my $MASKREG = $_[22]; # [clobbered] mask register + my $SHUFMASK = $_[23]; # [out] ZMM loaded with BE/LE shuffle mask + + if ($NUM_BLOCKS == 1) { + $code .= "vmovdqa64 SHUF_MASK(%rip),@{[XWORD($SHUFMASK)]}\n"; + } elsif ($NUM_BLOCKS == 2) { + $code .= "vmovdqa64 SHUF_MASK(%rip),@{[YWORD($SHUFMASK)]}\n"; + } else { + $code .= "vmovdqa64 SHUF_MASK(%rip),$SHUFMASK\n"; + } + + # ;; prepare AES counter blocks + if ($NUM_BLOCKS == 1) { + $code .= "vpaddd ONE(%rip),$CTR,@{[XWORD($CTR0)]}\n"; + } elsif ($NUM_BLOCKS == 2) { + $code .= <<___; + vshufi64x2 \$0,@{[YWORD($CTR)]},@{[YWORD($CTR)]},@{[YWORD($CTR0)]} + vpaddd ddq_add_1234(%rip),@{[YWORD($CTR0)]},@{[YWORD($CTR0)]} +___ + } else { + $code .= <<___; + vshufi64x2 \$0,@{[ZWORD($CTR)]},@{[ZWORD($CTR)]},@{[ZWORD($CTR)]} + vpaddd ddq_add_1234(%rip),@{[ZWORD($CTR)]},$CTR0 +___ + if ($NUM_BLOCKS > 4) { + $code .= "vpaddd ddq_add_5678(%rip),@{[ZWORD($CTR)]},$CTR1\n"; + } + if ($NUM_BLOCKS > 8) { + $code .= "vpaddd ddq_add_8888(%rip),$CTR0,$CTR2\n"; + } + if ($NUM_BLOCKS > 12) { + $code .= "vpaddd ddq_add_8888(%rip),$CTR1,$CTR3\n"; + } + } + + # ;; get load/store mask + $code .= <<___; + lea byte64_len_to_mask_table(%rip),$IA0 + mov $LENGTH,$IA1 +___ + if ($NUM_BLOCKS > 12) { + $code .= "sub \$`3*64`,$IA1\n"; + } elsif ($NUM_BLOCKS > 8) { + $code .= "sub \$`2*64`,$IA1\n"; + } elsif ($NUM_BLOCKS > 4) { + $code .= "sub \$`1*64`,$IA1\n"; + } + $code .= "kmovq ($IA0,$IA1,8),$MASKREG\n"; + + # ;; extract new counter value + # ;; shuffle the counters for AES rounds + if ($NUM_BLOCKS <= 4) { + $code .= "vextracti32x4 \$`($NUM_BLOCKS - 1)`,$CTR0,$CTR\n"; + } elsif ($NUM_BLOCKS <= 8) { + $code .= "vextracti32x4 \$`($NUM_BLOCKS - 5)`,$CTR1,$CTR\n"; + } elsif ($NUM_BLOCKS <= 12) { + $code .= "vextracti32x4 \$`($NUM_BLOCKS - 9)`,$CTR2,$CTR\n"; + } else { + $code .= "vextracti32x4 \$`($NUM_BLOCKS - 13)`,$CTR3,$CTR\n"; + } + &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16( + $NUM_BLOCKS, "vpshufb", $CTR0, $CTR1, $CTR2, $CTR3, $CTR0, + $CTR1, $CTR2, $CTR3, $SHUFMASK, $SHUFMASK, $SHUFMASK, $SHUFMASK); + + # ;; load plain/cipher text + &ZMM_LOAD_MASKED_BLOCKS_0_16($NUM_BLOCKS, $PLAIN_CIPH_IN, $DATA_OFFSET, $DAT0, $DAT1, $DAT2, $DAT3, $MASKREG); + + # ;; AES rounds and XOR with plain/cipher text + foreach my $j (0 .. ($NROUNDS + 1)) { + $code .= "vbroadcastf64x2 `($j * 16)`($AES_KEYS),$ZT1\n"; + &ZMM_AESENC_ROUND_BLOCKS_0_16($CTR0, $CTR1, $CTR2, $CTR3, $ZT1, $j, + $DAT0, $DAT1, $DAT2, $DAT3, $NUM_BLOCKS, $NROUNDS); + } + + # ;; retrieve the last cipher counter block (partially XOR'ed with text) + # ;; - this is needed for partial block cases + if ($NUM_BLOCKS <= 4) { + $code .= "vextracti32x4 \$`($NUM_BLOCKS - 1)`,$CTR0,$LAST_CIPHER_BLK\n"; + } elsif ($NUM_BLOCKS <= 8) { + $code .= "vextracti32x4 \$`($NUM_BLOCKS - 5)`,$CTR1,$LAST_CIPHER_BLK\n"; + } elsif ($NUM_BLOCKS <= 12) { + $code .= "vextracti32x4 \$`($NUM_BLOCKS - 9)`,$CTR2,$LAST_CIPHER_BLK\n"; + } else { + $code .= "vextracti32x4 \$`($NUM_BLOCKS - 13)`,$CTR3,$LAST_CIPHER_BLK\n"; + } + + # ;; write cipher/plain text back to output and + $code .= "mov $CIPH_PLAIN_OUT,$IA0\n"; + &ZMM_STORE_MASKED_BLOCKS_0_16($NUM_BLOCKS, $IA0, $DATA_OFFSET, $CTR0, $CTR1, $CTR2, $CTR3, $MASKREG); + + # ;; zero bytes outside the mask before hashing + if ($NUM_BLOCKS <= 4) { + $code .= "vmovdqu8 $CTR0,${CTR0}{$MASKREG}{z}\n"; + } elsif ($NUM_BLOCKS <= 8) { + $code .= "vmovdqu8 $CTR1,${CTR1}{$MASKREG}{z}\n"; + } elsif ($NUM_BLOCKS <= 12) { + $code .= "vmovdqu8 $CTR2,${CTR2}{$MASKREG}{z}\n"; + } else { + $code .= "vmovdqu8 $CTR3,${CTR3}{$MASKREG}{z}\n"; + } + + # ;; Shuffle the cipher text blocks for hashing part + # ;; ZT5 and ZT6 are expected outputs with blocks for hashing + if ($ENC_DEC eq "DEC") { + + # ;; Decrypt case + # ;; - cipher blocks are in ZT5 & ZT6 + &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16( + $NUM_BLOCKS, "vpshufb", $DAT0, $DAT1, $DAT2, $DAT3, $DAT0, + $DAT1, $DAT2, $DAT3, $SHUFMASK, $SHUFMASK, $SHUFMASK, $SHUFMASK); + } else { + + # ;; Encrypt case + # ;; - cipher blocks are in CTR0-CTR3 + &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16( + $NUM_BLOCKS, "vpshufb", $DAT0, $DAT1, $DAT2, $DAT3, $CTR0, + $CTR1, $CTR2, $CTR3, $SHUFMASK, $SHUFMASK, $SHUFMASK, $SHUFMASK); + } + + # ;; Extract the last block for partials and multi_call cases + if ($NUM_BLOCKS <= 4) { + $code .= "vextracti32x4 \$`($NUM_BLOCKS-1)`,$DAT0,$LAST_GHASH_BLK\n"; + } elsif ($NUM_BLOCKS <= 8) { + $code .= "vextracti32x4 \$`($NUM_BLOCKS-5)`,$DAT1,$LAST_GHASH_BLK\n"; + } elsif ($NUM_BLOCKS <= 12) { + $code .= "vextracti32x4 \$`($NUM_BLOCKS-9)`,$DAT2,$LAST_GHASH_BLK\n"; + } else { + $code .= "vextracti32x4 \$`($NUM_BLOCKS-13)`,$DAT3,$LAST_GHASH_BLK\n"; + } + +} + +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +# ;; Computes GHASH on 1 to 16 blocks +sub INITIAL_BLOCKS_PARTIAL_GHASH { + my $AES_KEYS = $_[0]; # [in] key pointer + my $GCM128_CTX = $_[1]; # [in] context pointer + my $LENGTH = $_[2]; # [in/clobbered] length in bytes + my $NUM_BLOCKS = $_[3]; # [in] can only be 1, 2, 3, 4, 5, ..., 15 or 16 (not 0) + my $HASH_IN_OUT = $_[4]; # [in/out] XMM ghash in/out value + my $ENC_DEC = $_[5]; # [in] cipher direction (ENC/DEC) + my $DAT0 = $_[6]; # [in] ZMM with cipher text shuffled for GHASH + my $DAT1 = $_[7]; # [in] ZMM with cipher text shuffled for GHASH + my $DAT2 = $_[8]; # [in] ZMM with cipher text shuffled for GHASH + my $DAT3 = $_[9]; # [in] ZMM with cipher text shuffled for GHASH + my $LAST_CIPHER_BLK = $_[10]; # [in] XMM with ciphered counter block partially xor'ed with text + my $LAST_GHASH_BLK = $_[11]; # [in] XMM with last cipher text block shuffled for GHASH + my $ZT0 = $_[12]; # [clobbered] ZMM temporary + my $ZT1 = $_[13]; # [clobbered] ZMM temporary + my $ZT2 = $_[14]; # [clobbered] ZMM temporary + my $ZT3 = $_[15]; # [clobbered] ZMM temporary + my $ZT4 = $_[16]; # [clobbered] ZMM temporary + my $ZT5 = $_[17]; # [clobbered] ZMM temporary + my $ZT6 = $_[18]; # [clobbered] ZMM temporary + my $ZT7 = $_[19]; # [clobbered] ZMM temporary + my $ZT8 = $_[20]; # [clobbered] ZMM temporary + my $PBLOCK_LEN = $_[21]; # [in] partial block length + my $GH = $_[22]; # [in] ZMM with hi product part + my $GM = $_[23]; # [in] ZMM with mid product part + my $GL = $_[24]; # [in] ZMM with lo product part + + my $label_suffix = $label_count++; + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;;; - Hash all but the last partial block of data + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + # ;; update data offset + if ($NUM_BLOCKS > 1) { + + # ;; The final block of data may be <16B + $code .= "sub \$16 * ($NUM_BLOCKS - 1),$LENGTH\n"; + } + + if ($NUM_BLOCKS < 16) { + $code .= <<___; + # ;; NOTE: the 'jl' is always taken for num_initial_blocks = 16. + # ;; This is run in the context of GCM_ENC_DEC_SMALL for length < 256. + cmp \$16,$LENGTH + jl .L_small_initial_partial_block_${label_suffix} + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;;; Handle a full length final block - encrypt and hash all blocks + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + sub \$16,$LENGTH + movq \$0,($PBLOCK_LEN) +___ + + # ;; Hash all of the data + if (scalar(@_) == 22) { + + # ;; start GHASH compute + &GHASH_1_TO_16($GCM128_CTX, $HASH_IN_OUT, $ZT0, $ZT1, $ZT2, $ZT3, $ZT4, + $ZT5, $ZT6, $ZT7, $ZT8, &ZWORD($HASH_IN_OUT), $DAT0, $DAT1, $DAT2, $DAT3, $NUM_BLOCKS); + } elsif (scalar(@_) == 25) { + + # ;; continue GHASH compute + &GHASH_1_TO_16($GCM128_CTX, $HASH_IN_OUT, $ZT0, $ZT1, $ZT2, $ZT3, $ZT4, + $ZT5, $ZT6, $ZT7, $ZT8, &ZWORD($HASH_IN_OUT), $DAT0, $DAT1, $DAT2, $DAT3, $NUM_BLOCKS, $GH, $GM, $GL); + } + $code .= "jmp .L_small_initial_compute_done_${label_suffix}\n"; + } + + $code .= <<___; +.L_small_initial_partial_block_${label_suffix}: + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;;; Handle ghash for a <16B final block + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + # ;; As it's an init / update / finalize series we need to leave the + # ;; last block if it's less than a full block of data. + + mov $LENGTH,($PBLOCK_LEN) + vmovdqu64 $LAST_CIPHER_BLK,$CTX_OFFSET_PEncBlock($GCM128_CTX) +___ + + my $k = ($NUM_BLOCKS - 1); + my $last_block_to_hash = 1; + if (($NUM_BLOCKS > $last_block_to_hash)) { + + # ;; ZT12-ZT20 - temporary registers + if (scalar(@_) == 22) { + + # ;; start GHASH compute + &GHASH_1_TO_16($GCM128_CTX, $HASH_IN_OUT, $ZT0, $ZT1, $ZT2, $ZT3, $ZT4, + $ZT5, $ZT6, $ZT7, $ZT8, &ZWORD($HASH_IN_OUT), $DAT0, $DAT1, $DAT2, $DAT3, $k); + } elsif (scalar(@_) == 25) { + + # ;; continue GHASH compute + &GHASH_1_TO_16($GCM128_CTX, $HASH_IN_OUT, $ZT0, $ZT1, $ZT2, $ZT3, $ZT4, + $ZT5, $ZT6, $ZT7, $ZT8, &ZWORD($HASH_IN_OUT), $DAT0, $DAT1, $DAT2, $DAT3, $k, $GH, $GM, $GL); + } + + # ;; just fall through no jmp needed + } else { + + if (scalar(@_) == 25) { + $code .= <<___; + # ;; Reduction is required in this case. + # ;; Integrate GM into GH and GL. + vpsrldq \$8,$GM,$ZT0 + vpslldq \$8,$GM,$ZT1 + vpxorq $ZT0,$GH,$GH + vpxorq $ZT1,$GL,$GL +___ + + # ;; Add GH and GL 128-bit words horizontally + &VHPXORI4x128($GH, $ZT0); + &VHPXORI4x128($GL, $ZT1); + + # ;; 256-bit to 128-bit reduction + $code .= "vmovdqa64 POLY2(%rip),@{[XWORD($ZT0)]}\n"; + &VCLMUL_REDUCE(&XWORD($HASH_IN_OUT), &XWORD($ZT0), &XWORD($GH), &XWORD($GL), &XWORD($ZT1), &XWORD($ZT2)); + } + $code .= <<___; + # ;; Record that a reduction is not needed - + # ;; In this case no hashes are computed because there + # ;; is only one initial block and it is < 16B in length. + # ;; We only need to check if a reduction is needed if + # ;; initial_blocks == 1 and init/update/final is being used. + # ;; In this case we may just have a partial block, and that + # ;; gets hashed in finalize. + + # ;; The hash should end up in HASH_IN_OUT. + # ;; The only way we should get here is if there is + # ;; a partial block of data, so xor that into the hash. + vpxorq $LAST_GHASH_BLK,$HASH_IN_OUT,$HASH_IN_OUT + # ;; The result is in $HASH_IN_OUT + jmp .L_after_reduction_${label_suffix} +___ + } + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;;; After GHASH reduction + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + $code .= ".L_small_initial_compute_done_${label_suffix}:\n"; + + # ;; If using init/update/finalize, we need to xor any partial block data + # ;; into the hash. + if ($NUM_BLOCKS > 1) { + + # ;; NOTE: for $NUM_BLOCKS = 0 the xor never takes place + if ($NUM_BLOCKS != 16) { + $code .= <<___; + # ;; NOTE: for $NUM_BLOCKS = 16, $LENGTH, stored in [PBlockLen] is never zero + or $LENGTH,$LENGTH + je .L_after_reduction_${label_suffix} +___ + } + $code .= "vpxorq $LAST_GHASH_BLK,$HASH_IN_OUT,$HASH_IN_OUT\n"; + } + + $code .= ".L_after_reduction_${label_suffix}:\n"; + + # ;; Final hash is now in HASH_IN_OUT +} + +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +# ;; INITIAL_BLOCKS_PARTIAL macro with support for a partial final block. +# ;; It may look similar to INITIAL_BLOCKS but its usage is different: +# ;; - first encrypts/decrypts required number of blocks and then +# ;; ghashes these blocks +# ;; - Small packets or left over data chunks (<256 bytes) +# ;; - Remaining data chunks below 256 bytes (multi buffer code) +# ;; +# ;; num_initial_blocks is expected to include the partial final block +# ;; in the count. +sub INITIAL_BLOCKS_PARTIAL { + my $AES_KEYS = $_[0]; # [in] key pointer + my $GCM128_CTX = $_[1]; # [in] context pointer + my $CIPH_PLAIN_OUT = $_[2]; # [in] text output pointer + my $PLAIN_CIPH_IN = $_[3]; # [in] text input pointer + my $LENGTH = $_[4]; # [in/clobbered] length in bytes + my $DATA_OFFSET = $_[5]; # [in/out] current data offset (updated) + my $NUM_BLOCKS = $_[6]; # [in] can only be 1, 2, 3, 4, 5, ..., 15 or 16 (not 0) + my $CTR = $_[7]; # [in/out] current counter value + my $HASH_IN_OUT = $_[8]; # [in/out] XMM ghash in/out value + my $ENC_DEC = $_[9]; # [in] cipher direction (ENC/DEC) + my $CTR0 = $_[10]; # [clobbered] ZMM temporary + my $CTR1 = $_[11]; # [clobbered] ZMM temporary + my $CTR2 = $_[12]; # [clobbered] ZMM temporary + my $CTR3 = $_[13]; # [clobbered] ZMM temporary + my $DAT0 = $_[14]; # [clobbered] ZMM temporary + my $DAT1 = $_[15]; # [clobbered] ZMM temporary + my $DAT2 = $_[16]; # [clobbered] ZMM temporary + my $DAT3 = $_[17]; # [clobbered] ZMM temporary + my $LAST_CIPHER_BLK = $_[18]; # [clobbered] ZMM temporary + my $LAST_GHASH_BLK = $_[19]; # [clobbered] ZMM temporary + my $ZT0 = $_[20]; # [clobbered] ZMM temporary + my $ZT1 = $_[21]; # [clobbered] ZMM temporary + my $ZT2 = $_[22]; # [clobbered] ZMM temporary + my $ZT3 = $_[23]; # [clobbered] ZMM temporary + my $ZT4 = $_[24]; # [clobbered] ZMM temporary + my $IA0 = $_[25]; # [clobbered] GP temporary + my $IA1 = $_[26]; # [clobbered] GP temporary + my $MASKREG = $_[27]; # [clobbered] mask register + my $SHUFMASK = $_[28]; # [clobbered] ZMM for BE/LE shuffle mask + my $PBLOCK_LEN = $_[29]; # [in] partial block length + + &INITIAL_BLOCKS_PARTIAL_CIPHER( + $AES_KEYS, $GCM128_CTX, $CIPH_PLAIN_OUT, $PLAIN_CIPH_IN, + $LENGTH, $DATA_OFFSET, $NUM_BLOCKS, $CTR, + $ENC_DEC, $DAT0, $DAT1, $DAT2, + $DAT3, &XWORD($LAST_CIPHER_BLK), &XWORD($LAST_GHASH_BLK), $CTR0, + $CTR1, $CTR2, $CTR3, $ZT0, + $IA0, $IA1, $MASKREG, $SHUFMASK); + + &INITIAL_BLOCKS_PARTIAL_GHASH($AES_KEYS, $GCM128_CTX, $LENGTH, $NUM_BLOCKS, $HASH_IN_OUT, $ENC_DEC, $DAT0, + $DAT1, $DAT2, $DAT3, &XWORD($LAST_CIPHER_BLK), + &XWORD($LAST_GHASH_BLK), $CTR0, $CTR1, $CTR2, $CTR3, $ZT0, $ZT1, $ZT2, $ZT3, $ZT4, $PBLOCK_LEN); +} + +# ;; =========================================================================== +# ;; Stitched GHASH of 16 blocks (with reduction) with encryption of N blocks +# ;; followed with GHASH of the N blocks. +sub GHASH_16_ENCRYPT_N_GHASH_N { + my $AES_KEYS = $_[0]; # [in] key pointer + my $GCM128_CTX = $_[1]; # [in] context pointer + my $CIPH_PLAIN_OUT = $_[2]; # [in] pointer to output buffer + my $PLAIN_CIPH_IN = $_[3]; # [in] pointer to input buffer + my $DATA_OFFSET = $_[4]; # [in] data offset + my $LENGTH = $_[5]; # [in] data length + my $CTR_BE = $_[6]; # [in/out] ZMM counter blocks (last 4) in big-endian + my $CTR_CHECK = $_[7]; # [in/out] GP with 8-bit counter for overflow check + my $HASHKEY_OFFSET = $_[8]; # [in] numerical offset for the highest hash key + # (can be in form of register or numerical value) + my $GHASHIN_BLK_OFFSET = $_[9]; # [in] numerical offset for GHASH blocks in + my $SHFMSK = $_[10]; # [in] ZMM with byte swap mask for pshufb + my $B00_03 = $_[11]; # [clobbered] temporary ZMM + my $B04_07 = $_[12]; # [clobbered] temporary ZMM + my $B08_11 = $_[13]; # [clobbered] temporary ZMM + my $B12_15 = $_[14]; # [clobbered] temporary ZMM + my $GH1H_UNUSED = $_[15]; # [clobbered] temporary ZMM + my $GH1L = $_[16]; # [clobbered] temporary ZMM + my $GH1M = $_[17]; # [clobbered] temporary ZMM + my $GH1T = $_[18]; # [clobbered] temporary ZMM + my $GH2H = $_[19]; # [clobbered] temporary ZMM + my $GH2L = $_[20]; # [clobbered] temporary ZMM + my $GH2M = $_[21]; # [clobbered] temporary ZMM + my $GH2T = $_[22]; # [clobbered] temporary ZMM + my $GH3H = $_[23]; # [clobbered] temporary ZMM + my $GH3L = $_[24]; # [clobbered] temporary ZMM + my $GH3M = $_[25]; # [clobbered] temporary ZMM + my $GH3T = $_[26]; # [clobbered] temporary ZMM + my $AESKEY1 = $_[27]; # [clobbered] temporary ZMM + my $AESKEY2 = $_[28]; # [clobbered] temporary ZMM + my $GHKEY1 = $_[29]; # [clobbered] temporary ZMM + my $GHKEY2 = $_[30]; # [clobbered] temporary ZMM + my $GHDAT1 = $_[31]; # [clobbered] temporary ZMM + my $GHDAT2 = $_[32]; # [clobbered] temporary ZMM + my $ZT01 = $_[33]; # [clobbered] temporary ZMM + my $ADDBE_4x4 = $_[34]; # [in] ZMM with 4x128bits 4 in big-endian + my $ADDBE_1234 = $_[35]; # [in] ZMM with 4x128bits 1, 2, 3 and 4 in big-endian + my $GHASH_TYPE = $_[36]; # [in] "start", "start_reduce", "mid", "end_reduce" + my $TO_REDUCE_L = $_[37]; # [in] ZMM for low 4x128-bit GHASH sum + my $TO_REDUCE_H = $_[38]; # [in] ZMM for hi 4x128-bit GHASH sum + my $TO_REDUCE_M = $_[39]; # [in] ZMM for medium 4x128-bit GHASH sum + my $ENC_DEC = $_[40]; # [in] cipher direction + my $HASH_IN_OUT = $_[41]; # [in/out] XMM ghash in/out value + my $IA0 = $_[42]; # [clobbered] GP temporary + my $IA1 = $_[43]; # [clobbered] GP temporary + my $MASKREG = $_[44]; # [clobbered] mask register + my $NUM_BLOCKS = $_[45]; # [in] numerical value with number of blocks to be encrypted/ghashed (1 to 16) + my $PBLOCK_LEN = $_[46]; # [in] partial block length + + die "GHASH_16_ENCRYPT_N_GHASH_N: num_blocks is out of bounds = $NUM_BLOCKS\n" + if ($NUM_BLOCKS > 16 || $NUM_BLOCKS < 0); + + my $label_suffix = $label_count++; + + my $GH1H = $HASH_IN_OUT; + + # ; this is to avoid additional move in do_reduction case + + my $LAST_GHASH_BLK = $GH1L; + my $LAST_CIPHER_BLK = $GH1T; + + my $RED_POLY = $GH2T; + my $RED_P1 = $GH2L; + my $RED_T1 = $GH2H; + my $RED_T2 = $GH2M; + + my $DATA1 = $GH3H; + my $DATA2 = $GH3L; + my $DATA3 = $GH3M; + my $DATA4 = $GH3T; + + # ;; do reduction after the 16 blocks ? + my $do_reduction = 0; + + # ;; is 16 block chunk a start? + my $is_start = 0; + + if ($GHASH_TYPE eq "start_reduce") { + $is_start = 1; + $do_reduction = 1; + } + + if ($GHASH_TYPE eq "start") { + $is_start = 1; + } + + if ($GHASH_TYPE eq "end_reduce") { + $do_reduction = 1; + } + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; - get load/store mask + # ;; - load plain/cipher text + # ;; get load/store mask + $code .= <<___; + lea byte64_len_to_mask_table(%rip),$IA0 + mov $LENGTH,$IA1 +___ + if ($NUM_BLOCKS > 12) { + $code .= "sub \$`3*64`,$IA1\n"; + } elsif ($NUM_BLOCKS > 8) { + $code .= "sub \$`2*64`,$IA1\n"; + } elsif ($NUM_BLOCKS > 4) { + $code .= "sub \$`1*64`,$IA1\n"; + } + $code .= "kmovq ($IA0,$IA1,8),$MASKREG\n"; + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; prepare counter blocks + + $code .= <<___; + cmp \$`(256 - $NUM_BLOCKS)`,@{[DWORD($CTR_CHECK)]} + jae .L_16_blocks_overflow_${label_suffix} +___ + + &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16( + $NUM_BLOCKS, "vpaddd", $B00_03, $B04_07, $B08_11, $B12_15, $CTR_BE, + $B00_03, $B04_07, $B08_11, $ADDBE_1234, $ADDBE_4x4, $ADDBE_4x4, $ADDBE_4x4); + $code .= <<___; + jmp .L_16_blocks_ok_${label_suffix} + +.L_16_blocks_overflow_${label_suffix}: + vpshufb $SHFMSK,$CTR_BE,$CTR_BE + vpaddd ddq_add_1234(%rip),$CTR_BE,$B00_03 +___ + if ($NUM_BLOCKS > 4) { + $code .= <<___; + vmovdqa64 ddq_add_4444(%rip),$B12_15 + vpaddd $B12_15,$B00_03,$B04_07 +___ + } + if ($NUM_BLOCKS > 8) { + $code .= "vpaddd $B12_15,$B04_07,$B08_11\n"; + } + if ($NUM_BLOCKS > 12) { + $code .= "vpaddd $B12_15,$B08_11,$B12_15\n"; + } + &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16( + $NUM_BLOCKS, "vpshufb", $B00_03, $B04_07, $B08_11, $B12_15, $B00_03, + $B04_07, $B08_11, $B12_15, $SHFMSK, $SHFMSK, $SHFMSK, $SHFMSK); + $code .= <<___; +.L_16_blocks_ok_${label_suffix}: + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; - pre-load constants + # ;; - add current hash into the 1st block + vbroadcastf64x2 `(16 * 0)`($AES_KEYS),$AESKEY1 +___ + if ($is_start != 0) { + $code .= "vpxorq `$GHASHIN_BLK_OFFSET + (0*64)`(%rsp),$HASH_IN_OUT,$GHDAT1\n"; + } else { + $code .= "vmovdqa64 `$GHASHIN_BLK_OFFSET + (0*64)`(%rsp),$GHDAT1\n"; + } + + $code .= "vmovdqu64 @{[EffectiveAddress(\"%rsp\",$HASHKEY_OFFSET,0*64)]},$GHKEY1\n"; + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; save counter for the next round + # ;; increment counter overflow check register + if ($NUM_BLOCKS <= 4) { + $code .= "vextracti32x4 \$`($NUM_BLOCKS - 1)`,$B00_03,@{[XWORD($CTR_BE)]}\n"; + } elsif ($NUM_BLOCKS <= 8) { + $code .= "vextracti32x4 \$`($NUM_BLOCKS - 5)`,$B04_07,@{[XWORD($CTR_BE)]}\n"; + } elsif ($NUM_BLOCKS <= 12) { + $code .= "vextracti32x4 \$`($NUM_BLOCKS - 9)`,$B08_11,@{[XWORD($CTR_BE)]}\n"; + } else { + $code .= "vextracti32x4 \$`($NUM_BLOCKS - 13)`,$B12_15,@{[XWORD($CTR_BE)]}\n"; + } + $code .= "vshufi64x2 \$0b00000000,$CTR_BE,$CTR_BE,$CTR_BE\n"; + + $code .= <<___; + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; pre-load constants + vbroadcastf64x2 `(16 * 1)`($AES_KEYS),$AESKEY2 + vmovdqu64 @{[EffectiveAddress("%rsp",$HASHKEY_OFFSET,1*64)]},$GHKEY2 + vmovdqa64 `$GHASHIN_BLK_OFFSET + (1*64)`(%rsp),$GHDAT2 +___ + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; stitch AES rounds with GHASH + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; AES round 0 - ARK + + &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16( + $NUM_BLOCKS, "vpxorq", $B00_03, $B04_07, $B08_11, $B12_15, $B00_03, + $B04_07, $B08_11, $B12_15, $AESKEY1, $AESKEY1, $AESKEY1, $AESKEY1); + $code .= "vbroadcastf64x2 `(16 * 2)`($AES_KEYS),$AESKEY1\n"; + + $code .= <<___; + # ;;================================================== + # ;; GHASH 4 blocks (15 to 12) + vpclmulqdq \$0x11,$GHKEY1,$GHDAT1,$GH1H # ; a1*b1 + vpclmulqdq \$0x00,$GHKEY1,$GHDAT1,$GH1L # ; a0*b0 + vpclmulqdq \$0x01,$GHKEY1,$GHDAT1,$GH1M # ; a1*b0 + vpclmulqdq \$0x10,$GHKEY1,$GHDAT1,$GH1T # ; a0*b1 + vmovdqu64 @{[EffectiveAddress("%rsp",$HASHKEY_OFFSET,2*64)]},$GHKEY1 + vmovdqa64 `$GHASHIN_BLK_OFFSET + (2*64)`(%rsp),$GHDAT1 +___ + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; AES round 1 + &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16( + $NUM_BLOCKS, "vaesenc", $B00_03, $B04_07, $B08_11, $B12_15, $B00_03, + $B04_07, $B08_11, $B12_15, $AESKEY2, $AESKEY2, $AESKEY2, $AESKEY2); + $code .= "vbroadcastf64x2 `(16 * 3)`($AES_KEYS),$AESKEY2\n"; + + $code .= <<___; + # ;; ================================================= + # ;; GHASH 4 blocks (11 to 8) + vpclmulqdq \$0x10,$GHKEY2,$GHDAT2,$GH2M # ; a0*b1 + vpclmulqdq \$0x01,$GHKEY2,$GHDAT2,$GH2T # ; a1*b0 + vpclmulqdq \$0x11,$GHKEY2,$GHDAT2,$GH2H # ; a1*b1 + vpclmulqdq \$0x00,$GHKEY2,$GHDAT2,$GH2L # ; a0*b0 + vmovdqu64 @{[EffectiveAddress("%rsp",$HASHKEY_OFFSET,3*64)]},$GHKEY2 + vmovdqa64 `$GHASHIN_BLK_OFFSET + (3*64)`(%rsp),$GHDAT2 +___ + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; AES round 2 + &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16( + $NUM_BLOCKS, "vaesenc", $B00_03, $B04_07, $B08_11, $B12_15, $B00_03, + $B04_07, $B08_11, $B12_15, $AESKEY1, $AESKEY1, $AESKEY1, $AESKEY1); + $code .= "vbroadcastf64x2 `(16 * 4)`($AES_KEYS),$AESKEY1\n"; + + $code .= <<___; + # ;; ================================================= + # ;; GHASH 4 blocks (7 to 4) + vpclmulqdq \$0x10,$GHKEY1,$GHDAT1,$GH3M # ; a0*b1 + vpclmulqdq \$0x01,$GHKEY1,$GHDAT1,$GH3T # ; a1*b0 + vpclmulqdq \$0x11,$GHKEY1,$GHDAT1,$GH3H # ; a1*b1 + vpclmulqdq \$0x00,$GHKEY1,$GHDAT1,$GH3L # ; a0*b0 +___ + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; AES rounds 3 + &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16( + $NUM_BLOCKS, "vaesenc", $B00_03, $B04_07, $B08_11, $B12_15, $B00_03, + $B04_07, $B08_11, $B12_15, $AESKEY2, $AESKEY2, $AESKEY2, $AESKEY2); + $code .= "vbroadcastf64x2 `(16 * 5)`($AES_KEYS),$AESKEY2\n"; + + $code .= <<___; + # ;; ================================================= + # ;; Gather (XOR) GHASH for 12 blocks + vpternlogq \$0x96,$GH3H,$GH2H,$GH1H + vpternlogq \$0x96,$GH3L,$GH2L,$GH1L + vpternlogq \$0x96,$GH3T,$GH2T,$GH1T + vpternlogq \$0x96,$GH3M,$GH2M,$GH1M +___ + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; AES rounds 4 + &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16( + $NUM_BLOCKS, "vaesenc", $B00_03, $B04_07, $B08_11, $B12_15, $B00_03, + $B04_07, $B08_11, $B12_15, $AESKEY1, $AESKEY1, $AESKEY1, $AESKEY1); + $code .= "vbroadcastf64x2 `(16 * 6)`($AES_KEYS),$AESKEY1\n"; + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; load plain/cipher text + &ZMM_LOAD_MASKED_BLOCKS_0_16($NUM_BLOCKS, $PLAIN_CIPH_IN, $DATA_OFFSET, $DATA1, $DATA2, $DATA3, $DATA4, $MASKREG); + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; AES rounds 5 + &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16( + $NUM_BLOCKS, "vaesenc", $B00_03, $B04_07, $B08_11, $B12_15, $B00_03, + $B04_07, $B08_11, $B12_15, $AESKEY2, $AESKEY2, $AESKEY2, $AESKEY2); + $code .= "vbroadcastf64x2 `(16 * 7)`($AES_KEYS),$AESKEY2\n"; + + $code .= <<___; + # ;; ================================================= + # ;; GHASH 4 blocks (3 to 0) + vpclmulqdq \$0x10,$GHKEY2,$GHDAT2,$GH2M # ; a0*b1 + vpclmulqdq \$0x01,$GHKEY2,$GHDAT2,$GH2T # ; a1*b0 + vpclmulqdq \$0x11,$GHKEY2,$GHDAT2,$GH2H # ; a1*b1 + vpclmulqdq \$0x00,$GHKEY2,$GHDAT2,$GH2L # ; a0*b0 +___ + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; AES round 6 + &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16( + $NUM_BLOCKS, "vaesenc", $B00_03, $B04_07, $B08_11, $B12_15, $B00_03, + $B04_07, $B08_11, $B12_15, $AESKEY1, $AESKEY1, $AESKEY1, $AESKEY1); + $code .= "vbroadcastf64x2 `(16 * 8)`($AES_KEYS),$AESKEY1\n"; + + # ;; ================================================= + # ;; gather GHASH in GH1L (low), GH1H (high), GH1M (mid) + # ;; - add GH2[MTLH] to GH1[MTLH] + $code .= "vpternlogq \$0x96,$GH2T,$GH1T,$GH1M\n"; + if ($do_reduction != 0) { + + if ($is_start != 0) { + $code .= "vpxorq $GH2M,$GH1M,$GH1M\n"; + } else { + $code .= <<___; + vpternlogq \$0x96,$GH2H,$TO_REDUCE_H,$GH1H + vpternlogq \$0x96,$GH2L,$TO_REDUCE_L,$GH1L + vpternlogq \$0x96,$GH2M,$TO_REDUCE_M,$GH1M +___ + } + + } else { + + # ;; Update H/M/L hash sums if not carrying reduction + if ($is_start != 0) { + $code .= <<___; + vpxorq $GH2H,$GH1H,$TO_REDUCE_H + vpxorq $GH2L,$GH1L,$TO_REDUCE_L + vpxorq $GH2M,$GH1M,$TO_REDUCE_M +___ + } else { + $code .= <<___; + vpternlogq \$0x96,$GH2H,$GH1H,$TO_REDUCE_H + vpternlogq \$0x96,$GH2L,$GH1L,$TO_REDUCE_L + vpternlogq \$0x96,$GH2M,$GH1M,$TO_REDUCE_M +___ + } + + } + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; AES round 7 + &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16( + $NUM_BLOCKS, "vaesenc", $B00_03, $B04_07, $B08_11, $B12_15, $B00_03, + $B04_07, $B08_11, $B12_15, $AESKEY2, $AESKEY2, $AESKEY2, $AESKEY2); + $code .= "vbroadcastf64x2 `(16 * 9)`($AES_KEYS),$AESKEY2\n"; + + # ;; ================================================= + # ;; prepare mid sum for adding to high & low + # ;; load polynomial constant for reduction + if ($do_reduction != 0) { + $code .= <<___; + vpsrldq \$8,$GH1M,$GH2M + vpslldq \$8,$GH1M,$GH1M + + vmovdqa64 POLY2(%rip),@{[XWORD($RED_POLY)]} +___ + } + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; AES round 8 + &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16( + $NUM_BLOCKS, "vaesenc", $B00_03, $B04_07, $B08_11, $B12_15, $B00_03, + $B04_07, $B08_11, $B12_15, $AESKEY1, $AESKEY1, $AESKEY1, $AESKEY1); + $code .= "vbroadcastf64x2 `(16 * 10)`($AES_KEYS),$AESKEY1\n"; + + # ;; ================================================= + # ;; Add mid product to high and low + if ($do_reduction != 0) { + if ($is_start != 0) { + $code .= <<___; + vpternlogq \$0x96,$GH2M,$GH2H,$GH1H # ; TH = TH1 + TH2 + TM>>64 + vpternlogq \$0x96,$GH1M,$GH2L,$GH1L # ; TL = TL1 + TL2 + TM<<64 +___ + } else { + $code .= <<___; + vpxorq $GH2M,$GH1H,$GH1H # ; TH = TH1 + TM>>64 + vpxorq $GH1M,$GH1L,$GH1L # ; TL = TL1 + TM<<64 +___ + } + } + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; AES round 9 + &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16( + $NUM_BLOCKS, "vaesenc", $B00_03, $B04_07, $B08_11, $B12_15, $B00_03, + $B04_07, $B08_11, $B12_15, $AESKEY2, $AESKEY2, $AESKEY2, $AESKEY2); + + # ;; ================================================= + # ;; horizontal xor of low and high 4x128 + if ($do_reduction != 0) { + &VHPXORI4x128($GH1H, $GH2H); + &VHPXORI4x128($GH1L, $GH2L); + } + + if (($NROUNDS >= 11)) { + $code .= "vbroadcastf64x2 `(16 * 11)`($AES_KEYS),$AESKEY2\n"; + } + + # ;; ================================================= + # ;; first phase of reduction + if ($do_reduction != 0) { + $code .= <<___; + vpclmulqdq \$0x01,@{[XWORD($GH1L)]},@{[XWORD($RED_POLY)]},@{[XWORD($RED_P1)]} + vpslldq \$8,@{[XWORD($RED_P1)]},@{[XWORD($RED_P1)]} # ; shift-L 2 DWs + vpxorq @{[XWORD($RED_P1)]},@{[XWORD($GH1L)]},@{[XWORD($RED_P1)]} # ; first phase of the reduct +___ + } + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; AES rounds up to 11 (AES192) or 13 (AES256) + # ;; AES128 is done + if (($NROUNDS >= 11)) { + &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16( + $NUM_BLOCKS, "vaesenc", $B00_03, $B04_07, $B08_11, $B12_15, $B00_03, + $B04_07, $B08_11, $B12_15, $AESKEY1, $AESKEY1, $AESKEY1, $AESKEY1); + $code .= "vbroadcastf64x2 `(16 * 12)`($AES_KEYS),$AESKEY1\n"; + + &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16( + $NUM_BLOCKS, "vaesenc", $B00_03, $B04_07, $B08_11, $B12_15, $B00_03, + $B04_07, $B08_11, $B12_15, $AESKEY2, $AESKEY2, $AESKEY2, $AESKEY2); + if (($NROUNDS == 13)) { + $code .= "vbroadcastf64x2 `(16 * 13)`($AES_KEYS),$AESKEY2\n"; + + &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16( + $NUM_BLOCKS, "vaesenc", $B00_03, $B04_07, $B08_11, $B12_15, $B00_03, + $B04_07, $B08_11, $B12_15, $AESKEY1, $AESKEY1, $AESKEY1, $AESKEY1); + $code .= "vbroadcastf64x2 `(16 * 14)`($AES_KEYS),$AESKEY1\n"; + + &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16( + $NUM_BLOCKS, "vaesenc", $B00_03, $B04_07, $B08_11, $B12_15, $B00_03, + $B04_07, $B08_11, $B12_15, $AESKEY2, $AESKEY2, $AESKEY2, $AESKEY2); + } + } + + # ;; ================================================= + # ;; second phase of the reduction + if ($do_reduction != 0) { + $code .= <<___; + vpclmulqdq \$0x00,@{[XWORD($RED_P1)]},@{[XWORD($RED_POLY)]},@{[XWORD($RED_T1)]} + vpsrldq \$4,@{[XWORD($RED_T1)]},@{[XWORD($RED_T1)]} # ; shift-R 1-DW to obtain 2-DWs shift-R + vpclmulqdq \$0x10,@{[XWORD($RED_P1)]},@{[XWORD($RED_POLY)]},@{[XWORD($RED_T2)]} + vpslldq \$4,@{[XWORD($RED_T2)]},@{[XWORD($RED_T2)]} # ; shift-L 1-DW for result without shifts + # ;; GH1H = GH1H + RED_T1 + RED_T2 + vpternlogq \$0x96,@{[XWORD($RED_T1)]},@{[XWORD($RED_T2)]},@{[XWORD($GH1H)]} +___ + } + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; the last AES round + &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16( + $NUM_BLOCKS, "vaesenclast", $B00_03, $B04_07, $B08_11, $B12_15, $B00_03, + $B04_07, $B08_11, $B12_15, $AESKEY1, $AESKEY1, $AESKEY1, $AESKEY1); + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; XOR against plain/cipher text + &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16( + $NUM_BLOCKS, "vpxorq", $B00_03, $B04_07, $B08_11, $B12_15, $B00_03, + $B04_07, $B08_11, $B12_15, $DATA1, $DATA2, $DATA3, $DATA4); + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; retrieve the last cipher counter block (partially XOR'ed with text) + # ;; - this is needed for partial block cases + if ($NUM_BLOCKS <= 4) { + $code .= "vextracti32x4 \$`($NUM_BLOCKS - 1)`,$B00_03,@{[XWORD($LAST_CIPHER_BLK)]}\n"; + } elsif ($NUM_BLOCKS <= 8) { + $code .= "vextracti32x4 \$`($NUM_BLOCKS - 5)`,$B04_07,@{[XWORD($LAST_CIPHER_BLK)]}\n"; + } elsif ($NUM_BLOCKS <= 12) { + $code .= "vextracti32x4 \$`($NUM_BLOCKS - 9)`,$B08_11,@{[XWORD($LAST_CIPHER_BLK)]}\n"; + } else { + $code .= "vextracti32x4 \$`($NUM_BLOCKS - 13)`,$B12_15,@{[XWORD($LAST_CIPHER_BLK)]}\n"; + } + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; store cipher/plain text + $code .= "mov $CIPH_PLAIN_OUT,$IA0\n"; + &ZMM_STORE_MASKED_BLOCKS_0_16($NUM_BLOCKS, $IA0, $DATA_OFFSET, $B00_03, $B04_07, $B08_11, $B12_15, $MASKREG); + + # ;; ================================================= + # ;; shuffle cipher text blocks for GHASH computation + if ($ENC_DEC eq "ENC") { + + # ;; zero bytes outside the mask before hashing + if ($NUM_BLOCKS <= 4) { + $code .= "vmovdqu8 $B00_03,${B00_03}{$MASKREG}{z}\n"; + } elsif ($NUM_BLOCKS <= 8) { + $code .= "vmovdqu8 $B04_07,${B04_07}{$MASKREG}{z}\n"; + } elsif ($NUM_BLOCKS <= 12) { + $code .= "vmovdqu8 $B08_11,${B08_11}{$MASKREG}{z}\n"; + } else { + $code .= "vmovdqu8 $B12_15,${B12_15}{$MASKREG}{z}\n"; + } + + &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16( + $NUM_BLOCKS, "vpshufb", $DATA1, $DATA2, $DATA3, $DATA4, $B00_03, + $B04_07, $B08_11, $B12_15, $SHFMSK, $SHFMSK, $SHFMSK, $SHFMSK); + } else { + + # ;; zero bytes outside the mask before hashing + if ($NUM_BLOCKS <= 4) { + $code .= "vmovdqu8 $DATA1,${DATA1}{$MASKREG}{z}\n"; + } elsif ($NUM_BLOCKS <= 8) { + $code .= "vmovdqu8 $DATA2,${DATA2}{$MASKREG}{z}\n"; + } elsif ($NUM_BLOCKS <= 12) { + $code .= "vmovdqu8 $DATA3,${DATA3}{$MASKREG}{z}\n"; + } else { + $code .= "vmovdqu8 $DATA4,${DATA4}{$MASKREG}{z}\n"; + } + + &ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16( + $NUM_BLOCKS, "vpshufb", $DATA1, $DATA2, $DATA3, $DATA4, $DATA1, + $DATA2, $DATA3, $DATA4, $SHFMSK, $SHFMSK, $SHFMSK, $SHFMSK); + } + + # ;; ================================================= + # ;; Extract the last block for partial / multi_call cases + if ($NUM_BLOCKS <= 4) { + $code .= "vextracti32x4 \$`($NUM_BLOCKS-1)`,$DATA1,@{[XWORD($LAST_GHASH_BLK)]}\n"; + } elsif ($NUM_BLOCKS <= 8) { + $code .= "vextracti32x4 \$`($NUM_BLOCKS-5)`,$DATA2,@{[XWORD($LAST_GHASH_BLK)]}\n"; + } elsif ($NUM_BLOCKS <= 12) { + $code .= "vextracti32x4 \$`($NUM_BLOCKS-9)`,$DATA3,@{[XWORD($LAST_GHASH_BLK)]}\n"; + } else { + $code .= "vextracti32x4 \$`($NUM_BLOCKS-13)`,$DATA4,@{[XWORD($LAST_GHASH_BLK)]}\n"; + } + + if ($do_reduction != 0) { + + # ;; GH1H holds reduced hash value + # ;; - normally do "vmovdqa64 &XWORD($GH1H), &XWORD($HASH_IN_OUT)" + # ;; - register rename trick obsoletes the above move + } + + # ;; ================================================= + # ;; GHASH last N blocks + # ;; - current hash value in HASH_IN_OUT or + # ;; product parts in TO_REDUCE_H/M/L + # ;; - DATA1-DATA4 include blocks for GHASH + + if ($do_reduction == 0) { + &INITIAL_BLOCKS_PARTIAL_GHASH( + $AES_KEYS, $GCM128_CTX, $LENGTH, $NUM_BLOCKS, + &XWORD($HASH_IN_OUT), $ENC_DEC, $DATA1, $DATA2, + $DATA3, $DATA4, &XWORD($LAST_CIPHER_BLK), &XWORD($LAST_GHASH_BLK), + $B00_03, $B04_07, $B08_11, $B12_15, + $GHDAT1, $GHDAT2, $AESKEY1, $AESKEY2, + $GHKEY1, $PBLOCK_LEN, $TO_REDUCE_H, $TO_REDUCE_M, + $TO_REDUCE_L); + } else { + &INITIAL_BLOCKS_PARTIAL_GHASH( + $AES_KEYS, $GCM128_CTX, $LENGTH, $NUM_BLOCKS, + &XWORD($HASH_IN_OUT), $ENC_DEC, $DATA1, $DATA2, + $DATA3, $DATA4, &XWORD($LAST_CIPHER_BLK), &XWORD($LAST_GHASH_BLK), + $B00_03, $B04_07, $B08_11, $B12_15, + $GHDAT1, $GHDAT2, $AESKEY1, $AESKEY2, + $GHKEY1, $PBLOCK_LEN); + } +} + +# ;; =========================================================================== +# ;; =========================================================================== +# ;; Stitched GHASH of 16 blocks (with reduction) with encryption of N blocks +# ;; followed with GHASH of the N blocks. +sub GCM_ENC_DEC_LAST { + my $AES_KEYS = $_[0]; # [in] key pointer + my $GCM128_CTX = $_[1]; # [in] context pointer + my $CIPH_PLAIN_OUT = $_[2]; # [in] pointer to output buffer + my $PLAIN_CIPH_IN = $_[3]; # [in] pointer to input buffer + my $DATA_OFFSET = $_[4]; # [in] data offset + my $LENGTH = $_[5]; # [in/clobbered] data length + my $CTR_BE = $_[6]; # [in/out] ZMM counter blocks (last 4) in big-endian + my $CTR_CHECK = $_[7]; # [in/out] GP with 8-bit counter for overflow check + my $HASHKEY_OFFSET = $_[8]; # [in] numerical offset for the highest hash key + # (can be register or numerical offset) + my $GHASHIN_BLK_OFFSET = $_[9]; # [in] numerical offset for GHASH blocks in + my $SHFMSK = $_[10]; # [in] ZMM with byte swap mask for pshufb + my $ZT00 = $_[11]; # [clobbered] temporary ZMM + my $ZT01 = $_[12]; # [clobbered] temporary ZMM + my $ZT02 = $_[13]; # [clobbered] temporary ZMM + my $ZT03 = $_[14]; # [clobbered] temporary ZMM + my $ZT04 = $_[15]; # [clobbered] temporary ZMM + my $ZT05 = $_[16]; # [clobbered] temporary ZMM + my $ZT06 = $_[17]; # [clobbered] temporary ZMM + my $ZT07 = $_[18]; # [clobbered] temporary ZMM + my $ZT08 = $_[19]; # [clobbered] temporary ZMM + my $ZT09 = $_[20]; # [clobbered] temporary ZMM + my $ZT10 = $_[21]; # [clobbered] temporary ZMM + my $ZT11 = $_[22]; # [clobbered] temporary ZMM + my $ZT12 = $_[23]; # [clobbered] temporary ZMM + my $ZT13 = $_[24]; # [clobbered] temporary ZMM + my $ZT14 = $_[25]; # [clobbered] temporary ZMM + my $ZT15 = $_[26]; # [clobbered] temporary ZMM + my $ZT16 = $_[27]; # [clobbered] temporary ZMM + my $ZT17 = $_[28]; # [clobbered] temporary ZMM + my $ZT18 = $_[29]; # [clobbered] temporary ZMM + my $ZT19 = $_[30]; # [clobbered] temporary ZMM + my $ZT20 = $_[31]; # [clobbered] temporary ZMM + my $ZT21 = $_[32]; # [clobbered] temporary ZMM + my $ZT22 = $_[33]; # [clobbered] temporary ZMM + my $ADDBE_4x4 = $_[34]; # [in] ZMM with 4x128bits 4 in big-endian + my $ADDBE_1234 = $_[35]; # [in] ZMM with 4x128bits 1, 2, 3 and 4 in big-endian + my $GHASH_TYPE = $_[36]; # [in] "start", "start_reduce", "mid", "end_reduce" + my $TO_REDUCE_L = $_[37]; # [in] ZMM for low 4x128-bit GHASH sum + my $TO_REDUCE_H = $_[38]; # [in] ZMM for hi 4x128-bit GHASH sum + my $TO_REDUCE_M = $_[39]; # [in] ZMM for medium 4x128-bit GHASH sum + my $ENC_DEC = $_[40]; # [in] cipher direction + my $HASH_IN_OUT = $_[41]; # [in/out] XMM ghash in/out value + my $IA0 = $_[42]; # [clobbered] GP temporary + my $IA1 = $_[43]; # [clobbered] GP temporary + my $MASKREG = $_[44]; # [clobbered] mask register + my $PBLOCK_LEN = $_[45]; # [in] partial block length + + my $label_suffix = $label_count++; + + $code .= <<___; + mov @{[DWORD($LENGTH)]},@{[DWORD($IA0)]} + add \$15,@{[DWORD($IA0)]} + shr \$4,@{[DWORD($IA0)]} + je .L_last_num_blocks_is_0_${label_suffix} + + cmp \$8,@{[DWORD($IA0)]} + je .L_last_num_blocks_is_8_${label_suffix} + jb .L_last_num_blocks_is_7_1_${label_suffix} + + + cmp \$12,@{[DWORD($IA0)]} + je .L_last_num_blocks_is_12_${label_suffix} + jb .L_last_num_blocks_is_11_9_${label_suffix} + + # ;; 16, 15, 14 or 13 + cmp \$15,@{[DWORD($IA0)]} + je .L_last_num_blocks_is_15_${label_suffix} + ja .L_last_num_blocks_is_16_${label_suffix} + cmp \$14,@{[DWORD($IA0)]} + je .L_last_num_blocks_is_14_${label_suffix} + jmp .L_last_num_blocks_is_13_${label_suffix} + +.L_last_num_blocks_is_11_9_${label_suffix}: + # ;; 11, 10 or 9 + cmp \$10,@{[DWORD($IA0)]} + je .L_last_num_blocks_is_10_${label_suffix} + ja .L_last_num_blocks_is_11_${label_suffix} + jmp .L_last_num_blocks_is_9_${label_suffix} + +.L_last_num_blocks_is_7_1_${label_suffix}: + cmp \$4,@{[DWORD($IA0)]} + je .L_last_num_blocks_is_4_${label_suffix} + jb .L_last_num_blocks_is_3_1_${label_suffix} + # ;; 7, 6 or 5 + cmp \$6,@{[DWORD($IA0)]} + ja .L_last_num_blocks_is_7_${label_suffix} + je .L_last_num_blocks_is_6_${label_suffix} + jmp .L_last_num_blocks_is_5_${label_suffix} + +.L_last_num_blocks_is_3_1_${label_suffix}: + # ;; 3, 2 or 1 + cmp \$2,@{[DWORD($IA0)]} + ja .L_last_num_blocks_is_3_${label_suffix} + je .L_last_num_blocks_is_2_${label_suffix} +___ + + # ;; fall through for `jmp .L_last_num_blocks_is_1` + + # ;; Use rep to generate different block size variants + # ;; - one block size has to be the first one + for my $num_blocks (1 .. 16) { + $code .= ".L_last_num_blocks_is_${num_blocks}_${label_suffix}:\n"; + &GHASH_16_ENCRYPT_N_GHASH_N( + $AES_KEYS, $GCM128_CTX, $CIPH_PLAIN_OUT, $PLAIN_CIPH_IN, $DATA_OFFSET, + $LENGTH, $CTR_BE, $CTR_CHECK, $HASHKEY_OFFSET, $GHASHIN_BLK_OFFSET, + $SHFMSK, $ZT00, $ZT01, $ZT02, $ZT03, + $ZT04, $ZT05, $ZT06, $ZT07, $ZT08, + $ZT09, $ZT10, $ZT11, $ZT12, $ZT13, + $ZT14, $ZT15, $ZT16, $ZT17, $ZT18, + $ZT19, $ZT20, $ZT21, $ZT22, $ADDBE_4x4, + $ADDBE_1234, $GHASH_TYPE, $TO_REDUCE_L, $TO_REDUCE_H, $TO_REDUCE_M, + $ENC_DEC, $HASH_IN_OUT, $IA0, $IA1, $MASKREG, + $num_blocks, $PBLOCK_LEN); + + $code .= "jmp .L_last_blocks_done_${label_suffix}\n"; + } + + $code .= ".L_last_num_blocks_is_0_${label_suffix}:\n"; + + # ;; if there is 0 blocks to cipher then there are only 16 blocks for ghash and reduction + # ;; - convert mid into end_reduce + # ;; - convert start into start_reduce + if ($GHASH_TYPE eq "mid") { + $GHASH_TYPE = "end_reduce"; + } + if ($GHASH_TYPE eq "start") { + $GHASH_TYPE = "start_reduce"; + } + + &GHASH_16($GHASH_TYPE, $TO_REDUCE_H, $TO_REDUCE_M, $TO_REDUCE_L, "%rsp", + $GHASHIN_BLK_OFFSET, 0, "%rsp", $HASHKEY_OFFSET, 0, $HASH_IN_OUT, $ZT00, $ZT01, + $ZT02, $ZT03, $ZT04, $ZT05, $ZT06, $ZT07, $ZT08, $ZT09); + + $code .= ".L_last_blocks_done_${label_suffix}:\n"; +} + +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +# ;; Main GCM macro stitching cipher with GHASH +# ;; - operates on single stream +# ;; - encrypts 16 blocks at a time +# ;; - ghash the 16 previously encrypted ciphertext blocks +# ;; - no partial block or multi_call handling here +sub GHASH_16_ENCRYPT_16_PARALLEL { + my $AES_KEYS = $_[0]; # [in] key pointer + my $CIPH_PLAIN_OUT = $_[1]; # [in] pointer to output buffer + my $PLAIN_CIPH_IN = $_[2]; # [in] pointer to input buffer + my $DATA_OFFSET = $_[3]; # [in] data offset + my $CTR_BE = $_[4]; # [in/out] ZMM counter blocks (last 4) in big-endian + my $CTR_CHECK = $_[5]; # [in/out] GP with 8-bit counter for overflow check + my $HASHKEY_OFFSET = $_[6]; # [in] numerical offset for the highest hash key (hash key index value) + my $AESOUT_BLK_OFFSET = $_[7]; # [in] numerical offset for AES-CTR out + my $GHASHIN_BLK_OFFSET = $_[8]; # [in] numerical offset for GHASH blocks in + my $SHFMSK = $_[9]; # [in] ZMM with byte swap mask for pshufb + my $ZT1 = $_[10]; # [clobbered] temporary ZMM (cipher) + my $ZT2 = $_[11]; # [clobbered] temporary ZMM (cipher) + my $ZT3 = $_[12]; # [clobbered] temporary ZMM (cipher) + my $ZT4 = $_[13]; # [clobbered] temporary ZMM (cipher) + my $ZT5 = $_[14]; # [clobbered/out] temporary ZMM or GHASH OUT (final_reduction) + my $ZT6 = $_[15]; # [clobbered] temporary ZMM (cipher) + my $ZT7 = $_[16]; # [clobbered] temporary ZMM (cipher) + my $ZT8 = $_[17]; # [clobbered] temporary ZMM (cipher) + my $ZT9 = $_[18]; # [clobbered] temporary ZMM (cipher) + my $ZT10 = $_[19]; # [clobbered] temporary ZMM (ghash) + my $ZT11 = $_[20]; # [clobbered] temporary ZMM (ghash) + my $ZT12 = $_[21]; # [clobbered] temporary ZMM (ghash) + my $ZT13 = $_[22]; # [clobbered] temporary ZMM (ghash) + my $ZT14 = $_[23]; # [clobbered] temporary ZMM (ghash) + my $ZT15 = $_[24]; # [clobbered] temporary ZMM (ghash) + my $ZT16 = $_[25]; # [clobbered] temporary ZMM (ghash) + my $ZT17 = $_[26]; # [clobbered] temporary ZMM (ghash) + my $ZT18 = $_[27]; # [clobbered] temporary ZMM (ghash) + my $ZT19 = $_[28]; # [clobbered] temporary ZMM + my $ZT20 = $_[29]; # [clobbered] temporary ZMM + my $ZT21 = $_[30]; # [clobbered] temporary ZMM + my $ZT22 = $_[31]; # [clobbered] temporary ZMM + my $ZT23 = $_[32]; # [clobbered] temporary ZMM + my $ADDBE_4x4 = $_[33]; # [in] ZMM with 4x128bits 4 in big-endian + my $ADDBE_1234 = $_[34]; # [in] ZMM with 4x128bits 1, 2, 3 and 4 in big-endian + my $TO_REDUCE_L = $_[35]; # [in/out] ZMM for low 4x128-bit GHASH sum + my $TO_REDUCE_H = $_[36]; # [in/out] ZMM for hi 4x128-bit GHASH sum + my $TO_REDUCE_M = $_[37]; # [in/out] ZMM for medium 4x128-bit GHASH sum + my $DO_REDUCTION = $_[38]; # [in] "no_reduction", "final_reduction", "first_time" + my $ENC_DEC = $_[39]; # [in] cipher direction + my $DATA_DISPL = $_[40]; # [in] fixed numerical data displacement/offset + my $GHASH_IN = $_[41]; # [in] current GHASH value or "no_ghash_in" + my $IA0 = $_[42]; # [clobbered] temporary GPR + + my $B00_03 = $ZT1; + my $B04_07 = $ZT2; + my $B08_11 = $ZT3; + my $B12_15 = $ZT4; + + my $GH1H = $ZT5; + + # ; @note: do not change this mapping + my $GH1L = $ZT6; + my $GH1M = $ZT7; + my $GH1T = $ZT8; + + my $GH2H = $ZT9; + my $GH2L = $ZT10; + my $GH2M = $ZT11; + my $GH2T = $ZT12; + + my $RED_POLY = $GH2T; + my $RED_P1 = $GH2L; + my $RED_T1 = $GH2H; + my $RED_T2 = $GH2M; + + my $GH3H = $ZT13; + my $GH3L = $ZT14; + my $GH3M = $ZT15; + my $GH3T = $ZT16; + + my $DATA1 = $ZT13; + my $DATA2 = $ZT14; + my $DATA3 = $ZT15; + my $DATA4 = $ZT16; + + my $AESKEY1 = $ZT17; + my $AESKEY2 = $ZT18; + + my $GHKEY1 = $ZT19; + my $GHKEY2 = $ZT20; + my $GHDAT1 = $ZT21; + my $GHDAT2 = $ZT22; + + my $label_suffix = $label_count++; + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; prepare counter blocks + + $code .= <<___; + cmpb \$`(256 - 16)`,@{[BYTE($CTR_CHECK)]} + jae .L_16_blocks_overflow_${label_suffix} + vpaddd $ADDBE_1234,$CTR_BE,$B00_03 + vpaddd $ADDBE_4x4,$B00_03,$B04_07 + vpaddd $ADDBE_4x4,$B04_07,$B08_11 + vpaddd $ADDBE_4x4,$B08_11,$B12_15 + jmp .L_16_blocks_ok_${label_suffix} +.L_16_blocks_overflow_${label_suffix}: + vpshufb $SHFMSK,$CTR_BE,$CTR_BE + vmovdqa64 ddq_add_4444(%rip),$B12_15 + vpaddd ddq_add_1234(%rip),$CTR_BE,$B00_03 + vpaddd $B12_15,$B00_03,$B04_07 + vpaddd $B12_15,$B04_07,$B08_11 + vpaddd $B12_15,$B08_11,$B12_15 + vpshufb $SHFMSK,$B00_03,$B00_03 + vpshufb $SHFMSK,$B04_07,$B04_07 + vpshufb $SHFMSK,$B08_11,$B08_11 + vpshufb $SHFMSK,$B12_15,$B12_15 +.L_16_blocks_ok_${label_suffix}: +___ + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; pre-load constants + $code .= "vbroadcastf64x2 `(16 * 0)`($AES_KEYS),$AESKEY1\n"; + if ($GHASH_IN ne "no_ghash_in") { + $code .= "vpxorq `$GHASHIN_BLK_OFFSET + (0*64)`(%rsp),$GHASH_IN,$GHDAT1\n"; + } else { + $code .= "vmovdqa64 `$GHASHIN_BLK_OFFSET + (0*64)`(%rsp),$GHDAT1\n"; + } + + $code .= <<___; + vmovdqu64 @{[HashKeyByIdx(($HASHKEY_OFFSET - (0*4)),"%rsp")]},$GHKEY1 + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; save counter for the next round + # ;; increment counter overflow check register + vshufi64x2 \$0b11111111,$B12_15,$B12_15,$CTR_BE + addb \$16,@{[BYTE($CTR_CHECK)]} + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; pre-load constants + vbroadcastf64x2 `(16 * 1)`($AES_KEYS),$AESKEY2 + vmovdqu64 @{[HashKeyByIdx(($HASHKEY_OFFSET - (1*4)),"%rsp")]},$GHKEY2 + vmovdqa64 `$GHASHIN_BLK_OFFSET + (1*64)`(%rsp),$GHDAT2 + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; stitch AES rounds with GHASH + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; AES round 0 - ARK + + vpxorq $AESKEY1,$B00_03,$B00_03 + vpxorq $AESKEY1,$B04_07,$B04_07 + vpxorq $AESKEY1,$B08_11,$B08_11 + vpxorq $AESKEY1,$B12_15,$B12_15 + vbroadcastf64x2 `(16 * 2)`($AES_KEYS),$AESKEY1 + + # ;;================================================== + # ;; GHASH 4 blocks (15 to 12) + vpclmulqdq \$0x11,$GHKEY1,$GHDAT1,$GH1H # ; a1*b1 + vpclmulqdq \$0x00,$GHKEY1,$GHDAT1,$GH1L # ; a0*b0 + vpclmulqdq \$0x01,$GHKEY1,$GHDAT1,$GH1M # ; a1*b0 + vpclmulqdq \$0x10,$GHKEY1,$GHDAT1,$GH1T # ; a0*b1 + vmovdqu64 @{[HashKeyByIdx(($HASHKEY_OFFSET - (2*4)),"%rsp")]},$GHKEY1 + vmovdqa64 `$GHASHIN_BLK_OFFSET + (2*64)`(%rsp),$GHDAT1 + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; AES round 1 + vaesenc $AESKEY2,$B00_03,$B00_03 + vaesenc $AESKEY2,$B04_07,$B04_07 + vaesenc $AESKEY2,$B08_11,$B08_11 + vaesenc $AESKEY2,$B12_15,$B12_15 + vbroadcastf64x2 `(16 * 3)`($AES_KEYS),$AESKEY2 + + # ;; ================================================= + # ;; GHASH 4 blocks (11 to 8) + vpclmulqdq \$0x10,$GHKEY2,$GHDAT2,$GH2M # ; a0*b1 + vpclmulqdq \$0x01,$GHKEY2,$GHDAT2,$GH2T # ; a1*b0 + vpclmulqdq \$0x11,$GHKEY2,$GHDAT2,$GH2H # ; a1*b1 + vpclmulqdq \$0x00,$GHKEY2,$GHDAT2,$GH2L # ; a0*b0 + vmovdqu64 @{[HashKeyByIdx(($HASHKEY_OFFSET - (3*4)),"%rsp")]},$GHKEY2 + vmovdqa64 `$GHASHIN_BLK_OFFSET + (3*64)`(%rsp),$GHDAT2 + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; AES round 2 + vaesenc $AESKEY1,$B00_03,$B00_03 + vaesenc $AESKEY1,$B04_07,$B04_07 + vaesenc $AESKEY1,$B08_11,$B08_11 + vaesenc $AESKEY1,$B12_15,$B12_15 + vbroadcastf64x2 `(16 * 4)`($AES_KEYS),$AESKEY1 + + # ;; ================================================= + # ;; GHASH 4 blocks (7 to 4) + vpclmulqdq \$0x10,$GHKEY1,$GHDAT1,$GH3M # ; a0*b1 + vpclmulqdq \$0x01,$GHKEY1,$GHDAT1,$GH3T # ; a1*b0 + vpclmulqdq \$0x11,$GHKEY1,$GHDAT1,$GH3H # ; a1*b1 + vpclmulqdq \$0x00,$GHKEY1,$GHDAT1,$GH3L # ; a0*b0 + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; AES rounds 3 + vaesenc $AESKEY2,$B00_03,$B00_03 + vaesenc $AESKEY2,$B04_07,$B04_07 + vaesenc $AESKEY2,$B08_11,$B08_11 + vaesenc $AESKEY2,$B12_15,$B12_15 + vbroadcastf64x2 `(16 * 5)`($AES_KEYS),$AESKEY2 + + # ;; ================================================= + # ;; Gather (XOR) GHASH for 12 blocks + vpternlogq \$0x96,$GH3H,$GH2H,$GH1H + vpternlogq \$0x96,$GH3L,$GH2L,$GH1L + vpternlogq \$0x96,$GH3T,$GH2T,$GH1T + vpternlogq \$0x96,$GH3M,$GH2M,$GH1M + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; AES rounds 4 + vaesenc $AESKEY1,$B00_03,$B00_03 + vaesenc $AESKEY1,$B04_07,$B04_07 + vaesenc $AESKEY1,$B08_11,$B08_11 + vaesenc $AESKEY1,$B12_15,$B12_15 + vbroadcastf64x2 `(16 * 6)`($AES_KEYS),$AESKEY1 + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; load plain/cipher text (recycle GH3xx registers) + vmovdqu8 `$DATA_DISPL + (0 * 64)`($PLAIN_CIPH_IN,$DATA_OFFSET),$DATA1 + vmovdqu8 `$DATA_DISPL + (1 * 64)`($PLAIN_CIPH_IN,$DATA_OFFSET),$DATA2 + vmovdqu8 `$DATA_DISPL + (2 * 64)`($PLAIN_CIPH_IN,$DATA_OFFSET),$DATA3 + vmovdqu8 `$DATA_DISPL + (3 * 64)`($PLAIN_CIPH_IN,$DATA_OFFSET),$DATA4 + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; AES rounds 5 + vaesenc $AESKEY2,$B00_03,$B00_03 + vaesenc $AESKEY2,$B04_07,$B04_07 + vaesenc $AESKEY2,$B08_11,$B08_11 + vaesenc $AESKEY2,$B12_15,$B12_15 + vbroadcastf64x2 `(16 * 7)`($AES_KEYS),$AESKEY2 + + # ;; ================================================= + # ;; GHASH 4 blocks (3 to 0) + vpclmulqdq \$0x10,$GHKEY2,$GHDAT2,$GH2M # ; a0*b1 + vpclmulqdq \$0x01,$GHKEY2,$GHDAT2,$GH2T # ; a1*b0 + vpclmulqdq \$0x11,$GHKEY2,$GHDAT2,$GH2H # ; a1*b1 + vpclmulqdq \$0x00,$GHKEY2,$GHDAT2,$GH2L # ; a0*b0 + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; AES round 6 + vaesenc $AESKEY1,$B00_03,$B00_03 + vaesenc $AESKEY1,$B04_07,$B04_07 + vaesenc $AESKEY1,$B08_11,$B08_11 + vaesenc $AESKEY1,$B12_15,$B12_15 + vbroadcastf64x2 `(16 * 8)`($AES_KEYS),$AESKEY1 +___ + + # ;; ================================================= + # ;; gather GHASH in GH1L (low) and GH1H (high) + if ($DO_REDUCTION eq "first_time") { + $code .= <<___; + vpternlogq \$0x96,$GH2T,$GH1T,$GH1M # ; TM + vpxorq $GH2M,$GH1M,$TO_REDUCE_M # ; TM + vpxorq $GH2H,$GH1H,$TO_REDUCE_H # ; TH + vpxorq $GH2L,$GH1L,$TO_REDUCE_L # ; TL +___ + } + if ($DO_REDUCTION eq "no_reduction") { + $code .= <<___; + vpternlogq \$0x96,$GH2T,$GH1T,$GH1M # ; TM + vpternlogq \$0x96,$GH2M,$GH1M,$TO_REDUCE_M # ; TM + vpternlogq \$0x96,$GH2H,$GH1H,$TO_REDUCE_H # ; TH + vpternlogq \$0x96,$GH2L,$GH1L,$TO_REDUCE_L # ; TL +___ + } + if ($DO_REDUCTION eq "final_reduction") { + $code .= <<___; + # ;; phase 1: add mid products together + # ;; also load polynomial constant for reduction + vpternlogq \$0x96,$GH2T,$GH1T,$GH1M # ; TM + vpternlogq \$0x96,$GH2M,$TO_REDUCE_M,$GH1M + + vpsrldq \$8,$GH1M,$GH2M + vpslldq \$8,$GH1M,$GH1M + + vmovdqa64 POLY2(%rip),@{[XWORD($RED_POLY)]} +___ + } + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; AES round 7 + $code .= <<___; + vaesenc $AESKEY2,$B00_03,$B00_03 + vaesenc $AESKEY2,$B04_07,$B04_07 + vaesenc $AESKEY2,$B08_11,$B08_11 + vaesenc $AESKEY2,$B12_15,$B12_15 + vbroadcastf64x2 `(16 * 9)`($AES_KEYS),$AESKEY2 +___ + + # ;; ================================================= + # ;; Add mid product to high and low + if ($DO_REDUCTION eq "final_reduction") { + $code .= <<___; + vpternlogq \$0x96,$GH2M,$GH2H,$GH1H # ; TH = TH1 + TH2 + TM>>64 + vpxorq $TO_REDUCE_H,$GH1H,$GH1H + vpternlogq \$0x96,$GH1M,$GH2L,$GH1L # ; TL = TL1 + TL2 + TM<<64 + vpxorq $TO_REDUCE_L,$GH1L,$GH1L +___ + } + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; AES round 8 + $code .= <<___; + vaesenc $AESKEY1,$B00_03,$B00_03 + vaesenc $AESKEY1,$B04_07,$B04_07 + vaesenc $AESKEY1,$B08_11,$B08_11 + vaesenc $AESKEY1,$B12_15,$B12_15 + vbroadcastf64x2 `(16 * 10)`($AES_KEYS),$AESKEY1 +___ + + # ;; ================================================= + # ;; horizontal xor of low and high 4x128 + if ($DO_REDUCTION eq "final_reduction") { + &VHPXORI4x128($GH1H, $GH2H); + &VHPXORI4x128($GH1L, $GH2L); + } + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; AES round 9 + $code .= <<___; + vaesenc $AESKEY2,$B00_03,$B00_03 + vaesenc $AESKEY2,$B04_07,$B04_07 + vaesenc $AESKEY2,$B08_11,$B08_11 + vaesenc $AESKEY2,$B12_15,$B12_15 +___ + if (($NROUNDS >= 11)) { + $code .= "vbroadcastf64x2 `(16 * 11)`($AES_KEYS),$AESKEY2\n"; + } + + # ;; ================================================= + # ;; first phase of reduction + if ($DO_REDUCTION eq "final_reduction") { + $code .= <<___; + vpclmulqdq \$0x01,@{[XWORD($GH1L)]},@{[XWORD($RED_POLY)]},@{[XWORD($RED_P1)]} + vpslldq \$8,@{[XWORD($RED_P1)]},@{[XWORD($RED_P1)]} # ; shift-L 2 DWs + vpxorq @{[XWORD($RED_P1)]},@{[XWORD($GH1L)]},@{[XWORD($RED_P1)]} # ; first phase of the reduct +___ + } + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; AES rounds up to 11 (AES192) or 13 (AES256) + # ;; AES128 is done + if (($NROUNDS >= 11)) { + $code .= <<___; + vaesenc $AESKEY1,$B00_03,$B00_03 + vaesenc $AESKEY1,$B04_07,$B04_07 + vaesenc $AESKEY1,$B08_11,$B08_11 + vaesenc $AESKEY1,$B12_15,$B12_15 + vbroadcastf64x2 `(16 * 12)`($AES_KEYS),$AESKEY1 + + vaesenc $AESKEY2,$B00_03,$B00_03 + vaesenc $AESKEY2,$B04_07,$B04_07 + vaesenc $AESKEY2,$B08_11,$B08_11 + vaesenc $AESKEY2,$B12_15,$B12_15 +___ + if (($NROUNDS == 13)) { + $code .= <<___; + vbroadcastf64x2 `(16 * 13)`($AES_KEYS),$AESKEY2 + + vaesenc $AESKEY1,$B00_03,$B00_03 + vaesenc $AESKEY1,$B04_07,$B04_07 + vaesenc $AESKEY1,$B08_11,$B08_11 + vaesenc $AESKEY1,$B12_15,$B12_15 + vbroadcastf64x2 `(16 * 14)`($AES_KEYS),$AESKEY1 + + vaesenc $AESKEY2,$B00_03,$B00_03 + vaesenc $AESKEY2,$B04_07,$B04_07 + vaesenc $AESKEY2,$B08_11,$B08_11 + vaesenc $AESKEY2,$B12_15,$B12_15 +___ + } + } + + # ;; ================================================= + # ;; second phase of the reduction + if ($DO_REDUCTION eq "final_reduction") { + $code .= <<___; + vpclmulqdq \$0x00,@{[XWORD($RED_P1)]},@{[XWORD($RED_POLY)]},@{[XWORD($RED_T1)]} + vpsrldq \$4,@{[XWORD($RED_T1)]},@{[XWORD($RED_T1)]} # ; shift-R 1-DW to obtain 2-DWs shift-R + vpclmulqdq \$0x10,@{[XWORD($RED_P1)]},@{[XWORD($RED_POLY)]},@{[XWORD($RED_T2)]} + vpslldq \$4,@{[XWORD($RED_T2)]},@{[XWORD($RED_T2)]} # ; shift-L 1-DW for result without shifts + # ;; GH1H = GH1H x RED_T1 x RED_T2 + vpternlogq \$0x96,@{[XWORD($RED_T1)]},@{[XWORD($RED_T2)]},@{[XWORD($GH1H)]} +___ + } + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; the last AES round + $code .= <<___; + vaesenclast $AESKEY1,$B00_03,$B00_03 + vaesenclast $AESKEY1,$B04_07,$B04_07 + vaesenclast $AESKEY1,$B08_11,$B08_11 + vaesenclast $AESKEY1,$B12_15,$B12_15 + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; XOR against plain/cipher text + vpxorq $DATA1,$B00_03,$B00_03 + vpxorq $DATA2,$B04_07,$B04_07 + vpxorq $DATA3,$B08_11,$B08_11 + vpxorq $DATA4,$B12_15,$B12_15 + + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; store cipher/plain text + mov $CIPH_PLAIN_OUT,$IA0 + vmovdqu8 $B00_03,`$DATA_DISPL + (0 * 64)`($IA0,$DATA_OFFSET,1) + vmovdqu8 $B04_07,`$DATA_DISPL + (1 * 64)`($IA0,$DATA_OFFSET,1) + vmovdqu8 $B08_11,`$DATA_DISPL + (2 * 64)`($IA0,$DATA_OFFSET,1) + vmovdqu8 $B12_15,`$DATA_DISPL + (3 * 64)`($IA0,$DATA_OFFSET,1) +___ + + # ;; ================================================= + # ;; shuffle cipher text blocks for GHASH computation + if ($ENC_DEC eq "ENC") { + $code .= <<___; + vpshufb $SHFMSK,$B00_03,$B00_03 + vpshufb $SHFMSK,$B04_07,$B04_07 + vpshufb $SHFMSK,$B08_11,$B08_11 + vpshufb $SHFMSK,$B12_15,$B12_15 +___ + } else { + $code .= <<___; + vpshufb $SHFMSK,$DATA1,$B00_03 + vpshufb $SHFMSK,$DATA2,$B04_07 + vpshufb $SHFMSK,$DATA3,$B08_11 + vpshufb $SHFMSK,$DATA4,$B12_15 +___ + } + + # ;; ================================================= + # ;; store shuffled cipher text for ghashing + $code .= <<___; + vmovdqa64 $B00_03,`$AESOUT_BLK_OFFSET + (0*64)`(%rsp) + vmovdqa64 $B04_07,`$AESOUT_BLK_OFFSET + (1*64)`(%rsp) + vmovdqa64 $B08_11,`$AESOUT_BLK_OFFSET + (2*64)`(%rsp) + vmovdqa64 $B12_15,`$AESOUT_BLK_OFFSET + (3*64)`(%rsp) +___ +} + +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +# ;;; Encryption of a single block +sub ENCRYPT_SINGLE_BLOCK { + my $AES_KEY = $_[0]; # ; [in] + my $XMM0 = $_[1]; # ; [in/out] + my $GPR1 = $_[2]; # ; [clobbered] + + my $label_suffix = $label_count++; + + $code .= <<___; + # ; load number of rounds from AES_KEY structure (offset in bytes is + # ; size of the |rd_key| buffer) + mov `4*15*4`($AES_KEY),@{[DWORD($GPR1)]} + cmp \$9,@{[DWORD($GPR1)]} + je .Laes_128_${label_suffix} + cmp \$11,@{[DWORD($GPR1)]} + je .Laes_192_${label_suffix} + cmp \$13,@{[DWORD($GPR1)]} + je .Laes_256_${label_suffix} + jmp .Lexit_aes_${label_suffix} +___ + for my $keylen (sort keys %aes_rounds) { + my $nr = $aes_rounds{$keylen}; + $code .= <<___; +.align 32 +.Laes_${keylen}_${label_suffix}: +___ + $code .= "vpxorq `16*0`($AES_KEY),$XMM0, $XMM0\n\n"; + for (my $i = 1; $i <= $nr; $i++) { + $code .= "vaesenc `16*$i`($AES_KEY),$XMM0,$XMM0\n\n"; + } + $code .= <<___; + vaesenclast `16*($nr+1)`($AES_KEY),$XMM0,$XMM0 + jmp .Lexit_aes_${label_suffix} +___ + } + $code .= ".Lexit_aes_${label_suffix}:\n\n"; +} + +sub CALC_J0 { + my $GCM128_CTX = $_[0]; #; [in] Pointer to GCM context + my $IV = $_[1]; #; [in] Pointer to IV + my $IV_LEN = $_[2]; #; [in] IV length + my $J0 = $_[3]; #; [out] XMM reg to contain J0 + my $ZT0 = $_[4]; #; [clobbered] ZMM register + my $ZT1 = $_[5]; #; [clobbered] ZMM register + my $ZT2 = $_[6]; #; [clobbered] ZMM register + my $ZT3 = $_[7]; #; [clobbered] ZMM register + my $ZT4 = $_[8]; #; [clobbered] ZMM register + my $ZT5 = $_[9]; #; [clobbered] ZMM register + my $ZT6 = $_[10]; #; [clobbered] ZMM register + my $ZT7 = $_[11]; #; [clobbered] ZMM register + my $ZT8 = $_[12]; #; [clobbered] ZMM register + my $ZT9 = $_[13]; #; [clobbered] ZMM register + my $ZT10 = $_[14]; #; [clobbered] ZMM register + my $ZT11 = $_[15]; #; [clobbered] ZMM register + my $ZT12 = $_[16]; #; [clobbered] ZMM register + my $ZT13 = $_[17]; #; [clobbered] ZMM register + my $ZT14 = $_[18]; #; [clobbered] ZMM register + my $ZT15 = $_[19]; #; [clobbered] ZMM register + my $ZT16 = $_[20]; #; [clobbered] ZMM register + my $T1 = $_[21]; #; [clobbered] GP register + my $T2 = $_[22]; #; [clobbered] GP register + my $T3 = $_[23]; #; [clobbered] GP register + my $MASKREG = $_[24]; #; [clobbered] mask register + + # ;; J0 = GHASH(IV || 0s+64 || len(IV)64) + # ;; s = 16 * RoundUp(len(IV)/16) - len(IV) */ + + # ;; Calculate GHASH of (IV || 0s) + $code .= "vpxor $J0,$J0,$J0\n"; + &CALC_AAD_HASH($IV, $IV_LEN, $J0, $GCM128_CTX, $ZT0, $ZT1, $ZT2, $ZT3, $ZT4, + $ZT5, $ZT6, $ZT7, $ZT8, $ZT9, $ZT10, $ZT11, $ZT12, $ZT13, $ZT14, $ZT15, $ZT16, $T1, $T2, $T3, $MASKREG); + + # ;; Calculate GHASH of last 16-byte block (0 || len(IV)64) + $code .= <<___; + mov $IV_LEN,$T1 + shl \$3,$T1 # ; IV length in bits + vmovq $T1,@{[XWORD($ZT2)]} + + # ;; Might need shuffle of ZT2 + vpxorq $J0,@{[XWORD($ZT2)]},$J0 + + vmovdqu64 @{[HashKeyByIdx(1,$GCM128_CTX)]},@{[XWORD($ZT0)]} +___ + &GHASH_MUL($J0, @{[XWORD($ZT0)]}, @{[XWORD($ZT1)]}, @{[XWORD($ZT2)]}, @{[XWORD($ZT3)]}); + + $code .= "vpshufb SHUF_MASK(%rip),$J0,$J0 # ; perform a 16Byte swap\n"; +} + +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +# ;;; GCM_INIT_IV performs an initialization of gcm128_ctx struct to prepare for +# ;;; encoding/decoding. +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +sub GCM_INIT_IV { + my $AES_KEYS = $_[0]; # [in] AES key schedule + my $GCM128_CTX = $_[1]; # [in/out] GCM context + my $IV = $_[2]; # [in] IV pointer + my $IV_LEN = $_[3]; # [in] IV length + my $GPR1 = $_[4]; # [clobbered] GP register + my $GPR2 = $_[5]; # [clobbered] GP register + my $GPR3 = $_[6]; # [clobbered] GP register + my $MASKREG = $_[7]; # [clobbered] mask register + my $CUR_COUNT = $_[8]; # [out] XMM with current counter + my $ZT0 = $_[9]; # [clobbered] ZMM register + my $ZT1 = $_[10]; # [clobbered] ZMM register + my $ZT2 = $_[11]; # [clobbered] ZMM register + my $ZT3 = $_[12]; # [clobbered] ZMM register + my $ZT4 = $_[13]; # [clobbered] ZMM register + my $ZT5 = $_[14]; # [clobbered] ZMM register + my $ZT6 = $_[15]; # [clobbered] ZMM register + my $ZT7 = $_[16]; # [clobbered] ZMM register + my $ZT8 = $_[17]; # [clobbered] ZMM register + my $ZT9 = $_[18]; # [clobbered] ZMM register + my $ZT10 = $_[19]; # [clobbered] ZMM register + my $ZT11 = $_[20]; # [clobbered] ZMM register + my $ZT12 = $_[21]; # [clobbered] ZMM register + my $ZT13 = $_[22]; # [clobbered] ZMM register + my $ZT14 = $_[23]; # [clobbered] ZMM register + my $ZT15 = $_[24]; # [clobbered] ZMM register + my $ZT16 = $_[25]; # [clobbered] ZMM register + + my $ZT0x = $ZT0; + $ZT0x =~ s/zmm/xmm/; + + $code .= <<___; + cmp \$12,$IV_LEN + je iv_len_12_init_IV +___ + + # ;; IV is different than 12 bytes + &CALC_J0($GCM128_CTX, $IV, $IV_LEN, $CUR_COUNT, $ZT0, $ZT1, $ZT2, $ZT3, $ZT4, $ZT5, $ZT6, $ZT7, + $ZT8, $ZT9, $ZT10, $ZT11, $ZT12, $ZT13, $ZT14, $ZT15, $ZT16, $GPR1, $GPR2, $GPR3, $MASKREG); + $code .= <<___; + jmp skip_iv_len_12_init_IV +iv_len_12_init_IV: # ;; IV is 12 bytes + # ;; read 12 IV bytes and pad with 0x00000001 + vmovdqu8 ONEf(%rip),$CUR_COUNT + mov $IV,$GPR2 + mov \$0x0000000000000fff,@{[DWORD($GPR1)]} + kmovq $GPR1,$MASKREG + vmovdqu8 ($GPR2),${CUR_COUNT}{$MASKREG} # ; ctr = IV | 0x1 +skip_iv_len_12_init_IV: + vmovdqu $CUR_COUNT,$ZT0x +___ + &ENCRYPT_SINGLE_BLOCK($AES_KEYS, "$ZT0x", "$GPR1"); # ; E(K, Y0) + $code .= <<___; + vmovdqu $ZT0x,`$CTX_OFFSET_EK0`($GCM128_CTX) # ; save EK0 for finalization stage + + # ;; store IV as counter in LE format + vpshufb SHUF_MASK(%rip),$CUR_COUNT,$CUR_COUNT + vmovdqu $CUR_COUNT,`$CTX_OFFSET_CurCount`($GCM128_CTX) # ; save current counter Yi +___ +} + +sub GCM_UPDATE_AAD { + my $GCM128_CTX = $_[0]; # [in] GCM context pointer + my $A_IN = $_[1]; # [in] AAD pointer + my $A_LEN = $_[2]; # [in] AAD length in bytes + my $GPR1 = $_[3]; # [clobbered] GP register + my $GPR2 = $_[4]; # [clobbered] GP register + my $GPR3 = $_[5]; # [clobbered] GP register + my $MASKREG = $_[6]; # [clobbered] mask register + my $AAD_HASH = $_[7]; # [out] XMM for AAD_HASH value + my $ZT0 = $_[8]; # [clobbered] ZMM register + my $ZT1 = $_[9]; # [clobbered] ZMM register + my $ZT2 = $_[10]; # [clobbered] ZMM register + my $ZT3 = $_[11]; # [clobbered] ZMM register + my $ZT4 = $_[12]; # [clobbered] ZMM register + my $ZT5 = $_[13]; # [clobbered] ZMM register + my $ZT6 = $_[14]; # [clobbered] ZMM register + my $ZT7 = $_[15]; # [clobbered] ZMM register + my $ZT8 = $_[16]; # [clobbered] ZMM register + my $ZT9 = $_[17]; # [clobbered] ZMM register + my $ZT10 = $_[18]; # [clobbered] ZMM register + my $ZT11 = $_[19]; # [clobbered] ZMM register + my $ZT12 = $_[20]; # [clobbered] ZMM register + my $ZT13 = $_[21]; # [clobbered] ZMM register + my $ZT14 = $_[22]; # [clobbered] ZMM register + my $ZT15 = $_[23]; # [clobbered] ZMM register + my $ZT16 = $_[24]; # [clobbered] ZMM register + + # ; load current hash + $code .= "vmovdqu64 $CTX_OFFSET_AadHash($GCM128_CTX),$AAD_HASH\n"; + + &CALC_AAD_HASH($A_IN, $A_LEN, $AAD_HASH, $GCM128_CTX, $ZT0, $ZT1, $ZT2, + $ZT3, $ZT4, $ZT5, $ZT6, $ZT7, $ZT8, $ZT9, $ZT10, $ZT11, $ZT12, $ZT13, + $ZT14, $ZT15, $ZT16, $GPR1, $GPR2, $GPR3, $MASKREG); + + # ; load current hash + $code .= "vmovdqu64 $AAD_HASH,$CTX_OFFSET_AadHash($GCM128_CTX)\n"; +} + +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +# ;;; Cipher and ghash of payloads shorter than 256 bytes +# ;;; - number of blocks in the message comes as argument +# ;;; - depending on the number of blocks an optimized variant of +# ;;; INITIAL_BLOCKS_PARTIAL is invoked +sub GCM_ENC_DEC_SMALL { + my $AES_KEYS = $_[0]; # [in] key pointer + my $GCM128_CTX = $_[1]; # [in] context pointer + my $CIPH_PLAIN_OUT = $_[2]; # [in] output buffer + my $PLAIN_CIPH_IN = $_[3]; # [in] input buffer + my $PLAIN_CIPH_LEN = $_[4]; # [in] buffer length + my $ENC_DEC = $_[5]; # [in] cipher direction + my $DATA_OFFSET = $_[6]; # [in] data offset + my $LENGTH = $_[7]; # [in] data length + my $NUM_BLOCKS = $_[8]; # [in] number of blocks to process 1 to 16 + my $CTR = $_[9]; # [in/out] XMM counter block + my $HASH_IN_OUT = $_[10]; # [in/out] XMM GHASH value + my $ZTMP0 = $_[11]; # [clobbered] ZMM register + my $ZTMP1 = $_[12]; # [clobbered] ZMM register + my $ZTMP2 = $_[13]; # [clobbered] ZMM register + my $ZTMP3 = $_[14]; # [clobbered] ZMM register + my $ZTMP4 = $_[15]; # [clobbered] ZMM register + my $ZTMP5 = $_[16]; # [clobbered] ZMM register + my $ZTMP6 = $_[17]; # [clobbered] ZMM register + my $ZTMP7 = $_[18]; # [clobbered] ZMM register + my $ZTMP8 = $_[19]; # [clobbered] ZMM register + my $ZTMP9 = $_[20]; # [clobbered] ZMM register + my $ZTMP10 = $_[21]; # [clobbered] ZMM register + my $ZTMP11 = $_[22]; # [clobbered] ZMM register + my $ZTMP12 = $_[23]; # [clobbered] ZMM register + my $ZTMP13 = $_[24]; # [clobbered] ZMM register + my $ZTMP14 = $_[25]; # [clobbered] ZMM register + my $IA0 = $_[26]; # [clobbered] GP register + my $IA1 = $_[27]; # [clobbered] GP register + my $MASKREG = $_[28]; # [clobbered] mask register + my $SHUFMASK = $_[29]; # [in] ZMM with BE/LE shuffle mask + my $PBLOCK_LEN = $_[30]; # [in] partial block length + + my $label_suffix = $label_count++; + + $code .= <<___; + cmp \$8,$NUM_BLOCKS + je .L_small_initial_num_blocks_is_8_${label_suffix} + jl .L_small_initial_num_blocks_is_7_1_${label_suffix} + + + cmp \$12,$NUM_BLOCKS + je .L_small_initial_num_blocks_is_12_${label_suffix} + jl .L_small_initial_num_blocks_is_11_9_${label_suffix} + + # ;; 16, 15, 14 or 13 + cmp \$16,$NUM_BLOCKS + je .L_small_initial_num_blocks_is_16_${label_suffix} + cmp \$15,$NUM_BLOCKS + je .L_small_initial_num_blocks_is_15_${label_suffix} + cmp \$14,$NUM_BLOCKS + je .L_small_initial_num_blocks_is_14_${label_suffix} + jmp .L_small_initial_num_blocks_is_13_${label_suffix} + +.L_small_initial_num_blocks_is_11_9_${label_suffix}: + # ;; 11, 10 or 9 + cmp \$11,$NUM_BLOCKS + je .L_small_initial_num_blocks_is_11_${label_suffix} + cmp \$10,$NUM_BLOCKS + je .L_small_initial_num_blocks_is_10_${label_suffix} + jmp .L_small_initial_num_blocks_is_9_${label_suffix} + +.L_small_initial_num_blocks_is_7_1_${label_suffix}: + cmp \$4,$NUM_BLOCKS + je .L_small_initial_num_blocks_is_4_${label_suffix} + jl .L_small_initial_num_blocks_is_3_1_${label_suffix} + # ;; 7, 6 or 5 + cmp \$7,$NUM_BLOCKS + je .L_small_initial_num_blocks_is_7_${label_suffix} + cmp \$6,$NUM_BLOCKS + je .L_small_initial_num_blocks_is_6_${label_suffix} + jmp .L_small_initial_num_blocks_is_5_${label_suffix} + +.L_small_initial_num_blocks_is_3_1_${label_suffix}: + # ;; 3, 2 or 1 + cmp \$3,$NUM_BLOCKS + je .L_small_initial_num_blocks_is_3_${label_suffix} + cmp \$2,$NUM_BLOCKS + je .L_small_initial_num_blocks_is_2_${label_suffix} + + # ;; for $NUM_BLOCKS == 1, just fall through and no 'jmp' needed + + # ;; Generation of different block size variants + # ;; - one block size has to be the first one +___ + + for (my $num_blocks = 1; $num_blocks <= 16; $num_blocks++) { + $code .= ".L_small_initial_num_blocks_is_${num_blocks}_${label_suffix}:\n"; + &INITIAL_BLOCKS_PARTIAL( + $AES_KEYS, $GCM128_CTX, $CIPH_PLAIN_OUT, $PLAIN_CIPH_IN, $LENGTH, $DATA_OFFSET, + $num_blocks, $CTR, $HASH_IN_OUT, $ENC_DEC, $ZTMP0, $ZTMP1, + $ZTMP2, $ZTMP3, $ZTMP4, $ZTMP5, $ZTMP6, $ZTMP7, + $ZTMP8, $ZTMP9, $ZTMP10, $ZTMP11, $ZTMP12, $ZTMP13, + $ZTMP14, $IA0, $IA1, $MASKREG, $SHUFMASK, $PBLOCK_LEN); + + if ($num_blocks != 16) { + $code .= "jmp .L_small_initial_blocks_encrypted_${label_suffix}\n"; + } + } + + $code .= ".L_small_initial_blocks_encrypted_${label_suffix}:\n"; +} + +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +# ; GCM_ENC_DEC Encrypts/Decrypts given data. Assumes that the passed gcm128_context +# ; struct has been initialized by GCM_INIT_IV +# ; Requires the input data be at least 1 byte long because of READ_SMALL_INPUT_DATA. +# ; Clobbers rax, r10-r15, and zmm0-zmm31, k1 +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +sub GCM_ENC_DEC { + my $AES_KEYS = $_[0]; # [in] AES Key schedule + my $GCM128_CTX = $_[1]; # [in] context pointer + my $PBLOCK_LEN = $_[2]; # [in] length of partial block at the moment of previous update + my $PLAIN_CIPH_IN = $_[3]; # [in] input buffer pointer + my $PLAIN_CIPH_LEN = $_[4]; # [in] buffer length + my $CIPH_PLAIN_OUT = $_[5]; # [in] output buffer pointer + my $ENC_DEC = $_[6]; # [in] cipher direction + + my $IA0 = "%r10"; + my $IA1 = "%r12"; + my $IA2 = "%r13"; + my $IA3 = "%r15"; + my $IA4 = "%r11"; + my $IA5 = "%rax"; + my $IA6 = "%rbx"; + my $IA7 = "%r14"; + + my $LENGTH = $win64 ? $IA2 : $PLAIN_CIPH_LEN; + + my $CTR_CHECK = $IA3; + my $DATA_OFFSET = $IA4; + my $HASHK_PTR = $IA6; + + my $HKEYS_READY = $IA7; + + my $CTR_BLOCKz = "%zmm2"; + my $CTR_BLOCKx = "%xmm2"; + + # ; hardcoded in GCM_INIT + + my $AAD_HASHz = "%zmm14"; + my $AAD_HASHx = "%xmm14"; + + # ; hardcoded in GCM_COMPLETE + + my $ZTMP0 = "%zmm0"; + my $ZTMP1 = "%zmm3"; + my $ZTMP2 = "%zmm4"; + my $ZTMP3 = "%zmm5"; + my $ZTMP4 = "%zmm6"; + my $ZTMP5 = "%zmm7"; + my $ZTMP6 = "%zmm10"; + my $ZTMP7 = "%zmm11"; + my $ZTMP8 = "%zmm12"; + my $ZTMP9 = "%zmm13"; + my $ZTMP10 = "%zmm15"; + my $ZTMP11 = "%zmm16"; + my $ZTMP12 = "%zmm17"; + + my $ZTMP13 = "%zmm19"; + my $ZTMP14 = "%zmm20"; + my $ZTMP15 = "%zmm21"; + my $ZTMP16 = "%zmm30"; + my $ZTMP17 = "%zmm31"; + my $ZTMP18 = "%zmm1"; + my $ZTMP19 = "%zmm18"; + my $ZTMP20 = "%zmm8"; + my $ZTMP21 = "%zmm22"; + my $ZTMP22 = "%zmm23"; + + my $GH = "%zmm24"; + my $GL = "%zmm25"; + my $GM = "%zmm26"; + my $SHUF_MASK = "%zmm29"; + + # ; Unused in the small packet path + my $ADDBE_4x4 = "%zmm27"; + my $ADDBE_1234 = "%zmm28"; + + my $MASKREG = "%k1"; + + my $label_suffix = $label_count++; + + # ;; reduction every 48 blocks, depth 32 blocks + # ;; @note 48 blocks is the maximum capacity of the stack frame + my $big_loop_nblocks = 48; + my $big_loop_depth = 32; + + # ;;; Macro flow depending on packet size + # ;;; - LENGTH <= 16 blocks + # ;;; - cipher followed by hashing (reduction) + # ;;; - 16 blocks < LENGTH < 32 blocks + # ;;; - cipher 16 blocks + # ;;; - cipher N blocks & hash 16 blocks, hash N blocks (reduction) + # ;;; - 32 blocks < LENGTH < 48 blocks + # ;;; - cipher 2 x 16 blocks + # ;;; - hash 16 blocks + # ;;; - cipher N blocks & hash 16 blocks, hash N blocks (reduction) + # ;;; - LENGTH >= 48 blocks + # ;;; - cipher 2 x 16 blocks + # ;;; - while (data_to_cipher >= 48 blocks): + # ;;; - cipher 16 blocks & hash 16 blocks + # ;;; - cipher 16 blocks & hash 16 blocks + # ;;; - cipher 16 blocks & hash 16 blocks (reduction) + # ;;; - if (data_to_cipher >= 32 blocks): + # ;;; - cipher 16 blocks & hash 16 blocks + # ;;; - cipher 16 blocks & hash 16 blocks + # ;;; - hash 16 blocks (reduction) + # ;;; - cipher N blocks & hash 16 blocks, hash N blocks (reduction) + # ;;; - elif (data_to_cipher >= 16 blocks): + # ;;; - cipher 16 blocks & hash 16 blocks + # ;;; - hash 16 blocks + # ;;; - cipher N blocks & hash 16 blocks, hash N blocks (reduction) + # ;;; - else: + # ;;; - hash 16 blocks + # ;;; - cipher N blocks & hash 16 blocks, hash N blocks (reduction) + + if ($win64) { + $code .= "cmpq \$0,$PLAIN_CIPH_LEN\n"; + } else { + $code .= "or $PLAIN_CIPH_LEN,$PLAIN_CIPH_LEN\n"; + } + $code .= "je .L_enc_dec_done_${label_suffix}\n"; + + # Length value from context $CTX_OFFSET_InLen`($GCM128_CTX) is updated in + # 'providers/implementations/ciphers/cipher_aes_gcm_hw_vaes_avx512.inc' + + $code .= "xor $HKEYS_READY, $HKEYS_READY\n"; + $code .= "vmovdqu64 `$CTX_OFFSET_AadHash`($GCM128_CTX),$AAD_HASHx\n"; + + # ;; Used for the update flow - if there was a previous partial + # ;; block fill the remaining bytes here. + &PARTIAL_BLOCK( + $GCM128_CTX, $PBLOCK_LEN, $CIPH_PLAIN_OUT, $PLAIN_CIPH_IN, $PLAIN_CIPH_LEN, + $DATA_OFFSET, $AAD_HASHx, $ENC_DEC, $IA0, $IA1, + $IA2, $ZTMP0, $ZTMP1, $ZTMP2, $ZTMP3, + $ZTMP4, $ZTMP5, $ZTMP6, $ZTMP7, $MASKREG); + + $code .= "vmovdqu64 `$CTX_OFFSET_CurCount`($GCM128_CTX),$CTR_BLOCKx\n"; + + # ;; Save the amount of data left to process in $LENGTH + # ;; NOTE: PLAIN_CIPH_LEN is a register on linux; + if ($win64) { + $code .= "mov $PLAIN_CIPH_LEN,$LENGTH\n"; + } + + # ;; There may be no more data if it was consumed in the partial block. + $code .= <<___; + sub $DATA_OFFSET,$LENGTH + je .L_enc_dec_done_${label_suffix} +___ + + $code .= <<___; + cmp \$`(16 * 16)`,$LENGTH + jbe .L_message_below_equal_16_blocks_${label_suffix} + + vmovdqa64 SHUF_MASK(%rip),$SHUF_MASK + vmovdqa64 ddq_addbe_4444(%rip),$ADDBE_4x4 + vmovdqa64 ddq_addbe_1234(%rip),$ADDBE_1234 + + # ;; start the pipeline + # ;; - 32 blocks aes-ctr + # ;; - 16 blocks ghash + aes-ctr + + # ;; set up CTR_CHECK + vmovd $CTR_BLOCKx,@{[DWORD($CTR_CHECK)]} + and \$255,@{[DWORD($CTR_CHECK)]} + # ;; in LE format after init, convert to BE + vshufi64x2 \$0,$CTR_BLOCKz,$CTR_BLOCKz,$CTR_BLOCKz + vpshufb $SHUF_MASK,$CTR_BLOCKz,$CTR_BLOCKz +___ + + # ;; ==== AES-CTR - first 16 blocks + my $aesout_offset = ($STACK_LOCAL_OFFSET + (0 * 16)); + my $data_in_out_offset = 0; + &INITIAL_BLOCKS_16( + $PLAIN_CIPH_IN, $CIPH_PLAIN_OUT, $AES_KEYS, $DATA_OFFSET, "no_ghash", $CTR_BLOCKz, + $CTR_CHECK, $ADDBE_4x4, $ADDBE_1234, $ZTMP0, $ZTMP1, $ZTMP2, + $ZTMP3, $ZTMP4, $ZTMP5, $ZTMP6, $ZTMP7, $ZTMP8, + $SHUF_MASK, $ENC_DEC, $aesout_offset, $data_in_out_offset, $IA0); + + &precompute_hkeys_on_stack($GCM128_CTX, $HKEYS_READY, $ZTMP0, $ZTMP1, $ZTMP2, $ZTMP3, $ZTMP4, $ZTMP5, $ZTMP6, + "first16"); + + $code .= <<___; + cmp \$`(32 * 16)`,$LENGTH + jb .L_message_below_32_blocks_${label_suffix} +___ + + # ;; ==== AES-CTR - next 16 blocks + $aesout_offset = ($STACK_LOCAL_OFFSET + (16 * 16)); + $data_in_out_offset = (16 * 16); + &INITIAL_BLOCKS_16( + $PLAIN_CIPH_IN, $CIPH_PLAIN_OUT, $AES_KEYS, $DATA_OFFSET, "no_ghash", $CTR_BLOCKz, + $CTR_CHECK, $ADDBE_4x4, $ADDBE_1234, $ZTMP0, $ZTMP1, $ZTMP2, + $ZTMP3, $ZTMP4, $ZTMP5, $ZTMP6, $ZTMP7, $ZTMP8, + $SHUF_MASK, $ENC_DEC, $aesout_offset, $data_in_out_offset, $IA0); + + &precompute_hkeys_on_stack($GCM128_CTX, $HKEYS_READY, $ZTMP0, $ZTMP1, $ZTMP2, $ZTMP3, $ZTMP4, $ZTMP5, $ZTMP6, + "last32"); + $code .= "mov \$1,$HKEYS_READY\n"; + + $code .= <<___; + add \$`(32 * 16)`,$DATA_OFFSET + sub \$`(32 * 16)`,$LENGTH + + cmp \$`($big_loop_nblocks * 16)`,$LENGTH + jb .L_no_more_big_nblocks_${label_suffix} +___ + + # ;; ==== + # ;; ==== AES-CTR + GHASH - 48 blocks loop + # ;; ==== + $code .= ".L_encrypt_big_nblocks_${label_suffix}:\n"; + + # ;; ==== AES-CTR + GHASH - 16 blocks, start + $aesout_offset = ($STACK_LOCAL_OFFSET + (32 * 16)); + $data_in_out_offset = (0 * 16); + my $ghashin_offset = ($STACK_LOCAL_OFFSET + (0 * 16)); + &GHASH_16_ENCRYPT_16_PARALLEL( + $AES_KEYS, $CIPH_PLAIN_OUT, $PLAIN_CIPH_IN, $DATA_OFFSET, $CTR_BLOCKz, $CTR_CHECK, + 48, $aesout_offset, $ghashin_offset, $SHUF_MASK, $ZTMP0, $ZTMP1, + $ZTMP2, $ZTMP3, $ZTMP4, $ZTMP5, $ZTMP6, $ZTMP7, + $ZTMP8, $ZTMP9, $ZTMP10, $ZTMP11, $ZTMP12, $ZTMP13, + $ZTMP14, $ZTMP15, $ZTMP16, $ZTMP17, $ZTMP18, $ZTMP19, + $ZTMP20, $ZTMP21, $ZTMP22, $ADDBE_4x4, $ADDBE_1234, $GL, + $GH, $GM, "first_time", $ENC_DEC, $data_in_out_offset, $AAD_HASHz, + $IA0); + + # ;; ==== AES-CTR + GHASH - 16 blocks, no reduction + $aesout_offset = ($STACK_LOCAL_OFFSET + (0 * 16)); + $data_in_out_offset = (16 * 16); + $ghashin_offset = ($STACK_LOCAL_OFFSET + (16 * 16)); + &GHASH_16_ENCRYPT_16_PARALLEL( + $AES_KEYS, $CIPH_PLAIN_OUT, $PLAIN_CIPH_IN, $DATA_OFFSET, $CTR_BLOCKz, $CTR_CHECK, + 32, $aesout_offset, $ghashin_offset, $SHUF_MASK, $ZTMP0, $ZTMP1, + $ZTMP2, $ZTMP3, $ZTMP4, $ZTMP5, $ZTMP6, $ZTMP7, + $ZTMP8, $ZTMP9, $ZTMP10, $ZTMP11, $ZTMP12, $ZTMP13, + $ZTMP14, $ZTMP15, $ZTMP16, $ZTMP17, $ZTMP18, $ZTMP19, + $ZTMP20, $ZTMP21, $ZTMP22, $ADDBE_4x4, $ADDBE_1234, $GL, + $GH, $GM, "no_reduction", $ENC_DEC, $data_in_out_offset, "no_ghash_in", + $IA0); + + # ;; ==== AES-CTR + GHASH - 16 blocks, reduction + $aesout_offset = ($STACK_LOCAL_OFFSET + (16 * 16)); + $data_in_out_offset = (32 * 16); + $ghashin_offset = ($STACK_LOCAL_OFFSET + (32 * 16)); + &GHASH_16_ENCRYPT_16_PARALLEL( + $AES_KEYS, $CIPH_PLAIN_OUT, $PLAIN_CIPH_IN, $DATA_OFFSET, $CTR_BLOCKz, $CTR_CHECK, + 16, $aesout_offset, $ghashin_offset, $SHUF_MASK, $ZTMP0, $ZTMP1, + $ZTMP2, $ZTMP3, $ZTMP4, $ZTMP5, $ZTMP6, $ZTMP7, + $ZTMP8, $ZTMP9, $ZTMP10, $ZTMP11, $ZTMP12, $ZTMP13, + $ZTMP14, $ZTMP15, $ZTMP16, $ZTMP17, $ZTMP18, $ZTMP19, + $ZTMP20, $ZTMP21, $ZTMP22, $ADDBE_4x4, $ADDBE_1234, $GL, + $GH, $GM, "final_reduction", $ENC_DEC, $data_in_out_offset, "no_ghash_in", + $IA0); + + # ;; === xor cipher block 0 with GHASH (ZT4) + $code .= <<___; + vmovdqa64 $ZTMP4,$AAD_HASHz + + add \$`($big_loop_nblocks * 16)`,$DATA_OFFSET + sub \$`($big_loop_nblocks * 16)`,$LENGTH + cmp \$`($big_loop_nblocks * 16)`,$LENGTH + jae .L_encrypt_big_nblocks_${label_suffix} + +.L_no_more_big_nblocks_${label_suffix}: + + cmp \$`(32 * 16)`,$LENGTH + jae .L_encrypt_32_blocks_${label_suffix} + + cmp \$`(16 * 16)`,$LENGTH + jae .L_encrypt_16_blocks_${label_suffix} +___ + + # ;; ===================================================== + # ;; ===================================================== + # ;; ==== GHASH 1 x 16 blocks + # ;; ==== GHASH 1 x 16 blocks (reduction) & encrypt N blocks + # ;; ==== then GHASH N blocks + $code .= ".L_encrypt_0_blocks_ghash_32_${label_suffix}:\n"; + + # ;; calculate offset to the right hash key + $code .= <<___; +mov @{[DWORD($LENGTH)]},@{[DWORD($IA0)]} +and \$~15,@{[DWORD($IA0)]} +mov \$`@{[HashKeyOffsetByIdx(32,"frame")]}`,@{[DWORD($HASHK_PTR)]} +sub @{[DWORD($IA0)]},@{[DWORD($HASHK_PTR)]} +___ + + # ;; ==== GHASH 32 blocks and follow with reduction + &GHASH_16("start", $GH, $GM, $GL, "%rsp", $STACK_LOCAL_OFFSET, (0 * 16), + "%rsp", $HASHK_PTR, 0, $AAD_HASHz, $ZTMP0, $ZTMP1, $ZTMP2, $ZTMP3, $ZTMP4, $ZTMP5, $ZTMP6, $ZTMP7, $ZTMP8, $ZTMP9); + + # ;; ==== GHASH 1 x 16 blocks with reduction + cipher and ghash on the reminder + $ghashin_offset = ($STACK_LOCAL_OFFSET + (16 * 16)); + $code .= "add \$`(16 * 16)`,@{[DWORD($HASHK_PTR)]}\n"; + &GCM_ENC_DEC_LAST( + $AES_KEYS, $GCM128_CTX, $CIPH_PLAIN_OUT, $PLAIN_CIPH_IN, $DATA_OFFSET, $LENGTH, + $CTR_BLOCKz, $CTR_CHECK, $HASHK_PTR, $ghashin_offset, $SHUF_MASK, $ZTMP0, + $ZTMP1, $ZTMP2, $ZTMP3, $ZTMP4, $ZTMP5, $ZTMP6, + $ZTMP7, $ZTMP8, $ZTMP9, $ZTMP10, $ZTMP11, $ZTMP12, + $ZTMP13, $ZTMP14, $ZTMP15, $ZTMP16, $ZTMP17, $ZTMP18, + $ZTMP19, $ZTMP20, $ZTMP21, $ZTMP22, $ADDBE_4x4, $ADDBE_1234, + "mid", $GL, $GH, $GM, $ENC_DEC, $AAD_HASHz, + $IA0, $IA5, $MASKREG, $PBLOCK_LEN); + + $code .= "vpshufb @{[XWORD($SHUF_MASK)]},$CTR_BLOCKx,$CTR_BLOCKx\n"; + $code .= "jmp .L_ghash_done_${label_suffix}\n"; + + # ;; ===================================================== + # ;; ===================================================== + # ;; ==== GHASH & encrypt 1 x 16 blocks + # ;; ==== GHASH & encrypt 1 x 16 blocks + # ;; ==== GHASH 1 x 16 blocks (reduction) + # ;; ==== GHASH 1 x 16 blocks (reduction) & encrypt N blocks + # ;; ==== then GHASH N blocks + $code .= ".L_encrypt_32_blocks_${label_suffix}:\n"; + + # ;; ==== AES-CTR + GHASH - 16 blocks, start + $aesout_offset = ($STACK_LOCAL_OFFSET + (32 * 16)); + $ghashin_offset = ($STACK_LOCAL_OFFSET + (0 * 16)); + $data_in_out_offset = (0 * 16); + &GHASH_16_ENCRYPT_16_PARALLEL( + $AES_KEYS, $CIPH_PLAIN_OUT, $PLAIN_CIPH_IN, $DATA_OFFSET, $CTR_BLOCKz, $CTR_CHECK, + 48, $aesout_offset, $ghashin_offset, $SHUF_MASK, $ZTMP0, $ZTMP1, + $ZTMP2, $ZTMP3, $ZTMP4, $ZTMP5, $ZTMP6, $ZTMP7, + $ZTMP8, $ZTMP9, $ZTMP10, $ZTMP11, $ZTMP12, $ZTMP13, + $ZTMP14, $ZTMP15, $ZTMP16, $ZTMP17, $ZTMP18, $ZTMP19, + $ZTMP20, $ZTMP21, $ZTMP22, $ADDBE_4x4, $ADDBE_1234, $GL, + $GH, $GM, "first_time", $ENC_DEC, $data_in_out_offset, $AAD_HASHz, + $IA0); + + # ;; ==== AES-CTR + GHASH - 16 blocks, no reduction + $aesout_offset = ($STACK_LOCAL_OFFSET + (0 * 16)); + $ghashin_offset = ($STACK_LOCAL_OFFSET + (16 * 16)); + $data_in_out_offset = (16 * 16); + &GHASH_16_ENCRYPT_16_PARALLEL( + $AES_KEYS, $CIPH_PLAIN_OUT, $PLAIN_CIPH_IN, $DATA_OFFSET, $CTR_BLOCKz, $CTR_CHECK, + 32, $aesout_offset, $ghashin_offset, $SHUF_MASK, $ZTMP0, $ZTMP1, + $ZTMP2, $ZTMP3, $ZTMP4, $ZTMP5, $ZTMP6, $ZTMP7, + $ZTMP8, $ZTMP9, $ZTMP10, $ZTMP11, $ZTMP12, $ZTMP13, + $ZTMP14, $ZTMP15, $ZTMP16, $ZTMP17, $ZTMP18, $ZTMP19, + $ZTMP20, $ZTMP21, $ZTMP22, $ADDBE_4x4, $ADDBE_1234, $GL, + $GH, $GM, "no_reduction", $ENC_DEC, $data_in_out_offset, "no_ghash_in", + $IA0); + + # ;; ==== GHASH 16 blocks with reduction + &GHASH_16( + "end_reduce", $GH, $GM, $GL, "%rsp", $STACK_LOCAL_OFFSET, (32 * 16), + "%rsp", &HashKeyOffsetByIdx(16, "frame"), + 0, $AAD_HASHz, $ZTMP0, $ZTMP1, $ZTMP2, $ZTMP3, $ZTMP4, $ZTMP5, $ZTMP6, $ZTMP7, $ZTMP8, $ZTMP9); + + # ;; ==== GHASH 1 x 16 blocks with reduction + cipher and ghash on the reminder + $ghashin_offset = ($STACK_LOCAL_OFFSET + (0 * 16)); + $code .= <<___; + sub \$`(32 * 16)`,$LENGTH + add \$`(32 * 16)`,$DATA_OFFSET +___ + + # ;; calculate offset to the right hash key + $code .= "mov @{[DWORD($LENGTH)]},@{[DWORD($IA0)]}\n"; + $code .= <<___; + and \$~15,@{[DWORD($IA0)]} + mov \$`@{[HashKeyOffsetByIdx(16,"frame")]}`,@{[DWORD($HASHK_PTR)]} + sub @{[DWORD($IA0)]},@{[DWORD($HASHK_PTR)]} +___ + &GCM_ENC_DEC_LAST( + $AES_KEYS, $GCM128_CTX, $CIPH_PLAIN_OUT, $PLAIN_CIPH_IN, $DATA_OFFSET, $LENGTH, + $CTR_BLOCKz, $CTR_CHECK, $HASHK_PTR, $ghashin_offset, $SHUF_MASK, $ZTMP0, + $ZTMP1, $ZTMP2, $ZTMP3, $ZTMP4, $ZTMP5, $ZTMP6, + $ZTMP7, $ZTMP8, $ZTMP9, $ZTMP10, $ZTMP11, $ZTMP12, + $ZTMP13, $ZTMP14, $ZTMP15, $ZTMP16, $ZTMP17, $ZTMP18, + $ZTMP19, $ZTMP20, $ZTMP21, $ZTMP22, $ADDBE_4x4, $ADDBE_1234, + "start", $GL, $GH, $GM, $ENC_DEC, $AAD_HASHz, + $IA0, $IA5, $MASKREG, $PBLOCK_LEN); + + $code .= "vpshufb @{[XWORD($SHUF_MASK)]},$CTR_BLOCKx,$CTR_BLOCKx\n"; + $code .= "jmp .L_ghash_done_${label_suffix}\n"; + + # ;; ===================================================== + # ;; ===================================================== + # ;; ==== GHASH & encrypt 16 blocks (done before) + # ;; ==== GHASH 1 x 16 blocks + # ;; ==== GHASH 1 x 16 blocks (reduction) & encrypt N blocks + # ;; ==== then GHASH N blocks + $code .= ".L_encrypt_16_blocks_${label_suffix}:\n"; + + # ;; ==== AES-CTR + GHASH - 16 blocks, start + $aesout_offset = ($STACK_LOCAL_OFFSET + (32 * 16)); + $ghashin_offset = ($STACK_LOCAL_OFFSET + (0 * 16)); + $data_in_out_offset = (0 * 16); + &GHASH_16_ENCRYPT_16_PARALLEL( + $AES_KEYS, $CIPH_PLAIN_OUT, $PLAIN_CIPH_IN, $DATA_OFFSET, $CTR_BLOCKz, $CTR_CHECK, + 48, $aesout_offset, $ghashin_offset, $SHUF_MASK, $ZTMP0, $ZTMP1, + $ZTMP2, $ZTMP3, $ZTMP4, $ZTMP5, $ZTMP6, $ZTMP7, + $ZTMP8, $ZTMP9, $ZTMP10, $ZTMP11, $ZTMP12, $ZTMP13, + $ZTMP14, $ZTMP15, $ZTMP16, $ZTMP17, $ZTMP18, $ZTMP19, + $ZTMP20, $ZTMP21, $ZTMP22, $ADDBE_4x4, $ADDBE_1234, $GL, + $GH, $GM, "first_time", $ENC_DEC, $data_in_out_offset, $AAD_HASHz, + $IA0); + + # ;; ==== GHASH 1 x 16 blocks + &GHASH_16( + "mid", $GH, $GM, $GL, "%rsp", $STACK_LOCAL_OFFSET, (16 * 16), + "%rsp", &HashKeyOffsetByIdx(32, "frame"), + 0, "no_hash_input", $ZTMP0, $ZTMP1, $ZTMP2, $ZTMP3, $ZTMP4, $ZTMP5, $ZTMP6, $ZTMP7, $ZTMP8, $ZTMP9); + + # ;; ==== GHASH 1 x 16 blocks with reduction + cipher and ghash on the reminder + $ghashin_offset = ($STACK_LOCAL_OFFSET + (32 * 16)); + $code .= <<___; + sub \$`(16 * 16)`,$LENGTH + add \$`(16 * 16)`,$DATA_OFFSET +___ + &GCM_ENC_DEC_LAST( + $AES_KEYS, $GCM128_CTX, $CIPH_PLAIN_OUT, $PLAIN_CIPH_IN, + $DATA_OFFSET, $LENGTH, $CTR_BLOCKz, $CTR_CHECK, + &HashKeyOffsetByIdx(16, "frame"), $ghashin_offset, $SHUF_MASK, $ZTMP0, + $ZTMP1, $ZTMP2, $ZTMP3, $ZTMP4, + $ZTMP5, $ZTMP6, $ZTMP7, $ZTMP8, + $ZTMP9, $ZTMP10, $ZTMP11, $ZTMP12, + $ZTMP13, $ZTMP14, $ZTMP15, $ZTMP16, + $ZTMP17, $ZTMP18, $ZTMP19, $ZTMP20, + $ZTMP21, $ZTMP22, $ADDBE_4x4, $ADDBE_1234, + "end_reduce", $GL, $GH, $GM, + $ENC_DEC, $AAD_HASHz, $IA0, $IA5, + $MASKREG, $PBLOCK_LEN); + + $code .= "vpshufb @{[XWORD($SHUF_MASK)]},$CTR_BLOCKx,$CTR_BLOCKx\n"; + $code .= <<___; + jmp .L_ghash_done_${label_suffix} + +.L_message_below_32_blocks_${label_suffix}: + # ;; 32 > number of blocks > 16 + + sub \$`(16 * 16)`,$LENGTH + add \$`(16 * 16)`,$DATA_OFFSET +___ + $ghashin_offset = ($STACK_LOCAL_OFFSET + (0 * 16)); + + # ;; calculate offset to the right hash key + $code .= "mov @{[DWORD($LENGTH)]},@{[DWORD($IA0)]}\n"; + + &precompute_hkeys_on_stack($GCM128_CTX, $HKEYS_READY, $ZTMP0, $ZTMP1, $ZTMP2, $ZTMP3, $ZTMP4, $ZTMP5, $ZTMP6, + "mid16"); + $code .= "mov \$1,$HKEYS_READY\n"; + + $code .= <<___; +and \$~15,@{[DWORD($IA0)]} +mov \$`@{[HashKeyOffsetByIdx(16,"frame")]}`,@{[DWORD($HASHK_PTR)]} +sub @{[DWORD($IA0)]},@{[DWORD($HASHK_PTR)]} +___ + + &GCM_ENC_DEC_LAST( + $AES_KEYS, $GCM128_CTX, $CIPH_PLAIN_OUT, $PLAIN_CIPH_IN, $DATA_OFFSET, $LENGTH, + $CTR_BLOCKz, $CTR_CHECK, $HASHK_PTR, $ghashin_offset, $SHUF_MASK, $ZTMP0, + $ZTMP1, $ZTMP2, $ZTMP3, $ZTMP4, $ZTMP5, $ZTMP6, + $ZTMP7, $ZTMP8, $ZTMP9, $ZTMP10, $ZTMP11, $ZTMP12, + $ZTMP13, $ZTMP14, $ZTMP15, $ZTMP16, $ZTMP17, $ZTMP18, + $ZTMP19, $ZTMP20, $ZTMP21, $ZTMP22, $ADDBE_4x4, $ADDBE_1234, + "start", $GL, $GH, $GM, $ENC_DEC, $AAD_HASHz, + $IA0, $IA5, $MASKREG, $PBLOCK_LEN); + + $code .= "vpshufb @{[XWORD($SHUF_MASK)]},$CTR_BLOCKx,$CTR_BLOCKx\n"; + $code .= <<___; + jmp .L_ghash_done_${label_suffix} + +.L_message_below_equal_16_blocks_${label_suffix}: + # ;; Determine how many blocks to process + # ;; - process one additional block if there is a partial block + mov @{[DWORD($LENGTH)]},@{[DWORD($IA1)]} + add \$15,@{[DWORD($IA1)]} + shr \$4, @{[DWORD($IA1)]} # ; $IA1 can be in the range from 0 to 16 +___ + &GCM_ENC_DEC_SMALL( + $AES_KEYS, $GCM128_CTX, $CIPH_PLAIN_OUT, $PLAIN_CIPH_IN, $PLAIN_CIPH_LEN, $ENC_DEC, + $DATA_OFFSET, $LENGTH, $IA1, $CTR_BLOCKx, $AAD_HASHx, $ZTMP0, + $ZTMP1, $ZTMP2, $ZTMP3, $ZTMP4, $ZTMP5, $ZTMP6, + $ZTMP7, $ZTMP8, $ZTMP9, $ZTMP10, $ZTMP11, $ZTMP12, + $ZTMP13, $ZTMP14, $IA0, $IA3, $MASKREG, $SHUF_MASK, + $PBLOCK_LEN); + + # ;; fall through to exit + + $code .= ".L_ghash_done_${label_suffix}:\n"; + + # ;; save the last counter block + $code .= "vmovdqu64 $CTR_BLOCKx,`$CTX_OFFSET_CurCount`($GCM128_CTX)\n"; + $code .= <<___; + vmovdqu64 $AAD_HASHx,`$CTX_OFFSET_AadHash`($GCM128_CTX) +.L_enc_dec_done_${label_suffix}: +___ +} + +# ;;; =========================================================================== +# ;;; Encrypt/decrypt the initial 16 blocks +sub INITIAL_BLOCKS_16 { + my $IN = $_[0]; # [in] input buffer + my $OUT = $_[1]; # [in] output buffer + my $AES_KEYS = $_[2]; # [in] pointer to expanded keys + my $DATA_OFFSET = $_[3]; # [in] data offset + my $GHASH = $_[4]; # [in] ZMM with AAD (low 128 bits) + my $CTR = $_[5]; # [in] ZMM with CTR BE blocks 4x128 bits + my $CTR_CHECK = $_[6]; # [in/out] GPR with counter overflow check + my $ADDBE_4x4 = $_[7]; # [in] ZMM 4x128bits with value 4 (big endian) + my $ADDBE_1234 = $_[8]; # [in] ZMM 4x128bits with values 1, 2, 3 & 4 (big endian) + my $T0 = $_[9]; # [clobered] temporary ZMM register + my $T1 = $_[10]; # [clobered] temporary ZMM register + my $T2 = $_[11]; # [clobered] temporary ZMM register + my $T3 = $_[12]; # [clobered] temporary ZMM register + my $T4 = $_[13]; # [clobered] temporary ZMM register + my $T5 = $_[14]; # [clobered] temporary ZMM register + my $T6 = $_[15]; # [clobered] temporary ZMM register + my $T7 = $_[16]; # [clobered] temporary ZMM register + my $T8 = $_[17]; # [clobered] temporary ZMM register + my $SHUF_MASK = $_[18]; # [in] ZMM with BE/LE shuffle mask + my $ENC_DEC = $_[19]; # [in] ENC (encrypt) or DEC (decrypt) selector + my $BLK_OFFSET = $_[20]; # [in] stack frame offset to ciphered blocks + my $DATA_DISPL = $_[21]; # [in] fixed numerical data displacement/offset + my $IA0 = $_[22]; # [clobered] temporary GP register + + my $B00_03 = $T5; + my $B04_07 = $T6; + my $B08_11 = $T7; + my $B12_15 = $T8; + + my $label_suffix = $label_count++; + + my $stack_offset = $BLK_OFFSET; + $code .= <<___; + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;; prepare counter blocks + + cmpb \$`(256 - 16)`,@{[BYTE($CTR_CHECK)]} + jae .L_next_16_overflow_${label_suffix} + vpaddd $ADDBE_1234,$CTR,$B00_03 + vpaddd $ADDBE_4x4,$B00_03,$B04_07 + vpaddd $ADDBE_4x4,$B04_07,$B08_11 + vpaddd $ADDBE_4x4,$B08_11,$B12_15 + jmp .L_next_16_ok_${label_suffix} +.L_next_16_overflow_${label_suffix}: + vpshufb $SHUF_MASK,$CTR,$CTR + vmovdqa64 ddq_add_4444(%rip),$B12_15 + vpaddd ddq_add_1234(%rip),$CTR,$B00_03 + vpaddd $B12_15,$B00_03,$B04_07 + vpaddd $B12_15,$B04_07,$B08_11 + vpaddd $B12_15,$B08_11,$B12_15 + vpshufb $SHUF_MASK,$B00_03,$B00_03 + vpshufb $SHUF_MASK,$B04_07,$B04_07 + vpshufb $SHUF_MASK,$B08_11,$B08_11 + vpshufb $SHUF_MASK,$B12_15,$B12_15 +.L_next_16_ok_${label_suffix}: + vshufi64x2 \$0b11111111,$B12_15,$B12_15,$CTR + addb \$16,@{[BYTE($CTR_CHECK)]} + # ;; === load 16 blocks of data + vmovdqu8 `$DATA_DISPL + (64*0)`($IN,$DATA_OFFSET,1),$T0 + vmovdqu8 `$DATA_DISPL + (64*1)`($IN,$DATA_OFFSET,1),$T1 + vmovdqu8 `$DATA_DISPL + (64*2)`($IN,$DATA_OFFSET,1),$T2 + vmovdqu8 `$DATA_DISPL + (64*3)`($IN,$DATA_OFFSET,1),$T3 + + # ;; move to AES encryption rounds + vbroadcastf64x2 `(16*0)`($AES_KEYS),$T4 + vpxorq $T4,$B00_03,$B00_03 + vpxorq $T4,$B04_07,$B04_07 + vpxorq $T4,$B08_11,$B08_11 + vpxorq $T4,$B12_15,$B12_15 +___ + foreach (1 .. ($NROUNDS)) { + $code .= <<___; + vbroadcastf64x2 `(16*$_)`($AES_KEYS),$T4 + vaesenc $T4,$B00_03,$B00_03 + vaesenc $T4,$B04_07,$B04_07 + vaesenc $T4,$B08_11,$B08_11 + vaesenc $T4,$B12_15,$B12_15 +___ + } + $code .= <<___; + vbroadcastf64x2 `(16*($NROUNDS+1))`($AES_KEYS),$T4 + vaesenclast $T4,$B00_03,$B00_03 + vaesenclast $T4,$B04_07,$B04_07 + vaesenclast $T4,$B08_11,$B08_11 + vaesenclast $T4,$B12_15,$B12_15 + + # ;; xor against text + vpxorq $T0,$B00_03,$B00_03 + vpxorq $T1,$B04_07,$B04_07 + vpxorq $T2,$B08_11,$B08_11 + vpxorq $T3,$B12_15,$B12_15 + + # ;; store + mov $OUT, $IA0 + vmovdqu8 $B00_03,`$DATA_DISPL + (64*0)`($IA0,$DATA_OFFSET,1) + vmovdqu8 $B04_07,`$DATA_DISPL + (64*1)`($IA0,$DATA_OFFSET,1) + vmovdqu8 $B08_11,`$DATA_DISPL + (64*2)`($IA0,$DATA_OFFSET,1) + vmovdqu8 $B12_15,`$DATA_DISPL + (64*3)`($IA0,$DATA_OFFSET,1) +___ + if ($ENC_DEC eq "DEC") { + $code .= <<___; + # ;; decryption - cipher text needs to go to GHASH phase + vpshufb $SHUF_MASK,$T0,$B00_03 + vpshufb $SHUF_MASK,$T1,$B04_07 + vpshufb $SHUF_MASK,$T2,$B08_11 + vpshufb $SHUF_MASK,$T3,$B12_15 +___ + } else { + $code .= <<___; + # ;; encryption + vpshufb $SHUF_MASK,$B00_03,$B00_03 + vpshufb $SHUF_MASK,$B04_07,$B04_07 + vpshufb $SHUF_MASK,$B08_11,$B08_11 + vpshufb $SHUF_MASK,$B12_15,$B12_15 +___ + } + + if ($GHASH ne "no_ghash") { + $code .= <<___; + # ;; === xor cipher block 0 with GHASH for the next GHASH round + vpxorq $GHASH,$B00_03,$B00_03 +___ + } + $code .= <<___; + vmovdqa64 $B00_03,`$stack_offset + (0 * 64)`(%rsp) + vmovdqa64 $B04_07,`$stack_offset + (1 * 64)`(%rsp) + vmovdqa64 $B08_11,`$stack_offset + (2 * 64)`(%rsp) + vmovdqa64 $B12_15,`$stack_offset + (3 * 64)`(%rsp) +___ +} + +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +# ; GCM_COMPLETE Finishes ghash calculation +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +sub GCM_COMPLETE { + my $GCM128_CTX = $_[0]; + my $PBLOCK_LEN = $_[1]; + + my $label_suffix = $label_count++; + + $code .= <<___; + vmovdqu @{[HashKeyByIdx(1,$GCM128_CTX)]},%xmm2 + vmovdqu $CTX_OFFSET_EK0($GCM128_CTX),%xmm3 # ; xmm3 = E(K,Y0) +___ + + $code .= <<___; + vmovdqu `$CTX_OFFSET_AadHash`($GCM128_CTX),%xmm4 + + # ;; Process the final partial block. + cmp \$0,$PBLOCK_LEN + je .L_partial_done_${label_suffix} +___ + + # ;GHASH computation for the last <16 Byte block + &GHASH_MUL("%xmm4", "%xmm2", "%xmm0", "%xmm16", "%xmm17"); + + $code .= <<___; +.L_partial_done_${label_suffix}: + vmovq `$CTX_OFFSET_InLen`($GCM128_CTX), %xmm5 + vpinsrq \$1, `$CTX_OFFSET_AadLen`($GCM128_CTX), %xmm5, %xmm5 # ; xmm5 = len(A)||len(C) + vpsllq \$3, %xmm5, %xmm5 # ; convert bytes into bits + + vpxor %xmm5,%xmm4,%xmm4 +___ + + &GHASH_MUL("%xmm4", "%xmm2", "%xmm0", "%xmm16", "%xmm17"); + + $code .= <<___; + vpshufb SHUF_MASK(%rip),%xmm4,%xmm4 # ; perform a 16Byte swap + vpxor %xmm4,%xmm3,%xmm3 + +.L_return_T_${label_suffix}: + vmovdqu %xmm3,`$CTX_OFFSET_AadHash`($GCM128_CTX) +___ +} + +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +# ;;; Functions definitions +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +$code .= ".text\n"; +{ + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + # ;void ossl_aes_gcm_init_avx512 / + # ; (const void *aes_keys, + # ; void *gcm128ctx) + # ; + # ; Precomputes hashkey table for GHASH optimization. + # ; Leaf function (does not allocate stack space, does not use non-volatile registers). + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + $code .= <<___; +.globl ossl_aes_gcm_init_avx512 +.type ossl_aes_gcm_init_avx512,\@abi-omnipotent +.align 32 +ossl_aes_gcm_init_avx512: +.cfi_startproc + endbranch +___ + if ($CHECK_FUNCTION_ARGUMENTS) { + $code .= <<___; + # ;; Check aes_keys != NULL + test $arg1,$arg1 + jz .Labort_init + + # ;; Check gcm128ctx != NULL + test $arg2,$arg2 + jz .Labort_init +___ + } + $code .= "vpxorq %xmm16,%xmm16,%xmm16\n"; + &ENCRYPT_SINGLE_BLOCK("$arg1", "%xmm16", "%rax"); # ; xmm16 = HashKey + $code .= <<___; + vpshufb SHUF_MASK(%rip),%xmm16,%xmm16 + # ;;; PRECOMPUTATION of HashKey<<1 mod poly from the HashKey ;;; + vmovdqa64 %xmm16,%xmm2 + vpsllq \$1,%xmm16,%xmm16 + vpsrlq \$63,%xmm2,%xmm2 + vmovdqa %xmm2,%xmm1 + vpslldq \$8,%xmm2,%xmm2 + vpsrldq \$8,%xmm1,%xmm1 + vporq %xmm2,%xmm16,%xmm16 + # ;reduction + vpshufd \$0b00100100,%xmm1,%xmm2 + vpcmpeqd TWOONE(%rip),%xmm2,%xmm2 + vpand POLY(%rip),%xmm2,%xmm2 + vpxorq %xmm2,%xmm16,%xmm16 # ; xmm16 holds the HashKey<<1 mod poly + # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + vmovdqu64 %xmm16,@{[HashKeyByIdx(1,$arg2)]} # ; store HashKey<<1 mod poly +___ + &PRECOMPUTE("$arg2", "%xmm16", "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5"); + if ($CLEAR_SCRATCH_REGISTERS) { + &clear_scratch_gps_asm(); + &clear_scratch_zmms_asm(); + } else { + $code .= "vzeroupper\n"; + } + $code .= <<___; +.Labort_init: +ret +.cfi_endproc +.size ossl_aes_gcm_init_avx512, .-ossl_aes_gcm_init_avx512 +___ +} + +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +# ;void ossl_aes_gcm_setiv_avx512 +# ; (const void *aes_keys, +# ; void *gcm128ctx, +# ; const unsigned char *iv, +# ; size_t ivlen) +# ; +# ; Computes E(K,Y0) for finalization, updates current counter Yi in gcm128_context structure. +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +$code .= <<___; +.globl ossl_aes_gcm_setiv_avx512 +.type ossl_aes_gcm_setiv_avx512,\@abi-omnipotent +.align 32 +ossl_aes_gcm_setiv_avx512: +.cfi_startproc +.Lsetiv_seh_begin: + endbranch +___ +if ($CHECK_FUNCTION_ARGUMENTS) { + $code .= <<___; + # ;; Check aes_keys != NULL + test $arg1,$arg1 + jz .Labort_setiv + + # ;; Check gcm128ctx != NULL + test $arg2,$arg2 + jz .Labort_setiv + + # ;; Check iv != NULL + test $arg3,$arg3 + jz .Labort_setiv + + # ;; Check ivlen != 0 + test $arg4,$arg4 + jz .Labort_setiv +___ +} + +# ; NOTE: code before PROLOG() must not modify any registers +&PROLOG( + 1, # allocate stack space for hkeys + 0, # do not allocate stack space for AES blocks + "setiv"); +&GCM_INIT_IV( + "$arg1", "$arg2", "$arg3", "$arg4", "%r10", "%r11", "%r12", "%k1", "%xmm2", "%zmm1", + "%zmm11", "%zmm3", "%zmm4", "%zmm5", "%zmm6", "%zmm7", "%zmm8", "%zmm9", "%zmm10", "%zmm12", + "%zmm13", "%zmm15", "%zmm16", "%zmm17", "%zmm18", "%zmm19"); +&EPILOG( + 1, # hkeys were allocated + $arg4); +$code .= <<___; +.Labort_setiv: +ret +.Lsetiv_seh_end: +.cfi_endproc +.size ossl_aes_gcm_setiv_avx512, .-ossl_aes_gcm_setiv_avx512 +___ + +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +# ;void ossl_aes_gcm_update_aad_avx512 +# ; (unsigned char *gcm128ctx, +# ; const unsigned char *aad, +# ; size_t aadlen) +# ; +# ; Updates AAD hash in gcm128_context structure. +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +$code .= <<___; +.globl ossl_aes_gcm_update_aad_avx512 +.type ossl_aes_gcm_update_aad_avx512,\@abi-omnipotent +.align 32 +ossl_aes_gcm_update_aad_avx512: +.cfi_startproc +.Lghash_seh_begin: + endbranch +___ +if ($CHECK_FUNCTION_ARGUMENTS) { + $code .= <<___; + # ;; Check gcm128ctx != NULL + test $arg1,$arg1 + jz .Lexit_update_aad + + # ;; Check aad != NULL + test $arg2,$arg2 + jz .Lexit_update_aad + + # ;; Check aadlen != 0 + test $arg3,$arg3 + jz .Lexit_update_aad +___ +} + +# ; NOTE: code before PROLOG() must not modify any registers +&PROLOG( + 1, # allocate stack space for hkeys, + 0, # do not allocate stack space for AES blocks + "ghash"); +&GCM_UPDATE_AAD( + "$arg1", "$arg2", "$arg3", "%r10", "%r11", "%r12", "%k1", "%xmm14", "%zmm1", "%zmm11", + "%zmm3", "%zmm4", "%zmm5", "%zmm6", "%zmm7", "%zmm8", "%zmm9", "%zmm10", "%zmm12", "%zmm13", + "%zmm15", "%zmm16", "%zmm17", "%zmm18", "%zmm19"); +&EPILOG( + 1, # hkeys were allocated + $arg3); +$code .= <<___; +.Lexit_update_aad: +ret +.Lghash_seh_end: +.cfi_endproc +.size ossl_aes_gcm_update_aad_avx512, .-ossl_aes_gcm_update_aad_avx512 +___ + +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +# ;void ossl_aes_gcm_encrypt_avx512 +# ; (const void* aes_keys, +# ; void *gcm128ctx, +# ; unsigned int *pblocklen, +# ; const unsigned char *in, +# ; size_t len, +# ; unsigned char *out); +# ; +# ; Performs encryption of data |in| of len |len|, and stores the output in |out|. +# ; Stores encrypted partial block (if any) in gcm128ctx and its length in |pblocklen|. +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +$code .= <<___; +.globl ossl_aes_gcm_encrypt_avx512 +.type ossl_aes_gcm_encrypt_avx512,\@abi-omnipotent +.align 32 +ossl_aes_gcm_encrypt_avx512: +.cfi_startproc +.Lencrypt_seh_begin: + endbranch +___ + +# ; NOTE: code before PROLOG() must not modify any registers +&PROLOG( + 1, # allocate stack space for hkeys + 1, # allocate stack space for AES blocks + "encrypt"); +if ($CHECK_FUNCTION_ARGUMENTS) { + $code .= <<___; + # ;; Check aes_keys != NULL + test $arg1,$arg1 + jz .Lexit_gcm_encrypt + + # ;; Check gcm128ctx != NULL + test $arg2,$arg2 + jz .Lexit_gcm_encrypt + + # ;; Check pblocklen != NULL + test $arg3,$arg3 + jz .Lexit_gcm_encrypt + + # ;; Check in != NULL + test $arg4,$arg4 + jz .Lexit_gcm_encrypt + + # ;; Check if len != 0 + cmp \$0,$arg5 + jz .Lexit_gcm_encrypt + + # ;; Check out != NULL + cmp \$0,$arg6 + jz .Lexit_gcm_encrypt +___ +} +$code .= <<___; + # ; load number of rounds from AES_KEY structure (offset in bytes is + # ; size of the |rd_key| buffer) + mov `4*15*4`($arg1),%eax + cmp \$9,%eax + je .Laes_gcm_encrypt_128_avx512 + cmp \$11,%eax + je .Laes_gcm_encrypt_192_avx512 + cmp \$13,%eax + je .Laes_gcm_encrypt_256_avx512 + xor %eax,%eax + jmp .Lexit_gcm_encrypt +___ +for my $keylen (sort keys %aes_rounds) { + $NROUNDS = $aes_rounds{$keylen}; + $code .= <<___; +.align 32 +.Laes_gcm_encrypt_${keylen}_avx512: +___ + &GCM_ENC_DEC("$arg1", "$arg2", "$arg3", "$arg4", "$arg5", "$arg6", "ENC"); + $code .= "jmp .Lexit_gcm_encrypt\n"; +} +$code .= ".Lexit_gcm_encrypt:\n"; +&EPILOG(1, $arg5); +$code .= <<___; +ret +.Lencrypt_seh_end: +.cfi_endproc +.size ossl_aes_gcm_encrypt_avx512, .-ossl_aes_gcm_encrypt_avx512 +___ + +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +# ;void ossl_aes_gcm_decrypt_avx512 +# ; (const void* keys, +# ; void *gcm128ctx, +# ; unsigned int *pblocklen, +# ; const unsigned char *in, +# ; size_t len, +# ; unsigned char *out); +# ; +# ; Performs decryption of data |in| of len |len|, and stores the output in |out|. +# ; Stores decrypted partial block (if any) in gcm128ctx and its length in |pblocklen|. +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +$code .= <<___; +.globl ossl_aes_gcm_decrypt_avx512 +.type ossl_aes_gcm_decrypt_avx512,\@abi-omnipotent +.align 32 +ossl_aes_gcm_decrypt_avx512: +.cfi_startproc +.Ldecrypt_seh_begin: + endbranch +___ + +# ; NOTE: code before PROLOG() must not modify any registers +&PROLOG( + 1, # allocate stack space for hkeys + 1, # allocate stack space for AES blocks + "decrypt"); +if ($CHECK_FUNCTION_ARGUMENTS) { + $code .= <<___; + # ;; Check keys != NULL + test $arg1,$arg1 + jz .Lexit_gcm_decrypt + + # ;; Check gcm128ctx != NULL + test $arg2,$arg2 + jz .Lexit_gcm_decrypt + + # ;; Check pblocklen != NULL + test $arg3,$arg3 + jz .Lexit_gcm_decrypt + + # ;; Check in != NULL + test $arg4,$arg4 + jz .Lexit_gcm_decrypt + + # ;; Check if len != 0 + cmp \$0,$arg5 + jz .Lexit_gcm_decrypt + + # ;; Check out != NULL + cmp \$0,$arg6 + jz .Lexit_gcm_decrypt +___ +} +$code .= <<___; + # ; load number of rounds from AES_KEY structure (offset in bytes is + # ; size of the |rd_key| buffer) + mov `4*15*4`($arg1),%eax + cmp \$9,%eax + je .Laes_gcm_decrypt_128_avx512 + cmp \$11,%eax + je .Laes_gcm_decrypt_192_avx512 + cmp \$13,%eax + je .Laes_gcm_decrypt_256_avx512 + xor %eax,%eax + jmp .Lexit_gcm_decrypt +___ +for my $keylen (sort keys %aes_rounds) { + $NROUNDS = $aes_rounds{$keylen}; + $code .= <<___; +.align 32 +.Laes_gcm_decrypt_${keylen}_avx512: +___ + &GCM_ENC_DEC("$arg1", "$arg2", "$arg3", "$arg4", "$arg5", "$arg6", "DEC"); + $code .= "jmp .Lexit_gcm_decrypt\n"; +} +$code .= ".Lexit_gcm_decrypt:\n"; +&EPILOG(1, $arg5); +$code .= <<___; +ret +.Ldecrypt_seh_end: +.cfi_endproc +.size ossl_aes_gcm_decrypt_avx512, .-ossl_aes_gcm_decrypt_avx512 +___ + +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +# ;void ossl_aes_gcm_finalize_vaes_avx512 +# ; (void *gcm128ctx, +# ; unsigned int pblocklen); +# ; +# ; Finalizes encryption / decryption +# ; Leaf function (does not allocate stack space, does not use non-volatile registers). +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +$code .= <<___; +.globl ossl_aes_gcm_finalize_avx512 +.type ossl_aes_gcm_finalize_avx512,\@abi-omnipotent +.align 32 +ossl_aes_gcm_finalize_avx512: +.cfi_startproc + endbranch +___ +if ($CHECK_FUNCTION_ARGUMENTS) { + $code .= <<___; + # ;; Check gcm128ctx != NULL + test $arg1,$arg1 + jz .Labort_finalize +___ +} + +&GCM_COMPLETE("$arg1", "$arg2"); + +$code .= <<___; +.Labort_finalize: +ret +.cfi_endproc +.size ossl_aes_gcm_finalize_avx512, .-ossl_aes_gcm_finalize_avx512 +___ + +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +# ;void ossl_gcm_gmult_avx512(u64 Xi[2], +# ; const void* gcm128ctx) +# ; +# ; Leaf function (does not allocate stack space, does not use non-volatile registers). +# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +$code .= <<___; +.globl ossl_gcm_gmult_avx512 +.hidden ossl_gcm_gmult_avx512 +.type ossl_gcm_gmult_avx512,\@abi-omnipotent +.align 32 +ossl_gcm_gmult_avx512: +.cfi_startproc + endbranch +___ +if ($CHECK_FUNCTION_ARGUMENTS) { + $code .= <<___; + # ;; Check Xi != NULL + test $arg1,$arg1 + jz .Labort_gmult + + # ;; Check gcm128ctx != NULL + test $arg2,$arg2 + jz .Labort_gmult +___ +} +$code .= "vmovdqu64 ($arg1),%xmm1\n"; +$code .= "vmovdqu64 @{[HashKeyByIdx(1,$arg2)]},%xmm2\n"; + +&GHASH_MUL("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5"); + +$code .= "vmovdqu64 %xmm1,($arg1)\n"; +if ($CLEAR_SCRATCH_REGISTERS) { + &clear_scratch_gps_asm(); + &clear_scratch_zmms_asm(); +} else { + $code .= "vzeroupper\n"; +} +$code .= <<___; +.Labort_gmult: +ret +.cfi_endproc +.size ossl_gcm_gmult_avx512, .-ossl_gcm_gmult_avx512 +___ + +if ($win64) { + + # Add unwind metadata for SEH. + + # See https://docs.microsoft.com/en-us/cpp/build/exception-handling-x64?view=msvc-160 + my $UWOP_PUSH_NONVOL = 0; + my $UWOP_ALLOC_LARGE = 1; + my $UWOP_SET_FPREG = 3; + my $UWOP_SAVE_XMM128 = 8; + my %UWOP_REG_NUMBER = ( + rax => 0, + rcx => 1, + rdx => 2, + rbx => 3, + rsp => 4, + rbp => 5, + rsi => 6, + rdi => 7, + map(("r$_" => $_), (8 .. 15))); + + $code .= <<___; +.section .pdata +.align 4 + .rva .Lsetiv_seh_begin + .rva .Lsetiv_seh_end + .rva .Lsetiv_seh_info + + .rva .Lghash_seh_begin + .rva .Lghash_seh_end + .rva .Lghash_seh_info + + .rva .Lencrypt_seh_begin + .rva .Lencrypt_seh_end + .rva .Lencrypt_seh_info + + .rva .Ldecrypt_seh_begin + .rva .Ldecrypt_seh_end + .rva .Ldecrypt_seh_info + +.section .xdata +___ + + foreach my $func_name ("setiv", "ghash", "encrypt", "decrypt") { + $code .= <<___; +.align 8 +.L${func_name}_seh_info: + .byte 1 # version 1, no flags + .byte .L${func_name}_seh_prolog_end-.L${func_name}_seh_begin + .byte 31 # num_slots = 1*8 + 2 + 1 + 2*10 + # FR = rbp; Offset from RSP = $XMM_STORAGE scaled on 16 + .byte @{[$UWOP_REG_NUMBER{rbp} | (($XMM_STORAGE / 16 ) << 4)]} +___ + + # Metadata for %xmm15-%xmm6 + # Occupy 2 slots each + for (my $reg_idx = 15; $reg_idx >= 6; $reg_idx--) { + + # Scaled-by-16 stack offset + my $xmm_reg_offset = ($reg_idx - 6); + $code .= <<___; + .byte .L${func_name}_seh_save_xmm${reg_idx}-.L${func_name}_seh_begin + .byte @{[$UWOP_SAVE_XMM128 | (${reg_idx} << 4)]} + .value $xmm_reg_offset +___ + } + + $code .= <<___; + # Frame pointer (occupy 1 slot) + .byte .L${func_name}_seh_setfp-.L${func_name}_seh_begin + .byte $UWOP_SET_FPREG + + # Occupy 2 slots, as stack allocation < 512K, but > 128 bytes + .byte .L${func_name}_seh_allocstack_xmm-.L${func_name}_seh_begin + .byte $UWOP_ALLOC_LARGE + .value `($XMM_STORAGE + 8) / 8` +___ + + # Metadata for GPR regs + # Occupy 1 slot each + foreach my $reg ("rsi", "rdi", "r15", "r14", "r13", "r12", "rbp", "rbx") { + $code .= <<___; + .byte .L${func_name}_seh_push_${reg}-.L${func_name}_seh_begin + .byte @{[$UWOP_PUSH_NONVOL | ($UWOP_REG_NUMBER{$reg} << 4)]} +___ + } + } +} + +$code .= <<___; +.section .rodata align=16 +.align 16 +POLY: .quad 0x0000000000000001, 0xC200000000000000 + +.align 64 +POLY2: + .quad 0x00000001C2000000, 0xC200000000000000 + .quad 0x00000001C2000000, 0xC200000000000000 + .quad 0x00000001C2000000, 0xC200000000000000 + .quad 0x00000001C2000000, 0xC200000000000000 + +.align 16 +TWOONE: .quad 0x0000000000000001, 0x0000000100000000 + +# ;;; Order of these constants should not change. +# ;;; More specifically, ALL_F should follow SHIFT_MASK, and ZERO should follow ALL_F +.align 64 +SHUF_MASK: + .quad 0x08090A0B0C0D0E0F, 0x0001020304050607 + .quad 0x08090A0B0C0D0E0F, 0x0001020304050607 + .quad 0x08090A0B0C0D0E0F, 0x0001020304050607 + .quad 0x08090A0B0C0D0E0F, 0x0001020304050607 + +.align 16 +SHIFT_MASK: + .quad 0x0706050403020100, 0x0f0e0d0c0b0a0908 + +ALL_F: + .quad 0xffffffffffffffff, 0xffffffffffffffff + +ZERO: + .quad 0x0000000000000000, 0x0000000000000000 + +.align 16 +ONE: + .quad 0x0000000000000001, 0x0000000000000000 + +.align 16 +ONEf: + .quad 0x0000000000000000, 0x0100000000000000 + +.align 64 +ddq_add_1234: + .quad 0x0000000000000001, 0x0000000000000000 + .quad 0x0000000000000002, 0x0000000000000000 + .quad 0x0000000000000003, 0x0000000000000000 + .quad 0x0000000000000004, 0x0000000000000000 + +.align 64 +ddq_add_5678: + .quad 0x0000000000000005, 0x0000000000000000 + .quad 0x0000000000000006, 0x0000000000000000 + .quad 0x0000000000000007, 0x0000000000000000 + .quad 0x0000000000000008, 0x0000000000000000 + +.align 64 +ddq_add_4444: + .quad 0x0000000000000004, 0x0000000000000000 + .quad 0x0000000000000004, 0x0000000000000000 + .quad 0x0000000000000004, 0x0000000000000000 + .quad 0x0000000000000004, 0x0000000000000000 + +.align 64 +ddq_add_8888: + .quad 0x0000000000000008, 0x0000000000000000 + .quad 0x0000000000000008, 0x0000000000000000 + .quad 0x0000000000000008, 0x0000000000000000 + .quad 0x0000000000000008, 0x0000000000000000 + +.align 64 +ddq_addbe_1234: + .quad 0x0000000000000000, 0x0100000000000000 + .quad 0x0000000000000000, 0x0200000000000000 + .quad 0x0000000000000000, 0x0300000000000000 + .quad 0x0000000000000000, 0x0400000000000000 + +.align 64 +ddq_addbe_4444: + .quad 0x0000000000000000, 0x0400000000000000 + .quad 0x0000000000000000, 0x0400000000000000 + .quad 0x0000000000000000, 0x0400000000000000 + .quad 0x0000000000000000, 0x0400000000000000 + +.align 64 +byte_len_to_mask_table: + .value 0x0000, 0x0001, 0x0003, 0x0007 + .value 0x000f, 0x001f, 0x003f, 0x007f + .value 0x00ff, 0x01ff, 0x03ff, 0x07ff + .value 0x0fff, 0x1fff, 0x3fff, 0x7fff + .value 0xffff + +.align 64 +byte64_len_to_mask_table: + .quad 0x0000000000000000, 0x0000000000000001 + .quad 0x0000000000000003, 0x0000000000000007 + .quad 0x000000000000000f, 0x000000000000001f + .quad 0x000000000000003f, 0x000000000000007f + .quad 0x00000000000000ff, 0x00000000000001ff + .quad 0x00000000000003ff, 0x00000000000007ff + .quad 0x0000000000000fff, 0x0000000000001fff + .quad 0x0000000000003fff, 0x0000000000007fff + .quad 0x000000000000ffff, 0x000000000001ffff + .quad 0x000000000003ffff, 0x000000000007ffff + .quad 0x00000000000fffff, 0x00000000001fffff + .quad 0x00000000003fffff, 0x00000000007fffff + .quad 0x0000000000ffffff, 0x0000000001ffffff + .quad 0x0000000003ffffff, 0x0000000007ffffff + .quad 0x000000000fffffff, 0x000000001fffffff + .quad 0x000000003fffffff, 0x000000007fffffff + .quad 0x00000000ffffffff, 0x00000001ffffffff + .quad 0x00000003ffffffff, 0x00000007ffffffff + .quad 0x0000000fffffffff, 0x0000001fffffffff + .quad 0x0000003fffffffff, 0x0000007fffffffff + .quad 0x000000ffffffffff, 0x000001ffffffffff + .quad 0x000003ffffffffff, 0x000007ffffffffff + .quad 0x00000fffffffffff, 0x00001fffffffffff + .quad 0x00003fffffffffff, 0x00007fffffffffff + .quad 0x0000ffffffffffff, 0x0001ffffffffffff + .quad 0x0003ffffffffffff, 0x0007ffffffffffff + .quad 0x000fffffffffffff, 0x001fffffffffffff + .quad 0x003fffffffffffff, 0x007fffffffffffff + .quad 0x00ffffffffffffff, 0x01ffffffffffffff + .quad 0x03ffffffffffffff, 0x07ffffffffffffff + .quad 0x0fffffffffffffff, 0x1fffffffffffffff + .quad 0x3fffffffffffffff, 0x7fffffffffffffff + .quad 0xffffffffffffffff +___ + +} else { +# Fallback for old assembler +$code .= <<___; +.text +.globl ossl_vaes_vpclmulqdq_capable +.type ossl_vaes_vpclmulqdq_capable,\@abi-omnipotent +ossl_vaes_vpclmulqdq_capable: + xor %eax,%eax + ret +.size ossl_vaes_vpclmulqdq_capable, .-ossl_vaes_vpclmulqdq_capable + +.globl ossl_aes_gcm_init_avx512 +.globl ossl_aes_gcm_setiv_avx512 +.globl ossl_aes_gcm_update_aad_avx512 +.globl ossl_aes_gcm_encrypt_avx512 +.globl ossl_aes_gcm_decrypt_avx512 +.globl ossl_aes_gcm_finalize_avx512 +.globl ossl_gcm_gmult_avx512 + +.type ossl_aes_gcm_init_avx512,\@abi-omnipotent +ossl_aes_gcm_init_avx512: +ossl_aes_gcm_setiv_avx512: +ossl_aes_gcm_update_aad_avx512: +ossl_aes_gcm_encrypt_avx512: +ossl_aes_gcm_decrypt_avx512: +ossl_aes_gcm_finalize_avx512: +ossl_gcm_gmult_avx512: + .byte 0x0f,0x0b # ud2 + ret +.size ossl_aes_gcm_init_avx512, .-ossl_aes_gcm_init_avx512 +___ +} + +$code =~ s/\`([^\`]*)\`/eval $1/gem; +print $code; +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/modes/asm/aes-gcm-ppc.pl b/crypto/openssl/crypto/modes/asm/aes-gcm-ppc.pl new file mode 100644 index 000000000000..e8a215027e83 --- /dev/null +++ b/crypto/openssl/crypto/modes/asm/aes-gcm-ppc.pl @@ -0,0 +1,1438 @@ +#! /usr/bin/env perl +# Copyright 2014-2022 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2021- IBM Inc. All rights reserved +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html +# +#=================================================================================== +# Written by Danny Tsen for OpenSSL Project, +# +# GHASH is based on the Karatsuba multiplication method. +# +# Xi xor X1 +# +# X1 * H^4 + X2 * H^3 + x3 * H^2 + X4 * H = +# (X1.h * H4.h + xX.l * H4.l + X1 * H4) + +# (X2.h * H3.h + X2.l * H3.l + X2 * H3) + +# (X3.h * H2.h + X3.l * H2.l + X3 * H2) + +# (X4.h * H.h + X4.l * H.l + X4 * H) +# +# Xi = v0 +# H Poly = v2 +# Hash keys = v3 - v14 +# ( H.l, H, H.h) +# ( H^2.l, H^2, H^2.h) +# ( H^3.l, H^3, H^3.h) +# ( H^4.l, H^4, H^4.h) +# +# v30 is IV +# v31 - counter 1 +# +# AES used, +# vs0 - vs14 for round keys +# v15, v16, v17, v18, v19, v20, v21, v22 for 8 blocks (encrypted) +# +# This implementation uses stitched AES-GCM approach to improve overall performance. +# AES is implemented with 8x blocks and GHASH is using 2 4x blocks. +# +# Current large block (16384 bytes) performance per second with 128 bit key -- +# +# Encrypt Decrypt +# Power10[le] (3.5GHz) 5.32G 5.26G +# +# =================================================================================== +# +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +if ($flavour =~ /64/) { + $SIZE_T=8; + $LRSAVE=2*$SIZE_T; + $STU="stdu"; + $POP="ld"; + $PUSH="std"; + $UCMP="cmpld"; + $SHRI="srdi"; +} elsif ($flavour =~ /32/) { + $SIZE_T=4; + $LRSAVE=$SIZE_T; + $STU="stwu"; + $POP="lwz"; + $PUSH="stw"; + $UCMP="cmplw"; + $SHRI="srwi"; +} else { die "nonsense $flavour"; } + +$sp="r1"; +$FRAME=6*$SIZE_T+13*16; # 13*16 is for v20-v31 offload + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or +die "can't locate ppc-xlate.pl"; + +open STDOUT,"| $^X $xlate $flavour \"$output\"" + or die "can't call $xlate: $!"; + +$code=<<___; +.machine "any" +.text + +# 4x loops +# v15 - v18 - input states +# vs1 - vs9 - round keys +# +.macro Loop_aes_middle4x + xxlor 19+32, 1, 1 + xxlor 20+32, 2, 2 + xxlor 21+32, 3, 3 + xxlor 22+32, 4, 4 + + vcipher 15, 15, 19 + vcipher 16, 16, 19 + vcipher 17, 17, 19 + vcipher 18, 18, 19 + + vcipher 15, 15, 20 + vcipher 16, 16, 20 + vcipher 17, 17, 20 + vcipher 18, 18, 20 + + vcipher 15, 15, 21 + vcipher 16, 16, 21 + vcipher 17, 17, 21 + vcipher 18, 18, 21 + + vcipher 15, 15, 22 + vcipher 16, 16, 22 + vcipher 17, 17, 22 + vcipher 18, 18, 22 + + xxlor 19+32, 5, 5 + xxlor 20+32, 6, 6 + xxlor 21+32, 7, 7 + xxlor 22+32, 8, 8 + + vcipher 15, 15, 19 + vcipher 16, 16, 19 + vcipher 17, 17, 19 + vcipher 18, 18, 19 + + vcipher 15, 15, 20 + vcipher 16, 16, 20 + vcipher 17, 17, 20 + vcipher 18, 18, 20 + + vcipher 15, 15, 21 + vcipher 16, 16, 21 + vcipher 17, 17, 21 + vcipher 18, 18, 21 + + vcipher 15, 15, 22 + vcipher 16, 16, 22 + vcipher 17, 17, 22 + vcipher 18, 18, 22 + + xxlor 23+32, 9, 9 + vcipher 15, 15, 23 + vcipher 16, 16, 23 + vcipher 17, 17, 23 + vcipher 18, 18, 23 +.endm + +# 8x loops +# v15 - v22 - input states +# vs1 - vs9 - round keys +# +.macro Loop_aes_middle8x + xxlor 23+32, 1, 1 + xxlor 24+32, 2, 2 + xxlor 25+32, 3, 3 + xxlor 26+32, 4, 4 + + vcipher 15, 15, 23 + vcipher 16, 16, 23 + vcipher 17, 17, 23 + vcipher 18, 18, 23 + vcipher 19, 19, 23 + vcipher 20, 20, 23 + vcipher 21, 21, 23 + vcipher 22, 22, 23 + + vcipher 15, 15, 24 + vcipher 16, 16, 24 + vcipher 17, 17, 24 + vcipher 18, 18, 24 + vcipher 19, 19, 24 + vcipher 20, 20, 24 + vcipher 21, 21, 24 + vcipher 22, 22, 24 + + vcipher 15, 15, 25 + vcipher 16, 16, 25 + vcipher 17, 17, 25 + vcipher 18, 18, 25 + vcipher 19, 19, 25 + vcipher 20, 20, 25 + vcipher 21, 21, 25 + vcipher 22, 22, 25 + + vcipher 15, 15, 26 + vcipher 16, 16, 26 + vcipher 17, 17, 26 + vcipher 18, 18, 26 + vcipher 19, 19, 26 + vcipher 20, 20, 26 + vcipher 21, 21, 26 + vcipher 22, 22, 26 + + xxlor 23+32, 5, 5 + xxlor 24+32, 6, 6 + xxlor 25+32, 7, 7 + xxlor 26+32, 8, 8 + + vcipher 15, 15, 23 + vcipher 16, 16, 23 + vcipher 17, 17, 23 + vcipher 18, 18, 23 + vcipher 19, 19, 23 + vcipher 20, 20, 23 + vcipher 21, 21, 23 + vcipher 22, 22, 23 + + vcipher 15, 15, 24 + vcipher 16, 16, 24 + vcipher 17, 17, 24 + vcipher 18, 18, 24 + vcipher 19, 19, 24 + vcipher 20, 20, 24 + vcipher 21, 21, 24 + vcipher 22, 22, 24 + + vcipher 15, 15, 25 + vcipher 16, 16, 25 + vcipher 17, 17, 25 + vcipher 18, 18, 25 + vcipher 19, 19, 25 + vcipher 20, 20, 25 + vcipher 21, 21, 25 + vcipher 22, 22, 25 + + vcipher 15, 15, 26 + vcipher 16, 16, 26 + vcipher 17, 17, 26 + vcipher 18, 18, 26 + vcipher 19, 19, 26 + vcipher 20, 20, 26 + vcipher 21, 21, 26 + vcipher 22, 22, 26 + + xxlor 23+32, 9, 9 + vcipher 15, 15, 23 + vcipher 16, 16, 23 + vcipher 17, 17, 23 + vcipher 18, 18, 23 + vcipher 19, 19, 23 + vcipher 20, 20, 23 + vcipher 21, 21, 23 + vcipher 22, 22, 23 +.endm + +# +# Compute 4x hash values based on Karatsuba method. +# +ppc_aes_gcm_ghash: + vxor 15, 15, 0 + + xxlxor 29, 29, 29 + + vpmsumd 23, 12, 15 # H4.L * X.L + vpmsumd 24, 9, 16 + vpmsumd 25, 6, 17 + vpmsumd 26, 3, 18 + + vxor 23, 23, 24 + vxor 23, 23, 25 + vxor 23, 23, 26 # L + + vpmsumd 24, 13, 15 # H4.L * X.H + H4.H * X.L + vpmsumd 25, 10, 16 # H3.L * X1.H + H3.H * X1.L + vpmsumd 26, 7, 17 + vpmsumd 27, 4, 18 + + vxor 24, 24, 25 + vxor 24, 24, 26 + vxor 24, 24, 27 # M + + # sum hash and reduction with H Poly + vpmsumd 28, 23, 2 # reduction + + xxlor 29+32, 29, 29 + vsldoi 26, 24, 29, 8 # mL + vsldoi 29, 29, 24, 8 # mH + vxor 23, 23, 26 # mL + L + + vsldoi 23, 23, 23, 8 # swap + vxor 23, 23, 28 + + vpmsumd 24, 14, 15 # H4.H * X.H + vpmsumd 25, 11, 16 + vpmsumd 26, 8, 17 + vpmsumd 27, 5, 18 + + vxor 24, 24, 25 + vxor 24, 24, 26 + vxor 24, 24, 27 + + vxor 24, 24, 29 + + # sum hash and reduction with H Poly + vsldoi 27, 23, 23, 8 # swap + vpmsumd 23, 23, 2 + vxor 27, 27, 24 + vxor 23, 23, 27 + + xxlor 32, 23+32, 23+32 # update hash + + blr + +# +# Combine two 4x ghash +# v15 - v22 - input blocks +# +.macro ppc_aes_gcm_ghash2_4x + # first 4x hash + vxor 15, 15, 0 # Xi + X + + xxlxor 29, 29, 29 + + vpmsumd 23, 12, 15 # H4.L * X.L + vpmsumd 24, 9, 16 + vpmsumd 25, 6, 17 + vpmsumd 26, 3, 18 + + vxor 23, 23, 24 + vxor 23, 23, 25 + vxor 23, 23, 26 # L + + vpmsumd 24, 13, 15 # H4.L * X.H + H4.H * X.L + vpmsumd 25, 10, 16 # H3.L * X1.H + H3.H * X1.L + vpmsumd 26, 7, 17 + vpmsumd 27, 4, 18 + + vxor 24, 24, 25 + vxor 24, 24, 26 + + # sum hash and reduction with H Poly + vpmsumd 28, 23, 2 # reduction + + xxlor 29+32, 29, 29 + + vxor 24, 24, 27 # M + vsldoi 26, 24, 29, 8 # mL + vsldoi 29, 29, 24, 8 # mH + vxor 23, 23, 26 # mL + L + + vsldoi 23, 23, 23, 8 # swap + vxor 23, 23, 28 + + vpmsumd 24, 14, 15 # H4.H * X.H + vpmsumd 25, 11, 16 + vpmsumd 26, 8, 17 + vpmsumd 27, 5, 18 + + vxor 24, 24, 25 + vxor 24, 24, 26 + vxor 24, 24, 27 # H + + vxor 24, 24, 29 # H + mH + + # sum hash and reduction with H Poly + vsldoi 27, 23, 23, 8 # swap + vpmsumd 23, 23, 2 + vxor 27, 27, 24 + vxor 27, 23, 27 # 1st Xi + + # 2nd 4x hash + vpmsumd 24, 9, 20 + vpmsumd 25, 6, 21 + vpmsumd 26, 3, 22 + vxor 19, 19, 27 # Xi + X + vpmsumd 23, 12, 19 # H4.L * X.L + + vxor 23, 23, 24 + vxor 23, 23, 25 + vxor 23, 23, 26 # L + + vpmsumd 24, 13, 19 # H4.L * X.H + H4.H * X.L + vpmsumd 25, 10, 20 # H3.L * X1.H + H3.H * X1.L + vpmsumd 26, 7, 21 + vpmsumd 27, 4, 22 + + vxor 24, 24, 25 + vxor 24, 24, 26 + + # sum hash and reduction with H Poly + vpmsumd 28, 23, 2 # reduction + + xxlor 29+32, 29, 29 + + vxor 24, 24, 27 # M + vsldoi 26, 24, 29, 8 # mL + vsldoi 29, 29, 24, 8 # mH + vxor 23, 23, 26 # mL + L + + vsldoi 23, 23, 23, 8 # swap + vxor 23, 23, 28 + + vpmsumd 24, 14, 19 # H4.H * X.H + vpmsumd 25, 11, 20 + vpmsumd 26, 8, 21 + vpmsumd 27, 5, 22 + + vxor 24, 24, 25 + vxor 24, 24, 26 + vxor 24, 24, 27 # H + + vxor 24, 24, 29 # H + mH + + # sum hash and reduction with H Poly + vsldoi 27, 23, 23, 8 # swap + vpmsumd 23, 23, 2 + vxor 27, 27, 24 + vxor 23, 23, 27 + + xxlor 32, 23+32, 23+32 # update hash + +.endm + +# +# Compute update single hash +# +.macro ppc_update_hash_1x + vxor 28, 28, 0 + + vxor 19, 19, 19 + + vpmsumd 22, 3, 28 # L + vpmsumd 23, 4, 28 # M + vpmsumd 24, 5, 28 # H + + vpmsumd 27, 22, 2 # reduction + + vsldoi 25, 23, 19, 8 # mL + vsldoi 26, 19, 23, 8 # mH + vxor 22, 22, 25 # LL + LL + vxor 24, 24, 26 # HH + HH + + vsldoi 22, 22, 22, 8 # swap + vxor 22, 22, 27 + + vsldoi 20, 22, 22, 8 # swap + vpmsumd 22, 22, 2 # reduction + vxor 20, 20, 24 + vxor 22, 22, 20 + + vmr 0, 22 # update hash + +.endm + +# +# ppc_aes_gcm_encrypt (const void *inp, void *out, size_t len, +# const AES_KEY *key, unsigned char iv[16], +# void *Xip); +# +# r3 - inp +# r4 - out +# r5 - len +# r6 - AES round keys +# r7 - iv +# r8 - Xi, HPoli, hash keys +# +.global ppc_aes_gcm_encrypt +.align 5 +ppc_aes_gcm_encrypt: +_ppc_aes_gcm_encrypt: + + stdu 1,-512(1) + mflr 0 + + std 14,112(1) + std 15,120(1) + std 16,128(1) + std 17,136(1) + std 18,144(1) + std 19,152(1) + std 20,160(1) + std 21,168(1) + li 9, 256 + stvx 20, 9, 1 + addi 9, 9, 16 + stvx 21, 9, 1 + addi 9, 9, 16 + stvx 22, 9, 1 + addi 9, 9, 16 + stvx 23, 9, 1 + addi 9, 9, 16 + stvx 24, 9, 1 + addi 9, 9, 16 + stvx 25, 9, 1 + addi 9, 9, 16 + stvx 26, 9, 1 + addi 9, 9, 16 + stvx 27, 9, 1 + addi 9, 9, 16 + stvx 28, 9, 1 + addi 9, 9, 16 + stvx 29, 9, 1 + addi 9, 9, 16 + stvx 30, 9, 1 + addi 9, 9, 16 + stvx 31, 9, 1 + std 0, 528(1) + + # Load Xi + lxvb16x 32, 0, 8 # load Xi + + # load Hash - h^4, h^3, h^2, h + li 10, 32 + lxvd2x 2+32, 10, 8 # H Poli + li 10, 48 + lxvd2x 3+32, 10, 8 # Hl + li 10, 64 + lxvd2x 4+32, 10, 8 # H + li 10, 80 + lxvd2x 5+32, 10, 8 # Hh + + li 10, 96 + lxvd2x 6+32, 10, 8 # H^2l + li 10, 112 + lxvd2x 7+32, 10, 8 # H^2 + li 10, 128 + lxvd2x 8+32, 10, 8 # H^2h + + li 10, 144 + lxvd2x 9+32, 10, 8 # H^3l + li 10, 160 + lxvd2x 10+32, 10, 8 # H^3 + li 10, 176 + lxvd2x 11+32, 10, 8 # H^3h + + li 10, 192 + lxvd2x 12+32, 10, 8 # H^4l + li 10, 208 + lxvd2x 13+32, 10, 8 # H^4 + li 10, 224 + lxvd2x 14+32, 10, 8 # H^4h + + # initialize ICB: GHASH( IV ), IV - r7 + lxvb16x 30+32, 0, 7 # load IV - v30 + + mr 12, 5 # length + li 11, 0 # block index + + # counter 1 + vxor 31, 31, 31 + vspltisb 22, 1 + vsldoi 31, 31, 22,1 # counter 1 + + # load round key to VSR + lxv 0, 0(6) + lxv 1, 0x10(6) + lxv 2, 0x20(6) + lxv 3, 0x30(6) + lxv 4, 0x40(6) + lxv 5, 0x50(6) + lxv 6, 0x60(6) + lxv 7, 0x70(6) + lxv 8, 0x80(6) + lxv 9, 0x90(6) + lxv 10, 0xa0(6) + + # load rounds - 10 (128), 12 (192), 14 (256) + lwz 9,240(6) + + # + # vxor state, state, w # addroundkey + xxlor 32+29, 0, 0 + vxor 15, 30, 29 # IV + round key - add round key 0 + + cmpdi 9, 10 + beq Loop_aes_gcm_8x + + # load 2 more round keys (v11, v12) + lxv 11, 0xb0(6) + lxv 12, 0xc0(6) + + cmpdi 9, 12 + beq Loop_aes_gcm_8x + + # load 2 more round keys (v11, v12, v13, v14) + lxv 13, 0xd0(6) + lxv 14, 0xe0(6) + cmpdi 9, 14 + beq Loop_aes_gcm_8x + + b aes_gcm_out + +.align 5 +Loop_aes_gcm_8x: + mr 14, 3 + mr 9, 4 + + # n blocks + li 10, 128 + divdu 10, 5, 10 # n 128 bytes-blocks + cmpdi 10, 0 + beq Loop_last_block + + vaddudm 30, 30, 31 # IV + counter + vxor 16, 30, 29 + vaddudm 30, 30, 31 + vxor 17, 30, 29 + vaddudm 30, 30, 31 + vxor 18, 30, 29 + vaddudm 30, 30, 31 + vxor 19, 30, 29 + vaddudm 30, 30, 31 + vxor 20, 30, 29 + vaddudm 30, 30, 31 + vxor 21, 30, 29 + vaddudm 30, 30, 31 + vxor 22, 30, 29 + + mtctr 10 + + li 15, 16 + li 16, 32 + li 17, 48 + li 18, 64 + li 19, 80 + li 20, 96 + li 21, 112 + + lwz 10, 240(6) + +Loop_8x_block: + + lxvb16x 15, 0, 14 # load block + lxvb16x 16, 15, 14 # load block + lxvb16x 17, 16, 14 # load block + lxvb16x 18, 17, 14 # load block + lxvb16x 19, 18, 14 # load block + lxvb16x 20, 19, 14 # load block + lxvb16x 21, 20, 14 # load block + lxvb16x 22, 21, 14 # load block + addi 14, 14, 128 + + Loop_aes_middle8x + + xxlor 23+32, 10, 10 + + cmpdi 10, 10 + beq Do_next_ghash + + # 192 bits + xxlor 24+32, 11, 11 + + vcipher 15, 15, 23 + vcipher 16, 16, 23 + vcipher 17, 17, 23 + vcipher 18, 18, 23 + vcipher 19, 19, 23 + vcipher 20, 20, 23 + vcipher 21, 21, 23 + vcipher 22, 22, 23 + + vcipher 15, 15, 24 + vcipher 16, 16, 24 + vcipher 17, 17, 24 + vcipher 18, 18, 24 + vcipher 19, 19, 24 + vcipher 20, 20, 24 + vcipher 21, 21, 24 + vcipher 22, 22, 24 + + xxlor 23+32, 12, 12 + + cmpdi 10, 12 + beq Do_next_ghash + + # 256 bits + xxlor 24+32, 13, 13 + + vcipher 15, 15, 23 + vcipher 16, 16, 23 + vcipher 17, 17, 23 + vcipher 18, 18, 23 + vcipher 19, 19, 23 + vcipher 20, 20, 23 + vcipher 21, 21, 23 + vcipher 22, 22, 23 + + vcipher 15, 15, 24 + vcipher 16, 16, 24 + vcipher 17, 17, 24 + vcipher 18, 18, 24 + vcipher 19, 19, 24 + vcipher 20, 20, 24 + vcipher 21, 21, 24 + vcipher 22, 22, 24 + + xxlor 23+32, 14, 14 + + cmpdi 10, 14 + beq Do_next_ghash + b aes_gcm_out + +Do_next_ghash: + + # + # last round + vcipherlast 15, 15, 23 + vcipherlast 16, 16, 23 + + xxlxor 47, 47, 15 + stxvb16x 47, 0, 9 # store output + xxlxor 48, 48, 16 + stxvb16x 48, 15, 9 # store output + + vcipherlast 17, 17, 23 + vcipherlast 18, 18, 23 + + xxlxor 49, 49, 17 + stxvb16x 49, 16, 9 # store output + xxlxor 50, 50, 18 + stxvb16x 50, 17, 9 # store output + + vcipherlast 19, 19, 23 + vcipherlast 20, 20, 23 + + xxlxor 51, 51, 19 + stxvb16x 51, 18, 9 # store output + xxlxor 52, 52, 20 + stxvb16x 52, 19, 9 # store output + + vcipherlast 21, 21, 23 + vcipherlast 22, 22, 23 + + xxlxor 53, 53, 21 + stxvb16x 53, 20, 9 # store output + xxlxor 54, 54, 22 + stxvb16x 54, 21, 9 # store output + + addi 9, 9, 128 + + # ghash here + ppc_aes_gcm_ghash2_4x + + xxlor 27+32, 0, 0 + vaddudm 30, 30, 31 # IV + counter + vmr 29, 30 + vxor 15, 30, 27 # add round key + vaddudm 30, 30, 31 + vxor 16, 30, 27 + vaddudm 30, 30, 31 + vxor 17, 30, 27 + vaddudm 30, 30, 31 + vxor 18, 30, 27 + vaddudm 30, 30, 31 + vxor 19, 30, 27 + vaddudm 30, 30, 31 + vxor 20, 30, 27 + vaddudm 30, 30, 31 + vxor 21, 30, 27 + vaddudm 30, 30, 31 + vxor 22, 30, 27 + + addi 12, 12, -128 + addi 11, 11, 128 + + bdnz Loop_8x_block + + vmr 30, 29 + +Loop_last_block: + cmpdi 12, 0 + beq aes_gcm_out + + # loop last few blocks + li 10, 16 + divdu 10, 12, 10 + + mtctr 10 + + lwz 10, 240(6) + + cmpdi 12, 16 + blt Final_block + +.macro Loop_aes_middle_1x + xxlor 19+32, 1, 1 + xxlor 20+32, 2, 2 + xxlor 21+32, 3, 3 + xxlor 22+32, 4, 4 + + vcipher 15, 15, 19 + vcipher 15, 15, 20 + vcipher 15, 15, 21 + vcipher 15, 15, 22 + + xxlor 19+32, 5, 5 + xxlor 20+32, 6, 6 + xxlor 21+32, 7, 7 + xxlor 22+32, 8, 8 + + vcipher 15, 15, 19 + vcipher 15, 15, 20 + vcipher 15, 15, 21 + vcipher 15, 15, 22 + + xxlor 19+32, 9, 9 + vcipher 15, 15, 19 +.endm + +Next_rem_block: + lxvb16x 15, 0, 14 # load block + + Loop_aes_middle_1x + + xxlor 23+32, 10, 10 + + cmpdi 10, 10 + beq Do_next_1x + + # 192 bits + xxlor 24+32, 11, 11 + + vcipher 15, 15, 23 + vcipher 15, 15, 24 + + xxlor 23+32, 12, 12 + + cmpdi 10, 12 + beq Do_next_1x + + # 256 bits + xxlor 24+32, 13, 13 + + vcipher 15, 15, 23 + vcipher 15, 15, 24 + + xxlor 23+32, 14, 14 + + cmpdi 10, 14 + beq Do_next_1x + +Do_next_1x: + vcipherlast 15, 15, 23 + + xxlxor 47, 47, 15 + stxvb16x 47, 0, 9 # store output + addi 14, 14, 16 + addi 9, 9, 16 + + vmr 28, 15 + ppc_update_hash_1x + + addi 12, 12, -16 + addi 11, 11, 16 + xxlor 19+32, 0, 0 + vaddudm 30, 30, 31 # IV + counter + vxor 15, 30, 19 # add round key + + bdnz Next_rem_block + + cmpdi 12, 0 + beq aes_gcm_out + +Final_block: + Loop_aes_middle_1x + + xxlor 23+32, 10, 10 + + cmpdi 10, 10 + beq Do_final_1x + + # 192 bits + xxlor 24+32, 11, 11 + + vcipher 15, 15, 23 + vcipher 15, 15, 24 + + xxlor 23+32, 12, 12 + + cmpdi 10, 12 + beq Do_final_1x + + # 256 bits + xxlor 24+32, 13, 13 + + vcipher 15, 15, 23 + vcipher 15, 15, 24 + + xxlor 23+32, 14, 14 + + cmpdi 10, 14 + beq Do_final_1x + +Do_final_1x: + vcipherlast 15, 15, 23 + + lxvb16x 15, 0, 14 # load last block + xxlxor 47, 47, 15 + + # create partial block mask + li 15, 16 + sub 15, 15, 12 # index to the mask + + vspltisb 16, -1 # first 16 bytes - 0xffff...ff + vspltisb 17, 0 # second 16 bytes - 0x0000...00 + li 10, 192 + stvx 16, 10, 1 + addi 10, 10, 16 + stvx 17, 10, 1 + + addi 10, 1, 192 + lxvb16x 16, 15, 10 # load partial block mask + xxland 47, 47, 16 + + vmr 28, 15 + ppc_update_hash_1x + + # * should store only the remaining bytes. + bl Write_partial_block + + b aes_gcm_out + +# +# Write partial block +# r9 - output +# r12 - remaining bytes +# v15 - partial input data +# +Write_partial_block: + li 10, 192 + stxvb16x 15+32, 10, 1 # last block + + #add 10, 9, 11 # Output + addi 10, 9, -1 + addi 16, 1, 191 + + mtctr 12 # remaining bytes + li 15, 0 + +Write_last_byte: + lbzu 14, 1(16) + stbu 14, 1(10) + bdnz Write_last_byte + blr + +aes_gcm_out: + # out = state + stxvb16x 32, 0, 8 # write out Xi + add 3, 11, 12 # return count + + li 9, 256 + lvx 20, 9, 1 + addi 9, 9, 16 + lvx 21, 9, 1 + addi 9, 9, 16 + lvx 22, 9, 1 + addi 9, 9, 16 + lvx 23, 9, 1 + addi 9, 9, 16 + lvx 24, 9, 1 + addi 9, 9, 16 + lvx 25, 9, 1 + addi 9, 9, 16 + lvx 26, 9, 1 + addi 9, 9, 16 + lvx 27, 9, 1 + addi 9, 9, 16 + lvx 28, 9, 1 + addi 9, 9, 16 + lvx 29, 9, 1 + addi 9, 9, 16 + lvx 30, 9, 1 + addi 9, 9, 16 + lvx 31, 9, 1 + + ld 0, 528(1) + ld 14,112(1) + ld 15,120(1) + ld 16,128(1) + ld 17,136(1) + ld 18,144(1) + ld 19,152(1) + ld 20,160(1) + ld 21,168(1) + + mtlr 0 + addi 1, 1, 512 + blr + +# +# 8x Decrypt +# +.global ppc_aes_gcm_decrypt +.align 5 +ppc_aes_gcm_decrypt: +_ppc_aes_gcm_decrypt: + + stdu 1,-512(1) + mflr 0 + + std 14,112(1) + std 15,120(1) + std 16,128(1) + std 17,136(1) + std 18,144(1) + std 19,152(1) + std 20,160(1) + std 21,168(1) + li 9, 256 + stvx 20, 9, 1 + addi 9, 9, 16 + stvx 21, 9, 1 + addi 9, 9, 16 + stvx 22, 9, 1 + addi 9, 9, 16 + stvx 23, 9, 1 + addi 9, 9, 16 + stvx 24, 9, 1 + addi 9, 9, 16 + stvx 25, 9, 1 + addi 9, 9, 16 + stvx 26, 9, 1 + addi 9, 9, 16 + stvx 27, 9, 1 + addi 9, 9, 16 + stvx 28, 9, 1 + addi 9, 9, 16 + stvx 29, 9, 1 + addi 9, 9, 16 + stvx 30, 9, 1 + addi 9, 9, 16 + stvx 31, 9, 1 + std 0, 528(1) + + # Load Xi + lxvb16x 32, 0, 8 # load Xi + + # load Hash - h^4, h^3, h^2, h + li 10, 32 + lxvd2x 2+32, 10, 8 # H Poli + li 10, 48 + lxvd2x 3+32, 10, 8 # Hl + li 10, 64 + lxvd2x 4+32, 10, 8 # H + li 10, 80 + lxvd2x 5+32, 10, 8 # Hh + + li 10, 96 + lxvd2x 6+32, 10, 8 # H^2l + li 10, 112 + lxvd2x 7+32, 10, 8 # H^2 + li 10, 128 + lxvd2x 8+32, 10, 8 # H^2h + + li 10, 144 + lxvd2x 9+32, 10, 8 # H^3l + li 10, 160 + lxvd2x 10+32, 10, 8 # H^3 + li 10, 176 + lxvd2x 11+32, 10, 8 # H^3h + + li 10, 192 + lxvd2x 12+32, 10, 8 # H^4l + li 10, 208 + lxvd2x 13+32, 10, 8 # H^4 + li 10, 224 + lxvd2x 14+32, 10, 8 # H^4h + + # initialize ICB: GHASH( IV ), IV - r7 + lxvb16x 30+32, 0, 7 # load IV - v30 + + mr 12, 5 # length + li 11, 0 # block index + + # counter 1 + vxor 31, 31, 31 + vspltisb 22, 1 + vsldoi 31, 31, 22,1 # counter 1 + + # load round key to VSR + lxv 0, 0(6) + lxv 1, 0x10(6) + lxv 2, 0x20(6) + lxv 3, 0x30(6) + lxv 4, 0x40(6) + lxv 5, 0x50(6) + lxv 6, 0x60(6) + lxv 7, 0x70(6) + lxv 8, 0x80(6) + lxv 9, 0x90(6) + lxv 10, 0xa0(6) + + # load rounds - 10 (128), 12 (192), 14 (256) + lwz 9,240(6) + + # + # vxor state, state, w # addroundkey + xxlor 32+29, 0, 0 + vxor 15, 30, 29 # IV + round key - add round key 0 + + cmpdi 9, 10 + beq Loop_aes_gcm_8x_dec + + # load 2 more round keys (v11, v12) + lxv 11, 0xb0(6) + lxv 12, 0xc0(6) + + cmpdi 9, 12 + beq Loop_aes_gcm_8x_dec + + # load 2 more round keys (v11, v12, v13, v14) + lxv 13, 0xd0(6) + lxv 14, 0xe0(6) + cmpdi 9, 14 + beq Loop_aes_gcm_8x_dec + + b aes_gcm_out + +.align 5 +Loop_aes_gcm_8x_dec: + mr 14, 3 + mr 9, 4 + + # n blocks + li 10, 128 + divdu 10, 5, 10 # n 128 bytes-blocks + cmpdi 10, 0 + beq Loop_last_block_dec + + vaddudm 30, 30, 31 # IV + counter + vxor 16, 30, 29 + vaddudm 30, 30, 31 + vxor 17, 30, 29 + vaddudm 30, 30, 31 + vxor 18, 30, 29 + vaddudm 30, 30, 31 + vxor 19, 30, 29 + vaddudm 30, 30, 31 + vxor 20, 30, 29 + vaddudm 30, 30, 31 + vxor 21, 30, 29 + vaddudm 30, 30, 31 + vxor 22, 30, 29 + + mtctr 10 + + li 15, 16 + li 16, 32 + li 17, 48 + li 18, 64 + li 19, 80 + li 20, 96 + li 21, 112 + + lwz 10, 240(6) + +Loop_8x_block_dec: + + lxvb16x 15, 0, 14 # load block + lxvb16x 16, 15, 14 # load block + lxvb16x 17, 16, 14 # load block + lxvb16x 18, 17, 14 # load block + lxvb16x 19, 18, 14 # load block + lxvb16x 20, 19, 14 # load block + lxvb16x 21, 20, 14 # load block + lxvb16x 22, 21, 14 # load block + addi 14, 14, 128 + + Loop_aes_middle8x + + xxlor 23+32, 10, 10 + + cmpdi 10, 10 + beq Do_last_aes_dec + + # 192 bits + xxlor 24+32, 11, 11 + + vcipher 15, 15, 23 + vcipher 16, 16, 23 + vcipher 17, 17, 23 + vcipher 18, 18, 23 + vcipher 19, 19, 23 + vcipher 20, 20, 23 + vcipher 21, 21, 23 + vcipher 22, 22, 23 + + vcipher 15, 15, 24 + vcipher 16, 16, 24 + vcipher 17, 17, 24 + vcipher 18, 18, 24 + vcipher 19, 19, 24 + vcipher 20, 20, 24 + vcipher 21, 21, 24 + vcipher 22, 22, 24 + + xxlor 23+32, 12, 12 + + cmpdi 10, 12 + beq Do_last_aes_dec + + # 256 bits + xxlor 24+32, 13, 13 + + vcipher 15, 15, 23 + vcipher 16, 16, 23 + vcipher 17, 17, 23 + vcipher 18, 18, 23 + vcipher 19, 19, 23 + vcipher 20, 20, 23 + vcipher 21, 21, 23 + vcipher 22, 22, 23 + + vcipher 15, 15, 24 + vcipher 16, 16, 24 + vcipher 17, 17, 24 + vcipher 18, 18, 24 + vcipher 19, 19, 24 + vcipher 20, 20, 24 + vcipher 21, 21, 24 + vcipher 22, 22, 24 + + xxlor 23+32, 14, 14 + + cmpdi 10, 14 + beq Do_last_aes_dec + b aes_gcm_out + +Do_last_aes_dec: + + # + # last round + vcipherlast 15, 15, 23 + vcipherlast 16, 16, 23 + + xxlxor 47, 47, 15 + stxvb16x 47, 0, 9 # store output + xxlxor 48, 48, 16 + stxvb16x 48, 15, 9 # store output + + vcipherlast 17, 17, 23 + vcipherlast 18, 18, 23 + + xxlxor 49, 49, 17 + stxvb16x 49, 16, 9 # store output + xxlxor 50, 50, 18 + stxvb16x 50, 17, 9 # store output + + vcipherlast 19, 19, 23 + vcipherlast 20, 20, 23 + + xxlxor 51, 51, 19 + stxvb16x 51, 18, 9 # store output + xxlxor 52, 52, 20 + stxvb16x 52, 19, 9 # store output + + vcipherlast 21, 21, 23 + vcipherlast 22, 22, 23 + + xxlxor 53, 53, 21 + stxvb16x 53, 20, 9 # store output + xxlxor 54, 54, 22 + stxvb16x 54, 21, 9 # store output + + addi 9, 9, 128 + + xxlor 15+32, 15, 15 + xxlor 16+32, 16, 16 + xxlor 17+32, 17, 17 + xxlor 18+32, 18, 18 + xxlor 19+32, 19, 19 + xxlor 20+32, 20, 20 + xxlor 21+32, 21, 21 + xxlor 22+32, 22, 22 + + # ghash here + ppc_aes_gcm_ghash2_4x + + xxlor 27+32, 0, 0 + vaddudm 30, 30, 31 # IV + counter + vmr 29, 30 + vxor 15, 30, 27 # add round key + vaddudm 30, 30, 31 + vxor 16, 30, 27 + vaddudm 30, 30, 31 + vxor 17, 30, 27 + vaddudm 30, 30, 31 + vxor 18, 30, 27 + vaddudm 30, 30, 31 + vxor 19, 30, 27 + vaddudm 30, 30, 31 + vxor 20, 30, 27 + vaddudm 30, 30, 31 + vxor 21, 30, 27 + vaddudm 30, 30, 31 + vxor 22, 30, 27 + addi 12, 12, -128 + addi 11, 11, 128 + + bdnz Loop_8x_block_dec + + vmr 30, 29 + +Loop_last_block_dec: + cmpdi 12, 0 + beq aes_gcm_out + + # loop last few blocks + li 10, 16 + divdu 10, 12, 10 + + mtctr 10 + + lwz 10,240(6) + + cmpdi 12, 16 + blt Final_block_dec + +Next_rem_block_dec: + lxvb16x 15, 0, 14 # load block + + Loop_aes_middle_1x + + xxlor 23+32, 10, 10 + + cmpdi 10, 10 + beq Do_next_1x_dec + + # 192 bits + xxlor 24+32, 11, 11 + + vcipher 15, 15, 23 + vcipher 15, 15, 24 + + xxlor 23+32, 12, 12 + + cmpdi 10, 12 + beq Do_next_1x_dec + + # 256 bits + xxlor 24+32, 13, 13 + + vcipher 15, 15, 23 + vcipher 15, 15, 24 + + xxlor 23+32, 14, 14 + + cmpdi 10, 14 + beq Do_next_1x_dec + +Do_next_1x_dec: + vcipherlast 15, 15, 23 + + xxlxor 47, 47, 15 + stxvb16x 47, 0, 9 # store output + addi 14, 14, 16 + addi 9, 9, 16 + + xxlor 28+32, 15, 15 + ppc_update_hash_1x + + addi 12, 12, -16 + addi 11, 11, 16 + xxlor 19+32, 0, 0 + vaddudm 30, 30, 31 # IV + counter + vxor 15, 30, 19 # add round key + + bdnz Next_rem_block_dec + + cmpdi 12, 0 + beq aes_gcm_out + +Final_block_dec: + Loop_aes_middle_1x + + xxlor 23+32, 10, 10 + + cmpdi 10, 10 + beq Do_final_1x_dec + + # 192 bits + xxlor 24+32, 11, 11 + + vcipher 15, 15, 23 + vcipher 15, 15, 24 + + xxlor 23+32, 12, 12 + + cmpdi 10, 12 + beq Do_final_1x_dec + + # 256 bits + xxlor 24+32, 13, 13 + + vcipher 15, 15, 23 + vcipher 15, 15, 24 + + xxlor 23+32, 14, 14 + + cmpdi 10, 14 + beq Do_final_1x_dec + +Do_final_1x_dec: + vcipherlast 15, 15, 23 + + lxvb16x 15, 0, 14 # load block + xxlxor 47, 47, 15 + + # create partial block mask + li 15, 16 + sub 15, 15, 12 # index to the mask + + vspltisb 16, -1 # first 16 bytes - 0xffff...ff + vspltisb 17, 0 # second 16 bytes - 0x0000...00 + li 10, 192 + stvx 16, 10, 1 + addi 10, 10, 16 + stvx 17, 10, 1 + + addi 10, 1, 192 + lxvb16x 16, 15, 10 # load block mask + xxland 47, 47, 16 + + xxlor 28+32, 15, 15 + ppc_update_hash_1x + + # * should store only the remaining bytes. + bl Write_partial_block + + b aes_gcm_out + + +___ + +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval $1/geo; + + if ($flavour =~ /le$/o) { # little-endian + s/le\?//o or + s/be\?/#be#/o; + } else { + s/le\?/#le#/o or + s/be\?//o; + } + print $_,"\n"; +} + +close STDOUT or die "error closing STDOUT: $!"; # enforce flush diff --git a/crypto/openssl/crypto/modes/asm/aes-gcm-riscv64-zvkb-zvkg-zvkned.pl b/crypto/openssl/crypto/modes/asm/aes-gcm-riscv64-zvkb-zvkg-zvkned.pl new file mode 100644 index 000000000000..84ecc65ab35c --- /dev/null +++ b/crypto/openssl/crypto/modes/asm/aes-gcm-riscv64-zvkb-zvkg-zvkned.pl @@ -0,0 +1,975 @@ +#! /usr/bin/env perl +# This file is dual-licensed, meaning that you can use it under your +# choice of either of the following two licenses: +# +# Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You can obtain +# a copy in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html +# +# or +# +# Copyright (c) 2023, Jerry Shih +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# - RV64I +# - RISC-V Vector ('V') with VLEN >= 128 +# - RISC-V Vector Cryptography Bit-manipulation extension ('Zvkb') +# - RISC-V Vector GCM/GMAC extension ('Zvkg') +# - RISC-V Vector AES Block Cipher extension ('Zvkned') +# - RISC-V Zicclsm(Main memory supports misaligned loads/stores) + +# Reference: https://github.com/riscv/riscv-crypto/issues/192#issuecomment-1270447575 +# +# Assume we have 12 GCM blocks and we try to parallelize GCM computation for 4 blocks. +# Tag = M0*H^12 + M1*H^11 + M2*H^10 + M3*H^9 + +# M4*H^8 + M5*H^7 + M6*H^6 + M7*H^5 + +# M8*H^4 + M9*H^3 + M10*H^2 + M11*H^1 +# We could rewrite the formula into: +# T0 = 0 +# T1 = (T0+M1)*H^4 T2 = (T0+M2)*H^4 T3 = (T0+M3)*H^4 T4 = (T0+M4)*H^4 +# T5 = (T1+M5)*H^4 T6 = (T2+M6)*H^4 T7 = (T3+M7)*H^4 T8 = (T4+M8)*H^4 +# T9 = (T5+M9)*H^4 T10 = (T6+M10)*H^3 T11 = (T7+M11)*H^2 T12 = (T8+M12)*H^1 +# +# We will multiply with [H^4, H^4, H^4, H^4] in each steps except the last iteration. +# The last iteration will multiply with [H^4, H^3, H^2, H^1]. + +use strict; +use warnings; + +use FindBin qw($Bin); +use lib "$Bin"; +use lib "$Bin/../../perlasm"; +use riscv; + +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$output and open STDOUT,">$output"; + +my $code=<<___; +.text +___ + +{ +my ($INP, $OUTP, $LEN, $KEYP, $IVP, $XIP) = ("a0", "a1", "a2", "a3", "a4", "a5"); +my ($T0, $T1, $T2, $T3) = ("t0", "t1", "t2", "t3"); +my ($PADDING_LEN32) = ("t4"); +my ($LEN32) = ("t5"); +my ($CTR) = ("t6"); +my ($FULL_BLOCK_LEN32) = ("a6"); +my ($ORIGINAL_LEN32) = ("a7"); +my ($PROCESSED_LEN) = ("a0"); +my ($CTR_MASK) = ("v0"); +my ($INPUT_PADDING_MASK) = ("v0"); +my ($V0, $V1, $V2, $V3, $V4, $V5, $V6, $V7, + $V8, $V9, $V10, $V11, $V12, $V13, $V14, $V15, + $V16, $V17, $V18, $V19, $V20, $V21, $V22, $V23, + $V24, $V25, $V26, $V27, $V28, $V29, $V30, $V31, +) = map("v$_",(0..31)); + +# Do aes-128 enc. +sub aes_128_cipher_body { + my $code=<<___; + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + @{[vaesz_vs $V28, $V1]} + @{[vaesem_vs $V28, $V2]} + @{[vaesem_vs $V28, $V3]} + @{[vaesem_vs $V28, $V4]} + @{[vaesem_vs $V28, $V5]} + @{[vaesem_vs $V28, $V6]} + @{[vaesem_vs $V28, $V7]} + @{[vaesem_vs $V28, $V8]} + @{[vaesem_vs $V28, $V9]} + @{[vaesem_vs $V28, $V10]} + @{[vaesef_vs $V28, $V11]} +___ + + return $code; +} + +# Do aes-192 enc. +sub aes_192_cipher_body { + my $TMP_REG = shift; + + my $code=<<___; + # Load key 4 + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + addi $TMP_REG, $KEYP, 48 + @{[vle32_v $V11, $TMP_REG]} + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + @{[vaesz_vs $V28, $V1]} + @{[vaesem_vs $V28, $V2]} + @{[vaesem_vs $V28, $V3]} + @{[vaesem_vs $V28, $V11]} + # Load key 8 + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + addi $TMP_REG, $KEYP, 112 + @{[vle32_v $V11, $TMP_REG]} + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + @{[vaesem_vs $V28, $V4]} + @{[vaesem_vs $V28, $V5]} + @{[vaesem_vs $V28, $V6]} + @{[vaesem_vs $V28, $V11]} + # Load key 13 + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + addi $TMP_REG, $KEYP, 192 + @{[vle32_v $V11, $TMP_REG]} + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + @{[vaesem_vs $V28, $V7]} + @{[vaesem_vs $V28, $V8]} + @{[vaesem_vs $V28, $V9]} + @{[vaesem_vs $V28, $V10]} + @{[vaesef_vs $V28, $V11]} +___ + + return $code; +} + +# Do aes-256 enc. +sub aes_256_cipher_body { + my $TMP_REG = shift; + + my $code=<<___; + # Load key 3 + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + addi $TMP_REG, $KEYP, 32 + @{[vle32_v $V11, $TMP_REG]} + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + @{[vaesz_vs $V28, $V1]} + @{[vaesem_vs $V28, $V2]} + @{[vaesem_vs $V28, $V11]} + # Load key 6 + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + addi $TMP_REG, $KEYP, 80 + @{[vle32_v $V11, $TMP_REG]} + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + @{[vaesem_vs $V28, $V3]} + @{[vaesem_vs $V28, $V4]} + @{[vaesem_vs $V28, $V11]} + # Load key 9 + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + addi $TMP_REG, $KEYP, 128 + @{[vle32_v $V11, $TMP_REG]} + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + @{[vaesem_vs $V28, $V5]} + @{[vaesem_vs $V28, $V6]} + @{[vaesem_vs $V28, $V11]} + # Load key 12 + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + addi $TMP_REG, $KEYP, 176 + @{[vle32_v $V11, $TMP_REG]} + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + @{[vaesem_vs $V28, $V7]} + @{[vaesem_vs $V28, $V8]} + @{[vaesem_vs $V28, $V11]} + # Load key 15 + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + addi $TMP_REG, $KEYP, 224 + @{[vle32_v $V11, $TMP_REG]} + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + @{[vaesem_vs $V28, $V9]} + @{[vaesem_vs $V28, $V10]} + @{[vaesef_vs $V28, $V11]} +___ + + return $code; +} + +sub handle_padding_in_first_round { + my $TMP_REG = shift; + + my $code=<<___; + bnez $PADDING_LEN32, 1f + + ## without padding + # Store ciphertext/plaintext + @{[vse32_v $V28, $OUTP]} + j 2f + + ## with padding +1: + # Store ciphertext/plaintext using mask + @{[vse32_v $V28, $OUTP, $INPUT_PADDING_MASK]} + + # Fill zero for the padding blocks + @{[vsetvli "zero", $PADDING_LEN32, "e32", "m4", "tu", "ma"]} + @{[vmv_v_i $V28, 0]} + + # We have used mask register for `INPUT_PADDING_MASK` before. We need to + # setup the ctr mask back. + # ctr mask : [000100010001....] + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e8", "m1", "ta", "ma"]} + li $TMP_REG, 0b10001000 + @{[vmv_v_x $CTR_MASK, $TMP_REG]} +2: + +___ + + return $code; +} + + +# Do aes-128 enc for first round. +sub aes_128_first_round { + my $PTR_OFFSET_REG = shift; + my $TMP_REG = shift; + + my $code=<<___; + # Load all 11 aes round keys to v1-v11 registers. + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vle32_v $V1, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V2, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V3, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V4, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V5, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V6, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V7, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V8, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V9, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V10, $KEYP]} + addi $KEYP, $KEYP, 16 + @{[vle32_v $V11, $KEYP]} + + # We already have the ciphertext/plaintext and ctr data for the first round. + @{[aes_128_cipher_body]} + + # Compute AES ctr result. + @{[vxor_vv $V28, $V28, $V24]} + + @{[handle_padding_in_first_round $TMP_REG]} + + add $INP, $INP, $PTR_OFFSET_REG + add $OUTP, $OUTP, $PTR_OFFSET_REG +___ + + return $code; +} + +# Do aes-192 enc for first round. +sub aes_192_first_round { + my $PTR_OFFSET_REG = shift; + my $TMP_REG = shift; + + my $code=<<___; + # We run out of 32 vector registers, so we just preserve some round keys + # and load the remaining round keys inside the aes body. + # We keep the round keys for: + # 1, 2, 3, 5, 6, 7, 9, 10, 11 and 12th keys. + # The following keys will be loaded in the aes body: + # 4, 8 and 13th keys. + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + # key 1 + @{[vle32_v $V1, $KEYP]} + # key 2 + addi $TMP_REG, $KEYP, 16 + @{[vle32_v $V2, $TMP_REG]} + # key 3 + addi $TMP_REG, $KEYP, 32 + @{[vle32_v $V3, $TMP_REG]} + # key 5 + addi $TMP_REG, $KEYP, 64 + @{[vle32_v $V4, $TMP_REG]} + # key 6 + addi $TMP_REG, $KEYP, 80 + @{[vle32_v $V5, $TMP_REG]} + # key 7 + addi $TMP_REG, $KEYP, 96 + @{[vle32_v $V6, $TMP_REG]} + # key 9 + addi $TMP_REG, $KEYP, 128 + @{[vle32_v $V7, $TMP_REG]} + # key 10 + addi $TMP_REG, $KEYP, 144 + @{[vle32_v $V8, $TMP_REG]} + # key 11 + addi $TMP_REG, $KEYP, 160 + @{[vle32_v $V9, $TMP_REG]} + # key 12 + addi $TMP_REG, $KEYP, 176 + @{[vle32_v $V10, $TMP_REG]} + + # We already have the ciphertext/plaintext and ctr data for the first round. + @{[aes_192_cipher_body $TMP_REG]} + + # Compute AES ctr result. + @{[vxor_vv $V28, $V28, $V24]} + + @{[handle_padding_in_first_round $TMP_REG]} + + add $INP, $INP, $PTR_OFFSET_REG + add $OUTP, $OUTP, $PTR_OFFSET_REG +___ + + return $code; +} + +# Do aes-256 enc for first round. +sub aes_256_first_round { + my $PTR_OFFSET_REG = shift; + my $TMP_REG = shift; + + my $code=<<___; + # We run out of 32 vector registers, so we just preserve some round keys + # and load the remaining round keys inside the aes body. + # We keep the round keys for: + # 1, 2, 4, 5, 7, 8, 10, 11, 13 and 14th keys. + # The following keys will be loaded in the aes body: + # 3, 6, 9, 12 and 15th keys. + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + # key 1 + @{[vle32_v $V1, $KEYP]} + # key 2 + addi $TMP_REG, $KEYP, 16 + @{[vle32_v $V2, $TMP_REG]} + # key 4 + addi $TMP_REG, $KEYP, 48 + @{[vle32_v $V3, $TMP_REG]} + # key 5 + addi $TMP_REG, $KEYP, 64 + @{[vle32_v $V4, $TMP_REG]} + # key 7 + addi $TMP_REG, $KEYP, 96 + @{[vle32_v $V5, $TMP_REG]} + # key 8 + addi $TMP_REG, $KEYP, 112 + @{[vle32_v $V6, $TMP_REG]} + # key 10 + addi $TMP_REG, $KEYP, 144 + @{[vle32_v $V7, $TMP_REG]} + # key 11 + addi $TMP_REG, $KEYP, 160 + @{[vle32_v $V8, $TMP_REG]} + # key 13 + addi $TMP_REG, $KEYP, 192 + @{[vle32_v $V9, $TMP_REG]} + # key 14 + addi $TMP_REG, $KEYP, 208 + @{[vle32_v $V10, $TMP_REG]} + + # We already have the ciphertext/plaintext and ctr data for the first round. + @{[aes_256_cipher_body $TMP_REG]} + + # Compute AES ctr result. + @{[vxor_vv $V28, $V28, $V24]} + + @{[handle_padding_in_first_round $TMP_REG]} + + add $INP, $INP, $PTR_OFFSET_REG + add $OUTP, $OUTP, $PTR_OFFSET_REG +___ + + return $code; +} + +sub aes_gcm_init { + my $code=<<___; + # Compute the AES-GCM full-block e32 length for `LMUL=4`. We will handle + # the multiple AES-GCM blocks at the same time within `LMUL=4` register. + # The AES-GCM's SEW is e32 and EGW is 128 bits. + # FULL_BLOCK_LEN32 = (VLEN*LMUL)/(EGW) * (EGW/SEW) = (VLEN*4)/(32*4) * 4 + # = (VLEN*4)/32 + # We could get the block_num using the VL value of `vsetvli with e32, m4`. + @{[vsetvli $FULL_BLOCK_LEN32, "zero", "e32", "m4", "ta", "ma"]} + # If `LEN32 % FULL_BLOCK_LEN32` is not equal to zero, we could fill the + # zero padding data to make sure we could always handle FULL_BLOCK_LEN32 + # blocks for all iterations. + + ## Prepare the H^n multiplier in v16 for GCM multiplier. The `n` is the gcm + ## block number in a LMUL=4 register group. + ## n = ((VLEN*LMUL)/(32*4)) = ((VLEN*4)/(32*4)) + ## = (VLEN/32) + ## We could use vsetvli with `e32, m1` to compute the `n` number. + @{[vsetvli $T0, "zero", "e32", "m1", "ta", "ma"]} + + # The H is at `gcm128_context.Htable[0]`(addr(Xi)+16*2). + addi $T1, $XIP, 32 + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vle32_v $V31, $T1]} + + # Compute the H^n + li $T1, 1 +1: + @{[vgmul_vv $V31, $V31]} + slli $T1, $T1, 1 + bltu $T1, $T0, 1b + + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + @{[vmv_v_i $V16, 0]} + @{[vaesz_vs $V16, $V31]} + + #### Load plaintext into v24 and handle padding. We also load the init tag + #### data into v20 and prepare the AES ctr input data into v12 and v28. + @{[vmv_v_i $V20, 0]} + + ## Prepare the AES ctr input data into v12. + # Setup ctr input mask. + # ctr mask : [000100010001....] + # Note: The actual vl should be `FULL_BLOCK_LEN32/4 * 2`, but we just use + # `FULL_BLOCK_LEN32` here. + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e8", "m1", "ta", "ma"]} + li $T0, 0b10001000 + @{[vmv_v_x $CTR_MASK, $T0]} + # Load IV. + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vle32_v $V31, $IVP]} + # Convert the big-endian counter into little-endian. + @{[vsetivli "zero", 4, "e32", "m1", "ta", "mu"]} + @{[vrev8_v $V31, $V31, $CTR_MASK]} + # Splat the `single block of IV` to v12 + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + @{[vmv_v_i $V12, 0]} + @{[vaesz_vs $V12, $V31]} + # Prepare the ctr counter into v8 + # v8: [x, x, x, 0, x, x, x, 1, x, x, x, 2, ...] + @{[viota_m $V8, $CTR_MASK, $CTR_MASK]} + # Merge IV and ctr counter into v12. + # v12:[x, x, x, count+0, x, x, x, count+1, ...] + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "mu"]} + @{[vadd_vv $V12, $V12, $V8, $CTR_MASK]} + + li $PADDING_LEN32, 0 + # Get the SEW32 size in the first round. + # If we have the non-zero value for `LEN32&(FULL_BLOCK_LEN32-1)`, then + # we will have the leading padding zero. + addi $T0, $FULL_BLOCK_LEN32, -1 + and $T0, $T0, $LEN32 + beqz $T0, 1f + + ## with padding + sub $LEN32, $LEN32, $T0 + sub $PADDING_LEN32, $FULL_BLOCK_LEN32, $T0 + # padding block size + srli $T1, $PADDING_LEN32, 2 + # padding byte size + slli $T2, $PADDING_LEN32, 2 + + # Adjust the ctr counter to make the counter start from `counter+0` for the + # first non-padding block. + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "mu"]} + @{[vsub_vx $V12, $V12, $T1, $CTR_MASK]} + # Prepare the AES ctr input into v28. + # The ctr data uses big-endian form. + @{[vmv_v_v $V28, $V12]} + @{[vrev8_v $V28, $V28, $CTR_MASK]} + + # Prepare the mask for input loading in the first round. We use + # `VL=FULL_BLOCK_LEN32` with the mask in the first round. + # Adjust input ptr. + sub $INP, $INP, $T2 + # Adjust output ptr. + sub $OUTP, $OUTP, $T2 + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e16", "m2", "ta", "ma"]} + @{[vid_v $V2]} + # We don't use the pseudo instruction `vmsgeu` here. Use `vmsgtu` instead. + # The original code is: + # vmsgeu.vx $INPUT_PADDING_MASK, $V2, $PADDING_LEN32 + addi $T0, $PADDING_LEN32, -1 + @{[vmsgtu_vx $INPUT_PADDING_MASK, $V2, $T0]} + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + @{[vmv_v_i $V24, 0]} + # Load the input for length FULL_BLOCK_LEN32 with mask. + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "mu"]} + @{[vle32_v $V24, $INP, $INPUT_PADDING_MASK]} + + # Load the init `Xi` data to v20 with preceding zero padding. + # Adjust Xi ptr. + sub $T0, $XIP, $T2 + # Load for length `zero-padding-e32-length + 4`. + addi $T1, $PADDING_LEN32, 4 + @{[vsetvli "zero", $T1, "e32", "m4", "tu", "mu"]} + @{[vle32_v $V20, $T0, $INPUT_PADDING_MASK]} + j 2f + +1: + ## without padding + sub $LEN32, $LEN32, $FULL_BLOCK_LEN32 + + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + @{[vle32_v $V24, $INP]} + + # Load the init Xi data to v20. + @{[vsetivli "zero", 4, "e32", "m1", "tu", "ma"]} + @{[vle32_v $V20, $XIP]} + + # Prepare the AES ctr input into v28. + # The ctr data uses big-endian form. + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "mu"]} + @{[vmv_v_v $V28, $V12]} + @{[vrev8_v $V28, $V28, $CTR_MASK]} +2: +___ + + return $code; +} + +sub prepare_input_and_ctr { + my $PTR_OFFSET_REG = shift; + + my $code=<<___; + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "mu"]} + # Increase ctr in v12. + @{[vadd_vx $V12, $V12, $CTR, $CTR_MASK]} + sub $LEN32, $LEN32, $FULL_BLOCK_LEN32 + # Load plaintext into v24 + @{[vsetvli "zero", "zero", "e32", "m4", "ta", "ma"]} + @{[vle32_v $V24, $INP]} + # Prepare the AES ctr input into v28. + # The ctr data uses big-endian form. + @{[vmv_v_v $V28, $V12]} + add $INP, $INP, $PTR_OFFSET_REG + @{[vsetvli "zero", "zero", "e32", "m4", "ta", "mu"]} + @{[vrev8_v $V28, $V28, $CTR_MASK]} +___ + + return $code; +} + +# Store the current CTR back to IV buffer. +sub store_current_ctr { + my $code=<<___; + @{[vsetivli "zero", 4, "e32", "m4", "ta", "ma"]} + # Update current ctr value to v12 + @{[vadd_vx $V12, $V12, $CTR, $CTR_MASK]} + # Convert ctr to big-endian counter. + @{[vrev8_v $V12, $V12, $CTR_MASK]} + @{[vse32_v $V12, $IVP, $CTR_MASK]} +___ + + return $code; +} + +# Compute the final tag into v0 from the partial tag v20. +sub compute_final_tag { + my $TMP_REG = shift; + + my $code=<<___; + # The H is at `gcm128_context.Htable[0]` (addr(Xi)+16*2). + # Load H to v1 + addi $TMP_REG, $XIP, 32 + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vle32_v $V1, $TMP_REG]} + # Multiply H for each partial tag and XOR them together. + # Handle 1st partial tag + @{[vmv_v_v $V0, $V20]} + @{[vgmul_vv $V0, $V1]} + # Handle 2nd to N-th partial tags + li $TMP_REG, 4 +1: + @{[vsetivli "zero", 4, "e32", "m4", "ta", "ma"]} + @{[vslidedown_vx $V4, $V20, $TMP_REG]} + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vghsh_vv $V0, $V1, $V4]} + addi $TMP_REG, $TMP_REG, 4 + blt $TMP_REG, $FULL_BLOCK_LEN32, 1b +___ + + return $code; +} + +################################################################################ +# size_t rv64i_zvkb_zvkg_zvkned_aes_gcm_encrypt(const unsigned char *in, +# unsigned char *out, size_t len, +# const void *key, +# unsigned char ivec[16], u64 *Xi); +{ +$code .= <<___; +.p2align 3 +.globl rv64i_zvkb_zvkg_zvkned_aes_gcm_encrypt +.type rv64i_zvkb_zvkg_zvkned_aes_gcm_encrypt,\@function +rv64i_zvkb_zvkg_zvkned_aes_gcm_encrypt: + srli $T0, $LEN, 4 + beqz $T0, .Lenc_end + slli $LEN32, $T0, 2 + + mv $ORIGINAL_LEN32, $LEN32 + + @{[aes_gcm_init]} + + # Load number of rounds + lwu $T0, 240($KEYP) + li $T1, 14 + li $T2, 12 + li $T3, 10 + + beq $T0, $T1, aes_gcm_enc_blocks_256 + beq $T0, $T2, aes_gcm_enc_blocks_192 + beq $T0, $T3, aes_gcm_enc_blocks_128 + +.Lenc_end: + li $PROCESSED_LEN, 0 + ret + +.size rv64i_zvkb_zvkg_zvkned_aes_gcm_encrypt,.-rv64i_zvkb_zvkg_zvkned_aes_gcm_encrypt +___ + +$code .= <<___; +.p2align 3 +aes_gcm_enc_blocks_128: + srli $CTR, $FULL_BLOCK_LEN32, 2 + slli $T0, $FULL_BLOCK_LEN32, 2 + + @{[aes_128_first_round $T0, $T1]} + + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + +.Lenc_blocks_128: + # Compute the partial tags. + # The partial tags will multiply with [H^n, H^n, ..., H^n] + # [tag0, tag1, ...] = + # ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n] + # We will skip the [H^n, H^n, ..., H^n] multiplication for the last round. + beqz $LEN32, .Lenc_blocks_128_end + @{[vghsh_vv $V20, $V16, $V28]} + + @{[prepare_input_and_ctr $T0]} + + @{[aes_128_cipher_body]} + + # Compute AES ctr ciphertext result. + @{[vxor_vv $V28, $V28, $V24]} + + # Store ciphertext + @{[vse32_v $V28, $OUTP]} + add $OUTP, $OUTP, $T0 + + j .Lenc_blocks_128 +.Lenc_blocks_128_end: + + # Add ciphertext into partial tag + @{[vxor_vv $V20, $V20, $V28]} + + @{[store_current_ctr]} + + @{[compute_final_tag $T1]} + + # Save the final tag + @{[vse32_v $V0, $XIP]} + + # return the processed size. + slli $PROCESSED_LEN, $ORIGINAL_LEN32, 2 + ret +.size aes_gcm_enc_blocks_128,.-aes_gcm_enc_blocks_128 +___ + +$code .= <<___; +.p2align 3 +aes_gcm_enc_blocks_192: + srli $CTR, $FULL_BLOCK_LEN32, 2 + slli $T0, $FULL_BLOCK_LEN32, 2 + + @{[aes_192_first_round $T0, $T1]} + + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + +.Lenc_blocks_192: + # Compute the partial tags. + # The partial tags will multiply with [H^n, H^n, ..., H^n] + # [tag0, tag1, ...] = + # ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n] + # We will skip the [H^n, H^n, ..., H^n] multiplication for the last round. + beqz $LEN32, .Lenc_blocks_192_end + @{[vghsh_vv $V20, $V16, $V28]} + + @{[prepare_input_and_ctr $T0]} + + @{[aes_192_cipher_body $T1]} + + # Compute AES ctr ciphertext result. + @{[vxor_vv $V28, $V28, $V24]} + + # Store ciphertext + @{[vse32_v $V28, $OUTP]} + add $OUTP, $OUTP, $T0 + + j .Lenc_blocks_192 +.Lenc_blocks_192_end: + + # Add ciphertext into partial tag + @{[vxor_vv $V20, $V20, $V28]} + + @{[store_current_ctr]} + + @{[compute_final_tag $T1]} + + # Save the final tag + @{[vse32_v $V0, $XIP]} + + # return the processed size. + slli $PROCESSED_LEN, $ORIGINAL_LEN32, 2 + ret +.size aes_gcm_enc_blocks_192,.-aes_gcm_enc_blocks_192 +___ + +$code .= <<___; +.p2align 3 +aes_gcm_enc_blocks_256: + srli $CTR, $FULL_BLOCK_LEN32, 2 + slli $T0, $FULL_BLOCK_LEN32, 2 + + @{[aes_256_first_round $T0, $T1]} + + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + +.Lenc_blocks_256: + # Compute the partial tags. + # The partial tags will multiply with [H^n, H^n, ..., H^n] + # [tag0, tag1, ...] = + # ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n] + # We will skip the [H^n, H^n, ..., H^n] multiplication for the last round. + beqz $LEN32, .Lenc_blocks_256_end + @{[vghsh_vv $V20, $V16, $V28]} + + @{[prepare_input_and_ctr $T0]} + + @{[aes_256_cipher_body $T1]} + + # Compute AES ctr ciphertext result. + @{[vxor_vv $V28, $V28, $V24]} + + # Store ciphertext + @{[vse32_v $V28, $OUTP]} + add $OUTP, $OUTP, $T0 + + j .Lenc_blocks_256 +.Lenc_blocks_256_end: + + # Add ciphertext into partial tag + @{[vxor_vv $V20, $V20, $V28]} + + @{[store_current_ctr]} + + @{[compute_final_tag $T1]} + + # Save the final tag + @{[vse32_v $V0, $XIP]} + + # return the processed size. + slli $PROCESSED_LEN, $ORIGINAL_LEN32, 2 + ret +.size aes_gcm_enc_blocks_256,.-aes_gcm_enc_blocks_256 +___ + +} + +################################################################################ +# size_t rv64i_zvkb_zvkg_zvkned_aes_gcm_decrypt(const unsigned char *in, +# unsigned char *out, size_t len, +# const void *key, +# unsigned char ivec[16], u64 *Xi); +{ +$code .= <<___; +.p2align 3 +.globl rv64i_zvkb_zvkg_zvkned_aes_gcm_decrypt +.type rv64i_zvkb_zvkg_zvkned_aes_gcm_decrypt,\@function +rv64i_zvkb_zvkg_zvkned_aes_gcm_decrypt: + srli $T0, $LEN, 4 + beqz $T0, .Ldec_end + slli $LEN32, $T0, 2 + + mv $ORIGINAL_LEN32, $LEN32 + + @{[aes_gcm_init]} + + # Load number of rounds + lwu $T0, 240($KEYP) + li $T1, 14 + li $T2, 12 + li $T3, 10 + + beq $T0, $T1, aes_gcm_dec_blocks_256 + beq $T0, $T2, aes_gcm_dec_blocks_192 + beq $T0, $T3, aes_gcm_dec_blocks_128 + +.Ldec_end: + li $PROCESSED_LEN, 0 + ret +.size rv64i_zvkb_zvkg_zvkned_aes_gcm_decrypt,.-rv64i_zvkb_zvkg_zvkned_aes_gcm_decrypt +___ + +$code .= <<___; +.p2align 3 +aes_gcm_dec_blocks_128: + srli $CTR, $FULL_BLOCK_LEN32, 2 + slli $T0, $FULL_BLOCK_LEN32, 2 + + @{[aes_128_first_round $T0, $T1]} + + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + +.Ldec_blocks_128: + # Compute the partial tags. + # The partial tags will multiply with [H^n, H^n, ..., H^n] + # [tag0, tag1, ...] = + # ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n] + # We will skip the [H^n, H^n, ..., H^n] multiplication for the last round. + beqz $LEN32, .Ldec_blocks_256_end + @{[vghsh_vv $V20, $V16, $V24]} + + @{[prepare_input_and_ctr $T0]} + + @{[aes_128_cipher_body]} + + # Compute AES ctr plaintext result. + @{[vxor_vv $V28, $V28, $V24]} + + # Store plaintext + @{[vse32_v $V28, $OUTP]} + add $OUTP, $OUTP, $T0 + + j .Ldec_blocks_128 +.Ldec_blocks_128_end: + + # Add ciphertext into partial tag + @{[vxor_vv $V20, $V20, $V24]} + + @{[store_current_ctr]} + + @{[compute_final_tag $T1]} + + # Save the final tag + @{[vse32_v $V0, $XIP]} + + # return the processed size. + slli $PROCESSED_LEN, $ORIGINAL_LEN32, 2 + ret +.size aes_gcm_dec_blocks_128,.-aes_gcm_dec_blocks_128 +___ + +$code .= <<___; +.p2align 3 +aes_gcm_dec_blocks_192: + srli $CTR, $FULL_BLOCK_LEN32, 2 + slli $T0, $FULL_BLOCK_LEN32, 2 + + @{[aes_192_first_round $T0, $T1]} + + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + +.Ldec_blocks_192: + # Compute the partial tags. + # The partial tags will multiply with [H^n, H^n, ..., H^n] + # [tag0, tag1, ...] = + # ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n] + # We will skip the [H^n, H^n, ..., H^n] multiplication for the last round. + beqz $LEN32, .Ldec_blocks_192_end + @{[vghsh_vv $V20, $V16, $V24]} + + @{[prepare_input_and_ctr $T0]} + + @{[aes_192_cipher_body $T1]} + + # Compute AES ctr plaintext result. + @{[vxor_vv $V28, $V28, $V24]} + + # Store plaintext + @{[vse32_v $V28, $OUTP]} + add $OUTP, $OUTP, $T0 + + j .Ldec_blocks_192 +.Ldec_blocks_192_end: + + # Add ciphertext into partial tag + @{[vxor_vv $V20, $V20, $V24]} + + @{[store_current_ctr]} + + @{[compute_final_tag $T1]} + + # Save the final tag + @{[vse32_v $V0, $XIP]} + + # return the processed size. + slli $PROCESSED_LEN, $ORIGINAL_LEN32, 2 + ret +.size aes_gcm_dec_blocks_192,.-aes_gcm_dec_blocks_192 +___ + +$code .= <<___; +.p2align 3 +aes_gcm_dec_blocks_256: + srli $CTR, $FULL_BLOCK_LEN32, 2 + slli $T0, $FULL_BLOCK_LEN32, 2 + + @{[aes_256_first_round $T0, $T1]} + + @{[vsetvli "zero", $FULL_BLOCK_LEN32, "e32", "m4", "ta", "ma"]} + +.Ldec_blocks_256: + # Compute the partial tags. + # The partial tags will multiply with [H^n, H^n, ..., H^n] + # [tag0, tag1, ...] = + # ([tag0, tag1, ...] + [ciphertext0, ciphertext1, ...] * [H^n, H^n, ..., H^n] + # We will skip the [H^n, H^n, ..., H^n] multiplication for the last round. + beqz $LEN32, .Ldec_blocks_256_end + @{[vghsh_vv $V20, $V16, $V24]} + + @{[prepare_input_and_ctr $T0]} + + @{[aes_256_cipher_body $T1]} + + # Compute AES ctr plaintext result. + @{[vxor_vv $V28, $V28, $V24]} + + # Store plaintext + @{[vse32_v $V28, $OUTP]} + add $OUTP, $OUTP, $T0 + + j .Ldec_blocks_256 +.Ldec_blocks_256_end: + + # Add ciphertext into partial tag + @{[vxor_vv $V20, $V20, $V24]} + + @{[store_current_ctr]} + + @{[compute_final_tag $T1]} + + # Save the final tag + @{[vse32_v $V0, $XIP]} + + # return the processed size. + slli $PROCESSED_LEN, $ORIGINAL_LEN32, 2 + ret +.size aes_gcm_dec_blocks_256,.-aes_gcm_dec_blocks_256 +___ + +} +} + +print $code; + +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/modes/asm/aesni-gcm-x86_64.pl b/crypto/openssl/crypto/modes/asm/aesni-gcm-x86_64.pl index eaf4d9c755fb..872e13f8f1a1 100755 --- a/crypto/openssl/crypto/modes/asm/aesni-gcm-x86_64.pl +++ b/crypto/openssl/crypto/modes/asm/aesni-gcm-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2013-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2013-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -943,6 +943,7 @@ $code.=<<___; ___ $code.=<<___; +.section .rodata align=64 .align 64 .Lbswap_mask: .byte 15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0 @@ -955,6 +956,7 @@ $code.=<<___; .Lone_lsb: .byte 1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 .asciz "AES-NI GCM module for x86_64, CRYPTOGAMS by " +.previous .align 64 ___ if ($win64) { diff --git a/crypto/openssl/crypto/modes/asm/ghash-riscv64-zvkb-zvbc.pl b/crypto/openssl/crypto/modes/asm/ghash-riscv64-zvkb-zvbc.pl new file mode 100644 index 000000000000..5eb748bdc2c3 --- /dev/null +++ b/crypto/openssl/crypto/modes/asm/ghash-riscv64-zvkb-zvbc.pl @@ -0,0 +1,378 @@ +#! /usr/bin/env perl +# This file is dual-licensed, meaning that you can use it under your +# choice of either of the following two licenses: +# +# Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You can obtain +# a copy in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html +# +# or +# +# Copyright (c) 2023, Christoph Müllner +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# - RV64I +# - RISC-V Vector ('V') with VLEN >= 128 +# - RISC-V Vector Cryptography Bit-manipulation extension ('Zvkb') +# - RISC-V Vector Carryless Multiplication extension ('Zvbc') + +use strict; +use warnings; + +use FindBin qw($Bin); +use lib "$Bin"; +use lib "$Bin/../../perlasm"; +use riscv; + +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$output and open STDOUT,">$output"; + +my $code=<<___; +.text +___ + +################################################################################ +# void gcm_init_rv64i_zvkb_zvbc(u128 Htable[16], const u64 H[2]); +# +# input: H: 128-bit H - secret parameter E(K, 0^128) +# output: Htable: Preprocessed key data for gcm_gmult_rv64i_zvkb_zvbc and +# gcm_ghash_rv64i_zvkb_zvbc +{ +my ($Htable,$H,$TMP0,$TMP1,$TMP2) = ("a0","a1","t0","t1","t2"); +my ($V0,$V1,$V2,$V3,$V4,$V5,$V6) = ("v0","v1","v2","v3","v4","v5","v6"); + +$code .= <<___; +.p2align 3 +.globl gcm_init_rv64i_zvkb_zvbc +.type gcm_init_rv64i_zvkb_zvbc,\@function +gcm_init_rv64i_zvkb_zvbc: + # Load/store data in reverse order. + # This is needed as a part of endianness swap. + add $H, $H, 8 + li $TMP0, -8 + li $TMP1, 63 + la $TMP2, Lpolymod + + @{[vsetivli__x0_2_e64_m1_tu_mu]} # vsetivli x0, 2, e64, m1, tu, mu + + @{[vlse64_v $V1, $H, $TMP0]} # vlse64.v v1, (a1), t0 + @{[vle64_v $V2, $TMP2]} # vle64.v v2, (t2) + + # Shift one left and get the carry bits. + @{[vsrl_vx $V3, $V1, $TMP1]} # vsrl.vx v3, v1, t1 + @{[vsll_vi $V1, $V1, 1]} # vsll.vi v1, v1, 1 + + # Use the fact that the polynomial degree is no more than 128, + # i.e. only the LSB of the upper half could be set. + # Thanks to this we don't need to do the full reduction here. + # Instead simply subtract the reduction polynomial. + # This idea was taken from x86 ghash implementation in OpenSSL. + @{[vslideup_vi $V4, $V3, 1]} # vslideup.vi v4, v3, 1 + @{[vslidedown_vi $V3, $V3, 1]} # vslidedown.vi v3, v3, 1 + + @{[vmv_v_i $V0, 2]} # vmv.v.i v0, 2 + @{[vor_vv_v0t $V1, $V1, $V4]} # vor.vv v1, v1, v4, v0.t + + # Need to set the mask to 3, if the carry bit is set. + @{[vmv_v_v $V0, $V3]} # vmv.v.v v0, v3 + @{[vmv_v_i $V3, 0]} # vmv.v.i v3, 0 + @{[vmerge_vim $V3, $V3, 3]} # vmerge.vim v3, v3, 3, v0 + @{[vmv_v_v $V0, $V3]} # vmv.v.v v0, v3 + + @{[vxor_vv_v0t $V1, $V1, $V2]} # vxor.vv v1, v1, v2, v0.t + + @{[vse64_v $V1, $Htable]} # vse64.v v1, (a0) + ret +.size gcm_init_rv64i_zvkb_zvbc,.-gcm_init_rv64i_zvkb_zvbc +___ +} + +################################################################################ +# void gcm_gmult_rv64i_zvkb_zvbc(u64 Xi[2], const u128 Htable[16]); +# +# input: Xi: current hash value +# Htable: preprocessed H +# output: Xi: next hash value Xi = (Xi * H mod f) +{ +my ($Xi,$Htable,$TMP0,$TMP1,$TMP2,$TMP3,$TMP4) = ("a0","a1","t0","t1","t2","t3","t4"); +my ($V0,$V1,$V2,$V3,$V4,$V5,$V6) = ("v0","v1","v2","v3","v4","v5","v6"); + +$code .= <<___; +.text +.p2align 3 +.globl gcm_gmult_rv64i_zvkb_zvbc +.type gcm_gmult_rv64i_zvkb_zvbc,\@function +gcm_gmult_rv64i_zvkb_zvbc: + ld $TMP0, ($Htable) + ld $TMP1, 8($Htable) + li $TMP2, 63 + la $TMP3, Lpolymod + ld $TMP3, 8($TMP3) + + # Load/store data in reverse order. + # This is needed as a part of endianness swap. + add $Xi, $Xi, 8 + li $TMP4, -8 + + @{[vsetivli__x0_2_e64_m1_tu_mu]} # vsetivli x0, 2, e64, m1, tu, mu + + @{[vlse64_v $V5, $Xi, $TMP4]} # vlse64.v v5, (a0), t4 + @{[vrev8_v $V5, $V5]} # vrev8.v v5, v5 + + # Multiplication + + # Do two 64x64 multiplications in one go to save some time + # and simplify things. + + # A = a1a0 (t1, t0) + # B = b1b0 (v5) + # C = c1c0 (256 bit) + # c1 = a1b1 + (a0b1)h + (a1b0)h + # c0 = a0b0 + (a0b1)l + (a1b0)h + + # v1 = (a0b1)l,(a0b0)l + @{[vclmul_vx $V1, $V5, $TMP0]} # vclmul.vx v1, v5, t0 + # v3 = (a0b1)h,(a0b0)h + @{[vclmulh_vx $V3, $V5, $TMP0]} # vclmulh.vx v3, v5, t0 + + # v4 = (a1b1)l,(a1b0)l + @{[vclmul_vx $V4, $V5, $TMP1]} # vclmul.vx v4, v5, t1 + # v2 = (a1b1)h,(a1b0)h + @{[vclmulh_vx $V2, $V5, $TMP1]} # vclmulh.vx v2, v5, t1 + + # Is there a better way to do this? + # Would need to swap the order of elements within a vector register. + @{[vslideup_vi $V5, $V3, 1]} # vslideup.vi v5, v3, 1 + @{[vslideup_vi $V6, $V4, 1]} # vslideup.vi v6, v4, 1 + @{[vslidedown_vi $V3, $V3, 1]} # vslidedown.vi v3, v3, 1 + @{[vslidedown_vi $V4, $V4, 1]} # vslidedown.vi v4, v4, 1 + + @{[vmv_v_i $V0, 1]} # vmv.v.i v0, 1 + # v2 += (a0b1)h + @{[vxor_vv_v0t $V2, $V2, $V3]} # vxor.vv v2, v2, v3, v0.t + # v2 += (a1b1)l + @{[vxor_vv_v0t $V2, $V2, $V4]} # vxor.vv v2, v2, v4, v0.t + + @{[vmv_v_i $V0, 2]} # vmv.v.i v0, 2 + # v1 += (a0b0)h,0 + @{[vxor_vv_v0t $V1, $V1, $V5]} # vxor.vv v1, v1, v5, v0.t + # v1 += (a1b0)l,0 + @{[vxor_vv_v0t $V1, $V1, $V6]} # vxor.vv v1, v1, v6, v0.t + + # Now the 256bit product should be stored in (v2,v1) + # v1 = (a0b1)l + (a0b0)h + (a1b0)l, (a0b0)l + # v2 = (a1b1)h, (a1b0)h + (a0b1)h + (a1b1)l + + # Reduction + # Let C := A*B = c3,c2,c1,c0 = v2[1],v2[0],v1[1],v1[0] + # This is a slight variation of the Gueron's Montgomery reduction. + # The difference being the order of some operations has been changed, + # to make a better use of vclmul(h) instructions. + + # First step: + # c1 += (c0 * P)l + # vmv.v.i v0, 2 + @{[vslideup_vi_v0t $V3, $V1, 1]} # vslideup.vi v3, v1, 1, v0.t + @{[vclmul_vx_v0t $V3, $V3, $TMP3]} # vclmul.vx v3, v3, t3, v0.t + @{[vxor_vv_v0t $V1, $V1, $V3]} # vxor.vv v1, v1, v3, v0.t + + # Second step: + # D = d1,d0 is final result + # We want: + # m1 = c1 + (c1 * P)h + # m0 = (c1 * P)l + (c0 * P)h + c0 + # d1 = c3 + m1 + # d0 = c2 + m0 + + #v3 = (c1 * P)l, 0 + @{[vclmul_vx_v0t $V3, $V1, $TMP3]} # vclmul.vx v3, v1, t3, v0.t + #v4 = (c1 * P)h, (c0 * P)h + @{[vclmulh_vx $V4, $V1, $TMP3]} # vclmulh.vx v4, v1, t3 + + @{[vmv_v_i $V0, 1]} # vmv.v.i v0, 1 + @{[vslidedown_vi $V3, $V3, 1]} # vslidedown.vi v3, v3, 1 + + @{[vxor_vv $V1, $V1, $V4]} # vxor.vv v1, v1, v4 + @{[vxor_vv_v0t $V1, $V1, $V3]} # vxor.vv v1, v1, v3, v0.t + + # XOR in the upper upper part of the product + @{[vxor_vv $V2, $V2, $V1]} # vxor.vv v2, v2, v1 + + @{[vrev8_v $V2, $V2]} # vrev8.v v2, v2 + @{[vsse64_v $V2, $Xi, $TMP4]} # vsse64.v v2, (a0), t4 + ret +.size gcm_gmult_rv64i_zvkb_zvbc,.-gcm_gmult_rv64i_zvkb_zvbc +___ +} + +################################################################################ +# void gcm_ghash_rv64i_zvkb_zvbc(u64 Xi[2], const u128 Htable[16], +# const u8 *inp, size_t len); +# +# input: Xi: current hash value +# Htable: preprocessed H +# inp: pointer to input data +# len: length of input data in bytes (multiple of block size) +# output: Xi: Xi+1 (next hash value Xi) +{ +my ($Xi,$Htable,$inp,$len,$TMP0,$TMP1,$TMP2,$TMP3,$M8,$TMP5,$TMP6) = ("a0","a1","a2","a3","t0","t1","t2","t3","t4","t5","t6"); +my ($V0,$V1,$V2,$V3,$V4,$V5,$V6,$Vinp) = ("v0","v1","v2","v3","v4","v5","v6","v7"); + +$code .= <<___; +.p2align 3 +.globl gcm_ghash_rv64i_zvkb_zvbc +.type gcm_ghash_rv64i_zvkb_zvbc,\@function +gcm_ghash_rv64i_zvkb_zvbc: + ld $TMP0, ($Htable) + ld $TMP1, 8($Htable) + li $TMP2, 63 + la $TMP3, Lpolymod + ld $TMP3, 8($TMP3) + + # Load/store data in reverse order. + # This is needed as a part of endianness swap. + add $Xi, $Xi, 8 + add $inp, $inp, 8 + li $M8, -8 + + @{[vsetivli__x0_2_e64_m1_tu_mu]} # vsetivli x0, 2, e64, m1, tu, mu + + @{[vlse64_v $V5, $Xi, $M8]} # vlse64.v v5, (a0), t4 + +Lstep: + # Read input data + @{[vlse64_v $Vinp, $inp, $M8]} # vle64.v v0, (a2) + add $inp, $inp, 16 + add $len, $len, -16 + # XOR them into Xi + @{[vxor_vv $V5, $V5, $Vinp]} # vxor.vv v0, v0, v1 + + @{[vrev8_v $V5, $V5]} # vrev8.v v5, v5 + + # Multiplication + + # Do two 64x64 multiplications in one go to save some time + # and simplify things. + + # A = a1a0 (t1, t0) + # B = b1b0 (v5) + # C = c1c0 (256 bit) + # c1 = a1b1 + (a0b1)h + (a1b0)h + # c0 = a0b0 + (a0b1)l + (a1b0)h + + # v1 = (a0b1)l,(a0b0)l + @{[vclmul_vx $V1, $V5, $TMP0]} # vclmul.vx v1, v5, t0 + # v3 = (a0b1)h,(a0b0)h + @{[vclmulh_vx $V3, $V5, $TMP0]} # vclmulh.vx v3, v5, t0 + + # v4 = (a1b1)l,(a1b0)l + @{[vclmul_vx $V4, $V5, $TMP1]} # vclmul.vx v4, v5, t1 + # v2 = (a1b1)h,(a1b0)h + @{[vclmulh_vx $V2, $V5, $TMP1]} # vclmulh.vx v2, v5, t1 + + # Is there a better way to do this? + # Would need to swap the order of elements within a vector register. + @{[vslideup_vi $V5, $V3, 1]} # vslideup.vi v5, v3, 1 + @{[vslideup_vi $V6, $V4, 1]} # vslideup.vi v6, v4, 1 + @{[vslidedown_vi $V3, $V3, 1]} # vslidedown.vi v3, v3, 1 + @{[vslidedown_vi $V4, $V4, 1]} # vslidedown.vi v4, v4, 1 + + @{[vmv_v_i $V0, 1]} # vmv.v.i v0, 1 + # v2 += (a0b1)h + @{[vxor_vv_v0t $V2, $V2, $V3]} # vxor.vv v2, v2, v3, v0.t + # v2 += (a1b1)l + @{[vxor_vv_v0t $V2, $V2, $V4]} # vxor.vv v2, v2, v4, v0.t + + @{[vmv_v_i $V0, 2]} # vmv.v.i v0, 2 + # v1 += (a0b0)h,0 + @{[vxor_vv_v0t $V1, $V1, $V5]} # vxor.vv v1, v1, v5, v0.t + # v1 += (a1b0)l,0 + @{[vxor_vv_v0t $V1, $V1, $V6]} # vxor.vv v1, v1, v6, v0.t + + # Now the 256bit product should be stored in (v2,v1) + # v1 = (a0b1)l + (a0b0)h + (a1b0)l, (a0b0)l + # v2 = (a1b1)h, (a1b0)h + (a0b1)h + (a1b1)l + + # Reduction + # Let C := A*B = c3,c2,c1,c0 = v2[1],v2[0],v1[1],v1[0] + # This is a slight variation of the Gueron's Montgomery reduction. + # The difference being the order of some operations has been changed, + # to make a better use of vclmul(h) instructions. + + # First step: + # c1 += (c0 * P)l + # vmv.v.i v0, 2 + @{[vslideup_vi_v0t $V3, $V1, 1]} # vslideup.vi v3, v1, 1, v0.t + @{[vclmul_vx_v0t $V3, $V3, $TMP3]} # vclmul.vx v3, v3, t3, v0.t + @{[vxor_vv_v0t $V1, $V1, $V3]} # vxor.vv v1, v1, v3, v0.t + + # Second step: + # D = d1,d0 is final result + # We want: + # m1 = c1 + (c1 * P)h + # m0 = (c1 * P)l + (c0 * P)h + c0 + # d1 = c3 + m1 + # d0 = c2 + m0 + + #v3 = (c1 * P)l, 0 + @{[vclmul_vx_v0t $V3, $V1, $TMP3]} # vclmul.vx v3, v1, t3, v0.t + #v4 = (c1 * P)h, (c0 * P)h + @{[vclmulh_vx $V4, $V1, $TMP3]} # vclmulh.vx v4, v1, t3 + + @{[vmv_v_i $V0, 1]} # vmv.v.i v0, 1 + @{[vslidedown_vi $V3, $V3, 1]} # vslidedown.vi v3, v3, 1 + + @{[vxor_vv $V1, $V1, $V4]} # vxor.vv v1, v1, v4 + @{[vxor_vv_v0t $V1, $V1, $V3]} # vxor.vv v1, v1, v3, v0.t + + # XOR in the upper upper part of the product + @{[vxor_vv $V2, $V2, $V1]} # vxor.vv v2, v2, v1 + + @{[vrev8_v $V5, $V2]} # vrev8.v v2, v2 + + bnez $len, Lstep + + @{[vsse64_v $V5, $Xi, $M8]} # vsse64.v v2, (a0), t4 + ret +.size gcm_ghash_rv64i_zvkb_zvbc,.-gcm_ghash_rv64i_zvkb_zvbc +___ +} + +$code .= <<___; +.p2align 4 +Lpolymod: + .dword 0x0000000000000001 + .dword 0xc200000000000000 +.size Lpolymod,.-Lpolymod +___ + +print $code; + +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/modes/asm/ghash-riscv64-zvkg.pl b/crypto/openssl/crypto/modes/asm/ghash-riscv64-zvkg.pl new file mode 100644 index 000000000000..8423ae9cf82f --- /dev/null +++ b/crypto/openssl/crypto/modes/asm/ghash-riscv64-zvkg.pl @@ -0,0 +1,169 @@ +#! /usr/bin/env perl +# This file is dual-licensed, meaning that you can use it under your +# choice of either of the following two licenses: +# +# Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You can obtain +# a copy in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html +# +# or +# +# Copyright (c) 2023, Christoph Müllner +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# - RV64I +# - RISC-V Vector ('V') with VLEN >= 128 +# - RISC-V Vector GCM/GMAC extension ('Zvkg') +# +# Optional: +# - RISC-V Vector Cryptography Bit-manipulation extension ('Zvkb') + +use strict; +use warnings; + +use FindBin qw($Bin); +use lib "$Bin"; +use lib "$Bin/../../perlasm"; +use riscv; + +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$output and open STDOUT,">$output"; + +my $code=<<___; +.text +___ + +################################################################################ +# void gcm_init_rv64i_zvkg(u128 Htable[16], const u64 H[2]); +# void gcm_init_rv64i_zvkg_zvkb(u128 Htable[16], const u64 H[2]); +# +# input: H: 128-bit H - secret parameter E(K, 0^128) +# output: Htable: Copy of secret parameter (in normalized byte order) +# +# All callers of this function revert the byte-order unconditionally +# on little-endian machines. So we need to revert the byte-order back. +{ +my ($Htable,$H,$VAL0,$VAL1,$TMP0) = ("a0","a1","a2","a3","t0"); + +$code .= <<___; +.p2align 3 +.globl gcm_init_rv64i_zvkg +.type gcm_init_rv64i_zvkg,\@function +gcm_init_rv64i_zvkg: + ld $VAL0, 0($H) + ld $VAL1, 8($H) + @{[sd_rev8_rv64i $VAL0, $Htable, 0, $TMP0]} + @{[sd_rev8_rv64i $VAL1, $Htable, 8, $TMP0]} + ret +.size gcm_init_rv64i_zvkg,.-gcm_init_rv64i_zvkg +___ +} + +{ +my ($Htable,$H,$V0) = ("a0","a1","v0"); + +$code .= <<___; +.p2align 3 +.globl gcm_init_rv64i_zvkg_zvkb +.type gcm_init_rv64i_zvkg_zvkb,\@function +gcm_init_rv64i_zvkg_zvkb: + @{[vsetivli__x0_2_e64_m1_tu_mu]} # vsetivli x0, 2, e64, m1, ta, ma + @{[vle64_v $V0, $H]} # vle64.v v0, (a1) + @{[vrev8_v $V0, $V0]} # vrev8.v v0, v0 + @{[vse64_v $V0, $Htable]} # vse64.v v0, (a0) + ret +.size gcm_init_rv64i_zvkg_zvkb,.-gcm_init_rv64i_zvkg_zvkb +___ +} + +################################################################################ +# void gcm_gmult_rv64i_zvkg(u64 Xi[2], const u128 Htable[16]); +# +# input: Xi: current hash value +# Htable: copy of H +# output: Xi: next hash value Xi +{ +my ($Xi,$Htable) = ("a0","a1"); +my ($VD,$VS2) = ("v1","v2"); + +$code .= <<___; +.p2align 3 +.globl gcm_gmult_rv64i_zvkg +.type gcm_gmult_rv64i_zvkg,\@function +gcm_gmult_rv64i_zvkg: + @{[vsetivli__x0_4_e32_m1_tu_mu]} + @{[vle32_v $VS2, $Htable]} + @{[vle32_v $VD, $Xi]} + @{[vgmul_vv $VD, $VS2]} + @{[vse32_v $VD, $Xi]} + ret +.size gcm_gmult_rv64i_zvkg,.-gcm_gmult_rv64i_zvkg +___ +} + +################################################################################ +# void gcm_ghash_rv64i_zvkg(u64 Xi[2], const u128 Htable[16], +# const u8 *inp, size_t len); +# +# input: Xi: current hash value +# Htable: copy of H +# inp: pointer to input data +# len: length of input data in bytes (multiple of block size) +# output: Xi: Xi+1 (next hash value Xi) +{ +my ($Xi,$Htable,$inp,$len) = ("a0","a1","a2","a3"); +my ($vXi,$vH,$vinp,$Vzero) = ("v1","v2","v3","v4"); + +$code .= <<___; +.p2align 3 +.globl gcm_ghash_rv64i_zvkg +.type gcm_ghash_rv64i_zvkg,\@function +gcm_ghash_rv64i_zvkg: + @{[vsetivli__x0_4_e32_m1_tu_mu]} + @{[vle32_v $vH, $Htable]} + @{[vle32_v $vXi, $Xi]} + +Lstep: + @{[vle32_v $vinp, $inp]} + add $inp, $inp, 16 + add $len, $len, -16 + @{[vghsh_vv $vXi, $vH, $vinp]} + bnez $len, Lstep + + @{[vse32_v $vXi, $Xi]} + ret + +.size gcm_ghash_rv64i_zvkg,.-gcm_ghash_rv64i_zvkg +___ +} + +print $code; + +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/modes/asm/ghash-riscv64.pl b/crypto/openssl/crypto/modes/asm/ghash-riscv64.pl new file mode 100644 index 000000000000..ee97ff0dd3f3 --- /dev/null +++ b/crypto/openssl/crypto/modes/asm/ghash-riscv64.pl @@ -0,0 +1,428 @@ +#! /usr/bin/env perl +# This file is dual-licensed, meaning that you can use it under your +# choice of either of the following two licenses: +# +# Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You can obtain +# a copy in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html +# +# or +# +# Copyright (c) 2023, Christoph Müllner +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +use strict; +use warnings; + +use FindBin qw($Bin); +use lib "$Bin"; +use lib "$Bin/../../perlasm"; +use riscv; + +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$output and open STDOUT,">$output"; + +my $code=<<___; +.text +___ + +################################################################################ +# void gcm_init_rv64i_zbc(u128 Htable[16], const u64 H[2]); +# void gcm_init_rv64i_zbc__zbb(u128 Htable[16], const u64 H[2]); +# void gcm_init_rv64i_zbc__zbkb(u128 Htable[16], const u64 H[2]); +# +# input: H: 128-bit H - secret parameter E(K, 0^128) +# output: Htable: Preprocessed key data for gcm_gmult_rv64i_zbc* and +# gcm_ghash_rv64i_zbc* +# +# All callers of this function revert the byte-order unconditionally +# on little-endian machines. So we need to revert the byte-order back. +# Additionally we reverse the bits of each byte. + +{ +my ($Htable,$H,$VAL0,$VAL1,$TMP0,$TMP1,$TMP2) = ("a0","a1","a2","a3","t0","t1","t2"); + +$code .= <<___; +.p2align 3 +.globl gcm_init_rv64i_zbc +.type gcm_init_rv64i_zbc,\@function +gcm_init_rv64i_zbc: + ld $VAL0,0($H) + ld $VAL1,8($H) + @{[brev8_rv64i $VAL0, $TMP0, $TMP1, $TMP2]} + @{[brev8_rv64i $VAL1, $TMP0, $TMP1, $TMP2]} + @{[sd_rev8_rv64i $VAL0, $Htable, 0, $TMP0]} + @{[sd_rev8_rv64i $VAL1, $Htable, 8, $TMP0]} + ret +.size gcm_init_rv64i_zbc,.-gcm_init_rv64i_zbc +___ +} + +{ +my ($Htable,$H,$VAL0,$VAL1,$TMP0,$TMP1,$TMP2) = ("a0","a1","a2","a3","t0","t1","t2"); + +$code .= <<___; +.p2align 3 +.globl gcm_init_rv64i_zbc__zbb +.type gcm_init_rv64i_zbc__zbb,\@function +gcm_init_rv64i_zbc__zbb: + ld $VAL0,0($H) + ld $VAL1,8($H) + @{[brev8_rv64i $VAL0, $TMP0, $TMP1, $TMP2]} + @{[brev8_rv64i $VAL1, $TMP0, $TMP1, $TMP2]} + @{[rev8 $VAL0, $VAL0]} + @{[rev8 $VAL1, $VAL1]} + sd $VAL0,0($Htable) + sd $VAL1,8($Htable) + ret +.size gcm_init_rv64i_zbc__zbb,.-gcm_init_rv64i_zbc__zbb +___ +} + +{ +my ($Htable,$H,$TMP0,$TMP1) = ("a0","a1","t0","t1"); + +$code .= <<___; +.p2align 3 +.globl gcm_init_rv64i_zbc__zbkb +.type gcm_init_rv64i_zbc__zbkb,\@function +gcm_init_rv64i_zbc__zbkb: + ld $TMP0,0($H) + ld $TMP1,8($H) + @{[brev8 $TMP0, $TMP0]} + @{[brev8 $TMP1, $TMP1]} + @{[rev8 $TMP0, $TMP0]} + @{[rev8 $TMP1, $TMP1]} + sd $TMP0,0($Htable) + sd $TMP1,8($Htable) + ret +.size gcm_init_rv64i_zbc__zbkb,.-gcm_init_rv64i_zbc__zbkb +___ +} + +################################################################################ +# void gcm_gmult_rv64i_zbc(u64 Xi[2], const u128 Htable[16]); +# void gcm_gmult_rv64i_zbc__zbkb(u64 Xi[2], const u128 Htable[16]); +# +# input: Xi: current hash value +# Htable: copy of H +# output: Xi: next hash value Xi +# +# Compute GMULT (Xi*H mod f) using the Zbc (clmul) and Zbb (basic bit manip) +# extensions. Using the no-Karatsuba approach and clmul for the final reduction. +# This results in an implementation with minimized number of instructions. +# HW with clmul latencies higher than 2 cycles might observe a performance +# improvement with Karatsuba. HW with clmul latencies higher than 6 cycles +# might observe a performance improvement with additionally converting the +# reduction to shift&xor. For a full discussion of this estimates see +# https://github.com/riscv/riscv-crypto/blob/master/doc/supp/gcm-mode-cmul.adoc +{ +my ($Xi,$Htable,$x0,$x1,$y0,$y1) = ("a0","a1","a4","a5","a6","a7"); +my ($z0,$z1,$z2,$z3,$t0,$t1,$polymod) = ("t0","t1","t2","t3","t4","t5","t6"); + +$code .= <<___; +.p2align 3 +.globl gcm_gmult_rv64i_zbc +.type gcm_gmult_rv64i_zbc,\@function +gcm_gmult_rv64i_zbc: + # Load Xi and bit-reverse it + ld $x0, 0($Xi) + ld $x1, 8($Xi) + @{[brev8_rv64i $x0, $z0, $z1, $z2]} + @{[brev8_rv64i $x1, $z0, $z1, $z2]} + + # Load the key (already bit-reversed) + ld $y0, 0($Htable) + ld $y1, 8($Htable) + + # Load the reduction constant + la $polymod, Lpolymod + lbu $polymod, 0($polymod) + + # Multiplication (without Karatsuba) + @{[clmulh $z3, $x1, $y1]} + @{[clmul $z2, $x1, $y1]} + @{[clmulh $t1, $x0, $y1]} + @{[clmul $z1, $x0, $y1]} + xor $z2, $z2, $t1 + @{[clmulh $t1, $x1, $y0]} + @{[clmul $t0, $x1, $y0]} + xor $z2, $z2, $t1 + xor $z1, $z1, $t0 + @{[clmulh $t1, $x0, $y0]} + @{[clmul $z0, $x0, $y0]} + xor $z1, $z1, $t1 + + # Reduction with clmul + @{[clmulh $t1, $z3, $polymod]} + @{[clmul $t0, $z3, $polymod]} + xor $z2, $z2, $t1 + xor $z1, $z1, $t0 + @{[clmulh $t1, $z2, $polymod]} + @{[clmul $t0, $z2, $polymod]} + xor $x1, $z1, $t1 + xor $x0, $z0, $t0 + + # Bit-reverse Xi back and store it + @{[brev8_rv64i $x0, $z0, $z1, $z2]} + @{[brev8_rv64i $x1, $z0, $z1, $z2]} + sd $x0, 0($Xi) + sd $x1, 8($Xi) + ret +.size gcm_gmult_rv64i_zbc,.-gcm_gmult_rv64i_zbc +___ +} + +{ +my ($Xi,$Htable,$x0,$x1,$y0,$y1) = ("a0","a1","a4","a5","a6","a7"); +my ($z0,$z1,$z2,$z3,$t0,$t1,$polymod) = ("t0","t1","t2","t3","t4","t5","t6"); + +$code .= <<___; +.p2align 3 +.globl gcm_gmult_rv64i_zbc__zbkb +.type gcm_gmult_rv64i_zbc__zbkb,\@function +gcm_gmult_rv64i_zbc__zbkb: + # Load Xi and bit-reverse it + ld $x0, 0($Xi) + ld $x1, 8($Xi) + @{[brev8 $x0, $x0]} + @{[brev8 $x1, $x1]} + + # Load the key (already bit-reversed) + ld $y0, 0($Htable) + ld $y1, 8($Htable) + + # Load the reduction constant + la $polymod, Lpolymod + lbu $polymod, 0($polymod) + + # Multiplication (without Karatsuba) + @{[clmulh $z3, $x1, $y1]} + @{[clmul $z2, $x1, $y1]} + @{[clmulh $t1, $x0, $y1]} + @{[clmul $z1, $x0, $y1]} + xor $z2, $z2, $t1 + @{[clmulh $t1, $x1, $y0]} + @{[clmul $t0, $x1, $y0]} + xor $z2, $z2, $t1 + xor $z1, $z1, $t0 + @{[clmulh $t1, $x0, $y0]} + @{[clmul $z0, $x0, $y0]} + xor $z1, $z1, $t1 + + # Reduction with clmul + @{[clmulh $t1, $z3, $polymod]} + @{[clmul $t0, $z3, $polymod]} + xor $z2, $z2, $t1 + xor $z1, $z1, $t0 + @{[clmulh $t1, $z2, $polymod]} + @{[clmul $t0, $z2, $polymod]} + xor $x1, $z1, $t1 + xor $x0, $z0, $t0 + + # Bit-reverse Xi back and store it + @{[brev8 $x0, $x0]} + @{[brev8 $x1, $x1]} + sd $x0, 0($Xi) + sd $x1, 8($Xi) + ret +.size gcm_gmult_rv64i_zbc__zbkb,.-gcm_gmult_rv64i_zbc__zbkb +___ +} + +################################################################################ +# void gcm_ghash_rv64i_zbc(u64 Xi[2], const u128 Htable[16], +# const u8 *inp, size_t len); +# void gcm_ghash_rv64i_zbc__zbkb(u64 Xi[2], const u128 Htable[16], +# const u8 *inp, size_t len); +# +# input: Xi: current hash value +# Htable: copy of H +# inp: pointer to input data +# len: length of input data in bytes (multiple of block size) +# output: Xi: Xi+1 (next hash value Xi) +{ +my ($Xi,$Htable,$inp,$len,$x0,$x1,$y0,$y1) = ("a0","a1","a2","a3","a4","a5","a6","a7"); +my ($z0,$z1,$z2,$z3,$t0,$t1,$polymod) = ("t0","t1","t2","t3","t4","t5","t6"); + +$code .= <<___; +.p2align 3 +.globl gcm_ghash_rv64i_zbc +.type gcm_ghash_rv64i_zbc,\@function +gcm_ghash_rv64i_zbc: + # Load Xi and bit-reverse it + ld $x0, 0($Xi) + ld $x1, 8($Xi) + @{[brev8_rv64i $x0, $z0, $z1, $z2]} + @{[brev8_rv64i $x1, $z0, $z1, $z2]} + + # Load the key (already bit-reversed) + ld $y0, 0($Htable) + ld $y1, 8($Htable) + + # Load the reduction constant + la $polymod, Lpolymod + lbu $polymod, 0($polymod) + +Lstep: + # Load the input data, bit-reverse them, and XOR them with Xi + ld $t0, 0($inp) + ld $t1, 8($inp) + add $inp, $inp, 16 + add $len, $len, -16 + @{[brev8_rv64i $t0, $z0, $z1, $z2]} + @{[brev8_rv64i $t1, $z0, $z1, $z2]} + xor $x0, $x0, $t0 + xor $x1, $x1, $t1 + + # Multiplication (without Karatsuba) + @{[clmulh $z3, $x1, $y1]} + @{[clmul $z2, $x1, $y1]} + @{[clmulh $t1, $x0, $y1]} + @{[clmul $z1, $x0, $y1]} + xor $z2, $z2, $t1 + @{[clmulh $t1, $x1, $y0]} + @{[clmul $t0, $x1, $y0]} + xor $z2, $z2, $t1 + xor $z1, $z1, $t0 + @{[clmulh $t1, $x0, $y0]} + @{[clmul $z0, $x0, $y0]} + xor $z1, $z1, $t1 + + # Reduction with clmul + @{[clmulh $t1, $z3, $polymod]} + @{[clmul $t0, $z3, $polymod]} + xor $z2, $z2, $t1 + xor $z1, $z1, $t0 + @{[clmulh $t1, $z2, $polymod]} + @{[clmul $t0, $z2, $polymod]} + xor $x1, $z1, $t1 + xor $x0, $z0, $t0 + + # Iterate over all blocks + bnez $len, Lstep + + # Bit-reverse final Xi back and store it + @{[brev8_rv64i $x0, $z0, $z1, $z2]} + @{[brev8_rv64i $x1, $z0, $z1, $z2]} + sd $x0, 0($Xi) + sd $x1, 8($Xi) + ret +.size gcm_ghash_rv64i_zbc,.-gcm_ghash_rv64i_zbc +___ +} + +{ +my ($Xi,$Htable,$inp,$len,$x0,$x1,$y0,$y1) = ("a0","a1","a2","a3","a4","a5","a6","a7"); +my ($z0,$z1,$z2,$z3,$t0,$t1,$polymod) = ("t0","t1","t2","t3","t4","t5","t6"); + +$code .= <<___; +.p2align 3 +.globl gcm_ghash_rv64i_zbc__zbkb +.type gcm_ghash_rv64i_zbc__zbkb,\@function +gcm_ghash_rv64i_zbc__zbkb: + # Load Xi and bit-reverse it + ld $x0, 0($Xi) + ld $x1, 8($Xi) + @{[brev8 $x0, $x0]} + @{[brev8 $x1, $x1]} + + # Load the key (already bit-reversed) + ld $y0, 0($Htable) + ld $y1, 8($Htable) + + # Load the reduction constant + la $polymod, Lpolymod + lbu $polymod, 0($polymod) + +Lstep_zkbk: + # Load the input data, bit-reverse them, and XOR them with Xi + ld $t0, 0($inp) + ld $t1, 8($inp) + add $inp, $inp, 16 + add $len, $len, -16 + @{[brev8 $t0, $t0]} + @{[brev8 $t1, $t1]} + xor $x0, $x0, $t0 + xor $x1, $x1, $t1 + + # Multiplication (without Karatsuba) + @{[clmulh $z3, $x1, $y1]} + @{[clmul $z2, $x1, $y1]} + @{[clmulh $t1, $x0, $y1]} + @{[clmul $z1, $x0, $y1]} + xor $z2, $z2, $t1 + @{[clmulh $t1, $x1, $y0]} + @{[clmul $t0, $x1, $y0]} + xor $z2, $z2, $t1 + xor $z1, $z1, $t0 + @{[clmulh $t1, $x0, $y0]} + @{[clmul $z0, $x0, $y0]} + xor $z1, $z1, $t1 + + # Reduction with clmul + @{[clmulh $t1, $z3, $polymod]} + @{[clmul $t0, $z3, $polymod]} + xor $z2, $z2, $t1 + xor $z1, $z1, $t0 + @{[clmulh $t1, $z2, $polymod]} + @{[clmul $t0, $z2, $polymod]} + xor $x1, $z1, $t1 + xor $x0, $z0, $t0 + + # Iterate over all blocks + bnez $len, Lstep_zkbk + + # Bit-reverse final Xi back and store it + @{[brev8 $x0, $x0]} + @{[brev8 $x1, $x1]} + sd $x0, 0($Xi) + sd $x1, 8($Xi) + ret +.size gcm_ghash_rv64i_zbc__zbkb,.-gcm_ghash_rv64i_zbc__zbkb +___ +} + +$code .= <<___; +.p2align 3 +Lbrev8_const: + .dword 0xAAAAAAAAAAAAAAAA + .dword 0xCCCCCCCCCCCCCCCC + .dword 0xF0F0F0F0F0F0F0F0 +.size Lbrev8_const,.-Lbrev8_const + +Lpolymod: + .byte 0x87 +.size Lpolymod,.-Lpolymod +___ + +print $code; + +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/modes/asm/ghash-s390x.pl b/crypto/openssl/crypto/modes/asm/ghash-s390x.pl index ba9c5b4a4f4f..0ae6adbf6146 100755 --- a/crypto/openssl/crypto/modes/asm/ghash-s390x.pl +++ b/crypto/openssl/crypto/modes/asm/ghash-s390x.pl @@ -90,25 +90,6 @@ $code.=<<___; .align 32 gcm_gmult_4bit: ___ -$code.=<<___ if(!$softonly && 0); # hardware is slow for single block... - larl %r1,OPENSSL_s390xcap_P - lghi %r0,0 - lg %r1,S390X_KIMD+8(%r1) # load second word of kimd capabilities - # vector - tmhh %r1,0x4000 # check for function 65 - jz .Lsoft_gmult - stg %r0,16($sp) # arrange 16 bytes of zero input - stg %r0,24($sp) - lghi %r0,S390X_GHASH # function 65 - la %r1,0($Xi) # H lies right after Xi in gcm128_context - la $inp,16($sp) - lghi $len,16 - .long 0xb93e0004 # kimd %r0,$inp - brc 1,.-4 # pay attention to "partial completion" - br %r14 -.align 32 -.Lsoft_gmult: -___ $code.=<<___; stm${g} %r6,%r14,6*$SIZE_T($sp) @@ -132,10 +113,21 @@ $code.=<<___ if(!$softonly); # vector tmhh %r0,0x4000 # check for function 65 jz .Lsoft_ghash + # Do not assume this function is called from a gcm128_context. + # This is not true, e.g., for AES-GCM-SIV. + # Parameter Block: + # Chaining Value (XI) 128byte + # Key (Htable[8]) 128byte + lmg %r0,%r1,0($Xi) + stmg %r0,%r1,8($sp) + lmg %r0,%r1,8*16($Htbl) + stmg %r0,%r1,24($sp) + la %r1,8($sp) lghi %r0,S390X_GHASH # function 65 - la %r1,0($Xi) # H lies right after Xi in gcm128_context .long 0xb93e0004 # kimd %r0,$inp brc 1,.-4 # pay attention to "partial completion" + lmg %r0,%r1,8($sp) + stmg %r0,%r1,0($Xi) br %r14 .align 32 .Lsoft_ghash: diff --git a/crypto/openssl/crypto/modes/asm/ghash-x86_64.pl b/crypto/openssl/crypto/modes/asm/ghash-x86_64.pl index 6709f96492ed..6ef8e555d0a2 100755 --- a/crypto/openssl/crypto/modes/asm/ghash-x86_64.pl +++ b/crypto/openssl/crypto/modes/asm/ghash-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2010-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2010-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -534,6 +534,7 @@ $code.=<<___; .align 16 gcm_init_clmul: .cfi_startproc + endbranch .L_init_clmul: ___ $code.=<<___ if ($win64); @@ -1027,6 +1028,7 @@ $code.=<<___; .align 32 gcm_init_avx: .cfi_startproc + endbranch ___ if ($avx) { my ($Htbl,$Xip)=@_4args; @@ -1609,6 +1611,7 @@ ___ } $code.=<<___; +.section .rodata align=64 .align 64 .Lbswap_mask: .byte 15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0 @@ -1662,6 +1665,7 @@ $code.=<<___; .asciz "GHASH for x86_64, CRYPTOGAMS by " .align 64 +.previous ___ # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, diff --git a/crypto/openssl/crypto/modes/asm/ghashv8-armx.pl b/crypto/openssl/crypto/modes/asm/ghashv8-armx.pl index b4862481222e..ac3ec558ef0f 100644 --- a/crypto/openssl/crypto/modes/asm/ghashv8-armx.pl +++ b/crypto/openssl/crypto/modes/asm/ghashv8-armx.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2014-2023 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2014-2025 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -161,6 +161,7 @@ $code.=<<___; ___ if ($flavour =~ /64/) { my ($t3,$Yl,$Ym,$Yh) = map("q$_",(4..7)); +my ($H3,$H34k,$H4,$H5,$H56k,$H6,$H7,$H78k,$H8) = map("q$_",(15..23)); $code.=<<___; @ calculate H^3 and H^4 @@ -195,15 +196,103 @@ $code.=<<___; vpmull.p64 $Yl,$Yl,$xC2 veor $t2,$t2,$Xh veor $t3,$t3,$Yh - veor $H, $Xl,$t2 @ H^3 - veor $H2,$Yl,$t3 @ H^4 - - vext.8 $t0,$H, $H,#8 @ Karatsuba pre-processing - vext.8 $t1,$H2,$H2,#8 - veor $t0,$t0,$H - veor $t1,$t1,$H2 - vext.8 $Hhl,$t0,$t1,#8 @ pack Karatsuba pre-processed - vst1.64 {$H-$H2},[PTR(0)] @ store Htable[3..5] + veor $H3, $Xl,$t2 @ H^3 + veor $H4,$Yl,$t3 @ H^4 + + vext.8 $t0,$H3, $H3,#8 @ Karatsuba pre-processing + vext.8 $t1,$H4,$H4,#8 + vext.8 $t2,$H2,$H2,#8 + veor $t0,$t0,$H3 + veor $t1,$t1,$H4 + veor $t2,$t2,$H2 + vext.8 $H34k,$t0,$t1,#8 @ pack Karatsuba pre-processed + vst1.64 {$H3-$H4},[PTR(0)],#48 @ store Htable[3..5] + + @ calculate H^5 and H^6 + vpmull.p64 $Xl,$H2, $H3 + vpmull.p64 $Yl,$H3,$H3 + vpmull2.p64 $Xh,$H2, $H3 + vpmull2.p64 $Yh,$H3,$H3 + vpmull.p64 $Xm,$t0,$t2 + vpmull.p64 $Ym,$t0,$t0 + + vext.8 $t0,$Xl,$Xh,#8 @ Karatsuba post-processing + vext.8 $t1,$Yl,$Yh,#8 + veor $t2,$Xl,$Xh + veor $Xm,$Xm,$t0 + veor $t3,$Yl,$Yh + veor $Ym,$Ym,$t1 + veor $Xm,$Xm,$t2 + vpmull.p64 $t2,$Xl,$xC2 @ 1st phase + veor $Ym,$Ym,$t3 + vpmull.p64 $t3,$Yl,$xC2 + + vmov $Xh#lo,$Xm#hi @ Xh|Xm - 256-bit result + vmov $Yh#lo,$Ym#hi + vmov $Xm#hi,$Xl#lo @ Xm is rotated Xl + vmov $Ym#hi,$Yl#lo + veor $Xl,$Xm,$t2 + veor $Yl,$Ym,$t3 + + vext.8 $t2,$Xl,$Xl,#8 @ 2nd phase + vext.8 $t3,$Yl,$Yl,#8 + vpmull.p64 $Xl,$Xl,$xC2 + vpmull.p64 $Yl,$Yl,$xC2 + veor $t2,$t2,$Xh + veor $t3,$t3,$Yh + veor $H5,$Xl,$t2 @ H^5 + veor $H6,$Yl,$t3 @ H^6 + + vext.8 $t0,$H5, $H5,#8 @ Karatsuba pre-processing + vext.8 $t1,$H6,$H6,#8 + vext.8 $t2,$H2,$H2,#8 + veor $t0,$t0,$H5 + veor $t1,$t1,$H6 + veor $t2,$t2,$H2 + vext.8 $H56k,$t0,$t1,#8 @ pack Karatsuba pre-processed + vst1.64 {$H5-$H6},[PTR(0)],#48 @ store Htable[6..8] + + @ calculate H^7 and H^8 + vpmull.p64 $Xl,$H2,$H5 + vpmull.p64 $Yl,$H2,$H6 + vpmull2.p64 $Xh,$H2,$H5 + vpmull2.p64 $Yh,$H2,$H6 + vpmull.p64 $Xm,$t0,$t2 + vpmull.p64 $Ym,$t1,$t2 + + vext.8 $t0,$Xl,$Xh,#8 @ Karatsuba post-processing + vext.8 $t1,$Yl,$Yh,#8 + veor $t2,$Xl,$Xh + veor $Xm,$Xm,$t0 + veor $t3,$Yl,$Yh + veor $Ym,$Ym,$t1 + veor $Xm,$Xm,$t2 + vpmull.p64 $t2,$Xl,$xC2 @ 1st phase + veor $Ym,$Ym,$t3 + vpmull.p64 $t3,$Yl,$xC2 + + vmov $Xh#lo,$Xm#hi @ Xh|Xm - 256-bit result + vmov $Yh#lo,$Ym#hi + vmov $Xm#hi,$Xl#lo @ Xm is rotated Xl + vmov $Ym#hi,$Yl#lo + veor $Xl,$Xm,$t2 + veor $Yl,$Ym,$t3 + + vext.8 $t2,$Xl,$Xl,#8 @ 2nd phase + vext.8 $t3,$Yl,$Yl,#8 + vpmull.p64 $Xl,$Xl,$xC2 + vpmull.p64 $Yl,$Yl,$xC2 + veor $t2,$t2,$Xh + veor $t3,$t3,$Yh + veor $H7,$Xl,$t2 @ H^7 + veor $H8,$Yl,$t3 @ H^8 + + vext.8 $t0,$H7,$H7,#8 @ Karatsuba pre-processing + vext.8 $t1,$H8,$H8,#8 + veor $t0,$t0,$H7 + veor $t1,$t1,$H8 + vext.8 $H78k,$t0,$t1,#8 @ pack Karatsuba pre-processed + vst1.64 {$H7-$H8},[PTR(0)] @ store Htable[9..11] ___ } $code.=<<___; @@ -724,6 +813,7 @@ ___ } $code.=<<___; +.rodata .asciz "GHASH for ARMv8, CRYPTOGAMS by " .align 2 #endif diff --git a/crypto/openssl/crypto/poly1305/asm/poly1305-armv4.pl b/crypto/openssl/crypto/poly1305/asm/poly1305-armv4.pl index 041bfd46e699..673ea62ec388 100755 --- a/crypto/openssl/crypto/poly1305/asm/poly1305-armv4.pl +++ b/crypto/openssl/crypto/poly1305/asm/poly1305-armv4.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2016-2023 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -1239,7 +1239,8 @@ $code.=<<___; .asciz "Poly1305 for ARMv4/NEON, CRYPTOGAMS by " .align 2 #if __ARM_MAX_ARCH__>=7 -.comm OPENSSL_armcap_P,4,4 +.extern OPENSSL_armcap_P +.hidden OPENSSL_armcap_P #endif ___ diff --git a/crypto/openssl/crypto/poly1305/asm/poly1305-armv8.pl b/crypto/openssl/crypto/poly1305/asm/poly1305-armv8.pl index 06434a9b60a5..749a5688fb6b 100755 --- a/crypto/openssl/crypto/poly1305/asm/poly1305-armv8.pl +++ b/crypto/openssl/crypto/poly1305/asm/poly1305-armv8.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2016-2023 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2016-2025 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -105,10 +105,14 @@ poly1305_init: tst w17,#ARMV7_NEON - adr $d0p,.Lpoly1305_blocks - adr $r0p,.Lpoly1305_blocks_neon - adr $d1p,.Lpoly1305_emit - adr $r1p,.Lpoly1305_emit_neon + adrp $d0p,poly1305_blocks + add $d0p,$d0p,#:lo12:.Lpoly1305_blocks + adrp $r0p,poly1305_blocks_neon + add $r0p,$r0p,#:lo12:.Lpoly1305_blocks_neon + adrp $d1p,poly1305_emit + add $d1p,$d1p,#:lo12:.Lpoly1305_emit + adrp $r1p,poly1305_emit_neon + add $r1p,$r1p,#:lo12:.Lpoly1305_emit_neon csel $d0p,$d0p,$r0p,eq csel $d1p,$d1p,$r1p,eq @@ -453,7 +457,8 @@ poly1305_blocks_neon: ldr PTR(30),[PTRN(sp),#PTR_WIDTH] add $in2,$inp,#32 - adr $zeros,.Lzeros + adrp $zeros,.Lzeros + add $zeros,$zeros,:lo12:.Lzeros subs $len,$len,#64 csel $in2,$zeros,$in2,lo @@ -465,7 +470,8 @@ poly1305_blocks_neon: .align 4 .Leven_neon: add $in2,$inp,#32 - adr $zeros,.Lzeros + adrp $zeros,.Lzeros + add $zeros,$zeros,:lo12:.Lzeros subs $len,$len,#64 csel $in2,$zeros,$in2,lo @@ -949,6 +955,8 @@ poly1305_emit_neon: ret .size poly1305_emit_neon,.-poly1305_emit_neon +.rodata + .align 5 .Lzeros: .long 0,0,0,0,0,0,0,0 diff --git a/crypto/openssl/crypto/poly1305/asm/poly1305-x86_64.pl b/crypto/openssl/crypto/poly1305/asm/poly1305-x86_64.pl index 4cddca1c514c..305099ca0308 100755 --- a/crypto/openssl/crypto/poly1305/asm/poly1305-x86_64.pl +++ b/crypto/openssl/crypto/poly1305/asm/poly1305-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2016-2023 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2016-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -229,6 +229,7 @@ $code.=<<___; .align 32 poly1305_blocks: .cfi_startproc + endbranch .Lblocks: shr \$4,$len jz .Lno_data # too short @@ -303,6 +304,7 @@ $code.=<<___; .align 32 poly1305_emit: .cfi_startproc + endbranch .Lemit: mov 0($ctx),%r8 # load hash value mov 8($ctx),%r9 @@ -524,6 +526,7 @@ __poly1305_init_avx: .align 32 poly1305_blocks_avx: .cfi_startproc + endbranch mov 20($ctx),%r8d # is_base2_26 cmp \$128,$len jae .Lblocks_avx @@ -1384,6 +1387,7 @@ $code.=<<___; .align 32 poly1305_emit_avx: .cfi_startproc + endbranch cmpl \$0,20($ctx) # is_base2_26? je .Lemit @@ -1448,6 +1452,7 @@ $code.=<<___; .align 32 poly1305_blocks_avx2: .cfi_startproc + endbranch mov 20($ctx),%r8d # is_base2_26 cmp \$128,$len jae .Lblocks_avx2 @@ -2144,6 +2149,7 @@ $code.=<<___; .align 32 poly1305_blocks_avx512: .cfi_startproc + endbranch .Lblocks_avx512: mov \$15,%eax kmovw %eax,%k2 @@ -3778,6 +3784,7 @@ poly1305_emit_base2_44: ___ } } } $code.=<<___; +.section .rodata align=64 .align 64 .Lconst: .Lmask24: @@ -3809,6 +3816,7 @@ $code.=<<___; .Lx_mask42: .quad 0x3ffffffffff,0x3ffffffffff,0x3ffffffffff,0x3ffffffffff .quad 0x3ffffffffff,0x3ffffffffff,0x3ffffffffff,0x3ffffffffff +.previous ___ } $code.=<<___; diff --git a/crypto/openssl/crypto/rc4/asm/rc4-x86_64.pl b/crypto/openssl/crypto/rc4/asm/rc4-x86_64.pl index 5a903f5739e1..83a1d13635fd 100755 --- a/crypto/openssl/crypto/rc4/asm/rc4-x86_64.pl +++ b/crypto/openssl/crypto/rc4/asm/rc4-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2005-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2005-2023 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -23,7 +23,7 @@ # Presumably it has everything to do with AMD cache architecture and # RAW or whatever penalties. Once again! The module *requires* config # line *without* RC4_CHAR! As for coding "secret," I bet on partial -# register arithmetics. For example instead of 'inc %r8; and $255,%r8' +# register arithmetic. For example instead of 'inc %r8; and $255,%r8' # I simply 'inc %r8b'. Even though optimization manual discourages # to operate on partial registers, it turned out to be the best bet. # At least for AMD... How IA32E would perform remains to be seen... diff --git a/crypto/openssl/crypto/sha/asm/keccak1600-armv4.pl b/crypto/openssl/crypto/sha/asm/keccak1600-armv4.pl index eaad86d39ddf..f9a8cd1f0046 100755 --- a/crypto/openssl/crypto/sha/asm/keccak1600-armv4.pl +++ b/crypto/openssl/crypto/sha/asm/keccak1600-armv4.pl @@ -932,7 +932,15 @@ SHA3_absorb: .size SHA3_absorb,.-SHA3_absorb ___ } -{ my ($out,$len,$A_flat,$bsz) = map("r$_", (4,5,10,12)); + +{ my ($out,$len,$A_flat,$bsz,$next) = map("r$_", (4,5,10,12,0)); + + +# void SHA3_squeeze(uint64_t A[5][5], +# unsigned char *out, size_t len, size_t r, int next) +# +# The first 4 parameters are passed in via r0..r3, +# next is passed on the stack [sp, #0] $code.=<<___; .global SHA3_squeeze @@ -945,6 +953,7 @@ SHA3_squeeze: mov $out,r1 mov $len,r2 mov $bsz,r3 + ldr $next, [sp, #40] @ next is after the 10 pushed registers (10*4) #ifdef __thumb2__ mov r9,#0x00ff00ff @@ -966,6 +975,8 @@ SHA3_squeeze: stmdb sp!,{r6-r9} mov r14,$A_flat + cmp $next, #1 + beq .Lnext_block b .Loop_squeeze .align 4 @@ -1037,7 +1048,7 @@ SHA3_squeeze: subs $bsz,$bsz,#8 @ bsz -= 8 bhi .Loop_squeeze - +.Lnext_block: mov r0,r14 @ original $A_flat bl KeccakF1600 diff --git a/crypto/openssl/crypto/sha/asm/keccak1600-armv8.pl b/crypto/openssl/crypto/sha/asm/keccak1600-armv8.pl index 1d6623fa5494..c28a2c554f35 100755 --- a/crypto/openssl/crypto/sha/asm/keccak1600-armv8.pl +++ b/crypto/openssl/crypto/sha/asm/keccak1600-armv8.pl @@ -1,5 +1,5 @@ #!/usr/bin/env perl -# Copyright 2017-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2017-2025 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -82,7 +82,7 @@ my @rhotates = ([ 0, 1, 62, 28, 27 ], $code.=<<___; #include "arm_arch.h" -.text +.rodata .align 8 // strategic alignment and padding that allows to use // address value as loop termination condition... @@ -124,13 +124,16 @@ my @C = map("x$_", (26,27,28,30)); my @P = map("PTR($_)", (26,27,28,30)); $code.=<<___; +.text + #define KeccakF1600_int_FRAMESZ (16 + 2*PTR_WIDTH) .type KeccakF1600_int,%function .align 5 KeccakF1600_int: AARCH64_SIGN_LINK_REGISTER - adr @P[2],iotas + adrp @P[2],iotas + add @P[2],@P[2],:lo12:iotas stp @P[2],PTR(30),[PTRN(sp),#16] // 32/48 bytes on top are mine b .Loop .align 4 @@ -487,6 +490,8 @@ SHA3_squeeze: mov $out,PTR(1) mov $len,x2 mov $bsz,x3 + cmp w4, #0 // w4 = 'next' argument + bne .Lnext_block .Loop_squeeze: ldr x4,[PTR(0)],#8 @@ -501,7 +506,7 @@ SHA3_squeeze: subs x3,x3,#8 bhi .Loop_squeeze - +.Lnext_block: mov PTR(0),$A_flat bl KeccakF1600 mov PTR(0),$A_flat @@ -558,7 +563,8 @@ $code.=<<___; .align 5 KeccakF1600_ce: mov x9,#24 - adr PTR(10),iotas + adrp PTR(10),iotas + add PTR(10),PTR(10),:lo12:iotas b .Loop_ce .align 4 .Loop_ce: diff --git a/crypto/openssl/crypto/sha/asm/keccak1600-ppc64.pl b/crypto/openssl/crypto/sha/asm/keccak1600-ppc64.pl index 83f8d8ef337b..54f32e8c9291 100755 --- a/crypto/openssl/crypto/sha/asm/keccak1600-ppc64.pl +++ b/crypto/openssl/crypto/sha/asm/keccak1600-ppc64.pl @@ -51,6 +51,16 @@ if ($flavour =~ /64/) { $PUSH ="std"; } else { die "nonsense $flavour"; } +$LITTLE_ENDIAN = ($flavour=~/le$/) ? 1 : 0; + +if ($LITTLE_ENDIAN) { + $DWORD_LE_LOAD = "ldu r0,8(r3)"; + $LE_LOAD_SIZE = "8"; +} else { + $DWORD_LE_LOAD = "bl dword_le_load"; + $LE_LOAD_SIZE = "1"; +} + $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or @@ -384,7 +394,9 @@ KeccakF1600: .byte 0,12,4,1,0x80,18,1,0 .long 0 .size KeccakF1600,.-KeccakF1600 - +___ +if (!$LITTLE_ENDIAN) { +$code.=<<___; .type dword_le_load,\@function .align 5 dword_le_load: @@ -408,7 +420,10 @@ dword_le_load: .byte 0,12,0x14,0,0,0,1,0 .long 0 .size dword_le_load,.-dword_le_load +___ +} +$code.=<<___; .globl SHA3_absorb .type SHA3_absorb,\@function .align 5 @@ -436,7 +451,7 @@ SHA3_absorb: $PUSH r0,`$FRAME+$LRSAVE`($sp) bl PICmeup - subi r4,r4,1 ; prepare for lbzu + subi r4,r4,$LE_LOAD_SIZE ; prepare for ldu or lbzu subi r12,r12,8 ; prepare for ldu $PUSH r3,`$LOCALS+0*$SIZE_T`($sp) ; save A[][] @@ -487,79 +502,79 @@ SHA3_absorb: srwi r5,r5,3 $PUSH r4,`$LOCALS+2*$SIZE_T`($sp) ; save len mtctr r5 - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[0][0],$A[0][0],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[0][1],$A[0][1],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[0][2],$A[0][2],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[0][3],$A[0][3],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[0][4],$A[0][4],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[1][0],$A[1][0],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[1][1],$A[1][1],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[1][2],$A[1][2],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[1][3],$A[1][3],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[1][4],$A[1][4],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[2][0],$A[2][0],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[2][1],$A[2][1],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[2][2],$A[2][2],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[2][3],$A[2][3],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[2][4],$A[2][4],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[3][0],$A[3][0],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[3][1],$A[3][1],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[3][2],$A[3][2],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[3][3],$A[3][3],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[3][4],$A[3][4],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[4][0],$A[4][0],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[4][1],$A[4][1],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[4][2],$A[4][2],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[4][3],$A[4][3],r0 bdz .Lprocess_block - bl dword_le_load ; *inp++ + $DWORD_LE_LOAD ; *inp++ xor $A[4][4],$A[4][4],r0 .Lprocess_block: @@ -653,6 +668,8 @@ SHA3_squeeze: subi $out,r4,1 ; prepare for stbu mr $len,r5 mr $bsz,r6 + cmplwi r7,0 ; r7 = 'next' argument + bne .Lnext_block b .Loop_squeeze .align 4 @@ -683,6 +700,7 @@ SHA3_squeeze: subic. r6,r6,8 bgt .Loop_squeeze +.Lnext_block: mr r3,$A_flat bl KeccakF1600 subi r3,$A_flat,8 ; prepare for ldu diff --git a/crypto/openssl/crypto/sha/asm/keccak1600-s390x.pl b/crypto/openssl/crypto/sha/asm/keccak1600-s390x.pl index 86233c7e38f2..7d5ebde11719 100755 --- a/crypto/openssl/crypto/sha/asm/keccak1600-s390x.pl +++ b/crypto/openssl/crypto/sha/asm/keccak1600-s390x.pl @@ -472,7 +472,7 @@ SHA3_absorb: .size SHA3_absorb,.-SHA3_absorb ___ } -{ my ($A_flat,$out,$len,$bsz) = map("%r$_",(2..5)); +{ my ($A_flat,$out,$len,$bsz,$next) = map("%r$_",(2..6)); $code.=<<___; .globl SHA3_squeeze @@ -484,6 +484,7 @@ SHA3_squeeze: lghi %r14,8 st${g} $bsz,5*$SIZE_T($sp) la %r1,0($A_flat) + cijne $next,0,.Lnext_block j .Loop_squeeze @@ -501,6 +502,7 @@ SHA3_squeeze: brct $bsz,.Loop_squeeze # bsz-- +.Lnext_block: stm${g} $out,$len,3*$SIZE_T($sp) bras %r14,.LKeccakF1600 lm${g} $out,$bsz,3*$SIZE_T($sp) diff --git a/crypto/openssl/crypto/sha/asm/keccak1600-x86_64.pl b/crypto/openssl/crypto/sha/asm/keccak1600-x86_64.pl index 02f0116014d6..3dbf17731e4d 100755 --- a/crypto/openssl/crypto/sha/asm/keccak1600-x86_64.pl +++ b/crypto/openssl/crypto/sha/asm/keccak1600-x86_64.pl @@ -1,5 +1,5 @@ #!/usr/bin/env perl -# Copyright 2017-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2017-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -503,12 +503,12 @@ SHA3_absorb: .size SHA3_absorb,.-SHA3_absorb ___ } -{ my ($A_flat,$out,$len,$bsz) = ("%rdi","%rsi","%rdx","%rcx"); +{ my ($A_flat,$out,$len,$bsz,$next) = ("%rdi","%rsi","%rdx","%rcx","%r8"); ($out,$len,$bsz) = ("%r12","%r13","%r14"); $code.=<<___; .globl SHA3_squeeze -.type SHA3_squeeze,\@function,4 +.type SHA3_squeeze,\@function,5 .align 32 SHA3_squeeze: .cfi_startproc @@ -520,10 +520,12 @@ SHA3_squeeze: .cfi_push %r14 shr \$3,%rcx - mov $A_flat,%r8 + mov $A_flat,%r9 mov %rsi,$out mov %rdx,$len mov %rcx,$bsz + bt \$0,${next}d + jc .Lnext_block jmp .Loop_squeeze .align 32 @@ -531,8 +533,8 @@ SHA3_squeeze: cmp \$8,$len jb .Ltail_squeeze - mov (%r8),%rax - lea 8(%r8),%r8 + mov (%r9),%rax + lea 8(%r9),%r9 mov %rax,($out) lea 8($out),$out sub \$8,$len # len -= 8 @@ -540,14 +542,14 @@ SHA3_squeeze: sub \$1,%rcx # bsz-- jnz .Loop_squeeze - +.Lnext_block: call KeccakF1600 - mov $A_flat,%r8 + mov $A_flat,%r9 mov $bsz,%rcx jmp .Loop_squeeze .Ltail_squeeze: - mov %r8, %rsi + mov %r9, %rsi mov $out,%rdi mov $len,%rcx .byte 0xf3,0xa4 # rep movsb @@ -565,6 +567,7 @@ SHA3_squeeze: ___ } $code.=<<___; +.section .rodata align=256 .align 256 .quad 0,0,0,0,0,0,0,0 .type iotas,\@object diff --git a/crypto/openssl/crypto/sha/asm/sha1-armv4-large.pl b/crypto/openssl/crypto/sha/asm/sha1-armv4-large.pl index 6a2e12bf17d7..b3a8ec694145 100755 --- a/crypto/openssl/crypto/sha/asm/sha1-armv4-large.pl +++ b/crypto/openssl/crypto/sha/asm/sha1-armv4-large.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2007-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2007-2023 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -707,7 +707,8 @@ ___ }}} $code.=<<___; #if __ARM_MAX_ARCH__>=7 -.comm OPENSSL_armcap_P,4,4 +.extern OPENSSL_armcap_P +.hidden OPENSSL_armcap_P #endif ___ diff --git a/crypto/openssl/crypto/sha/asm/sha1-armv8.pl b/crypto/openssl/crypto/sha/asm/sha1-armv8.pl index 0ffc9766b9f9..3a3c1d1806a8 100755 --- a/crypto/openssl/crypto/sha/asm/sha1-armv8.pl +++ b/crypto/openssl/crypto/sha/asm/sha1-armv8.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2014-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2014-2025 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -265,7 +265,8 @@ sha1_block_armv8: stp PTR(29),PTR(30),[PTRN(sp),#-(2*PTR_WIDTH)]! add PTR(29),PTRN(sp),#0 - adr PTR(4),.Lconst + adrp PTR(4),.Lconst + add PTR(4),PTR(4),:lo12:.Lconst eor $E,$E,$E ld1.32 {$ABCD},[$ctx],#16 ld1.32 {$E}[0],[$ctx] @@ -325,6 +326,9 @@ $code.=<<___; ldr PTR(29),[PTRN(sp)],#(2*PTR_WIDTH) ret .size sha1_block_armv8,.-sha1_block_armv8 + +.rodata + .align 6 .Lconst: .long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 //K_00_19 diff --git a/crypto/openssl/crypto/sha/asm/sha1-mb-x86_64.pl b/crypto/openssl/crypto/sha/asm/sha1-mb-x86_64.pl index 67faba136d05..d9d1630d16db 100755 --- a/crypto/openssl/crypto/sha/asm/sha1-mb-x86_64.pl +++ b/crypto/openssl/crypto/sha/asm/sha1-mb-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2013-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2013-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -1374,7 +1374,7 @@ $code.=<<___; ___ } }}} $code.=<<___; - +.section .rodata align=256 .align 256 .long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 # K_00_19 .long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 # K_00_19 @@ -1389,6 +1389,7 @@ K_XX_XX: .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f # pbswap .byte 0xf,0xe,0xd,0xc,0xb,0xa,0x9,0x8,0x7,0x6,0x5,0x4,0x3,0x2,0x1,0x0 .asciz "SHA1 multi-block transform for x86_64, CRYPTOGAMS by " +.previous ___ if ($win64) { diff --git a/crypto/openssl/crypto/sha/asm/sha1-x86_64.pl b/crypto/openssl/crypto/sha/asm/sha1-x86_64.pl index c66b8fca875e..30c545cf419a 100755 --- a/crypto/openssl/crypto/sha/asm/sha1-x86_64.pl +++ b/crypto/openssl/crypto/sha/asm/sha1-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2006-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2006-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -1832,6 +1832,7 @@ ___ } } $code.=<<___; +.section .rodata align=64 .align 64 K_XX_XX: .long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 # K_00_19 @@ -1845,6 +1846,7 @@ K_XX_XX: .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f # pbswap mask .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f # pbswap mask .byte 0xf,0xe,0xd,0xc,0xb,0xa,0x9,0x8,0x7,0x6,0x5,0x4,0x3,0x2,0x1,0x0 +.previous ___ }}} $code.=<<___; diff --git a/crypto/openssl/crypto/sha/asm/sha256-armv4.pl b/crypto/openssl/crypto/sha/asm/sha256-armv4.pl index e8af05c2981f..feb1f26109a9 100755 --- a/crypto/openssl/crypto/sha/asm/sha256-armv4.pl +++ b/crypto/openssl/crypto/sha/asm/sha256-armv4.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2007-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2007-2023 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -693,7 +693,8 @@ $code.=<<___; .asciz "SHA256 block transform for ARMv4/NEON/ARMv8, CRYPTOGAMS by " .align 2 #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) -.comm OPENSSL_armcap_P,4,4 +.extern OPENSSL_armcap_P +.hidden OPENSSL_armcap_P #endif ___ diff --git a/crypto/openssl/crypto/sha/asm/sha256-mb-x86_64.pl b/crypto/openssl/crypto/sha/asm/sha256-mb-x86_64.pl index fa2ca86046cc..9398b7954a7f 100755 --- a/crypto/openssl/crypto/sha/asm/sha256-mb-x86_64.pl +++ b/crypto/openssl/crypto/sha/asm/sha256-mb-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2013-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2013-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -1340,6 +1340,7 @@ $code.=<<___; ___ } }}} $code.=<<___; +.section .rodata align=256 .align 256 K256: ___ @@ -1389,6 +1390,7 @@ K256_shaext: .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 .asciz "SHA256 multi-block transform for x86_64, CRYPTOGAMS by " +.previous ___ if ($win64) { diff --git a/crypto/openssl/crypto/sha/asm/sha256-riscv64-zvkb-zvknha_or_zvknhb.pl b/crypto/openssl/crypto/sha/asm/sha256-riscv64-zvkb-zvknha_or_zvknhb.pl new file mode 100644 index 000000000000..5e4d6be34574 --- /dev/null +++ b/crypto/openssl/crypto/sha/asm/sha256-riscv64-zvkb-zvknha_or_zvknhb.pl @@ -0,0 +1,316 @@ +#! /usr/bin/env perl +# This file is dual-licensed, meaning that you can use it under your +# choice of either of the following two licenses: +# +# Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You can obtain +# a copy in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html +# +# or +# +# Copyright (c) 2023, Christoph Müllner +# Copyright (c) 2023, Phoebe Chen +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# The generated code of this file depends on the following RISC-V extensions: +# - RV64I +# - RISC-V Vector ('V') with VLEN >= 128 +# - RISC-V Vector Cryptography Bit-manipulation extension ('Zvkb') +# - RISC-V Vector SHA-2 Secure Hash extension ('Zvknha' or 'Zvknhb') + +use strict; +use warnings; + +use FindBin qw($Bin); +use lib "$Bin"; +use lib "$Bin/../../perlasm"; +use riscv; + +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$output and open STDOUT,">$output"; + +my $code=<<___; +.text +___ + +my ($V0, $V1, $V2, $V3, $V4, $V5, $V6, $V7, + $V8, $V9, $V10, $V11, $V12, $V13, $V14, $V15, + $V16, $V17, $V18, $V19, $V20, $V21, $V22, $V23, + $V24, $V25, $V26, $V27, $V28, $V29, $V30, $V31, +) = map("v$_",(0..31)); + +my $K256 = "K256"; + +# Function arguments +my ($H, $INP, $LEN, $KT, $H2, $INDEX_PATTERN) = ("a0", "a1", "a2", "a3", "t3", "t4"); + +sub sha_256_load_constant { + my $code=<<___; + la $KT, $K256 # Load round constants K256 + @{[vle32_v $V10, $KT]} + addi $KT, $KT, 16 + @{[vle32_v $V11, $KT]} + addi $KT, $KT, 16 + @{[vle32_v $V12, $KT]} + addi $KT, $KT, 16 + @{[vle32_v $V13, $KT]} + addi $KT, $KT, 16 + @{[vle32_v $V14, $KT]} + addi $KT, $KT, 16 + @{[vle32_v $V15, $KT]} + addi $KT, $KT, 16 + @{[vle32_v $V16, $KT]} + addi $KT, $KT, 16 + @{[vle32_v $V17, $KT]} + addi $KT, $KT, 16 + @{[vle32_v $V18, $KT]} + addi $KT, $KT, 16 + @{[vle32_v $V19, $KT]} + addi $KT, $KT, 16 + @{[vle32_v $V20, $KT]} + addi $KT, $KT, 16 + @{[vle32_v $V21, $KT]} + addi $KT, $KT, 16 + @{[vle32_v $V22, $KT]} + addi $KT, $KT, 16 + @{[vle32_v $V23, $KT]} + addi $KT, $KT, 16 + @{[vle32_v $V24, $KT]} + addi $KT, $KT, 16 + @{[vle32_v $V25, $KT]} +___ + + return $code; +} + +################################################################################ +# void sha256_block_data_order_zvkb_zvknha_or_zvknhb(void *c, const void *p, size_t len) +$code .= <<___; +.p2align 2 +.globl sha256_block_data_order_zvkb_zvknha_or_zvknhb +.type sha256_block_data_order_zvkb_zvknha_or_zvknhb,\@function +sha256_block_data_order_zvkb_zvknha_or_zvknhb: + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + + @{[sha_256_load_constant]} + + # H is stored as {a,b,c,d},{e,f,g,h}, but we need {f,e,b,a},{h,g,d,c} + # The dst vtype is e32m1 and the index vtype is e8mf4. + # We use index-load with the following index pattern at v26. + # i8 index: + # 20, 16, 4, 0 + # Instead of setting the i8 index, we could use a single 32bit + # little-endian value to cover the 4xi8 index. + # i32 value: + # 0x 00 04 10 14 + li $INDEX_PATTERN, 0x00041014 + @{[vsetivli "zero", 1, "e32", "m1", "ta", "ma"]} + @{[vmv_v_x $V26, $INDEX_PATTERN]} + + addi $H2, $H, 8 + + # Use index-load to get {f,e,b,a},{h,g,d,c} + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + @{[vluxei8_v $V6, $H, $V26]} + @{[vluxei8_v $V7, $H2, $V26]} + + # Setup v0 mask for the vmerge to replace the first word (idx==0) in key-scheduling. + # The AVL is 4 in SHA, so we could use a single e8(8 element masking) for masking. + @{[vsetivli "zero", 1, "e8", "m1", "ta", "ma"]} + @{[vmv_v_i $V0, 0x01]} + + @{[vsetivli "zero", 4, "e32", "m1", "ta", "ma"]} + +L_round_loop: + # Decrement length by 1 + add $LEN, $LEN, -1 + + # Keep the current state as we need it later: H' = H+{a',b',c',...,h'}. + @{[vmv_v_v $V30, $V6]} + @{[vmv_v_v $V31, $V7]} + + # Load the 512-bits of the message block in v1-v4 and perform + # an endian swap on each 4 bytes element. + @{[vle32_v $V1, $INP]} + @{[vrev8_v $V1, $V1]} + add $INP, $INP, 16 + @{[vle32_v $V2, $INP]} + @{[vrev8_v $V2, $V2]} + add $INP, $INP, 16 + @{[vle32_v $V3, $INP]} + @{[vrev8_v $V3, $V3]} + add $INP, $INP, 16 + @{[vle32_v $V4, $INP]} + @{[vrev8_v $V4, $V4]} + add $INP, $INP, 16 + + # Quad-round 0 (+0, Wt from oldest to newest in v1->v2->v3->v4) + @{[vadd_vv $V5, $V10, $V1]} + @{[vsha2cl_vv $V7, $V6, $V5]} + @{[vsha2ch_vv $V6, $V7, $V5]} + @{[vmerge_vvm $V5, $V3, $V2, $V0]} + @{[vsha2ms_vv $V1, $V5, $V4]} # Generate W[19:16] + + # Quad-round 1 (+1, v2->v3->v4->v1) + @{[vadd_vv $V5, $V11, $V2]} + @{[vsha2cl_vv $V7, $V6, $V5]} + @{[vsha2ch_vv $V6, $V7, $V5]} + @{[vmerge_vvm $V5, $V4, $V3, $V0]} + @{[vsha2ms_vv $V2, $V5, $V1]} # Generate W[23:20] + + # Quad-round 2 (+2, v3->v4->v1->v2) + @{[vadd_vv $V5, $V12, $V3]} + @{[vsha2cl_vv $V7, $V6, $V5]} + @{[vsha2ch_vv $V6, $V7, $V5]} + @{[vmerge_vvm $V5, $V1, $V4, $V0]} + @{[vsha2ms_vv $V3, $V5, $V2]} # Generate W[27:24] + + # Quad-round 3 (+3, v4->v1->v2->v3) + @{[vadd_vv $V5, $V13, $V4]} + @{[vsha2cl_vv $V7, $V6, $V5]} + @{[vsha2ch_vv $V6, $V7, $V5]} + @{[vmerge_vvm $V5, $V2, $V1, $V0]} + @{[vsha2ms_vv $V4, $V5, $V3]} # Generate W[31:28] + + # Quad-round 4 (+0, v1->v2->v3->v4) + @{[vadd_vv $V5, $V14, $V1]} + @{[vsha2cl_vv $V7, $V6, $V5]} + @{[vsha2ch_vv $V6, $V7, $V5]} + @{[vmerge_vvm $V5, $V3, $V2, $V0]} + @{[vsha2ms_vv $V1, $V5, $V4]} # Generate W[35:32] + + # Quad-round 5 (+1, v2->v3->v4->v1) + @{[vadd_vv $V5, $V15, $V2]} + @{[vsha2cl_vv $V7, $V6, $V5]} + @{[vsha2ch_vv $V6, $V7, $V5]} + @{[vmerge_vvm $V5, $V4, $V3, $V0]} + @{[vsha2ms_vv $V2, $V5, $V1]} # Generate W[39:36] + + # Quad-round 6 (+2, v3->v4->v1->v2) + @{[vadd_vv $V5, $V16, $V3]} + @{[vsha2cl_vv $V7, $V6, $V5]} + @{[vsha2ch_vv $V6, $V7, $V5]} + @{[vmerge_vvm $V5, $V1, $V4, $V0]} + @{[vsha2ms_vv $V3, $V5, $V2]} # Generate W[43:40] + + # Quad-round 7 (+3, v4->v1->v2->v3) + @{[vadd_vv $V5, $V17, $V4]} + @{[vsha2cl_vv $V7, $V6, $V5]} + @{[vsha2ch_vv $V6, $V7, $V5]} + @{[vmerge_vvm $V5, $V2, $V1, $V0]} + @{[vsha2ms_vv $V4, $V5, $V3]} # Generate W[47:44] + + # Quad-round 8 (+0, v1->v2->v3->v4) + @{[vadd_vv $V5, $V18, $V1]} + @{[vsha2cl_vv $V7, $V6, $V5]} + @{[vsha2ch_vv $V6, $V7, $V5]} + @{[vmerge_vvm $V5, $V3, $V2, $V0]} + @{[vsha2ms_vv $V1, $V5, $V4]} # Generate W[51:48] + + # Quad-round 9 (+1, v2->v3->v4->v1) + @{[vadd_vv $V5, $V19, $V2]} + @{[vsha2cl_vv $V7, $V6, $V5]} + @{[vsha2ch_vv $V6, $V7, $V5]} + @{[vmerge_vvm $V5, $V4, $V3, $V0]} + @{[vsha2ms_vv $V2, $V5, $V1]} # Generate W[55:52] + + # Quad-round 10 (+2, v3->v4->v1->v2) + @{[vadd_vv $V5, $V20, $V3]} + @{[vsha2cl_vv $V7, $V6, $V5]} + @{[vsha2ch_vv $V6, $V7, $V5]} + @{[vmerge_vvm $V5, $V1, $V4, $V0]} + @{[vsha2ms_vv $V3, $V5, $V2]} # Generate W[59:56] + + # Quad-round 11 (+3, v4->v1->v2->v3) + @{[vadd_vv $V5, $V21, $V4]} + @{[vsha2cl_vv $V7, $V6, $V5]} + @{[vsha2ch_vv $V6, $V7, $V5]} + @{[vmerge_vvm $V5, $V2, $V1, $V0]} + @{[vsha2ms_vv $V4, $V5, $V3]} # Generate W[63:60] + + # Quad-round 12 (+0, v1->v2->v3->v4) + # Note that we stop generating new message schedule words (Wt, v1-13) + # as we already generated all the words we end up consuming (i.e., W[63:60]). + @{[vadd_vv $V5, $V22, $V1]} + @{[vsha2cl_vv $V7, $V6, $V5]} + @{[vsha2ch_vv $V6, $V7, $V5]} + + # Quad-round 13 (+1, v2->v3->v4->v1) + @{[vadd_vv $V5, $V23, $V2]} + @{[vsha2cl_vv $V7, $V6, $V5]} + @{[vsha2ch_vv $V6, $V7, $V5]} + + # Quad-round 14 (+2, v3->v4->v1->v2) + @{[vadd_vv $V5, $V24, $V3]} + @{[vsha2cl_vv $V7, $V6, $V5]} + @{[vsha2ch_vv $V6, $V7, $V5]} + + # Quad-round 15 (+3, v4->v1->v2->v3) + @{[vadd_vv $V5, $V25, $V4]} + @{[vsha2cl_vv $V7, $V6, $V5]} + @{[vsha2ch_vv $V6, $V7, $V5]} + + # H' = H+{a',b',c',...,h'} + @{[vadd_vv $V6, $V30, $V6]} + @{[vadd_vv $V7, $V31, $V7]} + bnez $LEN, L_round_loop + + # Store {f,e,b,a},{h,g,d,c} back to {a,b,c,d},{e,f,g,h}. + @{[vsuxei8_v $V6, $H, $V26]} + @{[vsuxei8_v $V7, $H2, $V26]} + + ret +.size sha256_block_data_order_zvkb_zvknha_or_zvknhb,.-sha256_block_data_order_zvkb_zvknha_or_zvknhb + +.p2align 2 +.type $K256,\@object +$K256: + .word 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5 + .word 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5 + .word 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3 + .word 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174 + .word 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc + .word 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da + .word 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7 + .word 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967 + .word 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13 + .word 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85 + .word 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3 + .word 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070 + .word 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5 + .word 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3 + .word 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208 + .word 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 +.size $K256,.-$K256 +___ + +print $code; + +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/sha/asm/sha512-armv4.pl b/crypto/openssl/crypto/sha/asm/sha512-armv4.pl index 1751c671e6d8..9aa310ffd05f 100755 --- a/crypto/openssl/crypto/sha/asm/sha512-armv4.pl +++ b/crypto/openssl/crypto/sha/asm/sha512-armv4.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2007-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2007-2023 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -660,7 +660,8 @@ $code.=<<___; .asciz "SHA512 block transform for ARMv4/NEON, CRYPTOGAMS by " .align 2 #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) -.comm OPENSSL_armcap_P,4,4 +.extern OPENSSL_armcap_P +.hidden OPENSSL_armcap_P #endif ___ diff --git a/crypto/openssl/crypto/sha/asm/sha512-armv8.pl b/crypto/openssl/crypto/sha/asm/sha512-armv8.pl index 7358b4b51767..a3bbac159971 100755 --- a/crypto/openssl/crypto/sha/asm/sha512-armv8.pl +++ b/crypto/openssl/crypto/sha/asm/sha512-armv8.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2014-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2014-2025 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -247,7 +247,8 @@ $code.=<<___; add $num,$inp,$num,lsl#`log(16*$SZ)/log(2)` // end of input #endif ldp $G,$H,[$ctx,#6*$SZ] - adr $Ktbl,.LK$BITS + adrp $Ktbl,.LK$BITS + add $Ktbl,$Ktbl,:lo12:.LK$BITS stp $ctx,$num,[PTR(29),#(12*PTR_WIDTH)] .Loop: @@ -297,6 +298,8 @@ $code.=<<___; ret .size $func,.-$func +.rodata + .align 6 .type .LK$BITS,%object .LK$BITS: @@ -367,6 +370,8 @@ $code.=<<___; .size .LK$BITS,.-.LK$BITS .asciz "SHA$BITS block transform for ARMv8, CRYPTOGAMS by " .align 2 + +.text ___ if ($SZ==4) { @@ -388,7 +393,8 @@ sha256_block_armv8: add PTR(29),PTRN(sp),#0 ld1.32 {$ABCD,$EFGH},[$ctx] - adr $Ktbl,.LK256 + adrp $Ktbl,.LK256 + add $Ktbl,$Ktbl,:lo12:.LK256 .Loop_hw: ld1 {@MSG[0]-@MSG[3]},[$inp],#64 @@ -654,7 +660,8 @@ sha256_block_neon: mov PTR(29), PTRN(sp) sub PTRN(sp),PTRN(sp),#16*4 - adr $Ktbl,.LK256 + adrp $Ktbl,.LK256 + add $Ktbl,$Ktbl,:lo12:.LK256 #ifdef __CHERI_PURE_CAPABILITY__ lsl $temp,$numx,#6 add $num,$inp,$temp // len to point at the end of inp @@ -781,7 +788,8 @@ sha512_block_armv8: ld1 {@MSG[4]-@MSG[7]},[$inp],#64 ld1.64 {@H[0]-@H[3]},[$ctx] // load context - adr $Ktbl,.LK512 + adrp $Ktbl,.LK512 + add $Ktbl,$Ktbl,:lo12:.LK512 rev64 @MSG[0],@MSG[0] rev64 @MSG[1],@MSG[1] diff --git a/crypto/openssl/crypto/sha/asm/sha512-riscv64-zvkb-zvknhb.pl b/crypto/openssl/crypto/sha/asm/sha512-riscv64-zvkb-zvknhb.pl new file mode 100644 index 000000000000..c5df987296e2 --- /dev/null +++ b/crypto/openssl/crypto/sha/asm/sha512-riscv64-zvkb-zvknhb.pl @@ -0,0 +1,264 @@ +#! /usr/bin/env perl +# This file is dual-licensed, meaning that you can use it under your +# choice of either of the following two licenses: +# +# Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You can obtain +# a copy in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html +# +# or +# +# Copyright (c) 2023, Christoph Müllner +# Copyright (c) 2023, Phoebe Chen +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# The generated code of this file depends on the following RISC-V extensions: +# - RV64I +# - RISC-V vector ('V') with VLEN >= 128 +# - RISC-V Vector Cryptography Bit-manipulation extension ('Zvkb') +# - RISC-V Vector SHA-2 Secure Hash extension ('Zvknhb') + +use strict; +use warnings; + +use FindBin qw($Bin); +use lib "$Bin"; +use lib "$Bin/../../perlasm"; +use riscv; + +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$output and open STDOUT,">$output"; + +my $code=<<___; +.text +___ + +my ($V0, $V1, $V2, $V3, $V4, $V5, $V6, $V7, + $V8, $V9, $V10, $V11, $V12, $V13, $V14, $V15, + $V16, $V17, $V18, $V19, $V20, $V21, $V22, $V23, + $V24, $V25, $V26, $V27, $V28, $V29, $V30, $V31, +) = map("v$_",(0..31)); + +my $K512 = "K512"; + +# Function arguments +my ($H, $INP, $LEN, $KT, $H2, $INDEX_PATTERN) = ("a0", "a1", "a2", "a3", "t3", "t4"); + +################################################################################ +# void sha512_block_data_order_zvkb_zvknhb(void *c, const void *p, size_t len) +$code .= <<___; +.p2align 2 +.globl sha512_block_data_order_zvkb_zvknhb +.type sha512_block_data_order_zvkb_zvknhb,\@function +sha512_block_data_order_zvkb_zvknhb: + @{[vsetivli "zero", 4, "e64", "m2", "ta", "ma"]} + + # H is stored as {a,b,c,d},{e,f,g,h}, but we need {f,e,b,a},{h,g,d,c} + # The dst vtype is e64m2 and the index vtype is e8mf4. + # We use index-load with the following index pattern at v1. + # i8 index: + # 40, 32, 8, 0 + # Instead of setting the i8 index, we could use a single 32bit + # little-endian value to cover the 4xi8 index. + # i32 value: + # 0x 00 08 20 28 + li $INDEX_PATTERN, 0x00082028 + @{[vsetivli "zero", 1, "e32", "m1", "ta", "ma"]} + @{[vmv_v_x $V1, $INDEX_PATTERN]} + + addi $H2, $H, 16 + + # Use index-load to get {f,e,b,a},{h,g,d,c} + @{[vsetivli "zero", 4, "e64", "m2", "ta", "ma"]} + @{[vluxei8_v $V22, $H, $V1]} + @{[vluxei8_v $V24, $H2, $V1]} + + # Setup v0 mask for the vmerge to replace the first word (idx==0) in key-scheduling. + # The AVL is 4 in SHA, so we could use a single e8(8 element masking) for masking. + @{[vsetivli "zero", 1, "e8", "m1", "ta", "ma"]} + @{[vmv_v_i $V0, 0x01]} + + @{[vsetivli "zero", 4, "e64", "m2", "ta", "ma"]} + +L_round_loop: + # Load round constants K512 + la $KT, $K512 + + # Decrement length by 1 + addi $LEN, $LEN, -1 + + # Keep the current state as we need it later: H' = H+{a',b',c',...,h'}. + @{[vmv_v_v $V26, $V22]} + @{[vmv_v_v $V28, $V24]} + + # Load the 1024-bits of the message block in v10-v16 and perform the endian + # swap. + @{[vle64_v $V10, $INP]} + @{[vrev8_v $V10, $V10]} + addi $INP, $INP, 32 + @{[vle64_v $V12, $INP]} + @{[vrev8_v $V12, $V12]} + addi $INP, $INP, 32 + @{[vle64_v $V14, $INP]} + @{[vrev8_v $V14, $V14]} + addi $INP, $INP, 32 + @{[vle64_v $V16, $INP]} + @{[vrev8_v $V16, $V16]} + addi $INP, $INP, 32 + + .rept 4 + # Quad-round 0 (+0, v10->v12->v14->v16) + @{[vle64_v $V20, ($KT)]} + addi $KT, $KT, 32 + @{[vadd_vv $V18, $V20, $V10]} + @{[vsha2cl_vv $V24, $V22, $V18]} + @{[vsha2ch_vv $V22, $V24, $V18]} + @{[vmerge_vvm $V18, $V14, $V12, $V0]} + @{[vsha2ms_vv $V10, $V18, $V16]} + + # Quad-round 1 (+1, v12->v14->v16->v10) + @{[vle64_v $V20, ($KT)]} + addi $KT, $KT, 32 + @{[vadd_vv $V18, $V20, $V12]} + @{[vsha2cl_vv $V24, $V22, $V18]} + @{[vsha2ch_vv $V22, $V24, $V18]} + @{[vmerge_vvm $V18, $V16, $V14, $V0]} + @{[vsha2ms_vv $V12, $V18, $V10]} + + # Quad-round 2 (+2, v14->v16->v10->v12) + @{[vle64_v $V20, ($KT)]} + addi $KT, $KT, 32 + @{[vadd_vv $V18, $V20, $V14]} + @{[vsha2cl_vv $V24, $V22, $V18]} + @{[vsha2ch_vv $V22, $V24, $V18]} + @{[vmerge_vvm $V18, $V10, $V16, $V0]} + @{[vsha2ms_vv $V14, $V18, $V12]} + + # Quad-round 3 (+3, v16->v10->v12->v14) + @{[vle64_v $V20, ($KT)]} + addi $KT, $KT, 32 + @{[vadd_vv $V18, $V20, $V16]} + @{[vsha2cl_vv $V24, $V22, $V18]} + @{[vsha2ch_vv $V22, $V24, $V18]} + @{[vmerge_vvm $V18, $V12, $V10, $V0]} + @{[vsha2ms_vv $V16, $V18, $V14]} + .endr + + # Quad-round 16 (+0, v10->v12->v14->v16) + # Note that we stop generating new message schedule words (Wt, v10-16) + # as we already generated all the words we end up consuming (i.e., W[79:76]). + @{[vle64_v $V20, ($KT)]} + addi $KT, $KT, 32 + @{[vadd_vv $V18, $V20, $V10]} + @{[vsha2cl_vv $V24, $V22, $V18]} + @{[vsha2ch_vv $V22, $V24, $V18]} + + # Quad-round 17 (+1, v12->v14->v16->v10) + @{[vle64_v $V20, ($KT)]} + addi $KT, $KT, 32 + @{[vadd_vv $V18, $V20, $V12]} + @{[vsha2cl_vv $V24, $V22, $V18]} + @{[vsha2ch_vv $V22, $V24, $V18]} + + # Quad-round 18 (+2, v14->v16->v10->v12) + @{[vle64_v $V20, ($KT)]} + addi $KT, $KT, 32 + @{[vadd_vv $V18, $V20, $V14]} + @{[vsha2cl_vv $V24, $V22, $V18]} + @{[vsha2ch_vv $V22, $V24, $V18]} + + # Quad-round 19 (+3, v16->v10->v12->v14) + @{[vle64_v $V20, ($KT)]} + # No t1 increment needed. + @{[vadd_vv $V18, $V20, $V16]} + @{[vsha2cl_vv $V24, $V22, $V18]} + @{[vsha2ch_vv $V22, $V24, $V18]} + + # H' = H+{a',b',c',...,h'} + @{[vadd_vv $V22, $V26, $V22]} + @{[vadd_vv $V24, $V28, $V24]} + bnez $LEN, L_round_loop + + # Store {f,e,b,a},{h,g,d,c} back to {a,b,c,d},{e,f,g,h}. + @{[vsuxei8_v $V22, ($H), $V1]} + @{[vsuxei8_v $V24, ($H2), $V1]} + + ret +.size sha512_block_data_order_zvkb_zvknhb,.-sha512_block_data_order_zvkb_zvknhb + +.p2align 3 +.type $K512,\@object +$K512: + .dword 0x428a2f98d728ae22, 0x7137449123ef65cd + .dword 0xb5c0fbcfec4d3b2f, 0xe9b5dba58189dbbc + .dword 0x3956c25bf348b538, 0x59f111f1b605d019 + .dword 0x923f82a4af194f9b, 0xab1c5ed5da6d8118 + .dword 0xd807aa98a3030242, 0x12835b0145706fbe + .dword 0x243185be4ee4b28c, 0x550c7dc3d5ffb4e2 + .dword 0x72be5d74f27b896f, 0x80deb1fe3b1696b1 + .dword 0x9bdc06a725c71235, 0xc19bf174cf692694 + .dword 0xe49b69c19ef14ad2, 0xefbe4786384f25e3 + .dword 0x0fc19dc68b8cd5b5, 0x240ca1cc77ac9c65 + .dword 0x2de92c6f592b0275, 0x4a7484aa6ea6e483 + .dword 0x5cb0a9dcbd41fbd4, 0x76f988da831153b5 + .dword 0x983e5152ee66dfab, 0xa831c66d2db43210 + .dword 0xb00327c898fb213f, 0xbf597fc7beef0ee4 + .dword 0xc6e00bf33da88fc2, 0xd5a79147930aa725 + .dword 0x06ca6351e003826f, 0x142929670a0e6e70 + .dword 0x27b70a8546d22ffc, 0x2e1b21385c26c926 + .dword 0x4d2c6dfc5ac42aed, 0x53380d139d95b3df + .dword 0x650a73548baf63de, 0x766a0abb3c77b2a8 + .dword 0x81c2c92e47edaee6, 0x92722c851482353b + .dword 0xa2bfe8a14cf10364, 0xa81a664bbc423001 + .dword 0xc24b8b70d0f89791, 0xc76c51a30654be30 + .dword 0xd192e819d6ef5218, 0xd69906245565a910 + .dword 0xf40e35855771202a, 0x106aa07032bbd1b8 + .dword 0x19a4c116b8d2d0c8, 0x1e376c085141ab53 + .dword 0x2748774cdf8eeb99, 0x34b0bcb5e19b48a8 + .dword 0x391c0cb3c5c95a63, 0x4ed8aa4ae3418acb + .dword 0x5b9cca4f7763e373, 0x682e6ff3d6b2b8a3 + .dword 0x748f82ee5defb2fc, 0x78a5636f43172f60 + .dword 0x84c87814a1f0ab72, 0x8cc702081a6439ec + .dword 0x90befffa23631e28, 0xa4506cebde82bde9 + .dword 0xbef9a3f7b2c67915, 0xc67178f2e372532b + .dword 0xca273eceea26619c, 0xd186b8c721c0c207 + .dword 0xeada7dd6cde0eb1e, 0xf57d4f7fee6ed178 + .dword 0x06f067aa72176fba, 0x0a637dc5a2c898a6 + .dword 0x113f9804bef90dae, 0x1b710b35131c471b + .dword 0x28db77f523047d84, 0x32caab7b40c72493 + .dword 0x3c9ebe0a15c9bebc, 0x431d67c49c100d4c + .dword 0x4cc5d4becb3e42b6, 0x597f299cfc657e2a + .dword 0x5fcb6fab3ad6faec, 0x6c44198c4a475817 +.size $K512,.-$K512 +___ + +print $code; + +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/sha/asm/sha512-x86_64.pl b/crypto/openssl/crypto/sha/asm/sha512-x86_64.pl index 1bd0256954b3..b37058ae03fa 100755 --- a/crypto/openssl/crypto/sha/asm/sha512-x86_64.pl +++ b/crypto/openssl/crypto/sha/asm/sha512-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2005-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2005-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -418,6 +418,7 @@ ___ if ($SZ==4) { $code.=<<___; +.section .rodata align=64 .align 64 .type $TABLE,\@object $TABLE: @@ -461,9 +462,11 @@ $TABLE: .long 0xffffffff,0xffffffff,0x03020100,0x0b0a0908 .long 0xffffffff,0xffffffff,0x03020100,0x0b0a0908 .asciz "SHA256 block transform for x86_64, CRYPTOGAMS by " +.previous ___ } else { $code.=<<___; +.section .rodata align=64 .align 64 .type $TABLE,\@object $TABLE: @@ -551,6 +554,7 @@ $TABLE: .quad 0x0001020304050607,0x08090a0b0c0d0e0f .quad 0x0001020304050607,0x08090a0b0c0d0e0f .asciz "SHA512 block transform for x86_64, CRYPTOGAMS by " +.previous ___ } diff --git a/crypto/openssl/crypto/sm3/asm/sm3-armv8.pl b/crypto/openssl/crypto/sm3/asm/sm3-armv8.pl new file mode 100644 index 000000000000..f36e0e2be98c --- /dev/null +++ b/crypto/openssl/crypto/sm3/asm/sm3-armv8.pl @@ -0,0 +1,303 @@ +#! /usr/bin/env perl +# Copyright 2021-2025 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html +# +# This module implements support for Armv8 SM3 instructions + +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or +die "can't locate arm-xlate.pl"; + +open OUT,"| \"$^X\" $xlate $flavour \"$output\"" + or die "can't call $xlate: $!"; +*STDOUT=*OUT; + +$prefix="sm3"; +# Message expanding: +# Wj <- P1(W[j-16]^W[j-9]^(W[j-3]<<<15))^(W[j-13]<<<7)^W[j-6] +# Input: s0, s1, s2, s3 +# s0 = w0 | w1 | w2 | w3 +# s1 = w4 | w5 | w6 | w7 +# s2 = w8 | w9 | w10 | w11 +# s3 = w12 | w13 | w14 | w15 +# Output: s4 +sub msg_exp () { +my $s0 = shift; +my $s1 = shift; +my $s2 = shift; +my $s3 = shift; +my $s4 = shift; +my $vtmp1 = shift; +my $vtmp2 = shift; +$code.=<<___; + // s4 = w7 | w8 | w9 | w10 + ext $s4.16b, $s1.16b, $s2.16b, #12 + // vtmp1 = w3 | w4 | w5 | w6 + ext $vtmp1.16b, $s0.16b, $s1.16b, #12 + // vtmp2 = w10 | w11 | w12 | w13 + ext $vtmp2.16b, $s2.16b, $s3.16b, #8 + sm3partw1 $s4.4s, $s0.4s, $s3.4s + sm3partw2 $s4.4s, $vtmp2.4s, $vtmp1.4s +___ +} + +# A round of compresson function +# Input: +# ab - choose instruction among sm3tt1a, sm3tt1b, sm3tt2a, sm3tt2b +# vstate0 - vstate1, store digest status(A - H) +# vconst0 - vconst1, interleaved used to store Tj <<< j +# vtmp - temporary register +# vw - for sm3tt1ab, vw = s0 eor s1 +# s0 - for sm3tt2ab, just be s0 +# i, choose wj' or wj from vw +sub round () { +my $ab = shift; +my $vstate0 = shift; +my $vstate1 = shift; +my $vconst0 = shift; +my $vconst1 = shift; +my $vtmp = shift; +my $vw = shift; +my $s0 = shift; +my $i = shift; +$code.=<<___; + sm3ss1 $vtmp.4s, $vstate0.4s, $vconst0.4s, $vstate1.4s + shl $vconst1.4s, $vconst0.4s, #1 + sri $vconst1.4s, $vconst0.4s, #31 + sm3tt1$ab $vstate0.4s, $vtmp.4s, $vw.4s[$i] + sm3tt2$ab $vstate1.4s, $vtmp.4s, $s0.4s[$i] +___ +} + +sub qround () { +my $ab = shift; +my $vstate0 = shift; +my $vstate1 = shift; +my $vconst0 = shift; +my $vconst1 = shift; +my $vtmp1 = shift; +my $vtmp2 = shift; +my $s0 = shift; +my $s1 = shift; +my $s2 = shift; +my $s3 = shift; +my $s4 = shift; + if($s4) { + &msg_exp($s0, $s1, $s2, $s3, $s4, $vtmp1, $vtmp2); + } +$code.=<<___; + eor $vtmp1.16b, $s0.16b, $s1.16b +___ + &round($ab, $vstate0, $vstate1, $vconst0, $vconst1, $vtmp2, + $vtmp1, $s0, 0); + &round($ab, $vstate0, $vstate1, $vconst1, $vconst0, $vtmp2, + $vtmp1, $s0, 1); + &round($ab, $vstate0, $vstate1, $vconst0, $vconst1, $vtmp2, + $vtmp1, $s0, 2); + &round($ab, $vstate0, $vstate1, $vconst1, $vconst0, $vtmp2, + $vtmp1, $s0, 3); +} + +$code=<<___; +#include "arm_arch.h" +.text +___ + +{{{ +my ($pstate,$pdata,$num)=("x0","x1","w2"); +my ($state1,$state2)=("v5","v6"); +my ($sconst1, $sconst2)=("s16","s17"); +my ($vconst1, $vconst2)=("v16","v17"); +my ($s0,$s1,$s2,$s3,$s4)=map("v$_",(0..4)); +my ($bkstate1,$bkstate2)=("v18","v19"); +my ($vconst_tmp1,$vconst_tmp2)=("v20","v21"); +my ($vtmp1,$vtmp2)=("v22","v23"); +my $constaddr="x8"; +# void ossl_hwsm3_block_data_order(SM3_CTX *c, const void *p, size_t num) +$code.=<<___; +.globl ossl_hwsm3_block_data_order +.type ossl_hwsm3_block_data_order,%function +.align 5 +ossl_hwsm3_block_data_order: + AARCH64_VALID_CALL_TARGET + // load state + ld1 {$state1.4s-$state2.4s}, [$pstate] + rev64 $state1.4s, $state1.4s + rev64 $state2.4s, $state2.4s + ext $state1.16b, $state1.16b, $state1.16b, #8 + ext $state2.16b, $state2.16b, $state2.16b, #8 +___ +if ($flavour =~ /linux64/) +{ +$code.=<<___; + adrp $constaddr, .Tj + add $constaddr, $constaddr, #:lo12:.Tj +___ +} else { +$code.=<<___; + adr $constaddr, .Tj +___ +} +$code.=<<___; + ldp $sconst1, $sconst2, [$constaddr] + +.Loop: + // load input + ld1 {$s0.4s-$s3.4s}, [$pdata], #64 + sub $num, $num, #1 + + mov $bkstate1.16b, $state1.16b + mov $bkstate2.16b, $state2.16b + +#ifndef __AARCH64EB__ + rev32 $s0.16b, $s0.16b + rev32 $s1.16b, $s1.16b + rev32 $s2.16b, $s2.16b + rev32 $s3.16b, $s3.16b +#endif + + ext $vconst_tmp1.16b, $vconst1.16b, $vconst1.16b, #4 +___ + &qround("a",$state1,$state2,$vconst_tmp1,$vconst_tmp2,$vtmp1,$vtmp2, + $s0,$s1,$s2,$s3,$s4); + &qround("a",$state1,$state2,$vconst_tmp1,$vconst_tmp2,$vtmp1,$vtmp2, + $s1,$s2,$s3,$s4,$s0); + &qround("a",$state1,$state2,$vconst_tmp1,$vconst_tmp2,$vtmp1,$vtmp2, + $s2,$s3,$s4,$s0,$s1); + &qround("a",$state1,$state2,$vconst_tmp1,$vconst_tmp2,$vtmp1,$vtmp2, + $s3,$s4,$s0,$s1,$s2); + +$code.=<<___; + ext $vconst_tmp1.16b, $vconst2.16b, $vconst2.16b, #4 +___ + + &qround("b",$state1,$state2,$vconst_tmp1,$vconst_tmp2,$vtmp1,$vtmp2, + $s4,$s0,$s1,$s2,$s3); + &qround("b",$state1,$state2,$vconst_tmp1,$vconst_tmp2,$vtmp1,$vtmp2, + $s0,$s1,$s2,$s3,$s4); + &qround("b",$state1,$state2,$vconst_tmp1,$vconst_tmp2,$vtmp1,$vtmp2, + $s1,$s2,$s3,$s4,$s0); + &qround("b",$state1,$state2,$vconst_tmp1,$vconst_tmp2,$vtmp1,$vtmp2, + $s2,$s3,$s4,$s0,$s1); + &qround("b",$state1,$state2,$vconst_tmp1,$vconst_tmp2,$vtmp1,$vtmp2, + $s3,$s4,$s0,$s1,$s2); + &qround("b",$state1,$state2,$vconst_tmp1,$vconst_tmp2,$vtmp1,$vtmp2, + $s4,$s0,$s1,$s2,$s3); + &qround("b",$state1,$state2,$vconst_tmp1,$vconst_tmp2,$vtmp1,$vtmp2, + $s0,$s1,$s2,$s3,$s4); + &qround("b",$state1,$state2,$vconst_tmp1,$vconst_tmp2,$vtmp1,$vtmp2, + $s1,$s2,$s3,$s4,$s0); + &qround("b",$state1,$state2,$vconst_tmp1,$vconst_tmp2,$vtmp1,$vtmp2, + $s2,$s3,$s4,$s0,$s1); + &qround("b",$state1,$state2,$vconst_tmp1,$vconst_tmp2,$vtmp1,$vtmp2, + $s3,$s4); + &qround("b",$state1,$state2,$vconst_tmp1,$vconst_tmp2,$vtmp1,$vtmp2, + $s4,$s0); + &qround("b",$state1,$state2,$vconst_tmp1,$vconst_tmp2,$vtmp1,$vtmp2, + $s0,$s1); + +$code.=<<___; + eor $state1.16b, $state1.16b, $bkstate1.16b + eor $state2.16b, $state2.16b, $bkstate2.16b + + // any remained blocks? + cbnz $num, .Loop + + // save state + rev64 $state1.4s, $state1.4s + rev64 $state2.4s, $state2.4s + ext $state1.16b, $state1.16b, $state1.16b, #8 + ext $state2.16b, $state2.16b, $state2.16b, #8 + st1 {$state1.4s-$state2.4s}, [$pstate] + ret +.size ossl_hwsm3_block_data_order,.-ossl_hwsm3_block_data_order +___ + +$code.=".rodata\n" if ($flavour =~ /linux64/); + +$code.=<<___; + +.type _${prefix}_consts,%object +.align 3 +_${prefix}_consts: +.Tj: +.word 0x79cc4519, 0x9d8a7a87 +.size _${prefix}_consts,.-_${prefix}_consts +___ + +$code.=".previous\n" if ($flavour =~ /linux64/); + +}}} + +######################################### +my %sm3partopcode = ( + "sm3partw1" => 0xce60C000, + "sm3partw2" => 0xce60C400); + +my %sm3ss1opcode = ( + "sm3ss1" => 0xce400000); + +my %sm3ttopcode = ( + "sm3tt1a" => 0xce408000, + "sm3tt1b" => 0xce408400, + "sm3tt2a" => 0xce408800, + "sm3tt2b" => 0xce408C00); + +sub unsm3part { + my ($mnemonic,$arg)=@_; + + $arg=~ m/[qv](\d+)[^,]*,\s*[qv](\d+)[^,]*,\s*[qv](\d+)/o + && + sprintf ".inst\t0x%08x\t//%s %s", + $sm3partopcode{$mnemonic}|$1|($2<<5)|($3<<16), + $mnemonic,$arg; +} + +sub unsm3ss1 { + my ($mnemonic,$arg)=@_; + + $arg=~ m/[qv](\d+)[^,]*,\s*[qv](\d+)[^,]*,\s*[qv](\d+)[^,]*,\s*[qv](\d+)/o + && + sprintf ".inst\t0x%08x\t//%s %s", + $sm3ss1opcode{$mnemonic}|$1|($2<<5)|($3<<16)|($4<<10), + $mnemonic,$arg; +} + +sub unsm3tt { + my ($mnemonic,$arg)=@_; + + $arg=~ m/[qv](\d+)[^,]*,\s*[qv](\d+)[^,]*,\s*[qv](\d+)[^,]*\[([0-3])\]/o + && + sprintf ".inst\t0x%08x\t//%s %s", + $sm3ttopcode{$mnemonic}|$1|($2<<5)|($3<<16)|($4<<12), + $mnemonic,$arg; +} + +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/\/\// and !/^$/); + print; +} +close SELF; + +foreach(split("\n",$code)) { + s/\`([^\`]*)\`/eval($1)/ge; + + s/\b(sm3partw[1-2])\s+([qv].*)/unsm3part($1,$2)/ge; + s/\b(sm3ss1)\s+([qv].*)/unsm3ss1($1,$2)/ge; + s/\b(sm3tt[1-2][a-b])\s+([qv].*)/unsm3tt($1,$2)/ge; + print $_,"\n"; +} + +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/sm3/asm/sm3-riscv64-zvksh.pl b/crypto/openssl/crypto/sm3/asm/sm3-riscv64-zvksh.pl new file mode 100755 index 000000000000..1b8fd2eea0dc --- /dev/null +++ b/crypto/openssl/crypto/sm3/asm/sm3-riscv64-zvksh.pl @@ -0,0 +1,228 @@ +#! /usr/bin/env perl +# This file is dual-licensed, meaning that you can use it under your +# choice of either of the following two licenses: +# +# Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You can obtain +# a copy in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html +# +# or +# +# Copyright (c) 2023, Christoph Müllner +# Copyright (c) 2023, Jerry Shih +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# The generated code of this file depends on the following RISC-V extensions: +# - RV64I +# - RISC-V Vector ('V') with VLEN >= 128 +# - RISC-V Vector Cryptography Bit-manipulation extension ('Zvkb') +# - RISC-V Vector SM3 Secure Hash extension ('Zvksh') + +use strict; +use warnings; + +use FindBin qw($Bin); +use lib "$Bin"; +use lib "$Bin/../../perlasm"; +use riscv; + +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$output and open STDOUT,">$output"; + +my $code=<<___; +.text +___ + +################################################################################ +# ossl_hwsm3_block_data_order_zvksh(SM3_CTX *c, const void *p, size_t num); +{ +my ($CTX, $INPUT, $NUM) = ("a0", "a1", "a2"); +my ($V0, $V1, $V2, $V3, $V4, $V5, $V6, $V7, + $V8, $V9, $V10, $V11, $V12, $V13, $V14, $V15, + $V16, $V17, $V18, $V19, $V20, $V21, $V22, $V23, + $V24, $V25, $V26, $V27, $V28, $V29, $V30, $V31, +) = map("v$_",(0..31)); + +$code .= <<___; +.text +.p2align 3 +.globl ossl_hwsm3_block_data_order_zvksh +.type ossl_hwsm3_block_data_order_zvksh,\@function +ossl_hwsm3_block_data_order_zvksh: + @{[vsetivli "zero", 8, "e32", "m2", "ta", "ma"]} + + # Load initial state of hash context (c->A-H). + @{[vle32_v $V0, $CTX]} + @{[vrev8_v $V0, $V0]} + +L_sm3_loop: + # Copy the previous state to v2. + # It will be XOR'ed with the current state at the end of the round. + @{[vmv_v_v $V2, $V0]} + + # Load the 64B block in 2x32B chunks. + @{[vle32_v $V6, $INPUT]} # v6 := {w7, ..., w0} + addi $INPUT, $INPUT, 32 + + @{[vle32_v $V8, $INPUT]} # v8 := {w15, ..., w8} + addi $INPUT, $INPUT, 32 + + addi $NUM, $NUM, -1 + + # As vsm3c consumes only w0, w1, w4, w5 we need to slide the input + # 2 elements down so we process elements w2, w3, w6, w7 + # This will be repeated for each odd round. + @{[vslidedown_vi $V4, $V6, 2]} # v4 := {X, X, w7, ..., w2} + + @{[vsm3c_vi $V0, $V6, 0]} + @{[vsm3c_vi $V0, $V4, 1]} + + # Prepare a vector with {w11, ..., w4} + @{[vslidedown_vi $V4, $V4, 2]} # v4 := {X, X, X, X, w7, ..., w4} + @{[vslideup_vi $V4, $V8, 4]} # v4 := {w11, w10, w9, w8, w7, w6, w5, w4} + + @{[vsm3c_vi $V0, $V4, 2]} + @{[vslidedown_vi $V4, $V4, 2]} # v4 := {X, X, w11, w10, w9, w8, w7, w6} + @{[vsm3c_vi $V0, $V4, 3]} + + @{[vsm3c_vi $V0, $V8, 4]} + @{[vslidedown_vi $V4, $V8, 2]} # v4 := {X, X, w15, w14, w13, w12, w11, w10} + @{[vsm3c_vi $V0, $V4, 5]} + + @{[vsm3me_vv $V6, $V8, $V6]} # v6 := {w23, w22, w21, w20, w19, w18, w17, w16} + + # Prepare a register with {w19, w18, w17, w16, w15, w14, w13, w12} + @{[vslidedown_vi $V4, $V4, 2]} # v4 := {X, X, X, X, w15, w14, w13, w12} + @{[vslideup_vi $V4, $V6, 4]} # v4 := {w19, w18, w17, w16, w15, w14, w13, w12} + + @{[vsm3c_vi $V0, $V4, 6]} + @{[vslidedown_vi $V4, $V4, 2]} # v4 := {X, X, w19, w18, w17, w16, w15, w14} + @{[vsm3c_vi $V0, $V4, 7]} + + @{[vsm3c_vi $V0, $V6, 8]} + @{[vslidedown_vi $V4, $V6, 2]} # v4 := {X, X, w23, w22, w21, w20, w19, w18} + @{[vsm3c_vi $V0, $V4, 9]} + + @{[vsm3me_vv $V8, $V6, $V8]} # v8 := {w31, w30, w29, w28, w27, w26, w25, w24} + + # Prepare a register with {w27, w26, w25, w24, w23, w22, w21, w20} + @{[vslidedown_vi $V4, $V4, 2]} # v4 := {X, X, X, X, w23, w22, w21, w20} + @{[vslideup_vi $V4, $V8, 4]} # v4 := {w27, w26, w25, w24, w23, w22, w21, w20} + + @{[vsm3c_vi $V0, $V4, 10]} + @{[vslidedown_vi $V4, $V4, 2]} # v4 := {X, X, w27, w26, w25, w24, w23, w22} + @{[vsm3c_vi $V0, $V4, 11]} + + @{[vsm3c_vi $V0, $V8, 12]} + @{[vslidedown_vi $V4, $V8, 2]} # v4 := {x, X, w31, w30, w29, w28, w27, w26} + @{[vsm3c_vi $V0, $V4, 13]} + + @{[vsm3me_vv $V6, $V8, $V6]} # v6 := {w32, w33, w34, w35, w36, w37, w38, w39} + + # Prepare a register with {w35, w34, w33, w32, w31, w30, w29, w28} + @{[vslidedown_vi $V4, $V4, 2]} # v4 := {X, X, X, X, w31, w30, w29, w28} + @{[vslideup_vi $V4, $V6, 4]} # v4 := {w35, w34, w33, w32, w31, w30, w29, w28} + + @{[vsm3c_vi $V0, $V4, 14]} + @{[vslidedown_vi $V4, $V4, 2]} # v4 := {X, X, w35, w34, w33, w32, w31, w30} + @{[vsm3c_vi $V0, $V4, 15]} + + @{[vsm3c_vi $V0, $V6, 16]} + @{[vslidedown_vi $V4, $V6, 2]} # v4 := {X, X, w39, w38, w37, w36, w35, w34} + @{[vsm3c_vi $V0, $V4, 17]} + + @{[vsm3me_vv $V8, $V6, $V8]} # v8 := {w47, w46, w45, w44, w43, w42, w41, w40} + + # Prepare a register with {w43, w42, w41, w40, w39, w38, w37, w36} + @{[vslidedown_vi $V4, $V4, 2]} # v4 := {X, X, X, X, w39, w38, w37, w36} + @{[vslideup_vi $V4, $V8, 4]} # v4 := {w43, w42, w41, w40, w39, w38, w37, w36} + + @{[vsm3c_vi $V0, $V4, 18]} + @{[vslidedown_vi $V4, $V4, 2]} # v4 := {X, X, w43, w42, w41, w40, w39, w38} + @{[vsm3c_vi $V0, $V4, 19]} + + @{[vsm3c_vi $V0, $V8, 20]} + @{[vslidedown_vi $V4, $V8, 2]} # v4 := {X, X, w47, w46, w45, w44, w43, w42} + @{[vsm3c_vi $V0, $V4, 21]} + + @{[vsm3me_vv $V6, $V8, $V6]} # v6 := {w55, w54, w53, w52, w51, w50, w49, w48} + + # Prepare a register with {w51, w50, w49, w48, w47, w46, w45, w44} + @{[vslidedown_vi $V4, $V4, 2]} # v4 := {X, X, X, X, w47, w46, w45, w44} + @{[vslideup_vi $V4, $V6, 4]} # v4 := {w51, w50, w49, w48, w47, w46, w45, w44} + + @{[vsm3c_vi $V0, $V4, 22]} + @{[vslidedown_vi $V4, $V4, 2]} # v4 := {X, X, w51, w50, w49, w48, w47, w46} + @{[vsm3c_vi $V0, $V4, 23]} + + @{[vsm3c_vi $V0, $V6, 24]} + @{[vslidedown_vi $V4, $V6, 2]} # v4 := {X, X, w55, w54, w53, w52, w51, w50} + @{[vsm3c_vi $V0, $V4, 25]} + + @{[vsm3me_vv $V8, $V6, $V8]} # v8 := {w63, w62, w61, w60, w59, w58, w57, w56} + + # Prepare a register with {w59, w58, w57, w56, w55, w54, w53, w52} + @{[vslidedown_vi $V4, $V4, 2]} # v4 := {X, X, X, X, w55, w54, w53, w52} + @{[vslideup_vi $V4, $V8, 4]} # v4 := {w59, w58, w57, w56, w55, w54, w53, w52} + + @{[vsm3c_vi $V0, $V4, 26]} + @{[vslidedown_vi $V4, $V4, 2]} # v4 := {X, X, w59, w58, w57, w56, w55, w54} + @{[vsm3c_vi $V0, $V4, 27]} + + @{[vsm3c_vi $V0, $V8, 28]} + @{[vslidedown_vi $V4, $V8, 2]} # v4 := {X, X, w63, w62, w61, w60, w59, w58} + @{[vsm3c_vi $V0, $V4, 29]} + + @{[vsm3me_vv $V6, $V8, $V6]} # v6 := {w71, w70, w69, w68, w67, w66, w65, w64} + + # Prepare a register with {w67, w66, w65, w64, w63, w62, w61, w60} + @{[vslidedown_vi $V4, $V4, 2]} # v4 := {X, X, X, X, w63, w62, w61, w60} + @{[vslideup_vi $V4, $V6, 4]} # v4 := {w67, w66, w65, w64, w63, w62, w61, w60} + + @{[vsm3c_vi $V0, $V4, 30]} + @{[vslidedown_vi $V4, $V4, 2]} # v4 := {X, X, w67, w66, w65, w64, w63, w62} + @{[vsm3c_vi $V0, $V4, 31]} + + # XOR in the previous state. + @{[vxor_vv $V0, $V0, $V2]} + + bnez $NUM, L_sm3_loop # Check if there are any more block to process +L_sm3_end: + @{[vrev8_v $V0, $V0]} + @{[vse32_v $V0, $CTX]} + ret + +.size ossl_hwsm3_block_data_order_zvksh,.-ossl_hwsm3_block_data_order_zvksh +___ +} + +print $code; + +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/sm4/asm/sm4-armv8.pl b/crypto/openssl/crypto/sm4/asm/sm4-armv8.pl new file mode 100755 index 000000000000..1fe12dead49b --- /dev/null +++ b/crypto/openssl/crypto/sm4/asm/sm4-armv8.pl @@ -0,0 +1,646 @@ +#! /usr/bin/env perl +# Copyright 2022-2025 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +# +# This module implements support for SM4 hw support on aarch64 +# Oct 2021 +# + +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or +die "can't locate arm-xlate.pl"; + +open OUT,"| \"$^X\" $xlate $flavour \"$output\"" + or die "can't call $xlate: $!"; +*STDOUT=*OUT; + +$prefix="sm4_v8"; +my @rks=map("v$_",(0..7)); + +sub rev32() { +my $dst = shift; +my $src = shift; +$code.=<<___; +#ifndef __AARCH64EB__ + rev32 $dst.16b,$src.16b +#endif +___ +} + +sub enc_blk () { +my $data = shift; +$code.=<<___; + sm4e $data.4s,@rks[0].4s + sm4e $data.4s,@rks[1].4s + sm4e $data.4s,@rks[2].4s + sm4e $data.4s,@rks[3].4s + sm4e $data.4s,@rks[4].4s + sm4e $data.4s,@rks[5].4s + sm4e $data.4s,@rks[6].4s + sm4e $data.4s,@rks[7].4s + rev64 $data.4S,$data.4S + ext $data.16b,$data.16b,$data.16b,#8 +___ +} + +sub enc_4blks () { +my $data0 = shift; +my $data1 = shift; +my $data2 = shift; +my $data3 = shift; +$code.=<<___; + sm4e $data0.4s,@rks[0].4s + sm4e $data1.4s,@rks[0].4s + sm4e $data2.4s,@rks[0].4s + sm4e $data3.4s,@rks[0].4s + + sm4e $data0.4s,@rks[1].4s + sm4e $data1.4s,@rks[1].4s + sm4e $data2.4s,@rks[1].4s + sm4e $data3.4s,@rks[1].4s + + sm4e $data0.4s,@rks[2].4s + sm4e $data1.4s,@rks[2].4s + sm4e $data2.4s,@rks[2].4s + sm4e $data3.4s,@rks[2].4s + + sm4e $data0.4s,@rks[3].4s + sm4e $data1.4s,@rks[3].4s + sm4e $data2.4s,@rks[3].4s + sm4e $data3.4s,@rks[3].4s + + sm4e $data0.4s,@rks[4].4s + sm4e $data1.4s,@rks[4].4s + sm4e $data2.4s,@rks[4].4s + sm4e $data3.4s,@rks[4].4s + + sm4e $data0.4s,@rks[5].4s + sm4e $data1.4s,@rks[5].4s + sm4e $data2.4s,@rks[5].4s + sm4e $data3.4s,@rks[5].4s + + sm4e $data0.4s,@rks[6].4s + sm4e $data1.4s,@rks[6].4s + sm4e $data2.4s,@rks[6].4s + sm4e $data3.4s,@rks[6].4s + + sm4e $data0.4s,@rks[7].4s + rev64 $data0.4S,$data0.4S + sm4e $data1.4s,@rks[7].4s + ext $data0.16b,$data0.16b,$data0.16b,#8 + rev64 $data1.4S,$data1.4S + sm4e $data2.4s,@rks[7].4s + ext $data1.16b,$data1.16b,$data1.16b,#8 + rev64 $data2.4S,$data2.4S + sm4e $data3.4s,@rks[7].4s + ext $data2.16b,$data2.16b,$data2.16b,#8 + rev64 $data3.4S,$data3.4S + ext $data3.16b,$data3.16b,$data3.16b,#8 +___ +} + +$code=<<___; +#include "arm_arch.h" +.arch armv8-a+crypto +.text +___ + +{{{ +$code.=<<___; + +.rodata +.type _${prefix}_consts,%object +.align 6 +_${prefix}_consts: +.Lck: + .long 0x00070E15, 0x1C232A31, 0x383F464D, 0x545B6269 + .long 0x70777E85, 0x8C939AA1, 0xA8AFB6BD, 0xC4CBD2D9 + .long 0xE0E7EEF5, 0xFC030A11, 0x181F262D, 0x343B4249 + .long 0x50575E65, 0x6C737A81, 0x888F969D, 0xA4ABB2B9 + .long 0xC0C7CED5, 0xDCE3EAF1, 0xF8FF060D, 0x141B2229 + .long 0x30373E45, 0x4C535A61, 0x686F767D, 0x848B9299 + .long 0xA0A7AEB5, 0xBCC3CAD1, 0xD8DFE6ED, 0xF4FB0209 + .long 0x10171E25, 0x2C333A41, 0x484F565D, 0x646B7279 +.Lfk: + .long 0xa3b1bac6, 0x56aa3350, 0x677d9197, 0xb27022dc +.size _${prefix}_consts,.-_${prefix}_consts +.previous + +___ +}}} + +{{{ +my ($key,$keys)=("x0","x1"); +my ($tmp)=("x2"); +my ($key0,$key1,$key2,$key3,$key4,$key5,$key6,$key7)=map("v$_",(0..7)); +my ($const0,$const1,$const2,$const3,$const4,$const5,$const6,$const7)=map("v$_",(16..23)); +my ($fkconst) = ("v24"); +$code.=<<___; +.globl ${prefix}_set_encrypt_key +.type ${prefix}_set_encrypt_key,%function +.align 5 +${prefix}_set_encrypt_key: + AARCH64_VALID_CALL_TARGET + ld1 {$key0.4s},[$key] + adrp $tmp,.Lfk + add $tmp,$tmp,#:lo12:.Lfk + ld1 {$fkconst.4s},[$tmp] + adrp $tmp,.Lck + add $tmp,$tmp,#:lo12:.Lck + ld1 {$const0.4s,$const1.4s,$const2.4s,$const3.4s},[$tmp],64 +___ + &rev32($key0, $key0); +$code.=<<___; + ld1 {$const4.4s,$const5.4s,$const6.4s,$const7.4s},[$tmp] + eor $key0.16b,$key0.16b,$fkconst.16b; + sm4ekey $key0.4S,$key0.4S,$const0.4S + sm4ekey $key1.4S,$key0.4S,$const1.4S + sm4ekey $key2.4S,$key1.4S,$const2.4S + sm4ekey $key3.4S,$key2.4S,$const3.4S + sm4ekey $key4.4S,$key3.4S,$const4.4S + st1 {$key0.4s,$key1.4s,$key2.4s,$key3.4s},[$keys],64 + sm4ekey $key5.4S,$key4.4S,$const5.4S + sm4ekey $key6.4S,$key5.4S,$const6.4S + sm4ekey $key7.4S,$key6.4S,$const7.4S + st1 {$key4.4s,$key5.4s,$key6.4s,$key7.4s},[$keys] + ret +.size ${prefix}_set_encrypt_key,.-${prefix}_set_encrypt_key +___ +}}} + +{{{ +my ($key,$keys)=("x0","x1"); +my ($tmp)=("x2"); +my ($key7,$key6,$key5,$key4,$key3,$key2,$key1,$key0)=map("v$_",(0..7)); +my ($const0,$const1,$const2,$const3,$const4,$const5,$const6,$const7)=map("v$_",(16..23)); +my ($fkconst) = ("v24"); +$code.=<<___; +.globl ${prefix}_set_decrypt_key +.type ${prefix}_set_decrypt_key,%function +.align 5 +${prefix}_set_decrypt_key: + AARCH64_VALID_CALL_TARGET + ld1 {$key0.4s},[$key] + adrp $tmp,.Lfk + add $tmp,$tmp,#:lo12:.Lfk + ld1 {$fkconst.4s},[$tmp] + adrp $tmp,.Lck + add $tmp,$tmp,#:lo12:.Lck + ld1 {$const0.4s,$const1.4s,$const2.4s,$const3.4s},[$tmp],64 +___ + &rev32($key0, $key0); +$code.=<<___; + ld1 {$const4.4s,$const5.4s,$const6.4s,$const7.4s},[$tmp] + eor $key0.16b, $key0.16b,$fkconst.16b; + sm4ekey $key0.4S,$key0.4S,$const0.4S + sm4ekey $key1.4S,$key0.4S,$const1.4S + sm4ekey $key2.4S,$key1.4S,$const2.4S + rev64 $key0.4s,$key0.4s + rev64 $key1.4s,$key1.4s + ext $key0.16b,$key0.16b,$key0.16b,#8 + ext $key1.16b,$key1.16b,$key1.16b,#8 + sm4ekey $key3.4S,$key2.4S,$const3.4S + sm4ekey $key4.4S,$key3.4S,$const4.4S + rev64 $key2.4s,$key2.4s + rev64 $key3.4s,$key3.4s + ext $key2.16b,$key2.16b,$key2.16b,#8 + ext $key3.16b,$key3.16b,$key3.16b,#8 + sm4ekey $key5.4S,$key4.4S,$const5.4S + sm4ekey $key6.4S,$key5.4S,$const6.4S + rev64 $key4.4s,$key4.4s + rev64 $key5.4s,$key5.4s + ext $key4.16b,$key4.16b,$key4.16b,#8 + ext $key5.16b,$key5.16b,$key5.16b,#8 + sm4ekey $key7.4S,$key6.4S,$const7.4S + rev64 $key6.4s, $key6.4s + rev64 $key7.4s, $key7.4s + ext $key6.16b,$key6.16b,$key6.16b,#8 + ext $key7.16b,$key7.16b,$key7.16b,#8 + st1 {$key7.4s,$key6.4s,$key5.4s,$key4.4s},[$keys],64 + st1 {$key3.4s,$key2.4s,$key1.4s,$key0.4s},[$keys] + ret +.size ${prefix}_set_decrypt_key,.-${prefix}_set_decrypt_key +___ +}}} + +{{{ +sub gen_block () { +my $dir = shift; +my ($inp,$out,$rk)=map("x$_",(0..2)); +my ($data)=("v16"); +$code.=<<___; +.globl ${prefix}_${dir}crypt +.type ${prefix}_${dir}crypt,%function +.align 5 +${prefix}_${dir}crypt: + AARCH64_VALID_CALL_TARGET + ld1 {$data.4s},[$inp] + ld1 {@rks[0].4s,@rks[1].4s,@rks[2].4s,@rks[3].4s},[$rk],64 + ld1 {@rks[4].4s,@rks[5].4s,@rks[6].4s,@rks[7].4s},[$rk] +___ + &rev32($data,$data); + &enc_blk($data); + &rev32($data,$data); +$code.=<<___; + st1 {$data.4s},[$out] + ret +.size ${prefix}_${dir}crypt,.-${prefix}_${dir}crypt +___ +} + +&gen_block("en"); +&gen_block("de"); +}}} + +{{{ +my ($inp,$out,$len,$rk)=map("x$_",(0..3)); +my ($enc) = ("w4"); +my @dat=map("v$_",(16..23)); +$code.=<<___; +.globl ${prefix}_ecb_encrypt +.type ${prefix}_ecb_encrypt,%function +.align 5 +${prefix}_ecb_encrypt: + AARCH64_VALID_CALL_TARGET + ld1 {@rks[0].4s,@rks[1].4s,@rks[2].4s,@rks[3].4s},[$rk],#64 + ld1 {@rks[4].4s,@rks[5].4s,@rks[6].4s,@rks[7].4s},[$rk] +1: + cmp $len,#64 + b.lt 1f + ld1 {@dat[0].4s,@dat[1].4s,@dat[2].4s,@dat[3].4s},[$inp],#64 + cmp $len,#128 + b.lt 2f + ld1 {@dat[4].4s,@dat[5].4s,@dat[6].4s,@dat[7].4s},[$inp],#64 + // 8 blocks +___ + &rev32(@dat[0],@dat[0]); + &rev32(@dat[1],@dat[1]); + &rev32(@dat[2],@dat[2]); + &rev32(@dat[3],@dat[3]); + &rev32(@dat[4],@dat[4]); + &rev32(@dat[5],@dat[5]); + &rev32(@dat[6],@dat[6]); + &rev32(@dat[7],@dat[7]); + &enc_4blks(@dat[0],@dat[1],@dat[2],@dat[3]); + &enc_4blks(@dat[4],@dat[5],@dat[6],@dat[7]); + &rev32(@dat[0],@dat[0]); + &rev32(@dat[1],@dat[1]); + &rev32(@dat[2],@dat[2]); + &rev32(@dat[3],@dat[3]); + &rev32(@dat[4],@dat[4]); + &rev32(@dat[5],@dat[5]); +$code.=<<___; + st1 {@dat[0].4s,@dat[1].4s,@dat[2].4s,@dat[3].4s},[$out],#64 +___ + &rev32(@dat[6],@dat[6]); + &rev32(@dat[7],@dat[7]); +$code.=<<___; + st1 {@dat[4].4s,@dat[5].4s,@dat[6].4s,@dat[7].4s},[$out],#64 + subs $len,$len,#128 + b.gt 1b + ret + // 4 blocks +2: +___ + &rev32(@dat[0],@dat[0]); + &rev32(@dat[1],@dat[1]); + &rev32(@dat[2],@dat[2]); + &rev32(@dat[3],@dat[3]); + &enc_4blks(@dat[0],@dat[1],@dat[2],@dat[3]); + &rev32(@dat[0],@dat[0]); + &rev32(@dat[1],@dat[1]); + &rev32(@dat[2],@dat[2]); + &rev32(@dat[3],@dat[3]); +$code.=<<___; + st1 {@dat[0].4s,@dat[1].4s,@dat[2].4s,@dat[3].4s},[$out],#64 + subs $len,$len,#64 + b.gt 1b +1: + subs $len,$len,#16 + b.lt 1f + ld1 {@dat[0].4s},[$inp],#16 +___ + &rev32(@dat[0],@dat[0]); + &enc_blk(@dat[0]); + &rev32(@dat[0],@dat[0]); +$code.=<<___; + st1 {@dat[0].4s},[$out],#16 + b.ne 1b +1: + ret +.size ${prefix}_ecb_encrypt,.-${prefix}_ecb_encrypt +___ +}}} + +{{{ +my ($inp,$out,$len,$rk,$ivp)=map("x$_",(0..4)); +my ($enc) = ("w5"); +my @dat=map("v$_",(16..23)); +my @in=map("v$_",(24..31)); +my ($ivec) = ("v8"); +$code.=<<___; +.globl ${prefix}_cbc_encrypt +.type ${prefix}_cbc_encrypt,%function +.align 5 +${prefix}_cbc_encrypt: + AARCH64_VALID_CALL_TARGET + stp d8,d9,[sp, #-16]! + + ld1 {@rks[0].4s,@rks[1].4s,@rks[2].4s,@rks[3].4s},[$rk],#64 + ld1 {@rks[4].4s,@rks[5].4s,@rks[6].4s,@rks[7].4s},[$rk] + ld1 {$ivec.4s},[$ivp] + cmp $enc,#0 + b.eq .Ldec +1: + cmp $len, #64 + b.lt 1f + ld1 {@dat[0].4s,@dat[1].4s,@dat[2].4s,@dat[3].4s},[$inp],#64 + eor @dat[0].16b,@dat[0].16b,$ivec.16b +___ + &rev32(@dat[1],@dat[1]); + &rev32(@dat[0],@dat[0]); + &rev32(@dat[2],@dat[2]); + &rev32(@dat[3],@dat[3]); + &enc_blk(@dat[0]); +$code.=<<___; + eor @dat[1].16b,@dat[1].16b,@dat[0].16b +___ + &enc_blk(@dat[1]); + &rev32(@dat[0],@dat[0]); +$code.=<<___; + eor @dat[2].16b,@dat[2].16b,@dat[1].16b +___ + &enc_blk(@dat[2]); + &rev32(@dat[1],@dat[1]); +$code.=<<___; + eor @dat[3].16b,@dat[3].16b,@dat[2].16b +___ + &enc_blk(@dat[3]); + &rev32(@dat[2],@dat[2]); + &rev32(@dat[3],@dat[3]); +$code.=<<___; + mov $ivec.16b,@dat[3].16b + st1 {@dat[0].4s,@dat[1].4s,@dat[2].4s,@dat[3].4s},[$out],#64 + subs $len,$len,#64 + b.ne 1b +1: + subs $len,$len,#16 + b.lt 3f + ld1 {@dat[0].4s},[$inp],#16 + eor $ivec.16b,$ivec.16b,@dat[0].16b +___ + &rev32($ivec,$ivec); + &enc_blk($ivec); + &rev32($ivec,$ivec); +$code.=<<___; + st1 {$ivec.4s},[$out],#16 + b.ne 1b + b 3f +.Ldec: +1: + cmp $len, #64 + b.lt 1f + ld1 {@dat[0].4s,@dat[1].4s,@dat[2].4s,@dat[3].4s},[$inp] + ld1 {@in[0].4s,@in[1].4s,@in[2].4s,@in[3].4s},[$inp],#64 + cmp $len,#128 + b.lt 2f + // 8 blocks mode + ld1 {@dat[4].4s,@dat[5].4s,@dat[6].4s,@dat[7].4s},[$inp] + ld1 {@in[4].4s,@in[5].4s,@in[6].4s,@in[7].4s},[$inp],#64 +___ + &rev32(@dat[0],@dat[0]); + &rev32(@dat[1],@dat[1]); + &rev32(@dat[2],@dat[2]); + &rev32(@dat[3],$dat[3]); + &rev32(@dat[4],@dat[4]); + &rev32(@dat[5],@dat[5]); + &rev32(@dat[6],@dat[6]); + &rev32(@dat[7],$dat[7]); + &enc_4blks(@dat[0],@dat[1],@dat[2],@dat[3]); + &enc_4blks(@dat[4],@dat[5],@dat[6],@dat[7]); + &rev32(@dat[0],@dat[0]); + &rev32(@dat[1],@dat[1]); + &rev32(@dat[2],@dat[2]); + &rev32(@dat[3],@dat[3]); + &rev32(@dat[4],@dat[4]); + &rev32(@dat[5],@dat[5]); + &rev32(@dat[6],@dat[6]); + &rev32(@dat[7],@dat[7]); +$code.=<<___; + eor @dat[0].16b,@dat[0].16b,$ivec.16b + eor @dat[1].16b,@dat[1].16b,@in[0].16b + eor @dat[2].16b,@dat[2].16b,@in[1].16b + mov $ivec.16b,@in[7].16b + eor @dat[3].16b,$dat[3].16b,@in[2].16b + eor @dat[4].16b,$dat[4].16b,@in[3].16b + eor @dat[5].16b,$dat[5].16b,@in[4].16b + eor @dat[6].16b,$dat[6].16b,@in[5].16b + eor @dat[7].16b,$dat[7].16b,@in[6].16b + st1 {@dat[0].4s,@dat[1].4s,@dat[2].4s,@dat[3].4s},[$out],#64 + st1 {@dat[4].4s,@dat[5].4s,@dat[6].4s,@dat[7].4s},[$out],#64 + subs $len,$len,128 + b.gt 1b + b 3f + // 4 blocks mode +2: +___ + &rev32(@dat[0],@dat[0]); + &rev32(@dat[1],@dat[1]); + &rev32(@dat[2],@dat[2]); + &rev32(@dat[3],$dat[3]); + &enc_4blks(@dat[0],@dat[1],@dat[2],@dat[3]); + &rev32(@dat[0],@dat[0]); + &rev32(@dat[1],@dat[1]); + &rev32(@dat[2],@dat[2]); + &rev32(@dat[3],@dat[3]); +$code.=<<___; + eor @dat[0].16b,@dat[0].16b,$ivec.16b + eor @dat[1].16b,@dat[1].16b,@in[0].16b + mov $ivec.16b,@in[3].16b + eor @dat[2].16b,@dat[2].16b,@in[1].16b + eor @dat[3].16b,$dat[3].16b,@in[2].16b + st1 {@dat[0].4s,@dat[1].4s,@dat[2].4s,@dat[3].4s},[$out],#64 + subs $len,$len,#64 + b.gt 1b +1: + subs $len,$len,#16 + b.lt 3f + ld1 {@dat[0].4s},[$inp],#16 + mov @in[0].16b,@dat[0].16b +___ + &rev32(@dat[0],@dat[0]); + &enc_blk(@dat[0]); + &rev32(@dat[0],@dat[0]); +$code.=<<___; + eor @dat[0].16b,@dat[0].16b,$ivec.16b + mov $ivec.16b,@in[0].16b + st1 {@dat[0].4s},[$out],#16 + b.ne 1b +3: + // save back IV + st1 {$ivec.4s},[$ivp] + ldp d8,d9,[sp],#16 + ret +.size ${prefix}_cbc_encrypt,.-${prefix}_cbc_encrypt +___ +}}} + +{{{ +my ($inp,$out,$len,$rk,$ivp)=map("x$_",(0..4)); +my ($ctr)=("w5"); +my @dat=map("v$_",(16..23)); +my @in=map("v$_",(24..31)); +my ($ivec)=("v8"); +$code.=<<___; +.globl ${prefix}_ctr32_encrypt_blocks +.type ${prefix}_ctr32_encrypt_blocks,%function +.align 5 +${prefix}_ctr32_encrypt_blocks: + AARCH64_VALID_CALL_TARGET + stp d8,d9,[sp, #-16]! + + ld1 {$ivec.4s},[$ivp] + ld1 {@rks[0].4s,@rks[1].4s,@rks[2].4s,@rks[3].4s},[$rk],64 + ld1 {@rks[4].4s,@rks[5].4s,@rks[6].4s,@rks[7].4s},[$rk] +___ + &rev32($ivec,$ivec); +$code.=<<___; + mov $ctr,$ivec.s[3] +1: + cmp $len,#4 + b.lt 1f + ld1 {@in[0].4s,@in[1].4s,@in[2].4s,@in[3].4s},[$inp],#64 + mov @dat[0].16b,$ivec.16b + mov @dat[1].16b,$ivec.16b + mov @dat[2].16b,$ivec.16b + mov @dat[3].16b,$ivec.16b + add $ctr,$ctr,#1 + mov $dat[1].s[3],$ctr + add $ctr,$ctr,#1 + mov @dat[2].s[3],$ctr + add $ctr,$ctr,#1 + mov @dat[3].s[3],$ctr + cmp $len,#8 + b.lt 2f + ld1 {@in[4].4s,@in[5].4s,@in[6].4s,@in[7].4s},[$inp],#64 + mov @dat[4].16b,$ivec.16b + mov @dat[5].16b,$ivec.16b + mov @dat[6].16b,$ivec.16b + mov @dat[7].16b,$ivec.16b + add $ctr,$ctr,#1 + mov $dat[4].s[3],$ctr + add $ctr,$ctr,#1 + mov @dat[5].s[3],$ctr + add $ctr,$ctr,#1 + mov @dat[6].s[3],$ctr + add $ctr,$ctr,#1 + mov @dat[7].s[3],$ctr +___ + &enc_4blks(@dat[0],@dat[1],@dat[2],@dat[3]); + &enc_4blks(@dat[4],@dat[5],@dat[6],@dat[7]); + &rev32(@dat[0],@dat[0]); + &rev32(@dat[1],@dat[1]); + &rev32(@dat[2],@dat[2]); + &rev32(@dat[3],@dat[3]); + &rev32(@dat[4],@dat[4]); + &rev32(@dat[5],@dat[5]); + &rev32(@dat[6],@dat[6]); + &rev32(@dat[7],@dat[7]); +$code.=<<___; + eor @dat[0].16b,@dat[0].16b,@in[0].16b + eor @dat[1].16b,@dat[1].16b,@in[1].16b + eor @dat[2].16b,@dat[2].16b,@in[2].16b + eor @dat[3].16b,@dat[3].16b,@in[3].16b + eor @dat[4].16b,@dat[4].16b,@in[4].16b + eor @dat[5].16b,@dat[5].16b,@in[5].16b + eor @dat[6].16b,@dat[6].16b,@in[6].16b + eor @dat[7].16b,@dat[7].16b,@in[7].16b + st1 {@dat[0].4s,@dat[1].4s,@dat[2].4s,@dat[3].4s},[$out],#64 + st1 {@dat[4].4s,@dat[5].4s,@dat[6].4s,@dat[7].4s},[$out],#64 + subs $len,$len,#8 + b.eq 3f + add $ctr,$ctr,#1 + mov $ivec.s[3],$ctr + b 1b +2: +___ + &enc_4blks(@dat[0],@dat[1],@dat[2],@dat[3]); + &rev32(@dat[0],@dat[0]); + &rev32(@dat[1],@dat[1]); + &rev32(@dat[2],@dat[2]); + &rev32(@dat[3],@dat[3]); +$code.=<<___; + eor @dat[0].16b,@dat[0].16b,@in[0].16b + eor @dat[1].16b,@dat[1].16b,@in[1].16b + eor @dat[2].16b,@dat[2].16b,@in[2].16b + eor @dat[3].16b,@dat[3].16b,@in[3].16b + st1 {@dat[0].4s,@dat[1].4s,@dat[2].4s,@dat[3].4s},[$out],#64 + subs $len,$len,#4 + b.eq 3f + add $ctr,$ctr,#1 + mov $ivec.s[3],$ctr + b 1b +1: + subs $len,$len,#1 + b.lt 3f + mov $dat[0].16b,$ivec.16b + ld1 {@in[0].4s},[$inp],#16 +___ + &enc_blk(@dat[0]); + &rev32(@dat[0],@dat[0]); +$code.=<<___; + eor $dat[0].16b,$dat[0].16b,@in[0].16b + st1 {$dat[0].4s},[$out],#16 + b.eq 3f + add $ctr,$ctr,#1 + mov $ivec.s[3],$ctr + b 1b +3: + ldp d8,d9,[sp],#16 + ret +.size ${prefix}_ctr32_encrypt_blocks,.-${prefix}_ctr32_encrypt_blocks +___ +}}} +######################################## +{ my %opcode = ( + "sm4e" => 0xcec08400, + "sm4ekey" => 0xce60c800); + + sub unsm4 { + my ($mnemonic,$arg)=@_; + + $arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv]([0-9]+))?/o + && + sprintf ".inst\t0x%08x\t//%s %s", + $opcode{$mnemonic}|$1|($2<<5)|($3<<16), + $mnemonic,$arg; + } +} + +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/\/\// and !/^$/); + print; +} +close SELF; + +foreach(split("\n",$code)) { + s/\`([^\`]*)\`/eval($1)/ge; + + s/\b(sm4\w+)\s+([qv].*)/unsm4($1,$2)/ge; + print $_,"\n"; +} + +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/sm4/asm/sm4-riscv64-zvksed.pl b/crypto/openssl/crypto/sm4/asm/sm4-riscv64-zvksed.pl new file mode 100644 index 000000000000..0734e5fa4c70 --- /dev/null +++ b/crypto/openssl/crypto/sm4/asm/sm4-riscv64-zvksed.pl @@ -0,0 +1,298 @@ +#! /usr/bin/env perl +# This file is dual-licensed, meaning that you can use it under your +# choice of either of the following two licenses: +# +# Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You can obtain +# a copy in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html +# +# or +# +# Copyright (c) 2023, Christoph Müllner +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# The generated code of this file depends on the following RISC-V extensions: +# - RV64I +# - RISC-V Vector ('V') with VLEN >= 128 +# - RISC-V Vector Cryptography Bit-manipulation extension ('Zvkb') +# - RISC-V Vector SM4 Block Cipher extension ('Zvksed') + +use strict; +use warnings; + +use FindBin qw($Bin); +use lib "$Bin"; +use lib "$Bin/../../perlasm"; +use riscv; + +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$output and open STDOUT,">$output"; + +my $code=<<___; +.text +___ + +#### +# int rv64i_zvksed_sm4_set_encrypt_key(const unsigned char *userKey, +# SM4_KEY *key); +# +{ +my ($ukey,$keys,$fk)=("a0","a1","t0"); +my ($vukey,$vfk,$vk0,$vk1,$vk2,$vk3,$vk4,$vk5,$vk6,$vk7)=("v1","v2","v3","v4","v5","v6","v7","v8","v9","v10"); +$code .= <<___; +.p2align 3 +.globl rv64i_zvksed_sm4_set_encrypt_key +.type rv64i_zvksed_sm4_set_encrypt_key,\@function +rv64i_zvksed_sm4_set_encrypt_key: + @{[vsetivli__x0_4_e32_m1_tu_mu]} + + # Load the user key + @{[vle32_v $vukey, $ukey]} + @{[vrev8_v $vukey, $vukey]} + + # Load the FK. + la $fk, FK + @{[vle32_v $vfk, $fk]} + + # Generate round keys. + @{[vxor_vv $vukey, $vukey, $vfk]} + @{[vsm4k_vi $vk0, $vukey, 0]} # rk[0:3] + @{[vsm4k_vi $vk1, $vk0, 1]} # rk[4:7] + @{[vsm4k_vi $vk2, $vk1, 2]} # rk[8:11] + @{[vsm4k_vi $vk3, $vk2, 3]} # rk[12:15] + @{[vsm4k_vi $vk4, $vk3, 4]} # rk[16:19] + @{[vsm4k_vi $vk5, $vk4, 5]} # rk[20:23] + @{[vsm4k_vi $vk6, $vk5, 6]} # rk[24:27] + @{[vsm4k_vi $vk7, $vk6, 7]} # rk[28:31] + + # Store round keys + @{[vse32_v $vk0, $keys]} # rk[0:3] + addi $keys, $keys, 16 + @{[vse32_v $vk1, $keys]} # rk[4:7] + addi $keys, $keys, 16 + @{[vse32_v $vk2, $keys]} # rk[8:11] + addi $keys, $keys, 16 + @{[vse32_v $vk3, $keys]} # rk[12:15] + addi $keys, $keys, 16 + @{[vse32_v $vk4, $keys]} # rk[16:19] + addi $keys, $keys, 16 + @{[vse32_v $vk5, $keys]} # rk[20:23] + addi $keys, $keys, 16 + @{[vse32_v $vk6, $keys]} # rk[24:27] + addi $keys, $keys, 16 + @{[vse32_v $vk7, $keys]} # rk[28:31] + + li a0, 1 + ret +.size rv64i_zvksed_sm4_set_encrypt_key,.-rv64i_zvksed_sm4_set_encrypt_key +___ +} + +#### +# int rv64i_zvksed_sm4_set_decrypt_key(const unsigned char *userKey, +# SM4_KEY *key); +# +{ +my ($ukey,$keys,$fk,$stride)=("a0","a1","t0","t1"); +my ($vukey,$vfk,$vk0,$vk1,$vk2,$vk3,$vk4,$vk5,$vk6,$vk7)=("v1","v2","v3","v4","v5","v6","v7","v8","v9","v10"); +$code .= <<___; +.p2align 3 +.globl rv64i_zvksed_sm4_set_decrypt_key +.type rv64i_zvksed_sm4_set_decrypt_key,\@function +rv64i_zvksed_sm4_set_decrypt_key: + @{[vsetivli__x0_4_e32_m1_tu_mu]} + + # Load the user key + @{[vle32_v $vukey, $ukey]} + @{[vrev8_v $vukey, $vukey]} + + # Load the FK. + la $fk, FK + @{[vle32_v $vfk, $fk]} + + # Generate round keys. + @{[vxor_vv $vukey, $vukey, $vfk]} + @{[vsm4k_vi $vk0, $vukey, 0]} # rk[0:3] + @{[vsm4k_vi $vk1, $vk0, 1]} # rk[4:7] + @{[vsm4k_vi $vk2, $vk1, 2]} # rk[8:11] + @{[vsm4k_vi $vk3, $vk2, 3]} # rk[12:15] + @{[vsm4k_vi $vk4, $vk3, 4]} # rk[16:19] + @{[vsm4k_vi $vk5, $vk4, 5]} # rk[20:23] + @{[vsm4k_vi $vk6, $vk5, 6]} # rk[24:27] + @{[vsm4k_vi $vk7, $vk6, 7]} # rk[28:31] + + # Store round keys in reverse order + addi $keys, $keys, 12 + li $stride, -4 + @{[vsse32_v $vk7, $keys, $stride]} # rk[31:28] + addi $keys, $keys, 16 + @{[vsse32_v $vk6, $keys, $stride]} # rk[27:24] + addi $keys, $keys, 16 + @{[vsse32_v $vk5, $keys, $stride]} # rk[23:20] + addi $keys, $keys, 16 + @{[vsse32_v $vk4, $keys, $stride]} # rk[19:16] + addi $keys, $keys, 16 + @{[vsse32_v $vk3, $keys, $stride]} # rk[15:12] + addi $keys, $keys, 16 + @{[vsse32_v $vk2, $keys, $stride]} # rk[11:8] + addi $keys, $keys, 16 + @{[vsse32_v $vk1, $keys, $stride]} # rk[7:4] + addi $keys, $keys, 16 + @{[vsse32_v $vk0, $keys, $stride]} # rk[3:0] + + li a0, 1 + ret +.size rv64i_zvksed_sm4_set_decrypt_key,.-rv64i_zvksed_sm4_set_decrypt_key +___ +} + +#### +# void rv64i_zvksed_sm4_encrypt(const unsigned char *in, unsigned char *out, +# const SM4_KEY *key); +# +{ +my ($in,$out,$keys,$stride)=("a0","a1","a2","t0"); +my ($vdata,$vk0,$vk1,$vk2,$vk3,$vk4,$vk5,$vk6,$vk7,$vgen)=("v1","v2","v3","v4","v5","v6","v7","v8","v9","v10"); +$code .= <<___; +.p2align 3 +.globl rv64i_zvksed_sm4_encrypt +.type rv64i_zvksed_sm4_encrypt,\@function +rv64i_zvksed_sm4_encrypt: + @{[vsetivli__x0_4_e32_m1_tu_mu]} + + # Order of elements was adjusted in set_encrypt_key() + @{[vle32_v $vk0, $keys]} # rk[0:3] + addi $keys, $keys, 16 + @{[vle32_v $vk1, $keys]} # rk[4:7] + addi $keys, $keys, 16 + @{[vle32_v $vk2, $keys]} # rk[8:11] + addi $keys, $keys, 16 + @{[vle32_v $vk3, $keys]} # rk[12:15] + addi $keys, $keys, 16 + @{[vle32_v $vk4, $keys]} # rk[16:19] + addi $keys, $keys, 16 + @{[vle32_v $vk5, $keys]} # rk[20:23] + addi $keys, $keys, 16 + @{[vle32_v $vk6, $keys]} # rk[24:27] + addi $keys, $keys, 16 + @{[vle32_v $vk7, $keys]} # rk[28:31] + + # Load input data + @{[vle32_v $vdata, $in]} + @{[vrev8_v $vdata, $vdata]} + + # Encrypt with all keys + @{[vsm4r_vs $vdata, $vk0]} + @{[vsm4r_vs $vdata, $vk1]} + @{[vsm4r_vs $vdata, $vk2]} + @{[vsm4r_vs $vdata, $vk3]} + @{[vsm4r_vs $vdata, $vk4]} + @{[vsm4r_vs $vdata, $vk5]} + @{[vsm4r_vs $vdata, $vk6]} + @{[vsm4r_vs $vdata, $vk7]} + + # Save the ciphertext (in reverse element order) + @{[vrev8_v $vdata, $vdata]} + li $stride, -4 + addi $out, $out, 12 + @{[vsse32_v $vdata, $out, $stride]} + + ret +.size rv64i_zvksed_sm4_encrypt,.-rv64i_zvksed_sm4_encrypt +___ +} + +#### +# void rv64i_zvksed_sm4_decrypt(const unsigned char *in, unsigned char *out, +# const SM4_KEY *key); +# +{ +my ($in,$out,$keys,$stride)=("a0","a1","a2","t0"); +my ($vdata,$vk0,$vk1,$vk2,$vk3,$vk4,$vk5,$vk6,$vk7,$vgen)=("v1","v2","v3","v4","v5","v6","v7","v8","v9","v10"); +$code .= <<___; +.p2align 3 +.globl rv64i_zvksed_sm4_decrypt +.type rv64i_zvksed_sm4_decrypt,\@function +rv64i_zvksed_sm4_decrypt: + @{[vsetivli__x0_4_e32_m1_tu_mu]} + + # Order of elements was adjusted in set_decrypt_key() + @{[vle32_v $vk7, $keys]} # rk[31:28] + addi $keys, $keys, 16 + @{[vle32_v $vk6, $keys]} # rk[27:24] + addi $keys, $keys, 16 + @{[vle32_v $vk5, $keys]} # rk[23:20] + addi $keys, $keys, 16 + @{[vle32_v $vk4, $keys]} # rk[19:16] + addi $keys, $keys, 16 + @{[vle32_v $vk3, $keys]} # rk[15:11] + addi $keys, $keys, 16 + @{[vle32_v $vk2, $keys]} # rk[11:8] + addi $keys, $keys, 16 + @{[vle32_v $vk1, $keys]} # rk[7:4] + addi $keys, $keys, 16 + @{[vle32_v $vk0, $keys]} # rk[3:0] + + # Load input data + @{[vle32_v $vdata, $in]} + @{[vrev8_v $vdata, $vdata]} + + # Encrypt with all keys + @{[vsm4r_vs $vdata, $vk7]} + @{[vsm4r_vs $vdata, $vk6]} + @{[vsm4r_vs $vdata, $vk5]} + @{[vsm4r_vs $vdata, $vk4]} + @{[vsm4r_vs $vdata, $vk3]} + @{[vsm4r_vs $vdata, $vk2]} + @{[vsm4r_vs $vdata, $vk1]} + @{[vsm4r_vs $vdata, $vk0]} + + # Save the ciphertext (in reverse element order) + @{[vrev8_v $vdata, $vdata]} + li $stride, -4 + addi $out, $out, 12 + @{[vsse32_v $vdata, $out, $stride]} + + ret +.size rv64i_zvksed_sm4_decrypt,.-rv64i_zvksed_sm4_decrypt +___ +} + +$code .= <<___; +# Family Key (little-endian 32-bit chunks) +.p2align 3 +FK: + .word 0xA3B1BAC6, 0x56AA3350, 0x677D9197, 0xB27022DC +.size FK,.-FK +___ + +print $code; + +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/sm4/asm/vpsm4-armv8.pl b/crypto/openssl/crypto/sm4/asm/vpsm4-armv8.pl new file mode 100755 index 000000000000..c60ace15366f --- /dev/null +++ b/crypto/openssl/crypto/sm4/asm/vpsm4-armv8.pl @@ -0,0 +1,1587 @@ +#! /usr/bin/env perl +# Copyright 2020-2025 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +# +# This module implements SM4 with ASIMD on aarch64 +# +# Feb 2022 +# + +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or +die "can't locate arm-xlate.pl"; + +open OUT,"| \"$^X\" $xlate $flavour \"$output\"" + or die "can't call $xlate: $!"; +*STDOUT=*OUT; + +$prefix="vpsm4"; +my @vtmp=map("v$_",(0..3)); +my @qtmp=map("q$_",(0..3)); +my @data=map("v$_",(4..7)); +my @datax=map("v$_",(8..11)); +my ($rk0,$rk1)=("v12","v13"); +my ($rka,$rkb)=("v14","v15"); +my @vtmpx=map("v$_",(12..15)); +my @sbox=map("v$_",(16..31)); +my ($inp,$outp,$blocks,$rks)=("x0","x1","w2","x3"); +my ($tmpw,$tmp,$wtmp0,$wtmp1,$wtmp2)=("w6","x6","w7","w8","w9"); +my ($xtmp1,$xtmp2)=("x8","x9"); +my ($ptr,$counter)=("x10","w11"); +my ($word0,$word1,$word2,$word3)=("w12","w13","w14","w15"); + +sub rev32() { + my $dst = shift; + my $src = shift; + + if ($src and ("$src" ne "$dst")) { +$code.=<<___; +#ifndef __AARCH64EB__ + rev32 $dst.16b,$src.16b +#else + mov $dst.16b,$src.16b +#endif +___ + } else { +$code.=<<___; +#ifndef __AARCH64EB__ + rev32 $dst.16b,$dst.16b +#endif +___ + } +} + +sub rev32_armeb() { + my $dst = shift; + my $src = shift; + + if ($src and ("$src" ne "$dst")) { +$code.=<<___; +#ifdef __AARCH64EB__ + rev32 $dst.16b,$src.16b +#else + mov $dst.16b,$src.16b +#endif +___ + } else { +$code.=<<___; +#ifdef __AARCH64EB__ + rev32 $dst.16b,$dst.16b +#endif +___ + } +} + +sub rbit() { + my $dst = shift; + my $src = shift; + my $std = shift; + + if ($src and ("$src" ne "$dst")) { + if ($std eq "_gb") { +$code.=<<___; + rbit $dst.16b,$src.16b +___ + } else { +$code.=<<___; + mov $dst.16b,$src.16b +___ + } + } else { + if ($std eq "_gb") { +$code.=<<___; + rbit $dst.16b,$src.16b +___ + } + } +} + +sub transpose() { + my ($dat0,$dat1,$dat2,$dat3,$vt0,$vt1,$vt2,$vt3) = @_; + +$code.=<<___; + zip1 $vt0.4s,$dat0.4s,$dat1.4s + zip2 $vt1.4s,$dat0.4s,$dat1.4s + zip1 $vt2.4s,$dat2.4s,$dat3.4s + zip2 $vt3.4s,$dat2.4s,$dat3.4s + zip1 $dat0.2d,$vt0.2d,$vt2.2d + zip2 $dat1.2d,$vt0.2d,$vt2.2d + zip1 $dat2.2d,$vt1.2d,$vt3.2d + zip2 $dat3.2d,$vt1.2d,$vt3.2d +___ +} + +# sbox operations for 4-lane of words +sub sbox() { + my $dat = shift; + +$code.=<<___; + movi @vtmp[0].16b,#64 + movi @vtmp[1].16b,#128 + movi @vtmp[2].16b,#192 + sub @vtmp[0].16b,$dat.16b,@vtmp[0].16b + sub @vtmp[1].16b,$dat.16b,@vtmp[1].16b + sub @vtmp[2].16b,$dat.16b,@vtmp[2].16b + tbl $dat.16b,{@sbox[0].16b,@sbox[1].16b,@sbox[2].16b,@sbox[3].16b},$dat.16b + tbl @vtmp[0].16b,{@sbox[4].16b,@sbox[5].16b,@sbox[6].16b,@sbox[7].16b},@vtmp[0].16b + tbl @vtmp[1].16b,{@sbox[8].16b,@sbox[9].16b,@sbox[10].16b,@sbox[11].16b},@vtmp[1].16b + tbl @vtmp[2].16b,{@sbox[12].16b,@sbox[13].16b,@sbox[14].16b,@sbox[15].16b},@vtmp[2].16b + add @vtmp[0].2d,@vtmp[0].2d,@vtmp[1].2d + add @vtmp[2].2d,@vtmp[2].2d,$dat.2d + add $dat.2d,@vtmp[0].2d,@vtmp[2].2d + + ushr @vtmp[0].4s,$dat.4s,32-2 + sli @vtmp[0].4s,$dat.4s,2 + ushr @vtmp[2].4s,$dat.4s,32-10 + eor @vtmp[1].16b,@vtmp[0].16b,$dat.16b + sli @vtmp[2].4s,$dat.4s,10 + eor @vtmp[1].16b,@vtmp[2].16b,$vtmp[1].16b + ushr @vtmp[0].4s,$dat.4s,32-18 + sli @vtmp[0].4s,$dat.4s,18 + ushr @vtmp[2].4s,$dat.4s,32-24 + eor @vtmp[1].16b,@vtmp[0].16b,$vtmp[1].16b + sli @vtmp[2].4s,$dat.4s,24 + eor $dat.16b,@vtmp[2].16b,@vtmp[1].16b +___ +} + +# sbox operation for 8-lane of words +sub sbox_double() { + my $dat = shift; + my $datx = shift; + +$code.=<<___; + movi @vtmp[3].16b,#64 + sub @vtmp[0].16b,$dat.16b,@vtmp[3].16b + sub @vtmp[1].16b,@vtmp[0].16b,@vtmp[3].16b + sub @vtmp[2].16b,@vtmp[1].16b,@vtmp[3].16b + tbl $dat.16b,{@sbox[0].16b,@sbox[1].16b,@sbox[2].16b,@sbox[3].16b},$dat.16b + tbl @vtmp[0].16b,{@sbox[4].16b,@sbox[5].16b,@sbox[6].16b,@sbox[7].16b},@vtmp[0].16b + tbl @vtmp[1].16b,{@sbox[8].16b,@sbox[9].16b,@sbox[10].16b,@sbox[11].16b},@vtmp[1].16b + tbl @vtmp[2].16b,{@sbox[12].16b,@sbox[13].16b,@sbox[14].16b,@sbox[15].16b},@vtmp[2].16b + add @vtmp[1].2d,@vtmp[0].2d,@vtmp[1].2d + add $dat.2d,@vtmp[2].2d,$dat.2d + add $dat.2d,@vtmp[1].2d,$dat.2d + + sub @vtmp[0].16b,$datx.16b,@vtmp[3].16b + sub @vtmp[1].16b,@vtmp[0].16b,@vtmp[3].16b + sub @vtmp[2].16b,@vtmp[1].16b,@vtmp[3].16b + tbl $datx.16b,{@sbox[0].16b,@sbox[1].16b,@sbox[2].16b,@sbox[3].16b},$datx.16b + tbl @vtmp[0].16b,{@sbox[4].16b,@sbox[5].16b,@sbox[6].16b,@sbox[7].16b},@vtmp[0].16b + tbl @vtmp[1].16b,{@sbox[8].16b,@sbox[9].16b,@sbox[10].16b,@sbox[11].16b},@vtmp[1].16b + tbl @vtmp[2].16b,{@sbox[12].16b,@sbox[13].16b,@sbox[14].16b,@sbox[15].16b},@vtmp[2].16b + add @vtmp[1].2d,@vtmp[0].2d,@vtmp[1].2d + add $datx.2d,@vtmp[2].2d,$datx.2d + add $datx.2d,@vtmp[1].2d,$datx.2d + + ushr @vtmp[0].4s,$dat.4s,32-2 + sli @vtmp[0].4s,$dat.4s,2 + ushr @vtmp[2].4s,$datx.4s,32-2 + eor @vtmp[1].16b,@vtmp[0].16b,$dat.16b + sli @vtmp[2].4s,$datx.4s,2 + + ushr @vtmp[0].4s,$dat.4s,32-10 + eor @vtmp[3].16b,@vtmp[2].16b,$datx.16b + sli @vtmp[0].4s,$dat.4s,10 + ushr @vtmp[2].4s,$datx.4s,32-10 + eor @vtmp[1].16b,@vtmp[0].16b,$vtmp[1].16b + sli @vtmp[2].4s,$datx.4s,10 + + ushr @vtmp[0].4s,$dat.4s,32-18 + eor @vtmp[3].16b,@vtmp[2].16b,$vtmp[3].16b + sli @vtmp[0].4s,$dat.4s,18 + ushr @vtmp[2].4s,$datx.4s,32-18 + eor @vtmp[1].16b,@vtmp[0].16b,$vtmp[1].16b + sli @vtmp[2].4s,$datx.4s,18 + + ushr @vtmp[0].4s,$dat.4s,32-24 + eor @vtmp[3].16b,@vtmp[2].16b,$vtmp[3].16b + sli @vtmp[0].4s,$dat.4s,24 + ushr @vtmp[2].4s,$datx.4s,32-24 + eor $dat.16b,@vtmp[0].16b,@vtmp[1].16b + sli @vtmp[2].4s,$datx.4s,24 + eor $datx.16b,@vtmp[2].16b,@vtmp[3].16b +___ +} + +# sbox operation for one single word +sub sbox_1word () { + my $word = shift; + +$code.=<<___; + movi @vtmp[1].16b,#64 + movi @vtmp[2].16b,#128 + movi @vtmp[3].16b,#192 + mov @vtmp[0].s[0],$word + + sub @vtmp[1].16b,@vtmp[0].16b,@vtmp[1].16b + sub @vtmp[2].16b,@vtmp[0].16b,@vtmp[2].16b + sub @vtmp[3].16b,@vtmp[0].16b,@vtmp[3].16b + + tbl @vtmp[0].16b,{@sbox[0].16b,@sbox[1].16b,@sbox[2].16b,@sbox[3].16b},@vtmp[0].16b + tbl @vtmp[1].16b,{@sbox[4].16b,@sbox[5].16b,@sbox[6].16b,@sbox[7].16b},@vtmp[1].16b + tbl @vtmp[2].16b,{@sbox[8].16b,@sbox[9].16b,@sbox[10].16b,@sbox[11].16b},@vtmp[2].16b + tbl @vtmp[3].16b,{@sbox[12].16b,@sbox[13].16b,@sbox[14].16b,@sbox[15].16b},@vtmp[3].16b + + mov $word,@vtmp[0].s[0] + mov $wtmp0,@vtmp[1].s[0] + mov $wtmp2,@vtmp[2].s[0] + add $wtmp0,$word,$wtmp0 + mov $word,@vtmp[3].s[0] + add $wtmp0,$wtmp0,$wtmp2 + add $wtmp0,$wtmp0,$word + + eor $word,$wtmp0,$wtmp0,ror #32-2 + eor $word,$word,$wtmp0,ror #32-10 + eor $word,$word,$wtmp0,ror #32-18 + eor $word,$word,$wtmp0,ror #32-24 +___ +} + +# sm4 for one block of data, in scalar registers word0/word1/word2/word3 +sub sm4_1blk () { + my $kptr = shift; + +$code.=<<___; + ldp $wtmp0,$wtmp1,[$kptr],8 + // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) + eor $tmpw,$word2,$word3 + eor $wtmp2,$wtmp0,$word1 + eor $tmpw,$tmpw,$wtmp2 +___ + &sbox_1word($tmpw); +$code.=<<___; + eor $word0,$word0,$tmpw + // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) + eor $tmpw,$word2,$word3 + eor $wtmp2,$word0,$wtmp1 + eor $tmpw,$tmpw,$wtmp2 +___ + &sbox_1word($tmpw); +$code.=<<___; + ldp $wtmp0,$wtmp1,[$kptr],8 + eor $word1,$word1,$tmpw + // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) + eor $tmpw,$word0,$word1 + eor $wtmp2,$wtmp0,$word3 + eor $tmpw,$tmpw,$wtmp2 +___ + &sbox_1word($tmpw); +$code.=<<___; + eor $word2,$word2,$tmpw + // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) + eor $tmpw,$word0,$word1 + eor $wtmp2,$word2,$wtmp1 + eor $tmpw,$tmpw,$wtmp2 +___ + &sbox_1word($tmpw); +$code.=<<___; + eor $word3,$word3,$tmpw +___ +} + +# sm4 for 4-lanes of data, in neon registers data0/data1/data2/data3 +sub sm4_4blks () { + my $kptr = shift; + +$code.=<<___; + ldp $wtmp0,$wtmp1,[$kptr],8 + dup $rk0.4s,$wtmp0 + dup $rk1.4s,$wtmp1 + + // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) + eor $rka.16b,@data[2].16b,@data[3].16b + eor $rk0.16b,@data[1].16b,$rk0.16b + eor $rk0.16b,$rka.16b,$rk0.16b +___ + &sbox($rk0); +$code.=<<___; + eor @data[0].16b,@data[0].16b,$rk0.16b + + // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) + eor $rka.16b,$rka.16b,@data[0].16b + eor $rk1.16b,$rka.16b,$rk1.16b +___ + &sbox($rk1); +$code.=<<___; + ldp $wtmp0,$wtmp1,[$kptr],8 + eor @data[1].16b,@data[1].16b,$rk1.16b + + dup $rk0.4s,$wtmp0 + dup $rk1.4s,$wtmp1 + + // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) + eor $rka.16b,@data[0].16b,@data[1].16b + eor $rk0.16b,@data[3].16b,$rk0.16b + eor $rk0.16b,$rka.16b,$rk0.16b +___ + &sbox($rk0); +$code.=<<___; + eor @data[2].16b,@data[2].16b,$rk0.16b + + // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) + eor $rka.16b,$rka.16b,@data[2].16b + eor $rk1.16b,$rka.16b,$rk1.16b +___ + &sbox($rk1); +$code.=<<___; + eor @data[3].16b,@data[3].16b,$rk1.16b +___ +} + +# sm4 for 8 lanes of data, in neon registers +# data0/data1/data2/data3 datax0/datax1/datax2/datax3 +sub sm4_8blks () { + my $kptr = shift; + +$code.=<<___; + ldp $wtmp0,$wtmp1,[$kptr],8 + // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) + dup $rk0.4s,$wtmp0 + eor $rka.16b,@data[2].16b,@data[3].16b + eor $rkb.16b,@datax[2].16b,@datax[3].16b + eor @vtmp[0].16b,@data[1].16b,$rk0.16b + eor @vtmp[1].16b,@datax[1].16b,$rk0.16b + eor $rk0.16b,$rka.16b,@vtmp[0].16b + eor $rk1.16b,$rkb.16b,@vtmp[1].16b +___ + &sbox_double($rk0,$rk1); +$code.=<<___; + eor @data[0].16b,@data[0].16b,$rk0.16b + eor @datax[0].16b,@datax[0].16b,$rk1.16b + + // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) + dup $rk1.4s,$wtmp1 + eor $rka.16b,$rka.16b,@data[0].16b + eor $rkb.16b,$rkb.16b,@datax[0].16b + eor $rk0.16b,$rka.16b,$rk1.16b + eor $rk1.16b,$rkb.16b,$rk1.16b +___ + &sbox_double($rk0,$rk1); +$code.=<<___; + ldp $wtmp0,$wtmp1,[$kptr],8 + eor @data[1].16b,@data[1].16b,$rk0.16b + eor @datax[1].16b,@datax[1].16b,$rk1.16b + + // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) + dup $rk0.4s,$wtmp0 + eor $rka.16b,@data[0].16b,@data[1].16b + eor $rkb.16b,@datax[0].16b,@datax[1].16b + eor @vtmp[0].16b,@data[3].16b,$rk0.16b + eor @vtmp[1].16b,@datax[3].16b,$rk0.16b + eor $rk0.16b,$rka.16b,@vtmp[0].16b + eor $rk1.16b,$rkb.16b,@vtmp[1].16b +___ + &sbox_double($rk0,$rk1); +$code.=<<___; + eor @data[2].16b,@data[2].16b,$rk0.16b + eor @datax[2].16b,@datax[2].16b,$rk1.16b + + // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) + dup $rk1.4s,$wtmp1 + eor $rka.16b,$rka.16b,@data[2].16b + eor $rkb.16b,$rkb.16b,@datax[2].16b + eor $rk0.16b,$rka.16b,$rk1.16b + eor $rk1.16b,$rkb.16b,$rk1.16b +___ + &sbox_double($rk0,$rk1); +$code.=<<___; + eor @data[3].16b,@data[3].16b,$rk0.16b + eor @datax[3].16b,@datax[3].16b,$rk1.16b +___ +} + +sub encrypt_1blk_norev() { + my $dat = shift; + +$code.=<<___; + mov $ptr,$rks + mov $counter,#8 + mov $word0,$dat.s[0] + mov $word1,$dat.s[1] + mov $word2,$dat.s[2] + mov $word3,$dat.s[3] +10: +___ + &sm4_1blk($ptr); +$code.=<<___; + subs $counter,$counter,#1 + b.ne 10b + mov $dat.s[0],$word3 + mov $dat.s[1],$word2 + mov $dat.s[2],$word1 + mov $dat.s[3],$word0 +___ +} + +sub encrypt_1blk() { + my $dat = shift; + + &encrypt_1blk_norev($dat); + &rev32($dat,$dat); +} + +sub encrypt_4blks() { +$code.=<<___; + mov $ptr,$rks + mov $counter,#8 +10: +___ + &sm4_4blks($ptr); +$code.=<<___; + subs $counter,$counter,#1 + b.ne 10b +___ + &rev32(@vtmp[3],@data[0]); + &rev32(@vtmp[2],@data[1]); + &rev32(@vtmp[1],@data[2]); + &rev32(@vtmp[0],@data[3]); +} + +sub encrypt_8blks() { +$code.=<<___; + mov $ptr,$rks + mov $counter,#8 +10: +___ + &sm4_8blks($ptr); +$code.=<<___; + subs $counter,$counter,#1 + b.ne 10b +___ + &rev32(@vtmp[3],@data[0]); + &rev32(@vtmp[2],@data[1]); + &rev32(@vtmp[1],@data[2]); + &rev32(@vtmp[0],@data[3]); + &rev32(@data[3],@datax[0]); + &rev32(@data[2],@datax[1]); + &rev32(@data[1],@datax[2]); + &rev32(@data[0],@datax[3]); +} + +sub load_sbox () { + my $data = shift; + +$code.=<<___; + adrp $ptr,.Lsbox + add $ptr,$ptr,#:lo12:.Lsbox + ld1 {@sbox[0].16b,@sbox[1].16b,@sbox[2].16b,@sbox[3].16b},[$ptr],#64 + ld1 {@sbox[4].16b,@sbox[5].16b,@sbox[6].16b,@sbox[7].16b},[$ptr],#64 + ld1 {@sbox[8].16b,@sbox[9].16b,@sbox[10].16b,@sbox[11].16b},[$ptr],#64 + ld1 {@sbox[12].16b,@sbox[13].16b,@sbox[14].16b,@sbox[15].16b},[$ptr] +___ +} + + +sub mov_reg_to_vec() { + my $src0 = shift; + my $src1 = shift; + my $desv = shift; +$code.=<<___; + mov $desv.d[0],$src0 + mov $desv.d[1],$src1 +___ + &rev32_armeb($desv,$desv); +} + +sub mov_vec_to_reg() { + my $srcv = shift; + my $des0 = shift; + my $des1 = shift; +$code.=<<___; + mov $des0,$srcv.d[0] + mov $des1,$srcv.d[1] +___ +} + +sub compute_tweak() { + my $src0 = shift; + my $src1 = shift; + my $des0 = shift; + my $des1 = shift; +$code.=<<___; + mov $wtmp0,0x87 + extr $xtmp2,$src1,$src1,#32 + extr $des1,$src1,$src0,#63 + and $wtmp1,$wtmp0,$wtmp2,asr#31 + eor $des0,$xtmp1,$src0,lsl#1 +___ +} + +sub compute_tweak_vec() { + my $src = shift; + my $des = shift; + my $std = shift; + &rbit(@vtmp[2],$src,$std); +$code.=<<___; + adrp $ptr,.Lxts_magic + ldr @qtmp[0], [$ptr, #:lo12:.Lxts_magic] + shl $des.16b, @vtmp[2].16b, #1 + ext @vtmp[1].16b, @vtmp[2].16b, @vtmp[2].16b,#15 + ushr @vtmp[1].16b, @vtmp[1].16b, #7 + mul @vtmp[1].16b, @vtmp[1].16b, @vtmp[0].16b + eor $des.16b, $des.16b, @vtmp[1].16b +___ + &rbit($des,$des,$std); +} + +$code=<<___; +#include "arm_arch.h" +.arch armv8-a +.text + +.rodata +.type _${prefix}_consts,%object +.align 7 +_${prefix}_consts: +.Lsbox: + .byte 0xD6,0x90,0xE9,0xFE,0xCC,0xE1,0x3D,0xB7,0x16,0xB6,0x14,0xC2,0x28,0xFB,0x2C,0x05 + .byte 0x2B,0x67,0x9A,0x76,0x2A,0xBE,0x04,0xC3,0xAA,0x44,0x13,0x26,0x49,0x86,0x06,0x99 + .byte 0x9C,0x42,0x50,0xF4,0x91,0xEF,0x98,0x7A,0x33,0x54,0x0B,0x43,0xED,0xCF,0xAC,0x62 + .byte 0xE4,0xB3,0x1C,0xA9,0xC9,0x08,0xE8,0x95,0x80,0xDF,0x94,0xFA,0x75,0x8F,0x3F,0xA6 + .byte 0x47,0x07,0xA7,0xFC,0xF3,0x73,0x17,0xBA,0x83,0x59,0x3C,0x19,0xE6,0x85,0x4F,0xA8 + .byte 0x68,0x6B,0x81,0xB2,0x71,0x64,0xDA,0x8B,0xF8,0xEB,0x0F,0x4B,0x70,0x56,0x9D,0x35 + .byte 0x1E,0x24,0x0E,0x5E,0x63,0x58,0xD1,0xA2,0x25,0x22,0x7C,0x3B,0x01,0x21,0x78,0x87 + .byte 0xD4,0x00,0x46,0x57,0x9F,0xD3,0x27,0x52,0x4C,0x36,0x02,0xE7,0xA0,0xC4,0xC8,0x9E + .byte 0xEA,0xBF,0x8A,0xD2,0x40,0xC7,0x38,0xB5,0xA3,0xF7,0xF2,0xCE,0xF9,0x61,0x15,0xA1 + .byte 0xE0,0xAE,0x5D,0xA4,0x9B,0x34,0x1A,0x55,0xAD,0x93,0x32,0x30,0xF5,0x8C,0xB1,0xE3 + .byte 0x1D,0xF6,0xE2,0x2E,0x82,0x66,0xCA,0x60,0xC0,0x29,0x23,0xAB,0x0D,0x53,0x4E,0x6F + .byte 0xD5,0xDB,0x37,0x45,0xDE,0xFD,0x8E,0x2F,0x03,0xFF,0x6A,0x72,0x6D,0x6C,0x5B,0x51 + .byte 0x8D,0x1B,0xAF,0x92,0xBB,0xDD,0xBC,0x7F,0x11,0xD9,0x5C,0x41,0x1F,0x10,0x5A,0xD8 + .byte 0x0A,0xC1,0x31,0x88,0xA5,0xCD,0x7B,0xBD,0x2D,0x74,0xD0,0x12,0xB8,0xE5,0xB4,0xB0 + .byte 0x89,0x69,0x97,0x4A,0x0C,0x96,0x77,0x7E,0x65,0xB9,0xF1,0x09,0xC5,0x6E,0xC6,0x84 + .byte 0x18,0xF0,0x7D,0xEC,0x3A,0xDC,0x4D,0x20,0x79,0xEE,0x5F,0x3E,0xD7,0xCB,0x39,0x48 +.Lck: + .long 0x00070E15, 0x1C232A31, 0x383F464D, 0x545B6269 + .long 0x70777E85, 0x8C939AA1, 0xA8AFB6BD, 0xC4CBD2D9 + .long 0xE0E7EEF5, 0xFC030A11, 0x181F262D, 0x343B4249 + .long 0x50575E65, 0x6C737A81, 0x888F969D, 0xA4ABB2B9 + .long 0xC0C7CED5, 0xDCE3EAF1, 0xF8FF060D, 0x141B2229 + .long 0x30373E45, 0x4C535A61, 0x686F767D, 0x848B9299 + .long 0xA0A7AEB5, 0xBCC3CAD1, 0xD8DFE6ED, 0xF4FB0209 + .long 0x10171E25, 0x2C333A41, 0x484F565D, 0x646B7279 +.Lfk: + .quad 0x56aa3350a3b1bac6,0xb27022dc677d9197 +.Lshuffles: + .quad 0x0B0A090807060504,0x030201000F0E0D0C +.Lxts_magic: + .quad 0x0101010101010187,0x0101010101010101 + +.size _${prefix}_consts,.-_${prefix}_consts + +.previous + +___ + +{{{ +my ($key,$keys,$enc)=("x0","x1","w2"); +my ($pointer,$schedules,$wtmp,$roundkey)=("x5","x6","w7","w8"); +my ($vkey,$vfk,$vmap)=("v5","v6","v7"); +$code.=<<___; +.type _vpsm4_set_key,%function +.align 4 +_vpsm4_set_key: + AARCH64_VALID_CALL_TARGET + ld1 {$vkey.4s},[$key] +___ + &load_sbox(); + &rev32($vkey,$vkey); +$code.=<<___; + adrp $pointer,.Lshuffles + add $pointer,$pointer,#:lo12:.Lshuffles + ld1 {$vmap.2d},[$pointer] + adrp $pointer,.Lfk + add $pointer,$pointer,#:lo12:.Lfk + ld1 {$vfk.2d},[$pointer] + eor $vkey.16b,$vkey.16b,$vfk.16b + mov $schedules,#32 + adrp $pointer,.Lck + add $pointer,$pointer,#:lo12:.Lck + movi @vtmp[0].16b,#64 + cbnz $enc,1f + add $keys,$keys,124 +1: + mov $wtmp,$vkey.s[1] + ldr $roundkey,[$pointer],#4 + eor $roundkey,$roundkey,$wtmp + mov $wtmp,$vkey.s[2] + eor $roundkey,$roundkey,$wtmp + mov $wtmp,$vkey.s[3] + eor $roundkey,$roundkey,$wtmp + // sbox lookup + mov @data[0].s[0],$roundkey + tbl @vtmp[1].16b,{@sbox[0].16b,@sbox[1].16b,@sbox[2].16b,@sbox[3].16b},@data[0].16b + sub @data[0].16b,@data[0].16b,@vtmp[0].16b + tbx @vtmp[1].16b,{@sbox[4].16b,@sbox[5].16b,@sbox[6].16b,@sbox[7].16b},@data[0].16b + sub @data[0].16b,@data[0].16b,@vtmp[0].16b + tbx @vtmp[1].16b,{@sbox[8].16b,@sbox[9].16b,@sbox[10].16b,@sbox[11].16b},@data[0].16b + sub @data[0].16b,@data[0].16b,@vtmp[0].16b + tbx @vtmp[1].16b,{@sbox[12].16b,@sbox[13].16b,@sbox[14].16b,@sbox[15].16b},@data[0].16b + mov $wtmp,@vtmp[1].s[0] + eor $roundkey,$wtmp,$wtmp,ror #19 + eor $roundkey,$roundkey,$wtmp,ror #9 + mov $wtmp,$vkey.s[0] + eor $roundkey,$roundkey,$wtmp + mov $vkey.s[0],$roundkey + cbz $enc,2f + str $roundkey,[$keys],#4 + b 3f +2: + str $roundkey,[$keys],#-4 +3: + tbl $vkey.16b,{$vkey.16b},$vmap.16b + subs $schedules,$schedules,#1 + b.ne 1b + ret +.size _vpsm4_set_key,.-_vpsm4_set_key +___ +}}} + + +{{{ +$code.=<<___; +.type _vpsm4_enc_4blks,%function +.align 4 +_vpsm4_enc_4blks: + AARCH64_VALID_CALL_TARGET +___ + &encrypt_4blks(); +$code.=<<___; + ret +.size _vpsm4_enc_4blks,.-_vpsm4_enc_4blks +___ +}}} + +{{{ +$code.=<<___; +.type _vpsm4_enc_8blks,%function +.align 4 +_vpsm4_enc_8blks: + AARCH64_VALID_CALL_TARGET +___ + &encrypt_8blks(); +$code.=<<___; + ret +.size _vpsm4_enc_8blks,.-_vpsm4_enc_8blks +___ +}}} + + +{{{ +my ($key,$keys)=("x0","x1"); +$code.=<<___; +.globl ${prefix}_set_encrypt_key +.type ${prefix}_set_encrypt_key,%function +.align 5 +${prefix}_set_encrypt_key: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-16]! + mov w2,1 + bl _vpsm4_set_key + ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER + ret +.size ${prefix}_set_encrypt_key,.-${prefix}_set_encrypt_key +___ +}}} + +{{{ +my ($key,$keys)=("x0","x1"); +$code.=<<___; +.globl ${prefix}_set_decrypt_key +.type ${prefix}_set_decrypt_key,%function +.align 5 +${prefix}_set_decrypt_key: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-16]! + mov w2,0 + bl _vpsm4_set_key + ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER + ret +.size ${prefix}_set_decrypt_key,.-${prefix}_set_decrypt_key +___ +}}} + +{{{ +sub gen_block () { + my $dir = shift; + my ($inp,$outp,$rk)=map("x$_",(0..2)); + +$code.=<<___; +.globl ${prefix}_${dir}crypt +.type ${prefix}_${dir}crypt,%function +.align 5 +${prefix}_${dir}crypt: + AARCH64_VALID_CALL_TARGET + ld1 {@data[0].4s},[$inp] +___ + &load_sbox(); + &rev32(@data[0],@data[0]); +$code.=<<___; + mov $rks,x2 +___ + &encrypt_1blk(@data[0]); +$code.=<<___; + st1 {@data[0].4s},[$outp] + ret +.size ${prefix}_${dir}crypt,.-${prefix}_${dir}crypt +___ +} +&gen_block("en"); +&gen_block("de"); +}}} + +{{{ +my ($enc) = ("w4"); +my @dat=map("v$_",(16..23)); + +$code.=<<___; +.globl ${prefix}_ecb_encrypt +.type ${prefix}_ecb_encrypt,%function +.align 5 +${prefix}_ecb_encrypt: + AARCH64_SIGN_LINK_REGISTER + // convert length into blocks + lsr x2,x2,4 + stp d8,d9,[sp,#-80]! + stp d10,d11,[sp,#16] + stp d12,d13,[sp,#32] + stp d14,d15,[sp,#48] + stp x29,x30,[sp,#64] +___ + &load_sbox(); +$code.=<<___; +.Lecb_8_blocks_process: + cmp $blocks,#8 + b.lt .Lecb_4_blocks_process + ld4 {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$inp],#64 + ld4 {@datax[0].4s,$datax[1].4s,@datax[2].4s,@datax[3].4s},[$inp],#64 +___ + &rev32(@data[0],@data[0]); + &rev32(@data[1],@data[1]); + &rev32(@data[2],@data[2]); + &rev32(@data[3],@data[3]); + &rev32(@datax[0],@datax[0]); + &rev32(@datax[1],@datax[1]); + &rev32(@datax[2],@datax[2]); + &rev32(@datax[3],@datax[3]); +$code.=<<___; + bl _vpsm4_enc_8blks + st4 {@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s,@vtmp[3].4s},[$outp],#64 + st4 {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$outp],#64 + subs $blocks,$blocks,#8 + b.gt .Lecb_8_blocks_process + b 100f +.Lecb_4_blocks_process: + cmp $blocks,#4 + b.lt 1f + ld4 {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$inp],#64 +___ + &rev32(@data[0],@data[0]); + &rev32(@data[1],@data[1]); + &rev32(@data[2],@data[2]); + &rev32(@data[3],@data[3]); +$code.=<<___; + bl _vpsm4_enc_4blks + st4 {@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s,@vtmp[3].4s},[$outp],#64 + sub $blocks,$blocks,#4 +1: + // process last block + cmp $blocks,#1 + b.lt 100f + b.gt 1f + ld1 {@data[0].4s},[$inp] +___ + &rev32(@data[0],@data[0]); + &encrypt_1blk(@data[0]); +$code.=<<___; + st1 {@data[0].4s},[$outp] + b 100f +1: // process last 2 blocks + ld4 {@data[0].s,@data[1].s,@data[2].s,@data[3].s}[0],[$inp],#16 + ld4 {@data[0].s,@data[1].s,@data[2].s,@data[3].s}[1],[$inp],#16 + cmp $blocks,#2 + b.gt 1f +___ + &rev32(@data[0],@data[0]); + &rev32(@data[1],@data[1]); + &rev32(@data[2],@data[2]); + &rev32(@data[3],@data[3]); +$code.=<<___; + bl _vpsm4_enc_4blks + st4 {@vtmp[0].s-@vtmp[3].s}[0],[$outp],#16 + st4 {@vtmp[0].s-@vtmp[3].s}[1],[$outp] + b 100f +1: // process last 3 blocks + ld4 {@data[0].s,@data[1].s,@data[2].s,@data[3].s}[2],[$inp],#16 +___ + &rev32(@data[0],@data[0]); + &rev32(@data[1],@data[1]); + &rev32(@data[2],@data[2]); + &rev32(@data[3],@data[3]); +$code.=<<___; + bl _vpsm4_enc_4blks + st4 {@vtmp[0].s-@vtmp[3].s}[0],[$outp],#16 + st4 {@vtmp[0].s-@vtmp[3].s}[1],[$outp],#16 + st4 {@vtmp[0].s-@vtmp[3].s}[2],[$outp] +100: + ldp d10,d11,[sp,#16] + ldp d12,d13,[sp,#32] + ldp d14,d15,[sp,#48] + ldp x29,x30,[sp,#64] + ldp d8,d9,[sp],#80 + AARCH64_VALIDATE_LINK_REGISTER + ret +.size ${prefix}_ecb_encrypt,.-${prefix}_ecb_encrypt +___ +}}} + +{{{ +my ($len,$ivp,$enc)=("x2","x4","w5"); +my $ivec0=("v3"); +my $ivec1=("v15"); + +$code.=<<___; +.globl ${prefix}_cbc_encrypt +.type ${prefix}_cbc_encrypt,%function +.align 5 +${prefix}_cbc_encrypt: + AARCH64_VALID_CALL_TARGET + lsr $len,$len,4 +___ + &load_sbox(); +$code.=<<___; + cbz $enc,.Ldec + ld1 {$ivec0.4s},[$ivp] +.Lcbc_4_blocks_enc: + cmp $blocks,#4 + b.lt 1f + ld1 {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$inp],#64 + eor @data[0].16b,@data[0].16b,$ivec0.16b +___ + &rev32(@data[1],@data[1]); + &rev32(@data[0],@data[0]); + &rev32(@data[2],@data[2]); + &rev32(@data[3],@data[3]); + &encrypt_1blk_norev(@data[0]); +$code.=<<___; + eor @data[1].16b,@data[1].16b,@data[0].16b +___ + &encrypt_1blk_norev(@data[1]); + &rev32(@data[0],@data[0]); + +$code.=<<___; + eor @data[2].16b,@data[2].16b,@data[1].16b +___ + &encrypt_1blk_norev(@data[2]); + &rev32(@data[1],@data[1]); +$code.=<<___; + eor @data[3].16b,@data[3].16b,@data[2].16b +___ + &encrypt_1blk_norev(@data[3]); + &rev32(@data[2],@data[2]); + &rev32(@data[3],@data[3]); +$code.=<<___; + orr $ivec0.16b,@data[3].16b,@data[3].16b + st1 {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$outp],#64 + subs $blocks,$blocks,#4 + b.ne .Lcbc_4_blocks_enc + b 2f +1: + subs $blocks,$blocks,#1 + b.lt 2f + ld1 {@data[0].4s},[$inp],#16 + eor $ivec0.16b,$ivec0.16b,@data[0].16b +___ + &rev32($ivec0,$ivec0); + &encrypt_1blk($ivec0); +$code.=<<___; + st1 {$ivec0.4s},[$outp],#16 + b 1b +2: + // save back IV + st1 {$ivec0.4s},[$ivp] + ret + +.Ldec: + // decryption mode starts + AARCH64_SIGN_LINK_REGISTER + stp d8,d9,[sp,#-80]! + stp d10,d11,[sp,#16] + stp d12,d13,[sp,#32] + stp d14,d15,[sp,#48] + stp x29,x30,[sp,#64] +.Lcbc_8_blocks_dec: + cmp $blocks,#8 + b.lt 1f + ld4 {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$inp] + add $ptr,$inp,#64 + ld4 {@datax[0].4s,@datax[1].4s,@datax[2].4s,@datax[3].4s},[$ptr] +___ + &rev32(@data[0],@data[0]); + &rev32(@data[1],@data[1]); + &rev32(@data[2],@data[2]); + &rev32(@data[3],$data[3]); + &rev32(@datax[0],@datax[0]); + &rev32(@datax[1],@datax[1]); + &rev32(@datax[2],@datax[2]); + &rev32(@datax[3],$datax[3]); +$code.=<<___; + bl _vpsm4_enc_8blks +___ + &transpose(@vtmp,@datax); + &transpose(@data,@datax); +$code.=<<___; + ld1 {$ivec1.4s},[$ivp] + ld1 {@datax[0].4s,@datax[1].4s,@datax[2].4s,@datax[3].4s},[$inp],#64 + // note ivec1 and vtmpx[3] are reusing the same register + // care needs to be taken to avoid conflict + eor @vtmp[0].16b,@vtmp[0].16b,$ivec1.16b + ld1 {@vtmpx[0].4s,@vtmpx[1].4s,@vtmpx[2].4s,@vtmpx[3].4s},[$inp],#64 + eor @vtmp[1].16b,@vtmp[1].16b,@datax[0].16b + eor @vtmp[2].16b,@vtmp[2].16b,@datax[1].16b + eor @vtmp[3].16b,$vtmp[3].16b,@datax[2].16b + // save back IV + st1 {$vtmpx[3].4s}, [$ivp] + eor @data[0].16b,@data[0].16b,$datax[3].16b + eor @data[1].16b,@data[1].16b,@vtmpx[0].16b + eor @data[2].16b,@data[2].16b,@vtmpx[1].16b + eor @data[3].16b,$data[3].16b,@vtmpx[2].16b + st1 {@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s,@vtmp[3].4s},[$outp],#64 + st1 {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$outp],#64 + subs $blocks,$blocks,#8 + b.gt .Lcbc_8_blocks_dec + b.eq 100f +1: + ld1 {$ivec1.4s},[$ivp] +.Lcbc_4_blocks_dec: + cmp $blocks,#4 + b.lt 1f + ld4 {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$inp] +___ + &rev32(@data[0],@data[0]); + &rev32(@data[1],@data[1]); + &rev32(@data[2],@data[2]); + &rev32(@data[3],$data[3]); +$code.=<<___; + bl _vpsm4_enc_4blks + ld1 {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$inp],#64 +___ + &transpose(@vtmp,@datax); +$code.=<<___; + eor @vtmp[0].16b,@vtmp[0].16b,$ivec1.16b + eor @vtmp[1].16b,@vtmp[1].16b,@data[0].16b + orr $ivec1.16b,@data[3].16b,@data[3].16b + eor @vtmp[2].16b,@vtmp[2].16b,@data[1].16b + eor @vtmp[3].16b,$vtmp[3].16b,@data[2].16b + st1 {@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s,@vtmp[3].4s},[$outp],#64 + subs $blocks,$blocks,#4 + b.gt .Lcbc_4_blocks_dec + // save back IV + st1 {@data[3].4s}, [$ivp] + b 100f +1: // last block + subs $blocks,$blocks,#1 + b.lt 100f + b.gt 1f + ld1 {@data[0].4s},[$inp],#16 + // save back IV + st1 {$data[0].4s}, [$ivp] +___ + &rev32(@datax[0],@data[0]); + &encrypt_1blk(@datax[0]); +$code.=<<___; + eor @datax[0].16b,@datax[0].16b,$ivec1.16b + st1 {@datax[0].4s},[$outp],#16 + b 100f +1: // last two blocks + ld4 {@data[0].s,@data[1].s,@data[2].s,@data[3].s}[0],[$inp] + add $ptr,$inp,#16 + ld4 {@data[0].s,@data[1].s,@data[2].s,@data[3].s}[1],[$ptr],#16 + subs $blocks,$blocks,1 + b.gt 1f +___ + &rev32(@data[0],@data[0]); + &rev32(@data[1],@data[1]); + &rev32(@data[2],@data[2]); + &rev32(@data[3],@data[3]); +$code.=<<___; + bl _vpsm4_enc_4blks + ld1 {@data[0].4s,@data[1].4s},[$inp],#32 +___ + &transpose(@vtmp,@datax); +$code.=<<___; + eor @vtmp[0].16b,@vtmp[0].16b,$ivec1.16b + eor @vtmp[1].16b,@vtmp[1].16b,@data[0].16b + st1 {@vtmp[0].4s,@vtmp[1].4s},[$outp],#32 + // save back IV + st1 {@data[1].4s}, [$ivp] + b 100f +1: // last 3 blocks + ld4 {@data[0].s,@data[1].s,@data[2].s,@data[3].s}[2],[$ptr] +___ + &rev32(@data[0],@data[0]); + &rev32(@data[1],@data[1]); + &rev32(@data[2],@data[2]); + &rev32(@data[3],@data[3]); +$code.=<<___; + bl _vpsm4_enc_4blks + ld1 {@data[0].4s,@data[1].4s,@data[2].4s},[$inp],#48 +___ + &transpose(@vtmp,@datax); +$code.=<<___; + eor @vtmp[0].16b,@vtmp[0].16b,$ivec1.16b + eor @vtmp[1].16b,@vtmp[1].16b,@data[0].16b + eor @vtmp[2].16b,@vtmp[2].16b,@data[1].16b + st1 {@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s},[$outp],#48 + // save back IV + st1 {@data[2].4s}, [$ivp] +100: + ldp d10,d11,[sp,#16] + ldp d12,d13,[sp,#32] + ldp d14,d15,[sp,#48] + ldp x29,x30,[sp,#64] + ldp d8,d9,[sp],#80 + AARCH64_VALIDATE_LINK_REGISTER + ret +.size ${prefix}_cbc_encrypt,.-${prefix}_cbc_encrypt +___ +}}} + +{{{ +my ($ivp)=("x4"); +my ($ctr)=("w5"); +my $ivec=("v3"); + +$code.=<<___; +.globl ${prefix}_ctr32_encrypt_blocks +.type ${prefix}_ctr32_encrypt_blocks,%function +.align 5 +${prefix}_ctr32_encrypt_blocks: + AARCH64_VALID_CALL_TARGET + ld1 {$ivec.4s},[$ivp] +___ + &rev32($ivec,$ivec); + &load_sbox(); +$code.=<<___; + cmp $blocks,#1 + b.ne 1f + // fast processing for one single block without + // context saving overhead +___ + &encrypt_1blk($ivec); +$code.=<<___; + ld1 {@data[0].4s},[$inp] + eor @data[0].16b,@data[0].16b,$ivec.16b + st1 {@data[0].4s},[$outp] + ret +1: + AARCH64_SIGN_LINK_REGISTER + stp d8,d9,[sp,#-80]! + stp d10,d11,[sp,#16] + stp d12,d13,[sp,#32] + stp d14,d15,[sp,#48] + stp x29,x30,[sp,#64] + mov $word0,$ivec.s[0] + mov $word1,$ivec.s[1] + mov $word2,$ivec.s[2] + mov $ctr,$ivec.s[3] +.Lctr32_4_blocks_process: + cmp $blocks,#4 + b.lt 1f + dup @data[0].4s,$word0 + dup @data[1].4s,$word1 + dup @data[2].4s,$word2 + mov @data[3].s[0],$ctr + add $ctr,$ctr,#1 + mov $data[3].s[1],$ctr + add $ctr,$ctr,#1 + mov @data[3].s[2],$ctr + add $ctr,$ctr,#1 + mov @data[3].s[3],$ctr + add $ctr,$ctr,#1 + cmp $blocks,#8 + b.ge .Lctr32_8_blocks_process + bl _vpsm4_enc_4blks + ld4 {@vtmpx[0].4s,@vtmpx[1].4s,@vtmpx[2].4s,@vtmpx[3].4s},[$inp],#64 + eor @vtmp[0].16b,@vtmp[0].16b,@vtmpx[0].16b + eor @vtmp[1].16b,@vtmp[1].16b,@vtmpx[1].16b + eor @vtmp[2].16b,@vtmp[2].16b,@vtmpx[2].16b + eor @vtmp[3].16b,@vtmp[3].16b,@vtmpx[3].16b + st4 {@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s,@vtmp[3].4s},[$outp],#64 + subs $blocks,$blocks,#4 + b.ne .Lctr32_4_blocks_process + b 100f +.Lctr32_8_blocks_process: + dup @datax[0].4s,$word0 + dup @datax[1].4s,$word1 + dup @datax[2].4s,$word2 + mov @datax[3].s[0],$ctr + add $ctr,$ctr,#1 + mov $datax[3].s[1],$ctr + add $ctr,$ctr,#1 + mov @datax[3].s[2],$ctr + add $ctr,$ctr,#1 + mov @datax[3].s[3],$ctr + add $ctr,$ctr,#1 + bl _vpsm4_enc_8blks + ld4 {@vtmpx[0].4s,@vtmpx[1].4s,@vtmpx[2].4s,@vtmpx[3].4s},[$inp],#64 + ld4 {@datax[0].4s,@datax[1].4s,@datax[2].4s,@datax[3].4s},[$inp],#64 + eor @vtmp[0].16b,@vtmp[0].16b,@vtmpx[0].16b + eor @vtmp[1].16b,@vtmp[1].16b,@vtmpx[1].16b + eor @vtmp[2].16b,@vtmp[2].16b,@vtmpx[2].16b + eor @vtmp[3].16b,@vtmp[3].16b,@vtmpx[3].16b + eor @data[0].16b,@data[0].16b,@datax[0].16b + eor @data[1].16b,@data[1].16b,@datax[1].16b + eor @data[2].16b,@data[2].16b,@datax[2].16b + eor @data[3].16b,@data[3].16b,@datax[3].16b + st4 {@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s,@vtmp[3].4s},[$outp],#64 + st4 {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$outp],#64 + subs $blocks,$blocks,#8 + b.ne .Lctr32_4_blocks_process + b 100f +1: // last block processing + subs $blocks,$blocks,#1 + b.lt 100f + b.gt 1f + mov $ivec.s[0],$word0 + mov $ivec.s[1],$word1 + mov $ivec.s[2],$word2 + mov $ivec.s[3],$ctr +___ + &encrypt_1blk($ivec); +$code.=<<___; + ld1 {@data[0].4s},[$inp] + eor @data[0].16b,@data[0].16b,$ivec.16b + st1 {@data[0].4s},[$outp] + b 100f +1: // last 2 blocks processing + dup @data[0].4s,$word0 + dup @data[1].4s,$word1 + dup @data[2].4s,$word2 + mov @data[3].s[0],$ctr + add $ctr,$ctr,#1 + mov @data[3].s[1],$ctr + subs $blocks,$blocks,#1 + b.ne 1f + bl _vpsm4_enc_4blks + ld4 {@vtmpx[0].s,@vtmpx[1].s,@vtmpx[2].s,@vtmpx[3].s}[0],[$inp],#16 + ld4 {@vtmpx[0].s,@vtmpx[1].s,@vtmpx[2].s,@vtmpx[3].s}[1],[$inp],#16 + eor @vtmp[0].16b,@vtmp[0].16b,@vtmpx[0].16b + eor @vtmp[1].16b,@vtmp[1].16b,@vtmpx[1].16b + eor @vtmp[2].16b,@vtmp[2].16b,@vtmpx[2].16b + eor @vtmp[3].16b,@vtmp[3].16b,@vtmpx[3].16b + st4 {@vtmp[0].s,@vtmp[1].s,@vtmp[2].s,@vtmp[3].s}[0],[$outp],#16 + st4 {@vtmp[0].s,@vtmp[1].s,@vtmp[2].s,@vtmp[3].s}[1],[$outp],#16 + b 100f +1: // last 3 blocks processing + add $ctr,$ctr,#1 + mov @data[3].s[2],$ctr + bl _vpsm4_enc_4blks + ld4 {@vtmpx[0].s,@vtmpx[1].s,@vtmpx[2].s,@vtmpx[3].s}[0],[$inp],#16 + ld4 {@vtmpx[0].s,@vtmpx[1].s,@vtmpx[2].s,@vtmpx[3].s}[1],[$inp],#16 + ld4 {@vtmpx[0].s,@vtmpx[1].s,@vtmpx[2].s,@vtmpx[3].s}[2],[$inp],#16 + eor @vtmp[0].16b,@vtmp[0].16b,@vtmpx[0].16b + eor @vtmp[1].16b,@vtmp[1].16b,@vtmpx[1].16b + eor @vtmp[2].16b,@vtmp[2].16b,@vtmpx[2].16b + eor @vtmp[3].16b,@vtmp[3].16b,@vtmpx[3].16b + st4 {@vtmp[0].s,@vtmp[1].s,@vtmp[2].s,@vtmp[3].s}[0],[$outp],#16 + st4 {@vtmp[0].s,@vtmp[1].s,@vtmp[2].s,@vtmp[3].s}[1],[$outp],#16 + st4 {@vtmp[0].s,@vtmp[1].s,@vtmp[2].s,@vtmp[3].s}[2],[$outp],#16 +100: + ldp d10,d11,[sp,#16] + ldp d12,d13,[sp,#32] + ldp d14,d15,[sp,#48] + ldp x29,x30,[sp,#64] + ldp d8,d9,[sp],#80 + AARCH64_VALIDATE_LINK_REGISTER + ret +.size ${prefix}_ctr32_encrypt_blocks,.-${prefix}_ctr32_encrypt_blocks +___ +}}} + +{{{ +my ($blocks,$len)=("x2","x2"); +my $ivp=("x5"); +my @twx=map("x$_",(12..27)); +my ($rks1,$rks2)=("x26","x27"); +my $lastBlk=("x26"); +my $enc=("w28"); +my $remain=("x29"); + +my @tweak=@datax; + +sub gen_xts_cipher() { + my $std = shift; +$code.=<<___; +.globl ${prefix}_xts_encrypt${std} +.type ${prefix}_xts_encrypt${std},%function +.align 5 +${prefix}_xts_encrypt${std}: + AARCH64_SIGN_LINK_REGISTER + stp x15, x16, [sp, #-0x10]! + stp x17, x18, [sp, #-0x10]! + stp x19, x20, [sp, #-0x10]! + stp x21, x22, [sp, #-0x10]! + stp x23, x24, [sp, #-0x10]! + stp x25, x26, [sp, #-0x10]! + stp x27, x28, [sp, #-0x10]! + stp x29, x30, [sp, #-0x10]! + stp d8, d9, [sp, #-0x10]! + stp d10, d11, [sp, #-0x10]! + stp d12, d13, [sp, #-0x10]! + stp d14, d15, [sp, #-0x10]! + mov $rks1,x3 + mov $rks2,x4 + mov $enc,w6 + ld1 {@tweak[0].4s}, [$ivp] + mov $rks,$rks2 +___ + &load_sbox(); + &rev32(@tweak[0],@tweak[0]); + &encrypt_1blk(@tweak[0]); +$code.=<<___; + mov $rks,$rks1 + and $remain,$len,#0x0F + // convert length into blocks + lsr $blocks,$len,4 + cmp $blocks,#1 + b.lt .return${std} + + cmp $remain,0 + // If the encryption/decryption Length is N times of 16, + // the all blocks are encrypted/decrypted in .xts_encrypt_blocks${std} + b.eq .xts_encrypt_blocks${std} + + // If the encryption/decryption length is not N times of 16, + // the last two blocks are encrypted/decrypted in .last_2blks_tweak${std} or .only_2blks_tweak${std} + // the other blocks are encrypted/decrypted in .xts_encrypt_blocks${std} + subs $blocks,$blocks,#1 + b.eq .only_2blks_tweak${std} +.xts_encrypt_blocks${std}: +___ + &rbit(@tweak[0],@tweak[0],$std); + &rev32_armeb(@tweak[0],@tweak[0]); + &mov_vec_to_reg(@tweak[0],@twx[0],@twx[1]); + &compute_tweak(@twx[0],@twx[1],@twx[2],@twx[3]); + &compute_tweak(@twx[2],@twx[3],@twx[4],@twx[5]); + &compute_tweak(@twx[4],@twx[5],@twx[6],@twx[7]); + &compute_tweak(@twx[6],@twx[7],@twx[8],@twx[9]); + &compute_tweak(@twx[8],@twx[9],@twx[10],@twx[11]); + &compute_tweak(@twx[10],@twx[11],@twx[12],@twx[13]); + &compute_tweak(@twx[12],@twx[13],@twx[14],@twx[15]); +$code.=<<___; +.Lxts_8_blocks_process${std}: + cmp $blocks,#8 + b.lt .Lxts_4_blocks_process${std} +___ + &mov_reg_to_vec(@twx[0],@twx[1],@vtmp[0]); + &mov_reg_to_vec(@twx[2],@twx[3],@vtmp[1]); + &mov_reg_to_vec(@twx[4],@twx[5],@vtmp[2]); + &mov_reg_to_vec(@twx[6],@twx[7],@vtmp[3]); + &mov_reg_to_vec(@twx[8],@twx[9],@vtmpx[0]); + &mov_reg_to_vec(@twx[10],@twx[11],@vtmpx[1]); + &mov_reg_to_vec(@twx[12],@twx[13],@vtmpx[2]); + &mov_reg_to_vec(@twx[14],@twx[15],@vtmpx[3]); +$code.=<<___; + ld1 {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$inp],#64 +___ + &rbit(@vtmp[0],@vtmp[0],$std); + &rbit(@vtmp[1],@vtmp[1],$std); + &rbit(@vtmp[2],@vtmp[2],$std); + &rbit(@vtmp[3],@vtmp[3],$std); +$code.=<<___; + eor @data[0].16b, @data[0].16b, @vtmp[0].16b + eor @data[1].16b, @data[1].16b, @vtmp[1].16b + eor @data[2].16b, @data[2].16b, @vtmp[2].16b + eor @data[3].16b, @data[3].16b, @vtmp[3].16b + ld1 {@datax[0].4s,$datax[1].4s,@datax[2].4s,@datax[3].4s},[$inp],#64 +___ + &rbit(@vtmpx[0],@vtmpx[0],$std); + &rbit(@vtmpx[1],@vtmpx[1],$std); + &rbit(@vtmpx[2],@vtmpx[2],$std); + &rbit(@vtmpx[3],@vtmpx[3],$std); +$code.=<<___; + eor @datax[0].16b, @datax[0].16b, @vtmpx[0].16b + eor @datax[1].16b, @datax[1].16b, @vtmpx[1].16b + eor @datax[2].16b, @datax[2].16b, @vtmpx[2].16b + eor @datax[3].16b, @datax[3].16b, @vtmpx[3].16b +___ + &rev32(@data[0],@data[0]); + &rev32(@data[1],@data[1]); + &rev32(@data[2],@data[2]); + &rev32(@data[3],@data[3]); + &rev32(@datax[0],@datax[0]); + &rev32(@datax[1],@datax[1]); + &rev32(@datax[2],@datax[2]); + &rev32(@datax[3],@datax[3]); + &transpose(@data,@vtmp); + &transpose(@datax,@vtmp); +$code.=<<___; + bl _${prefix}_enc_8blks +___ + &transpose(@vtmp,@datax); + &transpose(@data,@datax); + + &mov_reg_to_vec(@twx[0],@twx[1],@vtmpx[0]); + &compute_tweak(@twx[14],@twx[15],@twx[0],@twx[1]); + &mov_reg_to_vec(@twx[2],@twx[3],@vtmpx[1]); + &compute_tweak(@twx[0],@twx[1],@twx[2],@twx[3]); + &mov_reg_to_vec(@twx[4],@twx[5],@vtmpx[2]); + &compute_tweak(@twx[2],@twx[3],@twx[4],@twx[5]); + &mov_reg_to_vec(@twx[6],@twx[7],@vtmpx[3]); + &compute_tweak(@twx[4],@twx[5],@twx[6],@twx[7]); + &mov_reg_to_vec(@twx[8],@twx[9],@tweak[0]); + &compute_tweak(@twx[6],@twx[7],@twx[8],@twx[9]); + &mov_reg_to_vec(@twx[10],@twx[11],@tweak[1]); + &compute_tweak(@twx[8],@twx[9],@twx[10],@twx[11]); + &mov_reg_to_vec(@twx[12],@twx[13],@tweak[2]); + &compute_tweak(@twx[10],@twx[11],@twx[12],@twx[13]); + &mov_reg_to_vec(@twx[14],@twx[15],@tweak[3]); + &compute_tweak(@twx[12],@twx[13],@twx[14],@twx[15]); +$code.=<<___; + eor @vtmp[0].16b, @vtmp[0].16b, @vtmpx[0].16b + eor @vtmp[1].16b, @vtmp[1].16b, @vtmpx[1].16b + eor @vtmp[2].16b, @vtmp[2].16b, @vtmpx[2].16b + eor @vtmp[3].16b, @vtmp[3].16b, @vtmpx[3].16b + eor @data[0].16b, @data[0].16b, @tweak[0].16b + eor @data[1].16b, @data[1].16b, @tweak[1].16b + eor @data[2].16b, @data[2].16b, @tweak[2].16b + eor @data[3].16b, @data[3].16b, @tweak[3].16b + + // save the last tweak + st1 {@tweak[3].4s},[$ivp] + st1 {@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s,@vtmp[3].4s},[$outp],#64 + st1 {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$outp],#64 + subs $blocks,$blocks,#8 + b.gt .Lxts_8_blocks_process${std} + b 100f +.Lxts_4_blocks_process${std}: +___ + &mov_reg_to_vec(@twx[0],@twx[1],@tweak[0]); + &mov_reg_to_vec(@twx[2],@twx[3],@tweak[1]); + &mov_reg_to_vec(@twx[4],@twx[5],@tweak[2]); + &mov_reg_to_vec(@twx[6],@twx[7],@tweak[3]); +$code.=<<___; + cmp $blocks,#4 + b.lt 1f + ld1 {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$inp],#64 +___ + &rbit(@tweak[0],@tweak[0],$std); + &rbit(@tweak[1],@tweak[1],$std); + &rbit(@tweak[2],@tweak[2],$std); + &rbit(@tweak[3],@tweak[3],$std); +$code.=<<___; + eor @data[0].16b, @data[0].16b, @tweak[0].16b + eor @data[1].16b, @data[1].16b, @tweak[1].16b + eor @data[2].16b, @data[2].16b, @tweak[2].16b + eor @data[3].16b, @data[3].16b, @tweak[3].16b +___ + &rev32(@data[0],@data[0]); + &rev32(@data[1],@data[1]); + &rev32(@data[2],@data[2]); + &rev32(@data[3],@data[3]); + &transpose(@data,@vtmp); +$code.=<<___; + bl _${prefix}_enc_4blks +___ + &transpose(@vtmp,@data); +$code.=<<___; + eor @vtmp[0].16b, @vtmp[0].16b, @tweak[0].16b + eor @vtmp[1].16b, @vtmp[1].16b, @tweak[1].16b + eor @vtmp[2].16b, @vtmp[2].16b, @tweak[2].16b + eor @vtmp[3].16b, @vtmp[3].16b, @tweak[3].16b + st1 {@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s,@vtmp[3].4s},[$outp],#64 + sub $blocks,$blocks,#4 +___ + &mov_reg_to_vec(@twx[8],@twx[9],@tweak[0]); + &mov_reg_to_vec(@twx[10],@twx[11],@tweak[1]); + &mov_reg_to_vec(@twx[12],@twx[13],@tweak[2]); +$code.=<<___; + // save the last tweak + st1 {@tweak[3].4s},[$ivp] +1: + // process last block + cmp $blocks,#1 + b.lt 100f + b.gt 1f + ld1 {@data[0].4s},[$inp],#16 +___ + &rbit(@tweak[0],@tweak[0],$std); +$code.=<<___; + eor @data[0].16b, @data[0].16b, @tweak[0].16b +___ + &rev32(@data[0],@data[0]); + &encrypt_1blk(@data[0]); +$code.=<<___; + eor @data[0].16b, @data[0].16b, @tweak[0].16b + st1 {@data[0].4s},[$outp],#16 + // save the last tweak + st1 {@tweak[0].4s},[$ivp] + b 100f +1: // process last 2 blocks + cmp $blocks,#2 + b.gt 1f + ld1 {@data[0].4s,@data[1].4s},[$inp],#32 +___ + &rbit(@tweak[0],@tweak[0],$std); + &rbit(@tweak[1],@tweak[1],$std); +$code.=<<___; + eor @data[0].16b, @data[0].16b, @tweak[0].16b + eor @data[1].16b, @data[1].16b, @tweak[1].16b +___ + &rev32(@data[0],@data[0]); + &rev32(@data[1],@data[1]); + &transpose(@data,@vtmp); +$code.=<<___; + bl _${prefix}_enc_4blks +___ + &transpose(@vtmp,@data); +$code.=<<___; + eor @vtmp[0].16b, @vtmp[0].16b, @tweak[0].16b + eor @vtmp[1].16b, @vtmp[1].16b, @tweak[1].16b + st1 {@vtmp[0].4s,@vtmp[1].4s},[$outp],#32 + // save the last tweak + st1 {@tweak[1].4s},[$ivp] + b 100f +1: // process last 3 blocks + ld1 {@data[0].4s,@data[1].4s,@data[2].4s},[$inp],#48 +___ + &rbit(@tweak[0],@tweak[0],$std); + &rbit(@tweak[1],@tweak[1],$std); + &rbit(@tweak[2],@tweak[2],$std); +$code.=<<___; + eor @data[0].16b, @data[0].16b, @tweak[0].16b + eor @data[1].16b, @data[1].16b, @tweak[1].16b + eor @data[2].16b, @data[2].16b, @tweak[2].16b +___ + &rev32(@data[0],@data[0]); + &rev32(@data[1],@data[1]); + &rev32(@data[2],@data[2]); + &transpose(@data,@vtmp); +$code.=<<___; + bl _${prefix}_enc_4blks +___ + &transpose(@vtmp,@data); +$code.=<<___; + eor @vtmp[0].16b, @vtmp[0].16b, @tweak[0].16b + eor @vtmp[1].16b, @vtmp[1].16b, @tweak[1].16b + eor @vtmp[2].16b, @vtmp[2].16b, @tweak[2].16b + st1 {@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s},[$outp],#48 + // save the last tweak + st1 {@tweak[2].4s},[$ivp] +100: + cmp $remain,0 + b.eq .return${std} + +// This branch calculates the last two tweaks, +// while the encryption/decryption length is larger than 32 +.last_2blks_tweak${std}: + ld1 {@tweak[0].4s},[$ivp] +___ + &rev32_armeb(@tweak[0],@tweak[0]); + &compute_tweak_vec(@tweak[0],@tweak[1],$std); + &compute_tweak_vec(@tweak[1],@tweak[2],$std); +$code.=<<___; + b .check_dec${std} + + +// This branch calculates the last two tweaks, +// while the encryption/decryption length is equal to 32, who only need two tweaks +.only_2blks_tweak${std}: + mov @tweak[1].16b,@tweak[0].16b +___ + &rev32_armeb(@tweak[1],@tweak[1]); + &compute_tweak_vec(@tweak[1],@tweak[2],$std); +$code.=<<___; + b .check_dec${std} + + +// Determine whether encryption or decryption is required. +// The last two tweaks need to be swapped for decryption. +.check_dec${std}: + // encryption:1 decryption:0 + cmp $enc,1 + b.eq .process_last_2blks${std} + mov @vtmp[0].16B,@tweak[1].16b + mov @tweak[1].16B,@tweak[2].16b + mov @tweak[2].16B,@vtmp[0].16b + +.process_last_2blks${std}: +___ + &rev32_armeb(@tweak[1],@tweak[1]); + &rev32_armeb(@tweak[2],@tweak[2]); +$code.=<<___; + ld1 {@data[0].4s},[$inp],#16 + eor @data[0].16b, @data[0].16b, @tweak[1].16b +___ + &rev32(@data[0],@data[0]); + &encrypt_1blk(@data[0]); +$code.=<<___; + eor @data[0].16b, @data[0].16b, @tweak[1].16b + st1 {@data[0].4s},[$outp],#16 + + sub $lastBlk,$outp,16 + .loop${std}: + subs $remain,$remain,1 + ldrb $wtmp0,[$lastBlk,$remain] + ldrb $wtmp1,[$inp,$remain] + strb $wtmp1,[$lastBlk,$remain] + strb $wtmp0,[$outp,$remain] + b.gt .loop${std} + ld1 {@data[0].4s}, [$lastBlk] + eor @data[0].16b, @data[0].16b, @tweak[2].16b +___ + &rev32(@data[0],@data[0]); + &encrypt_1blk(@data[0]); +$code.=<<___; + eor @data[0].16b, @data[0].16b, @tweak[2].16b + st1 {@data[0].4s}, [$lastBlk] +.return${std}: + ldp d14, d15, [sp], #0x10 + ldp d12, d13, [sp], #0x10 + ldp d10, d11, [sp], #0x10 + ldp d8, d9, [sp], #0x10 + ldp x29, x30, [sp], #0x10 + ldp x27, x28, [sp], #0x10 + ldp x25, x26, [sp], #0x10 + ldp x23, x24, [sp], #0x10 + ldp x21, x22, [sp], #0x10 + ldp x19, x20, [sp], #0x10 + ldp x17, x18, [sp], #0x10 + ldp x15, x16, [sp], #0x10 + AARCH64_VALIDATE_LINK_REGISTER + ret +.size ${prefix}_xts_encrypt${std},.-${prefix}_xts_encrypt${std} +___ +} # end of gen_xts_cipher +&gen_xts_cipher("_gb"); +&gen_xts_cipher(""); +}}} +######################################## +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/\/\// and !/^$/); + print; +} +close SELF; + +foreach(split("\n",$code)) { + s/\`([^\`]*)\`/eval($1)/ge; + print $_,"\n"; +} + +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/sm4/asm/vpsm4_ex-armv8.pl b/crypto/openssl/crypto/sm4/asm/vpsm4_ex-armv8.pl new file mode 100644 index 000000000000..2a1916268d16 --- /dev/null +++ b/crypto/openssl/crypto/sm4/asm/vpsm4_ex-armv8.pl @@ -0,0 +1,1558 @@ +#! /usr/bin/env perl +# Copyright 2022-2025 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +# +# This module implements SM4 with ASIMD and AESE on AARCH64 +# +# Dec 2022 +# + +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or +die "can't locate arm-xlate.pl"; + +open OUT,"| \"$^X\" $xlate $flavour \"$output\"" + or die "can't call $xlate: $!"; +*STDOUT=*OUT; + +$prefix="vpsm4_ex"; +my @vtmp=map("v$_",(0..3)); +my @qtmp=map("q$_",(0..3)); +my @data=map("v$_",(4..7)); +my @datax=map("v$_",(8..11)); +my ($rk0,$rk1)=("v12","v13"); +my ($rka,$rkb)=("v14","v15"); +my @vtmpx=map("v$_",(12..15)); +my ($vtmp4,$vtmp5)=("v24","v25"); +my ($MaskV,$TAHMatV,$TALMatV,$ATAHMatV,$ATALMatV,$ANDMaskV)=("v26","v27","v28","v29","v30","v31"); +my ($MaskQ,$TAHMatQ,$TALMatQ,$ATAHMatQ,$ATALMatQ,$ANDMaskQ)=("q26","q27","q28","q29","q30","q31"); + +my ($inp,$outp,$blocks,$rks)=("x0","x1","w2","x3"); +my ($tmpw,$tmp,$wtmp0,$wtmp1,$wtmp2)=("w6","x6","w7","w8","w9"); +my ($xtmp1,$xtmp2)=("x8","x9"); +my ($ptr,$counter)=("x10","w11"); +my ($word0,$word1,$word2,$word3)=("w12","w13","w14","w15"); + +sub rev32() { + my $dst = shift; + my $src = shift; + + if ($src and ("$src" ne "$dst")) { +$code.=<<___; +#ifndef __AARCH64EB__ + rev32 $dst.16b,$src.16b +#else + mov $dst.16b,$src.16b +#endif +___ + } else { +$code.=<<___; +#ifndef __AARCH64EB__ + rev32 $dst.16b,$dst.16b +#endif +___ + } +} + +sub rev32_armeb() { + my $dst = shift; + my $src = shift; + + if ($src and ("$src" ne "$dst")) { +$code.=<<___; +#ifdef __AARCH64EB__ + rev32 $dst.16b,$src.16b +#else + mov $dst.16b,$src.16b +#endif +___ + } else { +$code.=<<___; +#ifdef __AARCH64EB__ + rev32 $dst.16b,$dst.16b +#endif +___ + } +} + +sub rbit() { + my $dst = shift; + my $src = shift; + my $std = shift; + + if ($src and ("$src" ne "$dst")) { + if ($std eq "_gb") { +$code.=<<___; + rbit $dst.16b,$src.16b +___ + } else { +$code.=<<___; + mov $dst.16b,$src.16b +___ + } + } else { + if ($std eq "_gb") { +$code.=<<___; + rbit $dst.16b,$src.16b +___ + } + } +} + +sub transpose() { + my ($dat0,$dat1,$dat2,$dat3,$vt0,$vt1,$vt2,$vt3) = @_; + +$code.=<<___; + zip1 $vt0.4s,$dat0.4s,$dat1.4s + zip2 $vt1.4s,$dat0.4s,$dat1.4s + zip1 $vt2.4s,$dat2.4s,$dat3.4s + zip2 $vt3.4s,$dat2.4s,$dat3.4s + zip1 $dat0.2d,$vt0.2d,$vt2.2d + zip2 $dat1.2d,$vt0.2d,$vt2.2d + zip1 $dat2.2d,$vt1.2d,$vt3.2d + zip2 $dat3.2d,$vt1.2d,$vt3.2d +___ +} + +# matrix multiplication Mat*x = (lowerMat*x) ^ (higherMat*x) +sub mul_matrix() { + my $x = shift; + my $higherMat = shift; + my $lowerMat = shift; + my $tmp = shift; +$code.=<<___; + ushr $tmp.16b, $x.16b, 4 + and $x.16b, $x.16b, $ANDMaskV.16b + tbl $x.16b, {$lowerMat.16b}, $x.16b + tbl $tmp.16b, {$higherMat.16b}, $tmp.16b + eor $x.16b, $x.16b, $tmp.16b +___ +} + +# sbox operations for 4-lane of words +# sbox operation for 4-lane of words +sub sbox() { + my $dat = shift; + +$code.=<<___; + // optimize sbox using AESE instruction + tbl @vtmp[0].16b, {$dat.16b}, $MaskV.16b +___ + &mul_matrix(@vtmp[0], $TAHMatV, $TALMatV, $vtmp4); +$code.=<<___; + eor @vtmp[1].16b, @vtmp[1].16b, @vtmp[1].16b + aese @vtmp[0].16b,@vtmp[1].16b +___ + &mul_matrix(@vtmp[0], $ATAHMatV, $ATALMatV, $vtmp4); +$code.=<<___; + mov $dat.16b,@vtmp[0].16b + + // linear transformation + ushr @vtmp[0].4s,$dat.4s,32-2 + ushr @vtmp[1].4s,$dat.4s,32-10 + ushr @vtmp[2].4s,$dat.4s,32-18 + ushr @vtmp[3].4s,$dat.4s,32-24 + sli @vtmp[0].4s,$dat.4s,2 + sli @vtmp[1].4s,$dat.4s,10 + sli @vtmp[2].4s,$dat.4s,18 + sli @vtmp[3].4s,$dat.4s,24 + eor $vtmp4.16b,@vtmp[0].16b,$dat.16b + eor $vtmp4.16b,$vtmp4.16b,$vtmp[1].16b + eor $dat.16b,@vtmp[2].16b,@vtmp[3].16b + eor $dat.16b,$dat.16b,$vtmp4.16b +___ +} + +# sbox operation for 8-lane of words +sub sbox_double() { + my $dat = shift; + my $datx = shift; + +$code.=<<___; + // optimize sbox using AESE instruction + tbl @vtmp[0].16b, {$dat.16b}, $MaskV.16b + tbl @vtmp[1].16b, {$datx.16b}, $MaskV.16b +___ + &mul_matrix(@vtmp[0], $TAHMatV, $TALMatV, $vtmp4); + &mul_matrix(@vtmp[1], $TAHMatV, $TALMatV, $vtmp4); +$code.=<<___; + eor $vtmp5.16b, $vtmp5.16b, $vtmp5.16b + aese @vtmp[0].16b,$vtmp5.16b + aese @vtmp[1].16b,$vtmp5.16b +___ + &mul_matrix(@vtmp[0], $ATAHMatV, $ATALMatV,$vtmp4); + &mul_matrix(@vtmp[1], $ATAHMatV, $ATALMatV,$vtmp4); +$code.=<<___; + mov $dat.16b,@vtmp[0].16b + mov $datx.16b,@vtmp[1].16b + + // linear transformation + ushr @vtmp[0].4s,$dat.4s,32-2 + ushr $vtmp5.4s,$datx.4s,32-2 + ushr @vtmp[1].4s,$dat.4s,32-10 + ushr @vtmp[2].4s,$dat.4s,32-18 + ushr @vtmp[3].4s,$dat.4s,32-24 + sli @vtmp[0].4s,$dat.4s,2 + sli $vtmp5.4s,$datx.4s,2 + sli @vtmp[1].4s,$dat.4s,10 + sli @vtmp[2].4s,$dat.4s,18 + sli @vtmp[3].4s,$dat.4s,24 + eor $vtmp4.16b,@vtmp[0].16b,$dat.16b + eor $vtmp4.16b,$vtmp4.16b,@vtmp[1].16b + eor $dat.16b,@vtmp[2].16b,@vtmp[3].16b + eor $dat.16b,$dat.16b,$vtmp4.16b + ushr @vtmp[1].4s,$datx.4s,32-10 + ushr @vtmp[2].4s,$datx.4s,32-18 + ushr @vtmp[3].4s,$datx.4s,32-24 + sli @vtmp[1].4s,$datx.4s,10 + sli @vtmp[2].4s,$datx.4s,18 + sli @vtmp[3].4s,$datx.4s,24 + eor $vtmp4.16b,$vtmp5.16b,$datx.16b + eor $vtmp4.16b,$vtmp4.16b,@vtmp[1].16b + eor $datx.16b,@vtmp[2].16b,@vtmp[3].16b + eor $datx.16b,$datx.16b,$vtmp4.16b +___ +} + +# sbox operation for one single word +sub sbox_1word () { + my $word = shift; + +$code.=<<___; + mov @vtmp[3].s[0],$word + // optimize sbox using AESE instruction + tbl @vtmp[0].16b, {@vtmp[3].16b}, $MaskV.16b +___ + &mul_matrix(@vtmp[0], $TAHMatV, $TALMatV, @vtmp[2]); +$code.=<<___; + eor @vtmp[1].16b, @vtmp[1].16b, @vtmp[1].16b + aese @vtmp[0].16b,@vtmp[1].16b +___ + &mul_matrix(@vtmp[0], $ATAHMatV, $ATALMatV, @vtmp[2]); +$code.=<<___; + + mov $wtmp0,@vtmp[0].s[0] + eor $word,$wtmp0,$wtmp0,ror #32-2 + eor $word,$word,$wtmp0,ror #32-10 + eor $word,$word,$wtmp0,ror #32-18 + eor $word,$word,$wtmp0,ror #32-24 +___ +} + +# sm4 for one block of data, in scalar registers word0/word1/word2/word3 +sub sm4_1blk () { + my $kptr = shift; + +$code.=<<___; + ldp $wtmp0,$wtmp1,[$kptr],8 + // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) + eor $tmpw,$word2,$word3 + eor $wtmp2,$wtmp0,$word1 + eor $tmpw,$tmpw,$wtmp2 +___ + &sbox_1word($tmpw); +$code.=<<___; + eor $word0,$word0,$tmpw + // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) + eor $tmpw,$word2,$word3 + eor $wtmp2,$word0,$wtmp1 + eor $tmpw,$tmpw,$wtmp2 +___ + &sbox_1word($tmpw); +$code.=<<___; + ldp $wtmp0,$wtmp1,[$kptr],8 + eor $word1,$word1,$tmpw + // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) + eor $tmpw,$word0,$word1 + eor $wtmp2,$wtmp0,$word3 + eor $tmpw,$tmpw,$wtmp2 +___ + &sbox_1word($tmpw); +$code.=<<___; + eor $word2,$word2,$tmpw + // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) + eor $tmpw,$word0,$word1 + eor $wtmp2,$word2,$wtmp1 + eor $tmpw,$tmpw,$wtmp2 +___ + &sbox_1word($tmpw); +$code.=<<___; + eor $word3,$word3,$tmpw +___ +} + +# sm4 for 4-lanes of data, in neon registers data0/data1/data2/data3 +sub sm4_4blks () { + my $kptr = shift; + +$code.=<<___; + ldp $wtmp0,$wtmp1,[$kptr],8 + dup $rk0.4s,$wtmp0 + dup $rk1.4s,$wtmp1 + + // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) + eor $rka.16b,@data[2].16b,@data[3].16b + eor $rk0.16b,@data[1].16b,$rk0.16b + eor $rk0.16b,$rka.16b,$rk0.16b +___ + &sbox($rk0); +$code.=<<___; + eor @data[0].16b,@data[0].16b,$rk0.16b + + // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) + eor $rka.16b,$rka.16b,@data[0].16b + eor $rk1.16b,$rka.16b,$rk1.16b +___ + &sbox($rk1); +$code.=<<___; + ldp $wtmp0,$wtmp1,[$kptr],8 + eor @data[1].16b,@data[1].16b,$rk1.16b + + dup $rk0.4s,$wtmp0 + dup $rk1.4s,$wtmp1 + + // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) + eor $rka.16b,@data[0].16b,@data[1].16b + eor $rk0.16b,@data[3].16b,$rk0.16b + eor $rk0.16b,$rka.16b,$rk0.16b +___ + &sbox($rk0); +$code.=<<___; + eor @data[2].16b,@data[2].16b,$rk0.16b + + // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) + eor $rka.16b,$rka.16b,@data[2].16b + eor $rk1.16b,$rka.16b,$rk1.16b +___ + &sbox($rk1); +$code.=<<___; + eor @data[3].16b,@data[3].16b,$rk1.16b +___ +} + +# sm4 for 8 lanes of data, in neon registers +# data0/data1/data2/data3 datax0/datax1/datax2/datax3 +sub sm4_8blks () { + my $kptr = shift; + +$code.=<<___; + ldp $wtmp0,$wtmp1,[$kptr],8 + // B0 ^= SBOX(B1 ^ B2 ^ B3 ^ RK0) + dup $rk0.4s,$wtmp0 + eor $rka.16b,@data[2].16b,@data[3].16b + eor $rkb.16b,@datax[2].16b,@datax[3].16b + eor @vtmp[0].16b,@data[1].16b,$rk0.16b + eor @vtmp[1].16b,@datax[1].16b,$rk0.16b + eor $rk0.16b,$rka.16b,@vtmp[0].16b + eor $rk1.16b,$rkb.16b,@vtmp[1].16b +___ + &sbox_double($rk0,$rk1); +$code.=<<___; + eor @data[0].16b,@data[0].16b,$rk0.16b + eor @datax[0].16b,@datax[0].16b,$rk1.16b + + // B1 ^= SBOX(B0 ^ B2 ^ B3 ^ RK1) + dup $rk1.4s,$wtmp1 + eor $rka.16b,$rka.16b,@data[0].16b + eor $rkb.16b,$rkb.16b,@datax[0].16b + eor $rk0.16b,$rka.16b,$rk1.16b + eor $rk1.16b,$rkb.16b,$rk1.16b +___ + &sbox_double($rk0,$rk1); +$code.=<<___; + ldp $wtmp0,$wtmp1,[$kptr],8 + eor @data[1].16b,@data[1].16b,$rk0.16b + eor @datax[1].16b,@datax[1].16b,$rk1.16b + + // B2 ^= SBOX(B0 ^ B1 ^ B3 ^ RK2) + dup $rk0.4s,$wtmp0 + eor $rka.16b,@data[0].16b,@data[1].16b + eor $rkb.16b,@datax[0].16b,@datax[1].16b + eor @vtmp[0].16b,@data[3].16b,$rk0.16b + eor @vtmp[1].16b,@datax[3].16b,$rk0.16b + eor $rk0.16b,$rka.16b,@vtmp[0].16b + eor $rk1.16b,$rkb.16b,@vtmp[1].16b +___ + &sbox_double($rk0,$rk1); +$code.=<<___; + eor @data[2].16b,@data[2].16b,$rk0.16b + eor @datax[2].16b,@datax[2].16b,$rk1.16b + + // B3 ^= SBOX(B0 ^ B1 ^ B2 ^ RK3) + dup $rk1.4s,$wtmp1 + eor $rka.16b,$rka.16b,@data[2].16b + eor $rkb.16b,$rkb.16b,@datax[2].16b + eor $rk0.16b,$rka.16b,$rk1.16b + eor $rk1.16b,$rkb.16b,$rk1.16b +___ + &sbox_double($rk0,$rk1); +$code.=<<___; + eor @data[3].16b,@data[3].16b,$rk0.16b + eor @datax[3].16b,@datax[3].16b,$rk1.16b +___ +} + +sub encrypt_1blk_norev() { + my $dat = shift; + +$code.=<<___; + mov $ptr,$rks + mov $counter,#8 + mov $word0,$dat.s[0] + mov $word1,$dat.s[1] + mov $word2,$dat.s[2] + mov $word3,$dat.s[3] +10: +___ + &sm4_1blk($ptr); +$code.=<<___; + subs $counter,$counter,#1 + b.ne 10b + mov $dat.s[0],$word3 + mov $dat.s[1],$word2 + mov $dat.s[2],$word1 + mov $dat.s[3],$word0 +___ +} + +sub encrypt_1blk() { + my $dat = shift; + + &encrypt_1blk_norev($dat); + &rev32($dat,$dat); +} + +sub encrypt_4blks() { +$code.=<<___; + mov $ptr,$rks + mov $counter,#8 +10: +___ + &sm4_4blks($ptr); +$code.=<<___; + subs $counter,$counter,#1 + b.ne 10b +___ + &rev32(@vtmp[3],@data[0]); + &rev32(@vtmp[2],@data[1]); + &rev32(@vtmp[1],@data[2]); + &rev32(@vtmp[0],@data[3]); +} + +sub encrypt_8blks() { +$code.=<<___; + mov $ptr,$rks + mov $counter,#8 +10: +___ + &sm4_8blks($ptr); +$code.=<<___; + subs $counter,$counter,#1 + b.ne 10b +___ + &rev32(@vtmp[3],@data[0]); + &rev32(@vtmp[2],@data[1]); + &rev32(@vtmp[1],@data[2]); + &rev32(@vtmp[0],@data[3]); + &rev32(@data[3],@datax[0]); + &rev32(@data[2],@datax[1]); + &rev32(@data[1],@datax[2]); + &rev32(@data[0],@datax[3]); +} + +sub load_sbox () { + my $data = shift; + +$code.=<<___; + adrp $xtmp2, .Lsbox_magic + ldr $MaskQ, [$xtmp2, #:lo12:.Lsbox_magic] + ldr $TAHMatQ, [$xtmp2, #:lo12:.Lsbox_magic+16] + ldr $TALMatQ, [$xtmp2, #:lo12:.Lsbox_magic+32] + ldr $ATAHMatQ, [$xtmp2, #:lo12:.Lsbox_magic+48] + ldr $ATALMatQ, [$xtmp2, #:lo12:.Lsbox_magic+64] + ldr $ANDMaskQ, [$xtmp2, #:lo12:.Lsbox_magic+80] +___ +} + +sub mov_reg_to_vec() { + my $src0 = shift; + my $src1 = shift; + my $desv = shift; +$code.=<<___; + mov $desv.d[0],$src0 + mov $desv.d[1],$src1 +___ + &rev32_armeb($desv,$desv); +} + +sub mov_vec_to_reg() { + my $srcv = shift; + my $des0 = shift; + my $des1 = shift; +$code.=<<___; + mov $des0,$srcv.d[0] + mov $des1,$srcv.d[1] +___ +} + +sub compute_tweak() { + my $src0 = shift; + my $src1 = shift; + my $des0 = shift; + my $des1 = shift; +$code.=<<___; + mov $wtmp0,0x87 + extr $xtmp2,$src1,$src1,#32 + extr $des1,$src1,$src0,#63 + and $wtmp1,$wtmp0,$wtmp2,asr#31 + eor $des0,$xtmp1,$src0,lsl#1 +___ +} + +sub compute_tweak_vec() { + my $src = shift; + my $des = shift; + my $std = shift; + &rbit(@vtmp[2],$src,$std); +$code.=<<___; + adrp $xtmp2, .Lxts_magic + ldr @qtmp[0], [$xtmp2, #:lo12:.Lxts_magic] + shl $des.16b, @vtmp[2].16b, #1 + ext @vtmp[1].16b, @vtmp[2].16b, @vtmp[2].16b,#15 + ushr @vtmp[1].16b, @vtmp[1].16b, #7 + mul @vtmp[1].16b, @vtmp[1].16b, @vtmp[0].16b + eor $des.16b, $des.16b, @vtmp[1].16b +___ + &rbit($des,$des,$std); +} + +$code=<<___; +#include "arm_arch.h" +.arch armv8-a+crypto +.text + +.type _${prefix}_consts,%object +.align 7 +_${prefix}_consts: +.Lck: + .long 0x00070E15, 0x1C232A31, 0x383F464D, 0x545B6269 + .long 0x70777E85, 0x8C939AA1, 0xA8AFB6BD, 0xC4CBD2D9 + .long 0xE0E7EEF5, 0xFC030A11, 0x181F262D, 0x343B4249 + .long 0x50575E65, 0x6C737A81, 0x888F969D, 0xA4ABB2B9 + .long 0xC0C7CED5, 0xDCE3EAF1, 0xF8FF060D, 0x141B2229 + .long 0x30373E45, 0x4C535A61, 0x686F767D, 0x848B9299 + .long 0xA0A7AEB5, 0xBCC3CAD1, 0xD8DFE6ED, 0xF4FB0209 + .long 0x10171E25, 0x2C333A41, 0x484F565D, 0x646B7279 +.Lfk: + .quad 0x56aa3350a3b1bac6,0xb27022dc677d9197 +.Lshuffles: + .quad 0x0B0A090807060504,0x030201000F0E0D0C +.Lxts_magic: + .quad 0x0101010101010187,0x0101010101010101 +.Lsbox_magic: + .quad 0x0b0e0104070a0d00,0x0306090c0f020508 + .quad 0x62185a2042387a00,0x22581a6002783a40 + .quad 0x15df62a89e54e923,0xc10bb67c4a803df7 + .quad 0xb9aa6b78c1d21300,0x1407c6d56c7fbead + .quad 0x6404462679195b3b,0xe383c1a1fe9edcbc + .quad 0x0f0f0f0f0f0f0f0f,0x0f0f0f0f0f0f0f0f + +.size _${prefix}_consts,.-_${prefix}_consts +___ + +{{{ +my ($key,$keys,$enc)=("x0","x1","w2"); +my ($pointer,$schedules,$wtmp,$roundkey)=("x5","x6","w7","w8"); +my ($vkey,$vfk,$vmap)=("v5","v6","v7"); +$code.=<<___; +.type _${prefix}_set_key,%function +.align 4 +_${prefix}_set_key: + AARCH64_VALID_CALL_TARGET + ld1 {$vkey.4s},[$key] +___ + &load_sbox(); + &rev32($vkey,$vkey); +$code.=<<___; + adrp $pointer,.Lshuffles + add $pointer,$pointer,#:lo12:.Lshuffles + ld1 {$vmap.2d},[$pointer] + adrp $pointer,.Lfk + add $pointer,$pointer,#:lo12:.Lfk + ld1 {$vfk.2d},[$pointer] + eor $vkey.16b,$vkey.16b,$vfk.16b + mov $schedules,#32 + adrp $pointer,.Lck + add $pointer,$pointer,#:lo12:.Lck + movi @vtmp[0].16b,#64 + cbnz $enc,1f + add $keys,$keys,124 +1: + mov $wtmp,$vkey.s[1] + ldr $roundkey,[$pointer],#4 + eor $roundkey,$roundkey,$wtmp + mov $wtmp,$vkey.s[2] + eor $roundkey,$roundkey,$wtmp + mov $wtmp,$vkey.s[3] + eor $roundkey,$roundkey,$wtmp + // optimize sbox using AESE instruction + mov @data[0].s[0],$roundkey + tbl @vtmp[0].16b, {@data[0].16b}, $MaskV.16b +___ + &mul_matrix(@vtmp[0], $TAHMatV, $TALMatV, @vtmp[2]); +$code.=<<___; + eor @vtmp[1].16b, @vtmp[1].16b, @vtmp[1].16b + aese @vtmp[0].16b,@vtmp[1].16b +___ + &mul_matrix(@vtmp[0], $ATAHMatV, $ATALMatV, @vtmp[2]); +$code.=<<___; + mov $wtmp,@vtmp[0].s[0] + eor $roundkey,$wtmp,$wtmp,ror #19 + eor $roundkey,$roundkey,$wtmp,ror #9 + mov $wtmp,$vkey.s[0] + eor $roundkey,$roundkey,$wtmp + mov $vkey.s[0],$roundkey + cbz $enc,2f + str $roundkey,[$keys],#4 + b 3f +2: + str $roundkey,[$keys],#-4 +3: + tbl $vkey.16b,{$vkey.16b},$vmap.16b + subs $schedules,$schedules,#1 + b.ne 1b + ret +.size _${prefix}_set_key,.-_${prefix}_set_key +___ +}}} + + +{{{ +$code.=<<___; +.type _${prefix}_enc_4blks,%function +.align 4 +_${prefix}_enc_4blks: + AARCH64_VALID_CALL_TARGET +___ + &encrypt_4blks(); +$code.=<<___; + ret +.size _${prefix}_enc_4blks,.-_${prefix}_enc_4blks +___ +}}} + +{{{ +$code.=<<___; +.type _${prefix}_enc_8blks,%function +.align 4 +_${prefix}_enc_8blks: + AARCH64_VALID_CALL_TARGET +___ + &encrypt_8blks(); +$code.=<<___; + ret +.size _${prefix}_enc_8blks,.-_${prefix}_enc_8blks +___ +}}} + + +{{{ +my ($key,$keys)=("x0","x1"); +$code.=<<___; +.globl ${prefix}_set_encrypt_key +.type ${prefix}_set_encrypt_key,%function +.align 5 +${prefix}_set_encrypt_key: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-16]! + mov w2,1 + bl _${prefix}_set_key + ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER + ret +.size ${prefix}_set_encrypt_key,.-${prefix}_set_encrypt_key +___ +}}} + +{{{ +my ($key,$keys)=("x0","x1"); +$code.=<<___; +.globl ${prefix}_set_decrypt_key +.type ${prefix}_set_decrypt_key,%function +.align 5 +${prefix}_set_decrypt_key: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-16]! + mov w2,0 + bl _${prefix}_set_key + ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER + ret +.size ${prefix}_set_decrypt_key,.-${prefix}_set_decrypt_key +___ +}}} + +{{{ +sub gen_block () { + my $dir = shift; + my ($inp,$outp,$rk)=map("x$_",(0..2)); + +$code.=<<___; +.globl ${prefix}_${dir}crypt +.type ${prefix}_${dir}crypt,%function +.align 5 +${prefix}_${dir}crypt: + AARCH64_VALID_CALL_TARGET + ld1 {@data[0].4s},[$inp] +___ + &load_sbox(); + &rev32(@data[0],@data[0]); +$code.=<<___; + mov $rks,$rk +___ + &encrypt_1blk(@data[0]); +$code.=<<___; + st1 {@data[0].4s},[$outp] + ret +.size ${prefix}_${dir}crypt,.-${prefix}_${dir}crypt +___ +} +&gen_block("en"); +&gen_block("de"); +}}} + +{{{ +$code.=<<___; +.globl ${prefix}_ecb_encrypt +.type ${prefix}_ecb_encrypt,%function +.align 5 +${prefix}_ecb_encrypt: + AARCH64_SIGN_LINK_REGISTER + // convert length into blocks + lsr x2,x2,4 + stp d8,d9,[sp,#-80]! + stp d10,d11,[sp,#16] + stp d12,d13,[sp,#32] + stp d14,d15,[sp,#48] + stp x29,x30,[sp,#64] +___ + &load_sbox(); +$code.=<<___; +.Lecb_8_blocks_process: + cmp $blocks,#8 + b.lt .Lecb_4_blocks_process + ld4 {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$inp],#64 + ld4 {@datax[0].4s,$datax[1].4s,@datax[2].4s,@datax[3].4s},[$inp],#64 +___ + &rev32(@data[0],@data[0]); + &rev32(@data[1],@data[1]); + &rev32(@data[2],@data[2]); + &rev32(@data[3],@data[3]); + &rev32(@datax[0],@datax[0]); + &rev32(@datax[1],@datax[1]); + &rev32(@datax[2],@datax[2]); + &rev32(@datax[3],@datax[3]); +$code.=<<___; + bl _${prefix}_enc_8blks + st4 {@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s,@vtmp[3].4s},[$outp],#64 + st4 {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$outp],#64 + subs $blocks,$blocks,#8 + b.gt .Lecb_8_blocks_process + b 100f +.Lecb_4_blocks_process: + cmp $blocks,#4 + b.lt 1f + ld4 {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$inp],#64 +___ + &rev32(@data[0],@data[0]); + &rev32(@data[1],@data[1]); + &rev32(@data[2],@data[2]); + &rev32(@data[3],@data[3]); +$code.=<<___; + bl _${prefix}_enc_4blks + st4 {@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s,@vtmp[3].4s},[$outp],#64 + sub $blocks,$blocks,#4 +1: + // process last block + cmp $blocks,#1 + b.lt 100f + b.gt 1f + ld1 {@data[0].4s},[$inp] +___ + &rev32(@data[0],@data[0]); + &encrypt_1blk(@data[0]); +$code.=<<___; + st1 {@data[0].4s},[$outp] + b 100f +1: // process last 2 blocks + ld4 {@data[0].s,@data[1].s,@data[2].s,@data[3].s}[0],[$inp],#16 + ld4 {@data[0].s,@data[1].s,@data[2].s,@data[3].s}[1],[$inp],#16 + cmp $blocks,#2 + b.gt 1f +___ + &rev32(@data[0],@data[0]); + &rev32(@data[1],@data[1]); + &rev32(@data[2],@data[2]); + &rev32(@data[3],@data[3]); +$code.=<<___; + bl _${prefix}_enc_4blks + st4 {@vtmp[0].s-@vtmp[3].s}[0],[$outp],#16 + st4 {@vtmp[0].s-@vtmp[3].s}[1],[$outp] + b 100f +1: // process last 3 blocks + ld4 {@data[0].s,@data[1].s,@data[2].s,@data[3].s}[2],[$inp],#16 +___ + &rev32(@data[0],@data[0]); + &rev32(@data[1],@data[1]); + &rev32(@data[2],@data[2]); + &rev32(@data[3],@data[3]); +$code.=<<___; + bl _${prefix}_enc_4blks + st4 {@vtmp[0].s-@vtmp[3].s}[0],[$outp],#16 + st4 {@vtmp[0].s-@vtmp[3].s}[1],[$outp],#16 + st4 {@vtmp[0].s-@vtmp[3].s}[2],[$outp] +100: + ldp d10,d11,[sp,#16] + ldp d12,d13,[sp,#32] + ldp d14,d15,[sp,#48] + ldp x29,x30,[sp,#64] + ldp d8,d9,[sp],#80 + AARCH64_VALIDATE_LINK_REGISTER + ret +.size ${prefix}_ecb_encrypt,.-${prefix}_ecb_encrypt +___ +}}} + +{{{ +my ($len,$ivp,$enc)=("x2","x4","w5"); +my $ivec0=("v3"); +my $ivec1=("v15"); + +$code.=<<___; +.globl ${prefix}_cbc_encrypt +.type ${prefix}_cbc_encrypt,%function +.align 5 +${prefix}_cbc_encrypt: + AARCH64_VALID_CALL_TARGET + lsr $len,$len,4 +___ + &load_sbox(); +$code.=<<___; + cbz $enc,.Ldec + ld1 {$ivec0.4s},[$ivp] +.Lcbc_4_blocks_enc: + cmp $blocks,#4 + b.lt 1f + ld1 {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$inp],#64 + eor @data[0].16b,@data[0].16b,$ivec0.16b +___ + &rev32(@data[1],@data[1]); + &rev32(@data[0],@data[0]); + &rev32(@data[2],@data[2]); + &rev32(@data[3],@data[3]); + &encrypt_1blk_norev(@data[0]); +$code.=<<___; + eor @data[1].16b,@data[1].16b,@data[0].16b +___ + &encrypt_1blk_norev(@data[1]); + &rev32(@data[0],@data[0]); + +$code.=<<___; + eor @data[2].16b,@data[2].16b,@data[1].16b +___ + &encrypt_1blk_norev(@data[2]); + &rev32(@data[1],@data[1]); +$code.=<<___; + eor @data[3].16b,@data[3].16b,@data[2].16b +___ + &encrypt_1blk_norev(@data[3]); + &rev32(@data[2],@data[2]); + &rev32(@data[3],@data[3]); +$code.=<<___; + orr $ivec0.16b,@data[3].16b,@data[3].16b + st1 {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$outp],#64 + subs $blocks,$blocks,#4 + b.ne .Lcbc_4_blocks_enc + b 2f +1: + subs $blocks,$blocks,#1 + b.lt 2f + ld1 {@data[0].4s},[$inp],#16 + eor $ivec0.16b,$ivec0.16b,@data[0].16b +___ + &rev32($ivec0,$ivec0); + &encrypt_1blk($ivec0); +$code.=<<___; + st1 {$ivec0.4s},[$outp],#16 + b 1b +2: + // save back IV + st1 {$ivec0.4s},[$ivp] + ret + +.Ldec: + // decryption mode starts + AARCH64_SIGN_LINK_REGISTER + stp d8,d9,[sp,#-80]! + stp d10,d11,[sp,#16] + stp d12,d13,[sp,#32] + stp d14,d15,[sp,#48] + stp x29,x30,[sp,#64] +.Lcbc_8_blocks_dec: + cmp $blocks,#8 + b.lt 1f + ld4 {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$inp] + add $ptr,$inp,#64 + ld4 {@datax[0].4s,@datax[1].4s,@datax[2].4s,@datax[3].4s},[$ptr] +___ + &rev32(@data[0],@data[0]); + &rev32(@data[1],@data[1]); + &rev32(@data[2],@data[2]); + &rev32(@data[3],$data[3]); + &rev32(@datax[0],@datax[0]); + &rev32(@datax[1],@datax[1]); + &rev32(@datax[2],@datax[2]); + &rev32(@datax[3],$datax[3]); +$code.=<<___; + bl _${prefix}_enc_8blks +___ + &transpose(@vtmp,@datax); + &transpose(@data,@datax); +$code.=<<___; + ld1 {$ivec1.4s},[$ivp] + ld1 {@datax[0].4s,@datax[1].4s,@datax[2].4s,@datax[3].4s},[$inp],#64 + // note ivec1 and vtmpx[3] are reusing the same register + // care needs to be taken to avoid conflict + eor @vtmp[0].16b,@vtmp[0].16b,$ivec1.16b + ld1 {@vtmpx[0].4s,@vtmpx[1].4s,@vtmpx[2].4s,@vtmpx[3].4s},[$inp],#64 + eor @vtmp[1].16b,@vtmp[1].16b,@datax[0].16b + eor @vtmp[2].16b,@vtmp[2].16b,@datax[1].16b + eor @vtmp[3].16b,$vtmp[3].16b,@datax[2].16b + // save back IV + st1 {$vtmpx[3].4s}, [$ivp] + eor @data[0].16b,@data[0].16b,$datax[3].16b + eor @data[1].16b,@data[1].16b,@vtmpx[0].16b + eor @data[2].16b,@data[2].16b,@vtmpx[1].16b + eor @data[3].16b,$data[3].16b,@vtmpx[2].16b + st1 {@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s,@vtmp[3].4s},[$outp],#64 + st1 {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$outp],#64 + subs $blocks,$blocks,#8 + b.gt .Lcbc_8_blocks_dec + b.eq 100f +1: + ld1 {$ivec1.4s},[$ivp] +.Lcbc_4_blocks_dec: + cmp $blocks,#4 + b.lt 1f + ld4 {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$inp] +___ + &rev32(@data[0],@data[0]); + &rev32(@data[1],@data[1]); + &rev32(@data[2],@data[2]); + &rev32(@data[3],$data[3]); +$code.=<<___; + bl _${prefix}_enc_4blks + ld1 {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$inp],#64 +___ + &transpose(@vtmp,@datax); +$code.=<<___; + eor @vtmp[0].16b,@vtmp[0].16b,$ivec1.16b + eor @vtmp[1].16b,@vtmp[1].16b,@data[0].16b + orr $ivec1.16b,@data[3].16b,@data[3].16b + eor @vtmp[2].16b,@vtmp[2].16b,@data[1].16b + eor @vtmp[3].16b,$vtmp[3].16b,@data[2].16b + st1 {@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s,@vtmp[3].4s},[$outp],#64 + subs $blocks,$blocks,#4 + b.gt .Lcbc_4_blocks_dec + // save back IV + st1 {@data[3].4s}, [$ivp] + b 100f +1: // last block + subs $blocks,$blocks,#1 + b.lt 100f + b.gt 1f + ld1 {@data[0].4s},[$inp],#16 + // save back IV + st1 {$data[0].4s}, [$ivp] +___ + &rev32(@datax[0],@data[0]); + &encrypt_1blk(@datax[0]); +$code.=<<___; + eor @datax[0].16b,@datax[0].16b,$ivec1.16b + st1 {@datax[0].4s},[$outp],#16 + b 100f +1: // last two blocks + ld4 {@data[0].s,@data[1].s,@data[2].s,@data[3].s}[0],[$inp] + add $ptr,$inp,#16 + ld4 {@data[0].s,@data[1].s,@data[2].s,@data[3].s}[1],[$ptr],#16 + subs $blocks,$blocks,1 + b.gt 1f +___ + &rev32(@data[0],@data[0]); + &rev32(@data[1],@data[1]); + &rev32(@data[2],@data[2]); + &rev32(@data[3],@data[3]); +$code.=<<___; + bl _${prefix}_enc_4blks + ld1 {@data[0].4s,@data[1].4s},[$inp],#32 +___ + &transpose(@vtmp,@datax); +$code.=<<___; + eor @vtmp[0].16b,@vtmp[0].16b,$ivec1.16b + eor @vtmp[1].16b,@vtmp[1].16b,@data[0].16b + st1 {@vtmp[0].4s,@vtmp[1].4s},[$outp],#32 + // save back IV + st1 {@data[1].4s}, [$ivp] + b 100f +1: // last 3 blocks + ld4 {@data[0].s,@data[1].s,@data[2].s,@data[3].s}[2],[$ptr] +___ + &rev32(@data[0],@data[0]); + &rev32(@data[1],@data[1]); + &rev32(@data[2],@data[2]); + &rev32(@data[3],@data[3]); +$code.=<<___; + bl _${prefix}_enc_4blks + ld1 {@data[0].4s,@data[1].4s,@data[2].4s},[$inp],#48 +___ + &transpose(@vtmp,@datax); +$code.=<<___; + eor @vtmp[0].16b,@vtmp[0].16b,$ivec1.16b + eor @vtmp[1].16b,@vtmp[1].16b,@data[0].16b + eor @vtmp[2].16b,@vtmp[2].16b,@data[1].16b + st1 {@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s},[$outp],#48 + // save back IV + st1 {@data[2].4s}, [$ivp] +100: + ldp d10,d11,[sp,#16] + ldp d12,d13,[sp,#32] + ldp d14,d15,[sp,#48] + ldp x29,x30,[sp,#64] + ldp d8,d9,[sp],#80 + AARCH64_VALIDATE_LINK_REGISTER + ret +.size ${prefix}_cbc_encrypt,.-${prefix}_cbc_encrypt +___ +}}} + +{{{ +my ($ivp)=("x4"); +my ($ctr)=("w5"); +my $ivec=("v3"); + +$code.=<<___; +.globl ${prefix}_ctr32_encrypt_blocks +.type ${prefix}_ctr32_encrypt_blocks,%function +.align 5 +${prefix}_ctr32_encrypt_blocks: + AARCH64_VALID_CALL_TARGET + ld1 {$ivec.4s},[$ivp] +___ + &rev32($ivec,$ivec); + &load_sbox(); +$code.=<<___; + cmp $blocks,#1 + b.ne 1f + // fast processing for one single block without + // context saving overhead +___ + &encrypt_1blk($ivec); +$code.=<<___; + ld1 {@data[0].4s},[$inp] + eor @data[0].16b,@data[0].16b,$ivec.16b + st1 {@data[0].4s},[$outp] + ret +1: + AARCH64_SIGN_LINK_REGISTER + stp d8,d9,[sp,#-80]! + stp d10,d11,[sp,#16] + stp d12,d13,[sp,#32] + stp d14,d15,[sp,#48] + stp x29,x30,[sp,#64] + mov $word0,$ivec.s[0] + mov $word1,$ivec.s[1] + mov $word2,$ivec.s[2] + mov $ctr,$ivec.s[3] +.Lctr32_4_blocks_process: + cmp $blocks,#4 + b.lt 1f + dup @data[0].4s,$word0 + dup @data[1].4s,$word1 + dup @data[2].4s,$word2 + mov @data[3].s[0],$ctr + add $ctr,$ctr,#1 + mov $data[3].s[1],$ctr + add $ctr,$ctr,#1 + mov @data[3].s[2],$ctr + add $ctr,$ctr,#1 + mov @data[3].s[3],$ctr + add $ctr,$ctr,#1 + cmp $blocks,#8 + b.ge .Lctr32_8_blocks_process + bl _${prefix}_enc_4blks + ld4 {@vtmpx[0].4s,@vtmpx[1].4s,@vtmpx[2].4s,@vtmpx[3].4s},[$inp],#64 + eor @vtmp[0].16b,@vtmp[0].16b,@vtmpx[0].16b + eor @vtmp[1].16b,@vtmp[1].16b,@vtmpx[1].16b + eor @vtmp[2].16b,@vtmp[2].16b,@vtmpx[2].16b + eor @vtmp[3].16b,@vtmp[3].16b,@vtmpx[3].16b + st4 {@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s,@vtmp[3].4s},[$outp],#64 + subs $blocks,$blocks,#4 + b.ne .Lctr32_4_blocks_process + b 100f +.Lctr32_8_blocks_process: + dup @datax[0].4s,$word0 + dup @datax[1].4s,$word1 + dup @datax[2].4s,$word2 + mov @datax[3].s[0],$ctr + add $ctr,$ctr,#1 + mov $datax[3].s[1],$ctr + add $ctr,$ctr,#1 + mov @datax[3].s[2],$ctr + add $ctr,$ctr,#1 + mov @datax[3].s[3],$ctr + add $ctr,$ctr,#1 + bl _${prefix}_enc_8blks + ld4 {@vtmpx[0].4s,@vtmpx[1].4s,@vtmpx[2].4s,@vtmpx[3].4s},[$inp],#64 + ld4 {@datax[0].4s,@datax[1].4s,@datax[2].4s,@datax[3].4s},[$inp],#64 + eor @vtmp[0].16b,@vtmp[0].16b,@vtmpx[0].16b + eor @vtmp[1].16b,@vtmp[1].16b,@vtmpx[1].16b + eor @vtmp[2].16b,@vtmp[2].16b,@vtmpx[2].16b + eor @vtmp[3].16b,@vtmp[3].16b,@vtmpx[3].16b + eor @data[0].16b,@data[0].16b,@datax[0].16b + eor @data[1].16b,@data[1].16b,@datax[1].16b + eor @data[2].16b,@data[2].16b,@datax[2].16b + eor @data[3].16b,@data[3].16b,@datax[3].16b + st4 {@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s,@vtmp[3].4s},[$outp],#64 + st4 {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$outp],#64 + subs $blocks,$blocks,#8 + b.ne .Lctr32_4_blocks_process + b 100f +1: // last block processing + subs $blocks,$blocks,#1 + b.lt 100f + b.gt 1f + mov $ivec.s[0],$word0 + mov $ivec.s[1],$word1 + mov $ivec.s[2],$word2 + mov $ivec.s[3],$ctr +___ + &encrypt_1blk($ivec); +$code.=<<___; + ld1 {@data[0].4s},[$inp] + eor @data[0].16b,@data[0].16b,$ivec.16b + st1 {@data[0].4s},[$outp] + b 100f +1: // last 2 blocks processing + dup @data[0].4s,$word0 + dup @data[1].4s,$word1 + dup @data[2].4s,$word2 + mov @data[3].s[0],$ctr + add $ctr,$ctr,#1 + mov @data[3].s[1],$ctr + subs $blocks,$blocks,#1 + b.ne 1f + bl _${prefix}_enc_4blks + ld4 {@vtmpx[0].s,@vtmpx[1].s,@vtmpx[2].s,@vtmpx[3].s}[0],[$inp],#16 + ld4 {@vtmpx[0].s,@vtmpx[1].s,@vtmpx[2].s,@vtmpx[3].s}[1],[$inp],#16 + eor @vtmp[0].16b,@vtmp[0].16b,@vtmpx[0].16b + eor @vtmp[1].16b,@vtmp[1].16b,@vtmpx[1].16b + eor @vtmp[2].16b,@vtmp[2].16b,@vtmpx[2].16b + eor @vtmp[3].16b,@vtmp[3].16b,@vtmpx[3].16b + st4 {@vtmp[0].s,@vtmp[1].s,@vtmp[2].s,@vtmp[3].s}[0],[$outp],#16 + st4 {@vtmp[0].s,@vtmp[1].s,@vtmp[2].s,@vtmp[3].s}[1],[$outp],#16 + b 100f +1: // last 3 blocks processing + add $ctr,$ctr,#1 + mov @data[3].s[2],$ctr + bl _${prefix}_enc_4blks + ld4 {@vtmpx[0].s,@vtmpx[1].s,@vtmpx[2].s,@vtmpx[3].s}[0],[$inp],#16 + ld4 {@vtmpx[0].s,@vtmpx[1].s,@vtmpx[2].s,@vtmpx[3].s}[1],[$inp],#16 + ld4 {@vtmpx[0].s,@vtmpx[1].s,@vtmpx[2].s,@vtmpx[3].s}[2],[$inp],#16 + eor @vtmp[0].16b,@vtmp[0].16b,@vtmpx[0].16b + eor @vtmp[1].16b,@vtmp[1].16b,@vtmpx[1].16b + eor @vtmp[2].16b,@vtmp[2].16b,@vtmpx[2].16b + eor @vtmp[3].16b,@vtmp[3].16b,@vtmpx[3].16b + st4 {@vtmp[0].s,@vtmp[1].s,@vtmp[2].s,@vtmp[3].s}[0],[$outp],#16 + st4 {@vtmp[0].s,@vtmp[1].s,@vtmp[2].s,@vtmp[3].s}[1],[$outp],#16 + st4 {@vtmp[0].s,@vtmp[1].s,@vtmp[2].s,@vtmp[3].s}[2],[$outp],#16 +100: + ldp d10,d11,[sp,#16] + ldp d12,d13,[sp,#32] + ldp d14,d15,[sp,#48] + ldp x29,x30,[sp,#64] + ldp d8,d9,[sp],#80 + AARCH64_VALIDATE_LINK_REGISTER + ret +.size ${prefix}_ctr32_encrypt_blocks,.-${prefix}_ctr32_encrypt_blocks +___ +}}} + + +{{{ +my ($blocks,$len)=("x2","x2"); +my $ivp=("x5"); +my @twx=map("x$_",(12..27)); +my ($rks1,$rks2)=("x26","x27"); +my $lastBlk=("x26"); +my $enc=("w28"); +my $remain=("x29"); + +my @tweak=map("v$_",(16..23)); +my $lastTweak=("v25"); + +sub gen_xts_cipher() { + my $std = shift; +$code.=<<___; +.globl ${prefix}_xts_encrypt${std} +.type ${prefix}_xts_encrypt${std},%function +.align 5 +${prefix}_xts_encrypt${std}: + AARCH64_SIGN_LINK_REGISTER + stp x15, x16, [sp, #-0x10]! + stp x17, x18, [sp, #-0x10]! + stp x19, x20, [sp, #-0x10]! + stp x21, x22, [sp, #-0x10]! + stp x23, x24, [sp, #-0x10]! + stp x25, x26, [sp, #-0x10]! + stp x27, x28, [sp, #-0x10]! + stp x29, x30, [sp, #-0x10]! + stp d8, d9, [sp, #-0x10]! + stp d10, d11, [sp, #-0x10]! + stp d12, d13, [sp, #-0x10]! + stp d14, d15, [sp, #-0x10]! + mov $rks1,x3 + mov $rks2,x4 + mov $enc,w6 + ld1 {@tweak[0].4s}, [$ivp] + mov $rks,$rks2 +___ + &load_sbox(); + &rev32(@tweak[0],@tweak[0]); + &encrypt_1blk(@tweak[0]); +$code.=<<___; + mov $rks,$rks1 + and $remain,$len,#0x0F + // convert length into blocks + lsr $blocks,$len,4 + cmp $blocks,#1 + b.lt .return${std} + + cmp $remain,0 + // If the encryption/decryption Length is N times of 16, + // the all blocks are encrypted/decrypted in .xts_encrypt_blocks${std} + b.eq .xts_encrypt_blocks${std} + + // If the encryption/decryption length is not N times of 16, + // the last two blocks are encrypted/decrypted in .last_2blks_tweak${std} or .only_2blks_tweak${std} + // the other blocks are encrypted/decrypted in .xts_encrypt_blocks${std} + subs $blocks,$blocks,#1 + b.eq .only_2blks_tweak${std} +.xts_encrypt_blocks${std}: +___ + &rbit(@tweak[0],@tweak[0],$std); + &rev32_armeb(@tweak[0],@tweak[0]); + &mov_vec_to_reg(@tweak[0],@twx[0],@twx[1]); + &compute_tweak(@twx[0],@twx[1],@twx[2],@twx[3]); + &compute_tweak(@twx[2],@twx[3],@twx[4],@twx[5]); + &compute_tweak(@twx[4],@twx[5],@twx[6],@twx[7]); + &compute_tweak(@twx[6],@twx[7],@twx[8],@twx[9]); + &compute_tweak(@twx[8],@twx[9],@twx[10],@twx[11]); + &compute_tweak(@twx[10],@twx[11],@twx[12],@twx[13]); + &compute_tweak(@twx[12],@twx[13],@twx[14],@twx[15]); +$code.=<<___; +.Lxts_8_blocks_process${std}: + cmp $blocks,#8 +___ + &mov_reg_to_vec(@twx[0],@twx[1],@tweak[0]); + &compute_tweak(@twx[14],@twx[15],@twx[0],@twx[1]); + &mov_reg_to_vec(@twx[2],@twx[3],@tweak[1]); + &compute_tweak(@twx[0],@twx[1],@twx[2],@twx[3]); + &mov_reg_to_vec(@twx[4],@twx[5],@tweak[2]); + &compute_tweak(@twx[2],@twx[3],@twx[4],@twx[5]); + &mov_reg_to_vec(@twx[6],@twx[7],@tweak[3]); + &compute_tweak(@twx[4],@twx[5],@twx[6],@twx[7]); + &mov_reg_to_vec(@twx[8],@twx[9],@tweak[4]); + &compute_tweak(@twx[6],@twx[7],@twx[8],@twx[9]); + &mov_reg_to_vec(@twx[10],@twx[11],@tweak[5]); + &compute_tweak(@twx[8],@twx[9],@twx[10],@twx[11]); + &mov_reg_to_vec(@twx[12],@twx[13],@tweak[6]); + &compute_tweak(@twx[10],@twx[11],@twx[12],@twx[13]); + &mov_reg_to_vec(@twx[14],@twx[15],@tweak[7]); + &compute_tweak(@twx[12],@twx[13],@twx[14],@twx[15]); +$code.=<<___; + b.lt .Lxts_4_blocks_process${std} + ld1 {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$inp],#64 +___ + &rbit(@tweak[0],@tweak[0],$std); + &rbit(@tweak[1],@tweak[1],$std); + &rbit(@tweak[2],@tweak[2],$std); + &rbit(@tweak[3],@tweak[3],$std); +$code.=<<___; + eor @data[0].16b, @data[0].16b, @tweak[0].16b + eor @data[1].16b, @data[1].16b, @tweak[1].16b + eor @data[2].16b, @data[2].16b, @tweak[2].16b + eor @data[3].16b, @data[3].16b, @tweak[3].16b + ld1 {@datax[0].4s,$datax[1].4s,@datax[2].4s,@datax[3].4s},[$inp],#64 +___ + &rbit(@tweak[4],@tweak[4],$std); + &rbit(@tweak[5],@tweak[5],$std); + &rbit(@tweak[6],@tweak[6],$std); + &rbit(@tweak[7],@tweak[7],$std); +$code.=<<___; + eor @datax[0].16b, @datax[0].16b, @tweak[4].16b + eor @datax[1].16b, @datax[1].16b, @tweak[5].16b + eor @datax[2].16b, @datax[2].16b, @tweak[6].16b + eor @datax[3].16b, @datax[3].16b, @tweak[7].16b +___ + &rev32(@data[0],@data[0]); + &rev32(@data[1],@data[1]); + &rev32(@data[2],@data[2]); + &rev32(@data[3],@data[3]); + &rev32(@datax[0],@datax[0]); + &rev32(@datax[1],@datax[1]); + &rev32(@datax[2],@datax[2]); + &rev32(@datax[3],@datax[3]); + &transpose(@data,@vtmp); + &transpose(@datax,@vtmp); +$code.=<<___; + bl _${prefix}_enc_8blks +___ + &transpose(@vtmp,@datax); + &transpose(@data,@datax); +$code.=<<___; + eor @vtmp[0].16b, @vtmp[0].16b, @tweak[0].16b + eor @vtmp[1].16b, @vtmp[1].16b, @tweak[1].16b + eor @vtmp[2].16b, @vtmp[2].16b, @tweak[2].16b + eor @vtmp[3].16b, @vtmp[3].16b, @tweak[3].16b + eor @data[0].16b, @data[0].16b, @tweak[4].16b + eor @data[1].16b, @data[1].16b, @tweak[5].16b + eor @data[2].16b, @data[2].16b, @tweak[6].16b + eor @data[3].16b, @data[3].16b, @tweak[7].16b + + // save the last tweak + mov $lastTweak.16b,@tweak[7].16b + st1 {@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s,@vtmp[3].4s},[$outp],#64 + st1 {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$outp],#64 + subs $blocks,$blocks,#8 + b.gt .Lxts_8_blocks_process${std} + b 100f +.Lxts_4_blocks_process${std}: + cmp $blocks,#4 + b.lt 1f + ld1 {@data[0].4s,@data[1].4s,@data[2].4s,@data[3].4s},[$inp],#64 +___ + &rbit(@tweak[0],@tweak[0],$std); + &rbit(@tweak[1],@tweak[1],$std); + &rbit(@tweak[2],@tweak[2],$std); + &rbit(@tweak[3],@tweak[3],$std); +$code.=<<___; + eor @data[0].16b, @data[0].16b, @tweak[0].16b + eor @data[1].16b, @data[1].16b, @tweak[1].16b + eor @data[2].16b, @data[2].16b, @tweak[2].16b + eor @data[3].16b, @data[3].16b, @tweak[3].16b +___ + &rev32(@data[0],@data[0]); + &rev32(@data[1],@data[1]); + &rev32(@data[2],@data[2]); + &rev32(@data[3],@data[3]); + &transpose(@data,@vtmp); +$code.=<<___; + bl _${prefix}_enc_4blks +___ + &transpose(@vtmp,@data); +$code.=<<___; + eor @vtmp[0].16b, @vtmp[0].16b, @tweak[0].16b + eor @vtmp[1].16b, @vtmp[1].16b, @tweak[1].16b + eor @vtmp[2].16b, @vtmp[2].16b, @tweak[2].16b + eor @vtmp[3].16b, @vtmp[3].16b, @tweak[3].16b + st1 {@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s,@vtmp[3].4s},[$outp],#64 + sub $blocks,$blocks,#4 + mov @tweak[0].16b,@tweak[4].16b + mov @tweak[1].16b,@tweak[5].16b + mov @tweak[2].16b,@tweak[6].16b + // save the last tweak + mov $lastTweak.16b,@tweak[3].16b +1: + // process last block + cmp $blocks,#1 + b.lt 100f + b.gt 1f + ld1 {@data[0].4s},[$inp],#16 +___ + &rbit(@tweak[0],@tweak[0],$std); +$code.=<<___; + eor @data[0].16b, @data[0].16b, @tweak[0].16b +___ + &rev32(@data[0],@data[0]); + &encrypt_1blk(@data[0]); +$code.=<<___; + eor @data[0].16b, @data[0].16b, @tweak[0].16b + st1 {@data[0].4s},[$outp],#16 + // save the last tweak + mov $lastTweak.16b,@tweak[0].16b + b 100f +1: // process last 2 blocks + cmp $blocks,#2 + b.gt 1f + ld1 {@data[0].4s,@data[1].4s},[$inp],#32 +___ + &rbit(@tweak[0],@tweak[0],$std); + &rbit(@tweak[1],@tweak[1],$std); +$code.=<<___; + eor @data[0].16b, @data[0].16b, @tweak[0].16b + eor @data[1].16b, @data[1].16b, @tweak[1].16b +___ + &rev32(@data[0],@data[0]); + &rev32(@data[1],@data[1]); + &transpose(@data,@vtmp); +$code.=<<___; + bl _${prefix}_enc_4blks +___ + &transpose(@vtmp,@data); +$code.=<<___; + eor @vtmp[0].16b, @vtmp[0].16b, @tweak[0].16b + eor @vtmp[1].16b, @vtmp[1].16b, @tweak[1].16b + st1 {@vtmp[0].4s,@vtmp[1].4s},[$outp],#32 + // save the last tweak + mov $lastTweak.16b,@tweak[1].16b + b 100f +1: // process last 3 blocks + ld1 {@data[0].4s,@data[1].4s,@data[2].4s},[$inp],#48 +___ + &rbit(@tweak[0],@tweak[0],$std); + &rbit(@tweak[1],@tweak[1],$std); + &rbit(@tweak[2],@tweak[2],$std); +$code.=<<___; + eor @data[0].16b, @data[0].16b, @tweak[0].16b + eor @data[1].16b, @data[1].16b, @tweak[1].16b + eor @data[2].16b, @data[2].16b, @tweak[2].16b +___ + &rev32(@data[0],@data[0]); + &rev32(@data[1],@data[1]); + &rev32(@data[2],@data[2]); + &transpose(@data,@vtmp); +$code.=<<___; + bl _${prefix}_enc_4blks +___ + &transpose(@vtmp,@data); +$code.=<<___; + eor @vtmp[0].16b, @vtmp[0].16b, @tweak[0].16b + eor @vtmp[1].16b, @vtmp[1].16b, @tweak[1].16b + eor @vtmp[2].16b, @vtmp[2].16b, @tweak[2].16b + st1 {@vtmp[0].4s,@vtmp[1].4s,@vtmp[2].4s},[$outp],#48 + // save the last tweak + mov $lastTweak.16b,@tweak[2].16b +100: + cmp $remain,0 + b.eq .return${std} + +// This branch calculates the last two tweaks, +// while the encryption/decryption length is larger than 32 +.last_2blks_tweak${std}: +___ + &rev32_armeb($lastTweak,$lastTweak); + &compute_tweak_vec($lastTweak,@tweak[1],$std); + &compute_tweak_vec(@tweak[1],@tweak[2],$std); +$code.=<<___; + b .check_dec${std} + + +// This branch calculates the last two tweaks, +// while the encryption/decryption length is equal to 32, who only need two tweaks +.only_2blks_tweak${std}: + mov @tweak[1].16b,@tweak[0].16b +___ + &rev32_armeb(@tweak[1],@tweak[1]); + &compute_tweak_vec(@tweak[1],@tweak[2],$std); +$code.=<<___; + b .check_dec${std} + + +// Determine whether encryption or decryption is required. +// The last two tweaks need to be swapped for decryption. +.check_dec${std}: + // encryption:1 decryption:0 + cmp $enc,1 + b.eq .process_last_2blks${std} + mov @vtmp[0].16B,@tweak[1].16b + mov @tweak[1].16B,@tweak[2].16b + mov @tweak[2].16B,@vtmp[0].16b + +.process_last_2blks${std}: +___ + &rev32_armeb(@tweak[1],@tweak[1]); + &rev32_armeb(@tweak[2],@tweak[2]); +$code.=<<___; + ld1 {@data[0].4s},[$inp],#16 + eor @data[0].16b, @data[0].16b, @tweak[1].16b +___ + &rev32(@data[0],@data[0]); + &encrypt_1blk(@data[0]); +$code.=<<___; + eor @data[0].16b, @data[0].16b, @tweak[1].16b + st1 {@data[0].4s},[$outp],#16 + + sub $lastBlk,$outp,16 + .loop${std}: + subs $remain,$remain,1 + ldrb $wtmp0,[$lastBlk,$remain] + ldrb $wtmp1,[$inp,$remain] + strb $wtmp1,[$lastBlk,$remain] + strb $wtmp0,[$outp,$remain] + b.gt .loop${std} + ld1 {@data[0].4s}, [$lastBlk] + eor @data[0].16b, @data[0].16b, @tweak[2].16b +___ + &rev32(@data[0],@data[0]); + &encrypt_1blk(@data[0]); +$code.=<<___; + eor @data[0].16b, @data[0].16b, @tweak[2].16b + st1 {@data[0].4s}, [$lastBlk] +.return${std}: + ldp d14, d15, [sp], #0x10 + ldp d12, d13, [sp], #0x10 + ldp d10, d11, [sp], #0x10 + ldp d8, d9, [sp], #0x10 + ldp x29, x30, [sp], #0x10 + ldp x27, x28, [sp], #0x10 + ldp x25, x26, [sp], #0x10 + ldp x23, x24, [sp], #0x10 + ldp x21, x22, [sp], #0x10 + ldp x19, x20, [sp], #0x10 + ldp x17, x18, [sp], #0x10 + ldp x15, x16, [sp], #0x10 + AARCH64_VALIDATE_LINK_REGISTER + ret +.size ${prefix}_xts_encrypt${std},.-${prefix}_xts_encrypt${std} +___ +} # end of gen_xts_cipher +&gen_xts_cipher("_gb"); +&gen_xts_cipher(""); +}}} + +######################################## +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/\/\// and !/^$/); + print; +} +close SELF; + +foreach(split("\n",$code)) { + s/\`([^\`]*)\`/eval($1)/ge; + print $_,"\n"; +} + +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/openssl/crypto/whrlpool/asm/wp-x86_64.pl b/crypto/openssl/crypto/whrlpool/asm/wp-x86_64.pl index f94152af9769..e3421dcbdf18 100755 --- a/crypto/openssl/crypto/whrlpool/asm/wp-x86_64.pl +++ b/crypto/openssl/crypto/whrlpool/asm/wp-x86_64.pl @@ -1,5 +1,5 @@ #! /usr/bin/env perl -# Copyright 2005-2020 The OpenSSL Project Authors. All Rights Reserved. +# Copyright 2005-2024 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy @@ -236,6 +236,7 @@ $code.=<<___; .cfi_endproc .size $func,.-$func +.section .rodata align=64 .align 64 .type $table,\@object $table: