579 lines
13 KiB
NASM
579 lines
13 KiB
NASM
;---------------------
|
|
; https://software.intel.com/en-us/articles/improving-the-performance-of-the-secure-hash-algorithm-1
|
|
;
|
|
; License information:
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
; This implementation notably advances the performance of SHA-1 algorithm compared to existing
|
|
; implementations. We are encouraging all projects utilizing SHA-1 to integrate this new fast
|
|
; implementation and are ready to help if issues or concerns arise (you are welcome to leave
|
|
; a comment or write an email to the authors). It is provided 'as is' and free for either
|
|
; commercial or non-commercial use.
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
;
|
|
; This code implements two interfaces of SHA-1 update function: 1) working on a single
|
|
; 64-byte block and 2) working on a buffer of multiple 64-bit blocks. Multiple blocks
|
|
; version of code is software pipelined and faster overall, it is a default. Assemble
|
|
; with -DINTEL_SHA1_SINGLEBLOCK to select single 64-byte block function interface.
|
|
;
|
|
; C++ prototypes of implemented functions are below:
|
|
;
|
|
; #ifndef INTEL_SHA1_SINGLEBLOCK
|
|
; // Updates 20-byte SHA-1 record in 'hash' for 'num_blocks' consequtive 64-byte blocks
|
|
; extern "C" void sha1_update_intel(int *hash, const char* input, size_t num_blocks );
|
|
; #else
|
|
; // Updates 20-byte SHA-1 record in 'hash' for one 64-byte block pointed by 'input'
|
|
; extern "C" void sha1_update_intel(int *hash, const char* input);
|
|
; #endif
|
|
;
|
|
; Function name 'sha1_update_intel' can be changed in the source or via macro:
|
|
; -DINTEL_SHA1_UPDATE_FUNCNAME=my_sha1_update_func_name
|
|
;
|
|
; It implements both UNIX(default) and Windows ABIs, use -DWIN_ABI on Windows
|
|
;
|
|
; Code checks CPU for SSSE3 support via CPUID feature flag (CPUID.1.ECX.SSSE3[bit 9]==1),
|
|
; and performs dispatch. Since in most cases the functionality on non-SSSE3 supporting CPUs
|
|
; is also required, the default (e.g. one being replaced) function can be provided for
|
|
; dispatch on such CPUs, the name of old function can be changed in the source or via macro:
|
|
; -DINTEL_SHA1_UPDATE_DEFAULT_DISPATCH=default_sha1_update_function_name
|
|
;
|
|
; Authors: Maxim Locktyukhin and Ronen Zohar at Intel.com
|
|
;
|
|
|
|
%ifndef INTEL_SHA1_UPDATE_DEFAULT_DISPATCH
|
|
;; can be replaced with a default SHA-1 update function name
|
|
%define INTEL_SHA1_UPDATE_DEFAULT_DISPATCH sha1_intel_non_ssse3_cpu_stub_
|
|
%else
|
|
extern INTEL_SHA1_UPDATE_DEFAULT_DISPATCH
|
|
%endif
|
|
|
|
;; provide alternative SHA-1 update function's name here
|
|
%ifndef INTEL_SHA1_UPDATE_FUNCNAME
|
|
%define INTEL_SHA1_UPDATE_FUNCNAME sha1_update_intel
|
|
%endif
|
|
|
|
global INTEL_SHA1_UPDATE_FUNCNAME
|
|
|
|
|
|
%ifndef INTEL_SHA1_SINGLEBLOCK
|
|
%assign multiblock 1
|
|
%else
|
|
%assign multiblock 0
|
|
%endif
|
|
|
|
|
|
bits 64
|
|
default rel
|
|
|
|
%ifdef WIN_ABI
|
|
%xdefine arg1 rcx
|
|
%xdefine arg2 rdx
|
|
%xdefine arg3 r8
|
|
%else
|
|
%xdefine arg1 rdi
|
|
%xdefine arg2 rsi
|
|
%xdefine arg3 rdx
|
|
%endif
|
|
|
|
%xdefine ctx arg1
|
|
%xdefine buf arg2
|
|
%xdefine cnt arg3
|
|
|
|
%macro REGALLOC 0
|
|
%xdefine A ecx
|
|
%xdefine B esi
|
|
%xdefine C edi
|
|
%xdefine D ebp
|
|
%xdefine E edx
|
|
|
|
%xdefine T1 eax
|
|
%xdefine T2 ebx
|
|
%endmacro
|
|
|
|
%xdefine K_BASE r8
|
|
%xdefine HASH_PTR r9
|
|
%xdefine BUFFER_PTR r10
|
|
%xdefine BUFFER_END r11
|
|
|
|
%xdefine W_TMP xmm0
|
|
%xdefine W_TMP2 xmm9
|
|
|
|
%xdefine W0 xmm1
|
|
%xdefine W4 xmm2
|
|
%xdefine W8 xmm3
|
|
%xdefine W12 xmm4
|
|
%xdefine W16 xmm5
|
|
%xdefine W20 xmm6
|
|
%xdefine W24 xmm7
|
|
%xdefine W28 xmm8
|
|
|
|
%xdefine XMM_SHUFB_BSWAP xmm10
|
|
|
|
;; we keep window of 64 w[i]+K pre-calculated values in a circular buffer
|
|
%xdefine WK(t) (rsp + (t & 15)*4)
|
|
|
|
;------------------------------------------------------------------------------
|
|
;
|
|
; macro implements SHA-1 function's body for single or several 64-byte blocks
|
|
; first param: function's name
|
|
; second param: =0 - function implements single 64-byte block hash
|
|
; =1 - function implements multiple64-byte blocks hash
|
|
; 3rd function's argument is a number, greater 0, of 64-byte blocks to calc hash for
|
|
;
|
|
%macro SHA1_VECTOR_ASM 2
|
|
align 4096
|
|
%1:
|
|
push rbx
|
|
push rbp
|
|
|
|
%ifdef WIN_ABI
|
|
push rdi
|
|
push rsi
|
|
|
|
%xdefine stack_size (16*4 + 16*5 + 8)
|
|
%else
|
|
%xdefine stack_size (16*4 + 8)
|
|
%endif
|
|
|
|
sub rsp, stack_size
|
|
|
|
%ifdef WIN_ABI
|
|
%xdefine xmm_save_base (rsp + 16*4)
|
|
|
|
xmm_mov [xmm_save_base + 0*16], xmm6
|
|
xmm_mov [xmm_save_base + 1*16], xmm7
|
|
xmm_mov [xmm_save_base + 2*16], xmm8
|
|
xmm_mov [xmm_save_base + 3*16], xmm9
|
|
xmm_mov [xmm_save_base + 4*16], xmm10
|
|
%endif
|
|
|
|
mov HASH_PTR, ctx
|
|
mov BUFFER_PTR, buf
|
|
|
|
%if (%2 == 1)
|
|
shl cnt, 6 ;; mul by 64
|
|
add cnt, buf
|
|
mov BUFFER_END, cnt
|
|
%endif
|
|
|
|
lea K_BASE, [K_XMM_AR]
|
|
xmm_mov XMM_SHUFB_BSWAP, [bswap_shufb_ctl]
|
|
|
|
SHA1_PIPELINED_MAIN_BODY %2
|
|
|
|
%ifdef WIN_ABI
|
|
xmm_mov xmm6, [xmm_save_base + 0*16]
|
|
xmm_mov xmm7, [xmm_save_base + 1*16]
|
|
xmm_mov xmm8, [xmm_save_base + 2*16]
|
|
xmm_mov xmm9, [xmm_save_base + 3*16]
|
|
xmm_mov xmm10,[xmm_save_base + 4*16]
|
|
%endif
|
|
|
|
add rsp, stack_size
|
|
|
|
%ifdef WIN_ABI
|
|
pop rsi
|
|
pop rdi
|
|
%endif
|
|
|
|
pop rbp
|
|
pop rbx
|
|
|
|
ret
|
|
%endmacro
|
|
|
|
;--------------------------------------------
|
|
; macro implements 80 rounds of SHA-1, for one 64-byte block or multiple blocks with s/w pipelining
|
|
; macro param: =0 - process single 64-byte block
|
|
; =1 - multiple blocks
|
|
;
|
|
%macro SHA1_PIPELINED_MAIN_BODY 1
|
|
|
|
REGALLOC
|
|
|
|
mov A, [HASH_PTR ]
|
|
mov B, [HASH_PTR+ 4]
|
|
mov C, [HASH_PTR+ 8]
|
|
mov D, [HASH_PTR+12]
|
|
|
|
mov E, [HASH_PTR+16]
|
|
|
|
%assign i 0
|
|
%rep W_PRECALC_AHEAD
|
|
W_PRECALC i
|
|
%assign i i+1
|
|
%endrep
|
|
|
|
%xdefine F F1
|
|
|
|
%if (%1 == 1) ;; code loops through more than one block
|
|
%%_loop:
|
|
cmp BUFFER_PTR, K_BASE ;; we use K_BASE value as a signal of a last block,
|
|
jne %%_begin ;; it is set below by: cmovae BUFFER_PTR, K_BASE
|
|
jmp %%_end
|
|
|
|
align 32
|
|
%%_begin:
|
|
%endif
|
|
RR A,B,C,D,E,0
|
|
RR D,E,A,B,C,2
|
|
RR B,C,D,E,A,4
|
|
RR E,A,B,C,D,6
|
|
RR C,D,E,A,B,8
|
|
|
|
RR A,B,C,D,E,10
|
|
RR D,E,A,B,C,12
|
|
RR B,C,D,E,A,14
|
|
RR E,A,B,C,D,16
|
|
RR C,D,E,A,B,18
|
|
|
|
%xdefine F F2
|
|
|
|
RR A,B,C,D,E,20
|
|
RR D,E,A,B,C,22
|
|
RR B,C,D,E,A,24
|
|
RR E,A,B,C,D,26
|
|
RR C,D,E,A,B,28
|
|
|
|
RR A,B,C,D,E,30
|
|
RR D,E,A,B,C,32
|
|
RR B,C,D,E,A,34
|
|
RR E,A,B,C,D,36
|
|
RR C,D,E,A,B,38
|
|
|
|
%xdefine F F3
|
|
|
|
RR A,B,C,D,E,40
|
|
RR D,E,A,B,C,42
|
|
RR B,C,D,E,A,44
|
|
RR E,A,B,C,D,46
|
|
RR C,D,E,A,B,48
|
|
|
|
RR A,B,C,D,E,50
|
|
RR D,E,A,B,C,52
|
|
RR B,C,D,E,A,54
|
|
RR E,A,B,C,D,56
|
|
RR C,D,E,A,B,58
|
|
|
|
%xdefine F F4
|
|
|
|
%if (%1 == 1) ;; if code loops through more than one block
|
|
add BUFFER_PTR, 64 ;; move to next 64-byte block
|
|
cmp BUFFER_PTR, BUFFER_END ;; check if current block is the last one
|
|
cmovae BUFFER_PTR, K_BASE ;; smart way to signal the last iteration
|
|
%else
|
|
%xdefine W_NO_TAIL_PRECALC 1 ;; no software pipelining for single block interface
|
|
%endif
|
|
|
|
RR A,B,C,D,E,60
|
|
RR D,E,A,B,C,62
|
|
RR B,C,D,E,A,64
|
|
RR E,A,B,C,D,66
|
|
RR C,D,E,A,B,68
|
|
|
|
RR A,B,C,D,E,70
|
|
RR D,E,A,B,C,72
|
|
RR B,C,D,E,A,74
|
|
RR E,A,B,C,D,76
|
|
RR C,D,E,A,B,78
|
|
|
|
UPDATE_HASH [HASH_PTR ],A
|
|
UPDATE_HASH [HASH_PTR+ 4],B
|
|
UPDATE_HASH [HASH_PTR+ 8],C
|
|
UPDATE_HASH [HASH_PTR+12],D
|
|
UPDATE_HASH [HASH_PTR+16],E
|
|
|
|
%if (%1 == 1)
|
|
jmp %%_loop
|
|
|
|
align 32
|
|
%%_end:
|
|
%endif
|
|
|
|
|
|
%xdefine W_NO_TAIL_PRECALC 0
|
|
%xdefine F %error
|
|
|
|
%endmacro
|
|
|
|
|
|
%macro F1 3
|
|
mov T1,%2
|
|
xor T1,%3
|
|
and T1,%1
|
|
xor T1,%3
|
|
%endmacro
|
|
|
|
%macro F2 3
|
|
mov T1,%3
|
|
xor T1,%2
|
|
xor T1,%1
|
|
%endmacro
|
|
|
|
%macro F3 3
|
|
mov T1,%2
|
|
mov T2,%1
|
|
or T1,%1
|
|
and T2,%2
|
|
and T1,%3
|
|
or T1,T2
|
|
%endmacro
|
|
|
|
%define F4 F2
|
|
|
|
%macro UPDATE_HASH 2
|
|
add %2, %1
|
|
mov %1, %2
|
|
%endmacro
|
|
|
|
|
|
%macro W_PRECALC 1
|
|
%xdefine i (%1)
|
|
|
|
%if (i < 20)
|
|
%xdefine K_XMM 0
|
|
%elif (i < 40)
|
|
%xdefine K_XMM 16
|
|
%elif (i < 60)
|
|
%xdefine K_XMM 32
|
|
%else
|
|
%xdefine K_XMM 48
|
|
%endif
|
|
|
|
%if (i<16 || (i>=80 && i<(80 + W_PRECALC_AHEAD)))
|
|
|
|
%if (W_NO_TAIL_PRECALC == 0)
|
|
|
|
%xdefine i ((%1) % 80) ;; pre-compute for the next iteration
|
|
|
|
%if (i == 0)
|
|
W_PRECALC_RESET
|
|
%endif
|
|
|
|
|
|
W_PRECALC_00_15
|
|
%endif
|
|
|
|
%elif (i < 32)
|
|
W_PRECALC_16_31
|
|
%elif (i < 80) ;; rounds 32-79
|
|
W_PRECALC_32_79
|
|
%endif
|
|
%endmacro
|
|
|
|
%macro W_PRECALC_RESET 0
|
|
%xdefine W W0
|
|
%xdefine W_minus_04 W4
|
|
%xdefine W_minus_08 W8
|
|
%xdefine W_minus_12 W12
|
|
%xdefine W_minus_16 W16
|
|
%xdefine W_minus_20 W20
|
|
%xdefine W_minus_24 W24
|
|
%xdefine W_minus_28 W28
|
|
%xdefine W_minus_32 W
|
|
%endmacro
|
|
|
|
%macro W_PRECALC_ROTATE 0
|
|
%xdefine W_minus_32 W_minus_28
|
|
%xdefine W_minus_28 W_minus_24
|
|
%xdefine W_minus_24 W_minus_20
|
|
%xdefine W_minus_20 W_minus_16
|
|
%xdefine W_minus_16 W_minus_12
|
|
%xdefine W_minus_12 W_minus_08
|
|
%xdefine W_minus_08 W_minus_04
|
|
%xdefine W_minus_04 W
|
|
%xdefine W W_minus_32
|
|
%endmacro
|
|
|
|
%xdefine W_PRECALC_AHEAD 16
|
|
%xdefine W_NO_TAIL_PRECALC 0
|
|
|
|
|
|
%xdefine xmm_mov movdqa
|
|
|
|
%macro W_PRECALC_00_15 0
|
|
;; message scheduling pre-compute for rounds 0-15
|
|
%if ((i & 3) == 0) ;; blended SSE and ALU instruction scheduling, 1 vector iteration per 4 rounds
|
|
movdqu W_TMP, [BUFFER_PTR + (i * 4)]
|
|
%elif ((i & 3) == 1)
|
|
pshufb W_TMP, XMM_SHUFB_BSWAP
|
|
movdqa W, W_TMP
|
|
%elif ((i & 3) == 2)
|
|
paddd W_TMP, [K_BASE]
|
|
%elif ((i & 3) == 3)
|
|
movdqa [WK(i&~3)], W_TMP
|
|
|
|
W_PRECALC_ROTATE
|
|
%endif
|
|
%endmacro
|
|
|
|
%macro W_PRECALC_16_31 0
|
|
;; message scheduling pre-compute for rounds 16-31
|
|
;; calculating last 32 w[i] values in 8 XMM registers
|
|
;; pre-calculate K+w[i] values and store to mem, for later load by ALU add instruction
|
|
;;
|
|
;; "brute force" vectorization for rounds 16-31 only due to w[i]->w[i-3] dependency
|
|
;;
|
|
%if ((i & 3) == 0) ;; blended SSE and ALU instruction scheduling, 1 vector iteration per 4 rounds
|
|
movdqa W, W_minus_12
|
|
palignr W, W_minus_16, 8 ;; w[i-14]
|
|
movdqa W_TMP, W_minus_04
|
|
psrldq W_TMP, 4 ;; w[i-3]
|
|
pxor W, W_minus_08
|
|
%elif ((i & 3) == 1)
|
|
pxor W_TMP, W_minus_16
|
|
pxor W, W_TMP
|
|
movdqa W_TMP2, W
|
|
movdqa W_TMP, W
|
|
pslldq W_TMP2, 12
|
|
%elif ((i & 3) == 2)
|
|
psrld W, 31
|
|
pslld W_TMP, 1
|
|
por W_TMP, W
|
|
movdqa W, W_TMP2
|
|
psrld W_TMP2, 30
|
|
pslld W, 2
|
|
%elif ((i & 3) == 3)
|
|
pxor W_TMP, W
|
|
pxor W_TMP, W_TMP2
|
|
movdqa W, W_TMP
|
|
paddd W_TMP, [K_BASE + K_XMM]
|
|
movdqa [WK(i&~3)],W_TMP
|
|
|
|
W_PRECALC_ROTATE
|
|
%endif
|
|
%endmacro
|
|
|
|
%macro W_PRECALC_32_79 0
|
|
;; in SHA-1 specification: w[i] = (w[i-3] ^ w[i-8] ^ w[i-14] ^ w[i-16]) rol 1
|
|
;; instead we do equal: w[i] = (w[i-6] ^ w[i-16] ^ w[i-28] ^ w[i-32]) rol 2
|
|
;; allows more efficient vectorization since w[i]=>w[i-3] dependency is broken
|
|
;;
|
|
%if ((i & 3) == 0) ;; blended SSE and ALU instruction scheduling, 1 vector iteration per 4 rounds
|
|
movdqa W_TMP, W_minus_04
|
|
pxor W, W_minus_28 ;; W is W_minus_32 before xor
|
|
palignr W_TMP, W_minus_08, 8
|
|
%elif ((i & 3) == 1)
|
|
pxor W, W_minus_16
|
|
pxor W, W_TMP
|
|
movdqa W_TMP, W
|
|
%elif ((i & 3) == 2)
|
|
psrld W, 30
|
|
pslld W_TMP, 2
|
|
por W_TMP, W
|
|
%elif ((i & 3) == 3)
|
|
movdqa W, W_TMP
|
|
paddd W_TMP, [K_BASE + K_XMM]
|
|
movdqa [WK(i&~3)],W_TMP
|
|
|
|
W_PRECALC_ROTATE
|
|
%endif
|
|
%endmacro
|
|
|
|
%macro RR 6 ;; RR does two rounds of SHA-1 back to back with W pre-calculation
|
|
|
|
;; TEMP = A
|
|
;; A = F( i, B, C, D ) + E + ROTATE_LEFT( A, 5 ) + W[i] + K(i)
|
|
;; C = ROTATE_LEFT( B, 30 )
|
|
;; D = C
|
|
;; E = D
|
|
;; B = TEMP
|
|
|
|
W_PRECALC (%6 + W_PRECALC_AHEAD)
|
|
F %2, %3, %4 ;; F returns result in T1
|
|
add %5, [WK(%6)]
|
|
rol %2, 30
|
|
mov T2, %1
|
|
add %4, [WK(%6 + 1)]
|
|
rol T2, 5
|
|
add %5, T1
|
|
|
|
W_PRECALC (%6 + W_PRECALC_AHEAD + 1)
|
|
add T2, %5
|
|
mov %5, T2
|
|
rol T2, 5
|
|
add %4, T2
|
|
F %1, %2, %3 ;; F returns result in T1
|
|
add %4, T1
|
|
rol %1, 30
|
|
|
|
;; write: %1, %2
|
|
;; rotate: %1<=%4, %2<=%5, %3<=%1, %4<=%2, %5<=%3
|
|
%endmacro
|
|
|
|
|
|
|
|
;;----------------------
|
|
section .data align=128
|
|
|
|
%xdefine K1 0x5a827999
|
|
%xdefine K2 0x6ed9eba1
|
|
%xdefine K3 0x8f1bbcdc
|
|
%xdefine K4 0xca62c1d6
|
|
|
|
align 128
|
|
K_XMM_AR:
|
|
DD K1, K1, K1, K1
|
|
DD K2, K2, K2, K2
|
|
DD K3, K3, K3, K3
|
|
DD K4, K4, K4, K4
|
|
|
|
align 16
|
|
bswap_shufb_ctl:
|
|
DD 00010203h
|
|
DD 04050607h
|
|
DD 08090a0bh
|
|
DD 0c0d0e0fh
|
|
|
|
;; dispatch pointer, points to the init routine for the first invocation
|
|
sha1_update_intel_dispatched:
|
|
DQ sha1_update_intel_init_
|
|
|
|
;;----------------------
|
|
section .text align=4096
|
|
|
|
SHA1_VECTOR_ASM sha1_update_intel_ssse3_, multiblock
|
|
|
|
align 32
|
|
sha1_update_intel_init_: ;; we get here with the first time invocation
|
|
call sha1_update_intel_dispacth_init_
|
|
INTEL_SHA1_UPDATE_FUNCNAME: ;; we get here after init
|
|
jmp qword [sha1_update_intel_dispatched]
|
|
|
|
;; CPUID feature flag based dispatch
|
|
sha1_update_intel_dispacth_init_:
|
|
push rax
|
|
push rbx
|
|
push rcx
|
|
push rdx
|
|
push rsi
|
|
|
|
lea rsi, [INTEL_SHA1_UPDATE_DEFAULT_DISPATCH]
|
|
|
|
mov eax, 1
|
|
cpuid
|
|
|
|
test ecx, 0200h ;; SSSE3 support, CPUID.1.ECX[bit 9]
|
|
jz _done
|
|
|
|
lea rsi, [sha1_update_intel_ssse3_]
|
|
|
|
_done:
|
|
mov [sha1_update_intel_dispatched], rsi
|
|
|
|
pop rsi
|
|
pop rdx
|
|
pop rcx
|
|
pop rbx
|
|
pop rax
|
|
ret
|
|
|
|
;;----------------------
|
|
;; in the case a default SHA-1 update function implementation was not provided
|
|
;; and code was invoked on a non-SSSE3 supporting CPU, dispatch handles this
|
|
;; failure in a safest way - jumps to the stub function with UD2 instruction below
|
|
sha1_intel_non_ssse3_cpu_stub_:
|
|
ud2 ;; in the case no default SHA-1 was provided non-SSSE3 CPUs safely fail here
|
|
ret
|
|
|
|
; END
|
|
;----------------------
|