Feat: Added vaptvupt codec, fix jasmin tests

jasmin/zupt_aes_ctr.jazz

@@ -3,9 +3,10 @@
  *
  * CT-REQUIRED: AES-NI has no data-dependent timing.
  *
- * Uses reg ptr for read-only u128 inputs (key, counter, plaintext).
- * Uses reg u64 for write output (store infers width from reg u128 source).
- * C handles CTR loop and tail bytes.
+ * FIX v2.0.0: replaced `stack u128[15] rk` with 15 individual
+ * `stack u128` variables. The array form uses byte-offset indexing
+ * in jasminc (rk.[1] → [rsp+1] instead of [rsp+16]), producing
+ * incorrect round key loads. Individual variables avoid the issue.
  */
 
 inline fn key_expand_even(reg u128 t0, reg u128 assist) -> reg u128 {
@@ -41,84 +42,80 @@ export fn zupt_aes256_blk(
     reg ptr u128[2] key,
     reg ptr u128[1] ctr_blk)
 {
-    stack u128[15] rk;
+    stack u128 rk0 rk1 rk2 rk3 rk4 rk5 rk6 rk7;
+    stack u128 rk8 rk9 rk10 rk11 rk12 rk13 rk14;
     reg u128 t0 t1 assist b data;
 
-    /* Key expansion */
     t0 = key.[0];
     t1 = key.[1];
-    rk.[0] = t0;
-    rk.[1] = t1;
+    rk0 = t0;
+    rk1 = t1;
 
     assist = #VAESKEYGENASSIST(t1, 0x01);
     t0 = key_expand_even(t0, assist);
-    rk.[2] = t0;
+    rk2 = t0;
     t1 = key_expand_odd(t0, t1);
-    rk.[3] = t1;
+    rk3 = t1;
 
     assist = #VAESKEYGENASSIST(t1, 0x02);
     t0 = key_expand_even(t0, assist);
-    rk.[4] = t0;
+    rk4 = t0;
     t1 = key_expand_odd(t0, t1);
-    rk.[5] = t1;
+    rk5 = t1;
 
     assist = #VAESKEYGENASSIST(t1, 0x04);
     t0 = key_expand_even(t0, assist);
-    rk.[6] = t0;
+    rk6 = t0;
     t1 = key_expand_odd(t0, t1);
-    rk.[7] = t1;
+    rk7 = t1;
 
     assist = #VAESKEYGENASSIST(t1, 0x08);
     t0 = key_expand_even(t0, assist);
-    rk.[8] = t0;
+    rk8 = t0;
     t1 = key_expand_odd(t0, t1);
-    rk.[9] = t1;
+    rk9 = t1;
 
     assist = #VAESKEYGENASSIST(t1, 0x10);
     t0 = key_expand_even(t0, assist);
-    rk.[10] = t0;
+    rk10 = t0;
     t1 = key_expand_odd(t0, t1);
-    rk.[11] = t1;
+    rk11 = t1;
 
     assist = #VAESKEYGENASSIST(t1, 0x20);
     t0 = key_expand_even(t0, assist);
-    rk.[12] = t0;
+    rk12 = t0;
     t1 = key_expand_odd(t0, t1);
-    rk.[13] = t1;
+    rk13 = t1;
 
     assist = #VAESKEYGENASSIST(t1, 0x40);
     t0 = key_expand_even(t0, assist);
-    rk.[14] = t0;
+    rk14 = t0;
 
-    /* Encrypt counter block: 14 rounds AES-256 */
     b = ctr_blk.[0];
-    b ^= rk.[0];
-    b = #VAESENC(b, rk.[1]);
-    b = #VAESENC(b, rk.[2]);
-    b = #VAESENC(b, rk.[3]);
-    b = #VAESENC(b, rk.[4]);
-    b = #VAESENC(b, rk.[5]);
-    b = #VAESENC(b, rk.[6]);
-    b = #VAESENC(b, rk.[7]);
-    b = #VAESENC(b, rk.[8]);
-    b = #VAESENC(b, rk.[9]);
-    b = #VAESENC(b, rk.[10]);
-    b = #VAESENC(b, rk.[11]);
-    b = #VAESENC(b, rk.[12]);
-    b = #VAESENC(b, rk.[13]);
-    b = #VAESENCLAST(b, rk.[14]);
+    b ^= rk0;
+    b = #VAESENC(b, rk1);
+    b = #VAESENC(b, rk2);
+    b = #VAESENC(b, rk3);
+    b = #VAESENC(b, rk4);
+    b = #VAESENC(b, rk5);
+    b = #VAESENC(b, rk6);
+    b = #VAESENC(b, rk7);
+    b = #VAESENC(b, rk8);
+    b = #VAESENC(b, rk9);
+    b = #VAESENC(b, rk10);
+    b = #VAESENC(b, rk11);
+    b = #VAESENC(b, rk12);
+    b = #VAESENC(b, rk13);
+    b = #VAESENCLAST(b, rk14);
 
-    /* XOR keystream with plaintext, store result */
     data = in_blk.[0];
     b ^= data;
     [out_ptr + 0] = b;
 
-    /* Wipe round keys */
     reg u128 wipe;
-    inline int z;
-    for z = 0 to 15 {
-        wipe = rk.[z];
-        wipe ^= wipe;
-        rk.[z] = wipe;
-    }
+    wipe = rk0; wipe ^= wipe;
+    rk0 = wipe; rk1 = wipe; rk2 = wipe; rk3 = wipe;
+    rk4 = wipe; rk5 = wipe; rk6 = wipe; rk7 = wipe;
+    rk8 = wipe; rk9 = wipe; rk10 = wipe; rk11 = wipe;
+    rk12 = wipe; rk13 = wipe; rk14 = wipe;
 }

jasmin/zupt_aes_ctr.s

@@ -3,16 +3,30 @@
 	.p2align	5
 	.global	zupt_aes256_blk
 	.type	zupt_aes256_blk, %function
+/* zupt_aes256_blk(out_ptr=rdi, in_blk=rsi, key=rdx, ctr_blk=rcx)
+ * AES-256 single-block encrypt: out = AES(key, ctr) XOR in
+ *
+ * FIX v2.0.0: Round keys at [rsp+0], [rsp+16], [rsp+32], ..., [rsp+224]
+ * The previous version had [rsp+0], [rsp+1], ..., [rsp+14] (byte offsets).
+ *
+ * Stack layout: 15 × 16 bytes = 240 bytes for round keys, 16-byte aligned.
+ */
 zupt_aes256_blk:
 	mov 	r10, rsp
-	lea 	rsp, qword ptr[rsp + -240]
+	lea 	rsp, qword ptr[rsp - 256]
 	and 	rsp, -16
-	vmovdqu	xmm0, xmmword ptr[rdx]
-	vmovdqu	xmm1, xmmword ptr[rdx + 1]
-	vmovdqu	xmmword ptr[rsp], xmm0
-	vmovdqu	xmmword ptr[rsp + 1], xmm1
-	vaeskeygenassist	xmm2, xmm1, 1
-	vpshufd	xmm2, xmm2, 255
+
+	/* Load 256-bit key (two u128 halves) */
+	vmovdqu	xmm0, xmmword ptr[rdx]        /* key[0] = first 128 bits */
+	vmovdqu	xmm1, xmmword ptr[rdx + 16]   /* key[1] = second 128 bits */
+
+	/* Store round keys 0-1 (the raw key halves) */
+	vmovdqa	xmmword ptr[rsp + 0], xmm0     /* rk0 */
+	vmovdqa	xmmword ptr[rsp + 16], xmm1    /* rk1 */
+
+	/* Round key 2 (even): RCON=0x01 */
+	vaeskeygenassist	xmm2, xmm1, 0x01
+	vpshufd	xmm2, xmm2, 0xFF
 	vpslldq	xmm3, xmm0, 4
 	vpxor	xmm0, xmm0, xmm3
 	vpslldq	xmm3, xmm0, 4
@@ -20,9 +34,11 @@ zupt_aes256_blk:
 	vpslldq	xmm3, xmm0, 4
 	vpxor	xmm0, xmm0, xmm3
 	vpxor	xmm0, xmm0, xmm2
-	vmovdqu	xmmword ptr[rsp + 2], xmm0
+	vmovdqa	xmmword ptr[rsp + 32], xmm0    /* rk2 */
+
+	/* Round key 3 (odd) */
 	vaeskeygenassist	xmm2, xmm0, 0
-	vpshufd	xmm2, xmm2, 170
+	vpshufd	xmm2, xmm2, 0xAA
 	vpslldq	xmm3, xmm1, 4
 	vpxor	xmm1, xmm1, xmm3
 	vpslldq	xmm3, xmm1, 4
@@ -30,9 +46,11 @@ zupt_aes256_blk:
 	vpslldq	xmm3, xmm1, 4
 	vpxor	xmm1, xmm1, xmm3
 	vpxor	xmm1, xmm1, xmm2
-	vmovdqu	xmmword ptr[rsp + 3], xmm1
-	vaeskeygenassist	xmm2, xmm1, 2
-	vpshufd	xmm2, xmm2, 255
+	vmovdqa	xmmword ptr[rsp + 48], xmm1    /* rk3 */
+
+	/* Round key 4 (even): RCON=0x02 */
+	vaeskeygenassist	xmm2, xmm1, 0x02
+	vpshufd	xmm2, xmm2, 0xFF
 	vpslldq	xmm3, xmm0, 4
 	vpxor	xmm0, xmm0, xmm3
 	vpslldq	xmm3, xmm0, 4
@@ -40,9 +58,11 @@ zupt_aes256_blk:
 	vpslldq	xmm3, xmm0, 4
 	vpxor	xmm0, xmm0, xmm3
 	vpxor	xmm0, xmm0, xmm2
-	vmovdqu	xmmword ptr[rsp + 4], xmm0
+	vmovdqa	xmmword ptr[rsp + 64], xmm0    /* rk4 */
+
+	/* Round key 5 (odd) */
 	vaeskeygenassist	xmm2, xmm0, 0
-	vpshufd	xmm2, xmm2, 170
+	vpshufd	xmm2, xmm2, 0xAA
 	vpslldq	xmm3, xmm1, 4
 	vpxor	xmm1, xmm1, xmm3
 	vpslldq	xmm3, xmm1, 4
@@ -50,9 +70,11 @@ zupt_aes256_blk:
 	vpslldq	xmm3, xmm1, 4
 	vpxor	xmm1, xmm1, xmm3
 	vpxor	xmm1, xmm1, xmm2
-	vmovdqu	xmmword ptr[rsp + 5], xmm1
-	vaeskeygenassist	xmm2, xmm1, 4
-	vpshufd	xmm2, xmm2, 255
+	vmovdqa	xmmword ptr[rsp + 80], xmm1    /* rk5 */
+
+	/* Round key 6 (even): RCON=0x04 */
+	vaeskeygenassist	xmm2, xmm1, 0x04
+	vpshufd	xmm2, xmm2, 0xFF
 	vpslldq	xmm3, xmm0, 4
 	vpxor	xmm0, xmm0, xmm3
 	vpslldq	xmm3, xmm0, 4
@@ -60,9 +82,11 @@ zupt_aes256_blk:
 	vpslldq	xmm3, xmm0, 4
 	vpxor	xmm0, xmm0, xmm3
 	vpxor	xmm0, xmm0, xmm2
-	vmovdqu	xmmword ptr[rsp + 6], xmm0
+	vmovdqa	xmmword ptr[rsp + 96], xmm0    /* rk6 */
+
+	/* Round key 7 (odd) */
 	vaeskeygenassist	xmm2, xmm0, 0
-	vpshufd	xmm2, xmm2, 170
+	vpshufd	xmm2, xmm2, 0xAA
 	vpslldq	xmm3, xmm1, 4
 	vpxor	xmm1, xmm1, xmm3
 	vpslldq	xmm3, xmm1, 4
@@ -70,9 +94,11 @@ zupt_aes256_blk:
 	vpslldq	xmm3, xmm1, 4
 	vpxor	xmm1, xmm1, xmm3
 	vpxor	xmm1, xmm1, xmm2
-	vmovdqu	xmmword ptr[rsp + 7], xmm1
-	vaeskeygenassist	xmm2, xmm1, 8
-	vpshufd	xmm2, xmm2, 255
+	vmovdqa	xmmword ptr[rsp + 112], xmm1   /* rk7 */
+
+	/* Round key 8 (even): RCON=0x08 */
+	vaeskeygenassist	xmm2, xmm1, 0x08
+	vpshufd	xmm2, xmm2, 0xFF
 	vpslldq	xmm3, xmm0, 4
 	vpxor	xmm0, xmm0, xmm3
 	vpslldq	xmm3, xmm0, 4
@@ -80,9 +106,11 @@ zupt_aes256_blk:
 	vpslldq	xmm3, xmm0, 4
 	vpxor	xmm0, xmm0, xmm3
 	vpxor	xmm0, xmm0, xmm2
-	vmovdqu	xmmword ptr[rsp + 8], xmm0
+	vmovdqa	xmmword ptr[rsp + 128], xmm0   /* rk8 */
+
+	/* Round key 9 (odd) */
 	vaeskeygenassist	xmm2, xmm0, 0
-	vpshufd	xmm2, xmm2, 170
+	vpshufd	xmm2, xmm2, 0xAA
 	vpslldq	xmm3, xmm1, 4
 	vpxor	xmm1, xmm1, xmm3
 	vpslldq	xmm3, xmm1, 4
@@ -90,9 +118,11 @@ zupt_aes256_blk:
 	vpslldq	xmm3, xmm1, 4
 	vpxor	xmm1, xmm1, xmm3
 	vpxor	xmm1, xmm1, xmm2
-	vmovdqu	xmmword ptr[rsp + 9], xmm1
-	vaeskeygenassist	xmm2, xmm1, 16
-	vpshufd	xmm2, xmm2, 255
+	vmovdqa	xmmword ptr[rsp + 144], xmm1   /* rk9 */
+
+	/* Round key 10 (even): RCON=0x10 */
+	vaeskeygenassist	xmm2, xmm1, 0x10
+	vpshufd	xmm2, xmm2, 0xFF
 	vpslldq	xmm3, xmm0, 4
 	vpxor	xmm0, xmm0, xmm3
 	vpslldq	xmm3, xmm0, 4
@@ -100,9 +130,11 @@ zupt_aes256_blk:
 	vpslldq	xmm3, xmm0, 4
 	vpxor	xmm0, xmm0, xmm3
 	vpxor	xmm0, xmm0, xmm2
-	vmovdqu	xmmword ptr[rsp + 10], xmm0
+	vmovdqa	xmmword ptr[rsp + 160], xmm0   /* rk10 */
+
+	/* Round key 11 (odd) */
 	vaeskeygenassist	xmm2, xmm0, 0
-	vpshufd	xmm2, xmm2, 170
+	vpshufd	xmm2, xmm2, 0xAA
 	vpslldq	xmm3, xmm1, 4
 	vpxor	xmm1, xmm1, xmm3
 	vpslldq	xmm3, xmm1, 4
@@ -110,9 +142,11 @@ zupt_aes256_blk:
 	vpslldq	xmm3, xmm1, 4
 	vpxor	xmm1, xmm1, xmm3
 	vpxor	xmm1, xmm1, xmm2
-	vmovdqu	xmmword ptr[rsp + 11], xmm1
-	vaeskeygenassist	xmm2, xmm1, 32
-	vpshufd	xmm2, xmm2, 255
+	vmovdqa	xmmword ptr[rsp + 176], xmm1   /* rk11 */
+
+	/* Round key 12 (even): RCON=0x20 */
+	vaeskeygenassist	xmm2, xmm1, 0x20
+	vpshufd	xmm2, xmm2, 0xFF
 	vpslldq	xmm3, xmm0, 4
 	vpxor	xmm0, xmm0, xmm3
 	vpslldq	xmm3, xmm0, 4
@@ -120,9 +154,11 @@ zupt_aes256_blk:
 	vpslldq	xmm3, xmm0, 4
 	vpxor	xmm0, xmm0, xmm3
 	vpxor	xmm0, xmm0, xmm2
-	vmovdqu	xmmword ptr[rsp + 12], xmm0
+	vmovdqa	xmmword ptr[rsp + 192], xmm0   /* rk12 */
+
+	/* Round key 13 (odd) */
 	vaeskeygenassist	xmm2, xmm0, 0
-	vpshufd	xmm2, xmm2, 170
+	vpshufd	xmm2, xmm2, 0xAA
 	vpslldq	xmm3, xmm1, 4
 	vpxor	xmm1, xmm1, xmm3
 	vpslldq	xmm3, xmm1, 4
@@ -130,9 +166,11 @@ zupt_aes256_blk:
 	vpslldq	xmm3, xmm1, 4
 	vpxor	xmm1, xmm1, xmm3
 	vpxor	xmm1, xmm1, xmm2
-	vmovdqu	xmmword ptr[rsp + 13], xmm1
-	vaeskeygenassist	xmm2, xmm1, 64
-	vpshufd	xmm2, xmm2, 255
+	vmovdqa	xmmword ptr[rsp + 208], xmm1   /* rk13 */
+
+	/* Round key 14 (even): RCON=0x40 */
+	vaeskeygenassist	xmm2, xmm1, 0x40
+	vpshufd	xmm2, xmm2, 0xFF
 	vpslldq	xmm3, xmm0, 4
 	vpxor	xmm0, xmm0, xmm3
 	vpslldq	xmm3, xmm0, 4
@@ -140,27 +178,51 @@ zupt_aes256_blk:
 	vpslldq	xmm3, xmm0, 4
 	vpxor	xmm0, xmm0, xmm3
 	vpxor	xmm0, xmm0, xmm2
-	vmovdqu	xmmword ptr[rsp + 14], xmm0
-	vmovdqu	xmm0, xmmword ptr[rcx]
-	vpxor	xmm0, xmm0, xmmword ptr[rsp]
-	vaesenc	xmm0, xmm0, xmmword ptr[rsp + 1]
-	vaesenc	xmm0, xmm0, xmmword ptr[rsp + 2]
-	vaesenc	xmm0, xmm0, xmmword ptr[rsp + 3]
-	vaesenc	xmm0, xmm0, xmmword ptr[rsp + 4]
-	vaesenc	xmm0, xmm0, xmmword ptr[rsp + 5]
-	vaesenc	xmm0, xmm0, xmmword ptr[rsp + 6]
-	vaesenc	xmm0, xmm0, xmmword ptr[rsp + 7]
-	vaesenc	xmm0, xmm0, xmmword ptr[rsp + 8]
-	vaesenc	xmm0, xmm0, xmmword ptr[rsp + 9]
-	vaesenc	xmm0, xmm0, xmmword ptr[rsp + 10]
-	vaesenc	xmm0, xmm0, xmmword ptr[rsp + 11]
-	vaesenc	xmm0, xmm0, xmmword ptr[rsp + 12]
-	vaesenc	xmm0, xmm0, xmmword ptr[rsp + 13]
-	vaesenclast	xmm0, xmm0, xmmword ptr[rsp + 14]
-	vmovdqu	xmm1, xmmword ptr[rsi]
-	vpxor	xmm0, xmm0, xmm1
-	movq	qword ptr[rdi], xmm0
+	vmovdqa	xmmword ptr[rsp + 224], xmm0   /* rk14 */
+
+	/* ═══ Encrypt: AES-256 14 rounds ═══ */
+	vmovdqu	xmm4, xmmword ptr[rcx]         /* Load counter block */
+	vpxor	xmm4, xmm4, xmmword ptr[rsp + 0]    /* AddRoundKey(rk0) */
+	vaesenc	xmm4, xmm4, xmmword ptr[rsp + 16]   /* Round 1 */
+	vaesenc	xmm4, xmm4, xmmword ptr[rsp + 32]   /* Round 2 */
+	vaesenc	xmm4, xmm4, xmmword ptr[rsp + 48]   /* Round 3 */
+	vaesenc	xmm4, xmm4, xmmword ptr[rsp + 64]   /* Round 4 */
+	vaesenc	xmm4, xmm4, xmmword ptr[rsp + 80]   /* Round 5 */
+	vaesenc	xmm4, xmm4, xmmword ptr[rsp + 96]   /* Round 6 */
+	vaesenc	xmm4, xmm4, xmmword ptr[rsp + 112]  /* Round 7 */
+	vaesenc	xmm4, xmm4, xmmword ptr[rsp + 128]  /* Round 8 */
+	vaesenc	xmm4, xmm4, xmmword ptr[rsp + 144]  /* Round 9 */
+	vaesenc	xmm4, xmm4, xmmword ptr[rsp + 160]  /* Round 10 */
+	vaesenc	xmm4, xmm4, xmmword ptr[rsp + 176]  /* Round 11 */
+	vaesenc	xmm4, xmm4, xmmword ptr[rsp + 192]  /* Round 12 */
+	vaesenc	xmm4, xmm4, xmmword ptr[rsp + 208]  /* Round 13 */
+	vaesenclast	xmm4, xmm4, xmmword ptr[rsp + 224] /* Round 14 (final) */
+
+	/* XOR keystream with plaintext */
+	vmovdqu	xmm5, xmmword ptr[rsi]         /* Load plaintext block */
+	vpxor	xmm4, xmm4, xmm5
+	vmovdqu	xmmword ptr[rdi], xmm4         /* Store result */
+
+	/* Wipe round keys from stack */
+	vpxor	xmm0, xmm0, xmm0
+	vmovdqa	xmmword ptr[rsp + 0], xmm0
+	vmovdqa	xmmword ptr[rsp + 16], xmm0
+	vmovdqa	xmmword ptr[rsp + 32], xmm0
+	vmovdqa	xmmword ptr[rsp + 48], xmm0
+	vmovdqa	xmmword ptr[rsp + 64], xmm0
+	vmovdqa	xmmword ptr[rsp + 80], xmm0
+	vmovdqa	xmmword ptr[rsp + 96], xmm0
+	vmovdqa	xmmword ptr[rsp + 112], xmm0
+	vmovdqa	xmmword ptr[rsp + 128], xmm0
+	vmovdqa	xmmword ptr[rsp + 144], xmm0
+	vmovdqa	xmmword ptr[rsp + 160], xmm0
+	vmovdqa	xmmword ptr[rsp + 176], xmm0
+	vmovdqa	xmmword ptr[rsp + 192], xmm0
+	vmovdqa	xmmword ptr[rsp + 208], xmm0
+	vmovdqa	xmmword ptr[rsp + 224], xmm0
+
 	mov 	rsp, r10
 	ret
-	.ident	"Jasmin Compiler 2026.03.0"
-	.section	".note.GNU-stack", "", %progbits
+	.size	zupt_aes256_blk, . - zupt_aes256_blk
+
+	.section .note.GNU-stack,"",@progbits

jasmin/zupt_aes_ctr4.jazz

@@ -0,0 +1,24 @@
+/* Zupt — AES-256-CTR 4-Block Pipeline via AES-NI (Jasmin)
+ * Copyright (c) 2026 Cristian Cezar Moisés — MIT License
+ *
+ * CT-REQUIRED: AES-NI has no data-dependent timing.
+ *
+ * Interleaves 4 independent counter blocks through the AES round
+ * pipeline. AES-NI has 4-cycle latency, 1-cycle throughput — so
+ * 4 independent blocks saturate the pipeline for ~4× throughput.
+ *
+ * Expected: ~3.5 GB/s AES-256-CTR on modern x86-64 (Zen3/Alder Lake).
+ *
+ * Interface:
+ *   zupt_aes256_ctr4(out, in, key, ctr, nblocks)
+ *   Encrypts nblocks×16 bytes. Counter is incremented in the last 8 bytes
+ *   (big-endian) after each block. Processes 4 blocks per iteration;
+ *   remaining 1-3 blocks fall back to zupt_aes256_blk.
+ *
+ * NOTE: This is the Jasmin source for documentation. The actual linked
+ * assembly is in zupt_aes_ctr4.s (hand-written to match this logic).
+ */
+
+/* See zupt_aes_ctr4.s for the production assembly.
+ * This .jazz file documents the algorithm but is not compiled
+ * (jasminc is not required at build time). */

jasmin/zupt_aes_ctr4.s

@@ -0,0 +1,263 @@
+	.intel_syntax noprefix
+	.text
+	.p2align	5
+	.global	zupt_aes256_ctr4
+	.type	zupt_aes256_ctr4, %function
+/* zupt_aes256_ctr4(out=rdi, in=rsi, key=rdx, ctr=rcx, nblocks=r8)
+ *
+ * AES-256-CTR with 4-block interleaving for pipeline saturation.
+ * Processes 4 blocks per loop iteration. Remaining 1-3 blocks
+ * processed one at a time.
+ *
+ * AES-NI latency=4 cycles, throughput=1 cycle/block.
+ * 4 independent blocks → 4 AESENC in flight → ~4× throughput.
+ *
+ * Counter: big-endian increment in bytes [8..15] of the 16-byte block.
+ */
+zupt_aes256_ctr4:
+	push	rbx
+	push	r12
+	push	r13
+	mov	r12, r8			/* nblocks */
+	test	r12, r12
+	jz	.Ldone
+
+	/* Load 256-bit key into xmm14, xmm15 */
+	vmovdqu	xmm14, xmmword ptr[rdx]
+	vmovdqu	xmm15, xmmword ptr[rdx + 16]
+
+	/* Load counter template */
+	vmovdqu	xmm13, xmmword ptr[rcx]
+
+	/* Byte-swap mask for big-endian counter increment */
+	/* We increment a 64-bit big-endian value in bytes [8..15] */
+
+.Lloop4:
+	cmp	r12, 4
+	jb	.Lloop1
+
+	/* ═══ Generate 4 counter blocks with sequential values ═══ */
+	vmovdqa	xmm0, xmm13		/* ctr+0 */
+
+	/* Increment counter: byte-swap last 8 bytes, add 1, swap back */
+	/* Simple approach: store to stack, increment, reload */
+	sub	rsp, 64
+	vmovdqa	xmmword ptr[rsp], xmm13
+	/* Increment the big-endian counter in bytes [8..15] */
+	mov	rax, qword ptr[rsp + 8]
+	bswap	rax
+	lea	rbx, [rax + 1]
+	bswap	rbx
+	mov	qword ptr[rsp + 8], rbx
+	vmovdqa	xmm1, xmmword ptr[rsp]	/* ctr+1 */
+
+	bswap	rbx
+	lea	r13, [rbx + 1]
+	bswap	r13
+	mov	qword ptr[rsp + 8], r13
+	vmovdqa	xmm2, xmmword ptr[rsp]	/* ctr+2 */
+
+	bswap	r13
+	lea	rbx, [r13 + 1]
+	bswap	rbx
+	mov	qword ptr[rsp + 8], rbx
+	vmovdqa	xmm3, xmmword ptr[rsp]	/* ctr+3 */
+
+	/* Update counter template to ctr+4 */
+	bswap	rbx
+	add	rbx, 1
+	bswap	rbx
+	mov	qword ptr[rsp + 8], rbx
+	vmovdqa	xmm13, xmmword ptr[rsp]
+	add	rsp, 64
+
+	/* ═══ Key expansion + 14-round AES-256 on 4 blocks ═══ */
+	/* Round 0: AddRoundKey with key[0] */
+	vpxor	xmm0, xmm0, xmm14
+	vpxor	xmm1, xmm1, xmm14
+	vpxor	xmm2, xmm2, xmm14
+	vpxor	xmm3, xmm3, xmm14
+
+	/* We need round keys 1-14. For the 4-block pipeline, we compute
+	 * each round key once and apply it to all 4 blocks before moving
+	 * to the next round. This amortizes key expansion cost. */
+
+	/* For simplicity and correctness, we expand all 15 round keys
+	 * on the stack first, then apply them to all 4 blocks. */
+	sub	rsp, 240
+
+	/* Store rk0 = key[0], rk1 = key[1] */
+	vmovdqa	xmmword ptr[rsp + 0], xmm14
+	vmovdqa	xmmword ptr[rsp + 16], xmm15
+
+	/* Expand remaining round keys (same logic as zupt_aes_ctr.s) */
+	vmovdqa	xmm4, xmm14		/* t0 */
+	vmovdqa	xmm5, xmm15		/* t1 */
+
+	.macro EXPAND_EVEN rcon, offset
+	vaeskeygenassist	xmm6, xmm5, \rcon
+	vpshufd	xmm6, xmm6, 0xFF
+	vpslldq	xmm7, xmm4, 4
+	vpxor	xmm4, xmm4, xmm7
+	vpslldq	xmm7, xmm4, 4
+	vpxor	xmm4, xmm4, xmm7
+	vpslldq	xmm7, xmm4, 4
+	vpxor	xmm4, xmm4, xmm7
+	vpxor	xmm4, xmm4, xmm6
+	vmovdqa	xmmword ptr[rsp + \offset], xmm4
+	.endm
+
+	.macro EXPAND_ODD offset
+	vaeskeygenassist	xmm6, xmm4, 0
+	vpshufd	xmm6, xmm6, 0xAA
+	vpslldq	xmm7, xmm5, 4
+	vpxor	xmm5, xmm5, xmm7
+	vpslldq	xmm7, xmm5, 4
+	vpxor	xmm5, xmm5, xmm7
+	vpslldq	xmm7, xmm5, 4
+	vpxor	xmm5, xmm5, xmm7
+	vpxor	xmm5, xmm5, xmm6
+	vmovdqa	xmmword ptr[rsp + \offset], xmm5
+	.endm
+
+	EXPAND_EVEN 0x01, 32
+	EXPAND_ODD 48
+	EXPAND_EVEN 0x02, 64
+	EXPAND_ODD 80
+	EXPAND_EVEN 0x04, 96
+	EXPAND_ODD 112
+	EXPAND_EVEN 0x08, 128
+	EXPAND_ODD 144
+	EXPAND_EVEN 0x10, 160
+	EXPAND_ODD 176
+	EXPAND_EVEN 0x20, 192
+	EXPAND_ODD 208
+	EXPAND_EVEN 0x40, 224
+
+	/* ═══ Apply rounds 1-13 to all 4 blocks (interleaved) ═══ */
+	.macro ROUND4 offset
+	vmovdqa	xmm8, xmmword ptr[rsp + \offset]
+	vaesenc	xmm0, xmm0, xmm8
+	vaesenc	xmm1, xmm1, xmm8
+	vaesenc	xmm2, xmm2, xmm8
+	vaesenc	xmm3, xmm3, xmm8
+	.endm
+
+	ROUND4 16	/* Round 1 */
+	ROUND4 32	/* Round 2 */
+	ROUND4 48	/* Round 3 */
+	ROUND4 64	/* Round 4 */
+	ROUND4 80	/* Round 5 */
+	ROUND4 96	/* Round 6 */
+	ROUND4 112	/* Round 7 */
+	ROUND4 128	/* Round 8 */
+	ROUND4 144	/* Round 9 */
+	ROUND4 160	/* Round 10 */
+	ROUND4 176	/* Round 11 */
+	ROUND4 192	/* Round 12 */
+	ROUND4 208	/* Round 13 */
+
+	/* Round 14 (final) */
+	vmovdqa	xmm8, xmmword ptr[rsp + 224]
+	vaesenclast	xmm0, xmm0, xmm8
+	vaesenclast	xmm1, xmm1, xmm8
+	vaesenclast	xmm2, xmm2, xmm8
+	vaesenclast	xmm3, xmm3, xmm8
+
+	/* Wipe round keys */
+	vpxor	xmm8, xmm8, xmm8
+	.irp off, 0,16,32,48,64,80,96,112,128,144,160,176,192,208,224
+	vmovdqa	xmmword ptr[rsp + \off], xmm8
+	.endr
+	add	rsp, 240
+
+	/* XOR keystreams with plaintext */
+	vpxor	xmm0, xmm0, xmmword ptr[rsi]
+	vpxor	xmm1, xmm1, xmmword ptr[rsi + 16]
+	vpxor	xmm2, xmm2, xmmword ptr[rsi + 32]
+	vpxor	xmm3, xmm3, xmmword ptr[rsi + 48]
+
+	/* Store results */
+	vmovdqu	xmmword ptr[rdi], xmm0
+	vmovdqu	xmmword ptr[rdi + 16], xmm1
+	vmovdqu	xmmword ptr[rdi + 32], xmm2
+	vmovdqu	xmmword ptr[rdi + 48], xmm3
+
+	add	rsi, 64
+	add	rdi, 64
+	sub	r12, 4
+	jmp	.Lloop4
+
+.Lloop1:
+	test	r12, r12
+	jz	.Ldone
+
+	/* Single-block fallback for remaining 1-3 blocks */
+	/* Expand keys on stack (reuse zupt_aes256_blk logic) */
+	sub	rsp, 256
+	and	rsp, -16
+
+	vmovdqa	xmm4, xmm14
+	vmovdqa	xmm5, xmm15
+	vmovdqa	xmmword ptr[rsp + 0], xmm4
+	vmovdqa	xmmword ptr[rsp + 16], xmm5
+
+	EXPAND_EVEN 0x01, 32
+	EXPAND_ODD 48
+	EXPAND_EVEN 0x02, 64
+	EXPAND_ODD 80
+	EXPAND_EVEN 0x04, 96
+	EXPAND_ODD 112
+	EXPAND_EVEN 0x08, 128
+	EXPAND_ODD 144
+	EXPAND_EVEN 0x10, 160
+	EXPAND_ODD 176
+	EXPAND_EVEN 0x20, 192
+	EXPAND_ODD 208
+	EXPAND_EVEN 0x40, 224
+
+.Lsingle:
+	vmovdqa	xmm0, xmm13
+	vpxor	xmm0, xmm0, xmmword ptr[rsp + 0]
+	.irp off, 16,32,48,64,80,96,112,128,144,160,176,192,208
+	vaesenc	xmm0, xmm0, xmmword ptr[rsp + \off]
+	.endr
+	vaesenclast	xmm0, xmm0, xmmword ptr[rsp + 224]
+
+	vpxor	xmm0, xmm0, xmmword ptr[rsi]
+	vmovdqu	xmmword ptr[rdi], xmm0
+
+	/* Increment counter */
+	sub	rsp, 16
+	vmovdqa	xmmword ptr[rsp], xmm13
+	mov	rax, qword ptr[rsp + 8]
+	bswap	rax
+	add	rax, 1
+	bswap	rax
+	mov	qword ptr[rsp + 8], rax
+	vmovdqa	xmm13, xmmword ptr[rsp]
+	add	rsp, 16
+
+	add	rsi, 16
+	add	rdi, 16
+	dec	r12
+	jnz	.Lsingle
+
+	/* Wipe round keys */
+	vpxor	xmm8, xmm8, xmm8
+	.irp off, 0,16,32,48,64,80,96,112,128,144,160,176,192,208,224
+	vmovdqa	xmmword ptr[rsp + \off], xmm8
+	.endr
+	add	rsp, 256
+
+.Ldone:
+	/* Store updated counter back */
+	vmovdqu	xmmword ptr[rcx], xmm13
+
+	pop	r13
+	pop	r12
+	pop	rbx
+	ret
+	.size	zupt_aes256_ctr4, . - zupt_aes256_ctr4
+
+	.section .note.GNU-stack,"",@progbits