aarch64: Add support for FEAT_SME_TMOP

2025-11-16 04:24:43 +00:00 · 2025-10-07 23:38:01 +01:00
parent 3b957f92de
commit 0787e01a25
23 changed files with 967 additions and 462 deletions
--- a/gas/config/tc-aarch64.c
+++ b/gas/config/tc-aarch64.c
@@ -6937,6 +6937,7 @@ parse_operands (char *str, const aarch64_opcode *opcode)
 	case AARCH64_OPND_SVE_Zm4_INDEX:
 	case AARCH64_OPND_SVE_Zn_INDEX:
 	case AARCH64_OPND_SVE_Zn_5_INDEX:
+	case AARCH64_OPND_SME_Zk_INDEX:
 	case AARCH64_OPND_SME_Zm_INDEX1:
 	case AARCH64_OPND_SME_Zm_INDEX2:
 	case AARCH64_OPND_SME_Zm_INDEX2_3:
@@ -10893,6 +10894,7 @@ static const struct aarch64_option_cpu_value_table aarch64_features[] = {
  {"sve2p2",		AARCH64_FEATURE (SVE2p2), AARCH64_FEATURE (SVE2p1)},
  {"sme2p2",		AARCH64_FEATURE (SME2p2), AARCH64_FEATURE (SME2p1)},
  {"gcie",		AARCH64_FEATURE (GCIE), AARCH64_NO_FEATURES},
+  {"sme-tmop",		AARCH64_FEATURE (SME_TMOP), AARCH64_FEATURE (SME2)},
  {NULL,		AARCH64_NO_FEATURES, AARCH64_NO_FEATURES},
 };

--- a/gas/doc/c-aarch64.texi
+++ b/gas/doc/c-aarch64.texi
@@ -309,6 +309,8 @@ automatically cause those extensions to be disabled.
 @tab Enable SME I16I64 Extension.
@item @code{sme-lutv2} @tab
 @tab Enable SME Lookup Table v2 (LUTv2) extension.
+@item @code{sme-tmop} @tab @code{sme2}
+ @tab Enable SME structured sparsity outer product instructions.
@item @code{sme2} @tab @code{sme}
 @tab Enable SME2.
@item @code{sme2p1} @tab @code{sme2}
--- a/gas/testsuite/gas/aarch64/sme-tmop-b16b16.d
+++ b/gas/testsuite/gas/aarch64/sme-tmop-b16b16.d
@@ -0,0 +1,17 @@
+#as: -march=armv8-a+sme-tmop+sme-b16b16
+#objdump: -dr
+
+.*:     file format .*
+
+
+Disassembly of section \.text:
+
+0+ <\.text>:
+ *[0-9a-f]+:	81600008 	bftmopa	za0\.h, {z0\.h-z1\.h}, z0\.h, z20\[0\]
+ *[0-9a-f]+:	81600009 	bftmopa	za1\.h, {z0\.h-z1\.h}, z0\.h, z20\[0\]
+ *[0-9a-f]+:	816003c8 	bftmopa	za0\.h, {z30\.h-z31\.h}, z0\.h, z20\[0\]
+ *[0-9a-f]+:	817f0008 	bftmopa	za0\.h, {z0\.h-z1\.h}, z31\.h, z20\[0\]
+ *[0-9a-f]+:	81600c08 	bftmopa	za0\.h, {z0\.h-z1\.h}, z0\.h, z23\[0\]
+ *[0-9a-f]+:	81601008 	bftmopa	za0\.h, {z0\.h-z1\.h}, z0\.h, z28\[0\]
+ *[0-9a-f]+:	81601c08 	bftmopa	za0\.h, {z0\.h-z1\.h}, z0\.h, z31\[0\]
+ *[0-9a-f]+:	81600038 	bftmopa	za0\.h, {z0\.h-z1\.h}, z0\.h, z20\[3\]
--- a/gas/testsuite/gas/aarch64/sme-tmop-b16b16.s
+++ b/gas/testsuite/gas/aarch64/sme-tmop-b16b16.s
@@ -0,0 +1,8 @@
+	bftmopa	za0.h, { z0.h-z1.h }, z0.h, z20[0]
+	bftmopa	za1.h, { z0.h-z1.h }, z0.h, z20[0]
+	bftmopa	za0.h, { z30.h-z31.h }, z0.h, z20[0]
+	bftmopa	za0.h, { z0.h-z1.h }, z31.h, z20[0]
+	bftmopa	za0.h, { z0.h-z1.h }, z0.h, z23[0]
+	bftmopa	za0.h, { z0.h-z1.h }, z0.h, z28[0]
+	bftmopa	za0.h, { z0.h-z1.h }, z0.h, z31[0]
+	bftmopa	za0.h, { z0.h-z1.h }, z0.h, z20[3]
--- a/gas/testsuite/gas/aarch64/sme-tmop-f16f16.d
+++ b/gas/testsuite/gas/aarch64/sme-tmop-f16f16.d
@@ -0,0 +1,17 @@
+#as: -march=armv8-a+sme-tmop+sme-f16f16
+#objdump: -dr
+
+.*:     file format .*
+
+
+Disassembly of section \.text:
+
+0+ <\.text>:
+ *[0-9a-f]+:	81400008 	ftmopa	za0\.h, {z0\.h-z1\.h}, z0\.h, z20\[0\]
+ *[0-9a-f]+:	81400009 	ftmopa	za1\.h, {z0\.h-z1\.h}, z0\.h, z20\[0\]
+ *[0-9a-f]+:	814003c8 	ftmopa	za0\.h, {z30\.h-z31\.h}, z0\.h, z20\[0\]
+ *[0-9a-f]+:	815f0008 	ftmopa	za0\.h, {z0\.h-z1\.h}, z31\.h, z20\[0\]
+ *[0-9a-f]+:	81400c08 	ftmopa	za0\.h, {z0\.h-z1\.h}, z0\.h, z23\[0\]
+ *[0-9a-f]+:	81401008 	ftmopa	za0\.h, {z0\.h-z1\.h}, z0\.h, z28\[0\]
+ *[0-9a-f]+:	81401c08 	ftmopa	za0\.h, {z0\.h-z1\.h}, z0\.h, z31\[0\]
+ *[0-9a-f]+:	81400038 	ftmopa	za0\.h, {z0\.h-z1\.h}, z0\.h, z20\[3\]
--- a/gas/testsuite/gas/aarch64/sme-tmop-f16f16.s
+++ b/gas/testsuite/gas/aarch64/sme-tmop-f16f16.s
@@ -0,0 +1,8 @@
+	ftmopa	za0.h, { z0.h-z1.h }, z0.h, z20[0]
+	ftmopa	za1.h, { z0.h-z1.h }, z0.h, z20[0]
+	ftmopa	za0.h, { z30.h-z31.h }, z0.h, z20[0]
+	ftmopa	za0.h, { z0.h-z1.h }, z31.h, z20[0]
+	ftmopa	za0.h, { z0.h-z1.h }, z0.h, z23[0]
+	ftmopa	za0.h, { z0.h-z1.h }, z0.h, z28[0]
+	ftmopa	za0.h, { z0.h-z1.h }, z0.h, z31[0]
+	ftmopa	za0.h, { z0.h-z1.h }, z0.h, z20[3]
--- a/gas/testsuite/gas/aarch64/sme-tmop-f8f16.d
+++ b/gas/testsuite/gas/aarch64/sme-tmop-f8f16.d
@@ -0,0 +1,17 @@
+#as: -march=armv8-a+sme-tmop+sme-f8f16
+#objdump: -dr
+
+.*:     file format .*
+
+
+Disassembly of section \.text:
+
+0+ <\.text>:
+ *[0-9a-f]+:	80600008 	ftmopa	za0\.h, {z0\.b-z1\.b}, z0\.b, z20\[0\]
+ *[0-9a-f]+:	80600009 	ftmopa	za1\.h, {z0\.b-z1\.b}, z0\.b, z20\[0\]
+ *[0-9a-f]+:	806003c8 	ftmopa	za0\.h, {z30\.b-z31\.b}, z0\.b, z20\[0\]
+ *[0-9a-f]+:	807f0008 	ftmopa	za0\.h, {z0\.b-z1\.b}, z31\.b, z20\[0\]
+ *[0-9a-f]+:	80600c08 	ftmopa	za0\.h, {z0\.b-z1\.b}, z0\.b, z23\[0\]
+ *[0-9a-f]+:	80601008 	ftmopa	za0\.h, {z0\.b-z1\.b}, z0\.b, z28\[0\]
+ *[0-9a-f]+:	80601c08 	ftmopa	za0\.h, {z0\.b-z1\.b}, z0\.b, z31\[0\]
+ *[0-9a-f]+:	80600038 	ftmopa	za0\.h, {z0\.b-z1\.b}, z0\.b, z20\[3\]
--- a/gas/testsuite/gas/aarch64/sme-tmop-f8f16.s
+++ b/gas/testsuite/gas/aarch64/sme-tmop-f8f16.s
@@ -0,0 +1,8 @@
+	ftmopa	za0.h, { z0.b-z1.b }, z0.b, z20[0]
+	ftmopa	za1.h, { z0.b-z1.b }, z0.b, z20[0]
+	ftmopa	za0.h, { z30.b-z31.b }, z0.b, z20[0]
+	ftmopa	za0.h, { z0.b-z1.b }, z31.b, z20[0]
+	ftmopa	za0.h, { z0.b-z1.b }, z0.b, z23[0]
+	ftmopa	za0.h, { z0.b-z1.b }, z0.b, z28[0]
+	ftmopa	za0.h, { z0.b-z1.b }, z0.b, z31[0]
+	ftmopa	za0.h, { z0.b-z1.b }, z0.b, z20[3]
--- a/gas/testsuite/gas/aarch64/sme-tmop-f8f32.d
+++ b/gas/testsuite/gas/aarch64/sme-tmop-f8f32.d
@@ -0,0 +1,17 @@
+#as: -march=armv8-a+sme-tmop+sme-f8f32
+#objdump: -dr
+
+.*:     file format .*
+
+
+Disassembly of section \.text:
+
+0+ <\.text>:
+ *[0-9a-f]+:	80600000 	ftmopa	za0\.s, {z0\.b-z1\.b}, z0\.b, z20\[0\]
+ *[0-9a-f]+:	80600003 	ftmopa	za3\.s, {z0\.b-z1\.b}, z0\.b, z20\[0\]
+ *[0-9a-f]+:	806003c0 	ftmopa	za0\.s, {z30\.b-z31\.b}, z0\.b, z20\[0\]
+ *[0-9a-f]+:	807f0000 	ftmopa	za0\.s, {z0\.b-z1\.b}, z31\.b, z20\[0\]
+ *[0-9a-f]+:	80600c00 	ftmopa	za0\.s, {z0\.b-z1\.b}, z0\.b, z23\[0\]
+ *[0-9a-f]+:	80601000 	ftmopa	za0\.s, {z0\.b-z1\.b}, z0\.b, z28\[0\]
+ *[0-9a-f]+:	80601c00 	ftmopa	za0\.s, {z0\.b-z1\.b}, z0\.b, z31\[0\]
+ *[0-9a-f]+:	80600030 	ftmopa	za0\.s, {z0\.b-z1\.b}, z0\.b, z20\[3\]
--- a/gas/testsuite/gas/aarch64/sme-tmop-f8f32.s
+++ b/gas/testsuite/gas/aarch64/sme-tmop-f8f32.s
@@ -0,0 +1,8 @@
+	ftmopa	za0.s, { z0.b-z1.b }, z0.b, z20[0]
+	ftmopa	za3.s, { z0.b-z1.b }, z0.b, z20[0]
+	ftmopa	za0.s, { z30.b-z31.b }, z0.b, z20[0]
+	ftmopa	za0.s, { z0.b-z1.b }, z31.b, z20[0]
+	ftmopa	za0.s, { z0.b-z1.b }, z0.b, z23[0]
+	ftmopa	za0.s, { z0.b-z1.b }, z0.b, z28[0]
+	ftmopa	za0.s, { z0.b-z1.b }, z0.b, z31[0]
+	ftmopa	za0.s, { z0.b-z1.b }, z0.b, z20[3]
--- a/gas/testsuite/gas/aarch64/sme-tmop-invalid.d
+++ b/gas/testsuite/gas/aarch64/sme-tmop-invalid.d
@@ -0,0 +1,2 @@
+#as: -march=armv8-a+sme-tmop
+#error_output: sme-tmop-invalid.l
--- a/gas/testsuite/gas/aarch64/sme-tmop-invalid.l
+++ b/gas/testsuite/gas/aarch64/sme-tmop-invalid.l
@@ -0,0 +1,6 @@
+[^:]*: Assembler messages:
+[^:]*:\d+: Error: register out of range at operand 4 -- `ftmopa za0\.s,{z0\.s-z1\.s},z0\.s,z0\[0\]'
+[^:]*:\d+: Error: register out of range at operand 4 -- `stmopa za0\.s,{z0\.h-z1\.h},z0\.h,z7\[0\]'
+[^:]*:\d+: Error: register out of range at operand 4 -- `utmopa za0\.s,{z0\.b-z1\.b},z0\.b,z19\[0\]'
+[^:]*:\d+: Error: register out of range at operand 4 -- `bftmopa za0\.s,{z0\.h-z1\.h},z0\.h,z27\[0\]'
+[^:]*:\d+: Error: register element index out of range 0 to 3 at operand 4 -- `ftmopa za0\.s,{z0\.s-z1\.s},z0\.s,z20\[4\]'
--- a/gas/testsuite/gas/aarch64/sme-tmop-invalid.s
+++ b/gas/testsuite/gas/aarch64/sme-tmop-invalid.s
@@ -0,0 +1,5 @@
+	ftmopa	za0.s, { z0.s-z1.s }, z0.s, z0[0]
+	stmopa	za0.s, { z0.h-z1.h }, z0.h, z7[0]
+	utmopa	za0.s, { z0.b-z1.b }, z0.b, z19[0]
+	bftmopa	za0.s, { z0.h-z1.h }, z0.h, z27[0]
+	ftmopa	za0.s, { z0.s-z1.s }, z0.s, z20[4]
--- a/gas/testsuite/gas/aarch64/sme-tmop.d
+++ b/gas/testsuite/gas/aarch64/sme-tmop.d
@@ -0,0 +1,81 @@
+#as: -march=armv8-a+sme-tmop
+#objdump: -dr
+
+.*:     file format .*
+
+
+Disassembly of section \.text:
+
+0+ <\.text>:
+ *[0-9a-f]+:	81400000 	bftmopa	za0\.s, {z0\.h-z1\.h}, z0\.h, z20\[0\]
+ *[0-9a-f]+:	81400003 	bftmopa	za3\.s, {z0\.h-z1\.h}, z0\.h, z20\[0\]
+ *[0-9a-f]+:	814003c0 	bftmopa	za0\.s, {z30\.h-z31\.h}, z0\.h, z20\[0\]
+ *[0-9a-f]+:	815f0000 	bftmopa	za0\.s, {z0\.h-z1\.h}, z31\.h, z20\[0\]
+ *[0-9a-f]+:	81400c00 	bftmopa	za0\.s, {z0\.h-z1\.h}, z0\.h, z23\[0\]
+ *[0-9a-f]+:	81401000 	bftmopa	za0\.s, {z0\.h-z1\.h}, z0\.h, z28\[0\]
+ *[0-9a-f]+:	81401c00 	bftmopa	za0\.s, {z0\.h-z1\.h}, z0\.h, z31\[0\]
+ *[0-9a-f]+:	81400030 	bftmopa	za0\.s, {z0\.h-z1\.h}, z0\.h, z20\[3\]
+ *[0-9a-f]+:	80400000 	ftmopa	za0\.s, {z0\.s-z1\.s}, z0\.s, z20\[0\]
+ *[0-9a-f]+:	80400003 	ftmopa	za3\.s, {z0\.s-z1\.s}, z0\.s, z20\[0\]
+ *[0-9a-f]+:	804003c0 	ftmopa	za0\.s, {z30\.s-z31\.s}, z0\.s, z20\[0\]
+ *[0-9a-f]+:	805f0000 	ftmopa	za0\.s, {z0\.s-z1\.s}, z31\.s, z20\[0\]
+ *[0-9a-f]+:	80400c00 	ftmopa	za0\.s, {z0\.s-z1\.s}, z0\.s, z23\[0\]
+ *[0-9a-f]+:	80401000 	ftmopa	za0\.s, {z0\.s-z1\.s}, z0\.s, z28\[0\]
+ *[0-9a-f]+:	80401c00 	ftmopa	za0\.s, {z0\.s-z1\.s}, z0\.s, z31\[0\]
+ *[0-9a-f]+:	80400030 	ftmopa	za0\.s, {z0\.s-z1\.s}, z0\.s, z20\[3\]
+ *[0-9a-f]+:	81600000 	ftmopa	za0\.s, {z0\.h-z1\.h}, z0\.h, z20\[0\]
+ *[0-9a-f]+:	81600003 	ftmopa	za3\.s, {z0\.h-z1\.h}, z0\.h, z20\[0\]
+ *[0-9a-f]+:	816003c0 	ftmopa	za0\.s, {z30\.h-z31\.h}, z0\.h, z20\[0\]
+ *[0-9a-f]+:	817f0000 	ftmopa	za0\.s, {z0\.h-z1\.h}, z31\.h, z20\[0\]
+ *[0-9a-f]+:	81600c00 	ftmopa	za0\.s, {z0\.h-z1\.h}, z0\.h, z23\[0\]
+ *[0-9a-f]+:	81601000 	ftmopa	za0\.s, {z0\.h-z1\.h}, z0\.h, z28\[0\]
+ *[0-9a-f]+:	81601c00 	ftmopa	za0\.s, {z0\.h-z1\.h}, z0\.h, z31\[0\]
+ *[0-9a-f]+:	81600030 	ftmopa	za0\.s, {z0\.h-z1\.h}, z0\.h, z20\[3\]
+ *[0-9a-f]+:	80408008 	stmopa	za0\.s, {z0\.h-z1\.h}, z0\.h, z20\[0\]
+ *[0-9a-f]+:	8040800b 	stmopa	za3\.s, {z0\.h-z1\.h}, z0\.h, z20\[0\]
+ *[0-9a-f]+:	804083c8 	stmopa	za0\.s, {z30\.h-z31\.h}, z0\.h, z20\[0\]
+ *[0-9a-f]+:	805f8008 	stmopa	za0\.s, {z0\.h-z1\.h}, z31\.h, z20\[0\]
+ *[0-9a-f]+:	80408c08 	stmopa	za0\.s, {z0\.h-z1\.h}, z0\.h, z23\[0\]
+ *[0-9a-f]+:	80409008 	stmopa	za0\.s, {z0\.h-z1\.h}, z0\.h, z28\[0\]
+ *[0-9a-f]+:	80409c08 	stmopa	za0\.s, {z0\.h-z1\.h}, z0\.h, z31\[0\]
+ *[0-9a-f]+:	80408038 	stmopa	za0\.s, {z0\.h-z1\.h}, z0\.h, z20\[3\]
+ *[0-9a-f]+:	80408000 	stmopa	za0\.s, {z0\.b-z1\.b}, z0\.b, z20\[0\]
+ *[0-9a-f]+:	80408003 	stmopa	za3\.s, {z0\.b-z1\.b}, z0\.b, z20\[0\]
+ *[0-9a-f]+:	804083c0 	stmopa	za0\.s, {z30\.b-z31\.b}, z0\.b, z20\[0\]
+ *[0-9a-f]+:	805f8000 	stmopa	za0\.s, {z0\.b-z1\.b}, z31\.b, z20\[0\]
+ *[0-9a-f]+:	80408c00 	stmopa	za0\.s, {z0\.b-z1\.b}, z0\.b, z23\[0\]
+ *[0-9a-f]+:	80409000 	stmopa	za0\.s, {z0\.b-z1\.b}, z0\.b, z28\[0\]
+ *[0-9a-f]+:	80409c00 	stmopa	za0\.s, {z0\.b-z1\.b}, z0\.b, z31\[0\]
+ *[0-9a-f]+:	80408030 	stmopa	za0\.s, {z0\.b-z1\.b}, z0\.b, z20\[3\]
+ *[0-9a-f]+:	80608000 	sutmopa	za0\.s, {z0\.b-z1\.b}, z0\.b, z20\[0\]
+ *[0-9a-f]+:	80608003 	sutmopa	za3\.s, {z0\.b-z1\.b}, z0\.b, z20\[0\]
+ *[0-9a-f]+:	806083c0 	sutmopa	za0\.s, {z30\.b-z31\.b}, z0\.b, z20\[0\]
+ *[0-9a-f]+:	807f8000 	sutmopa	za0\.s, {z0\.b-z1\.b}, z31\.b, z20\[0\]
+ *[0-9a-f]+:	80608c00 	sutmopa	za0\.s, {z0\.b-z1\.b}, z0\.b, z23\[0\]
+ *[0-9a-f]+:	80609000 	sutmopa	za0\.s, {z0\.b-z1\.b}, z0\.b, z28\[0\]
+ *[0-9a-f]+:	80609c00 	sutmopa	za0\.s, {z0\.b-z1\.b}, z0\.b, z31\[0\]
+ *[0-9a-f]+:	80608030 	sutmopa	za0\.s, {z0\.b-z1\.b}, z0\.b, z20\[3\]
+ *[0-9a-f]+:	81408000 	ustmopa	za0\.s, {z0\.b-z1\.b}, z0\.b, z20\[0\]
+ *[0-9a-f]+:	81408003 	ustmopa	za3\.s, {z0\.b-z1\.b}, z0\.b, z20\[0\]
+ *[0-9a-f]+:	814083c0 	ustmopa	za0\.s, {z30\.b-z31\.b}, z0\.b, z20\[0\]
+ *[0-9a-f]+:	815f8000 	ustmopa	za0\.s, {z0\.b-z1\.b}, z31\.b, z20\[0\]
+ *[0-9a-f]+:	81408c00 	ustmopa	za0\.s, {z0\.b-z1\.b}, z0\.b, z23\[0\]
+ *[0-9a-f]+:	81409000 	ustmopa	za0\.s, {z0\.b-z1\.b}, z0\.b, z28\[0\]
+ *[0-9a-f]+:	81409c00 	ustmopa	za0\.s, {z0\.b-z1\.b}, z0\.b, z31\[0\]
+ *[0-9a-f]+:	81408030 	ustmopa	za0\.s, {z0\.b-z1\.b}, z0\.b, z20\[3\]
+ *[0-9a-f]+:	81408008 	utmopa	za0\.s, {z0\.h-z1\.h}, z0\.h, z20\[0\]
+ *[0-9a-f]+:	8140800b 	utmopa	za3\.s, {z0\.h-z1\.h}, z0\.h, z20\[0\]
+ *[0-9a-f]+:	814083c8 	utmopa	za0\.s, {z30\.h-z31\.h}, z0\.h, z20\[0\]
+ *[0-9a-f]+:	815f8008 	utmopa	za0\.s, {z0\.h-z1\.h}, z31\.h, z20\[0\]
+ *[0-9a-f]+:	81408c08 	utmopa	za0\.s, {z0\.h-z1\.h}, z0\.h, z23\[0\]
+ *[0-9a-f]+:	81409008 	utmopa	za0\.s, {z0\.h-z1\.h}, z0\.h, z28\[0\]
+ *[0-9a-f]+:	81409c08 	utmopa	za0\.s, {z0\.h-z1\.h}, z0\.h, z31\[0\]
+ *[0-9a-f]+:	81408038 	utmopa	za0\.s, {z0\.h-z1\.h}, z0\.h, z20\[3\]
+ *[0-9a-f]+:	81608000 	utmopa	za0\.s, {z0\.b-z1\.b}, z0\.b, z20\[0\]
+ *[0-9a-f]+:	81608003 	utmopa	za3\.s, {z0\.b-z1\.b}, z0\.b, z20\[0\]
+ *[0-9a-f]+:	816083c0 	utmopa	za0\.s, {z30\.b-z31\.b}, z0\.b, z20\[0\]
+ *[0-9a-f]+:	817f8000 	utmopa	za0\.s, {z0\.b-z1\.b}, z31\.b, z20\[0\]
+ *[0-9a-f]+:	81608c00 	utmopa	za0\.s, {z0\.b-z1\.b}, z0\.b, z23\[0\]
+ *[0-9a-f]+:	81609000 	utmopa	za0\.s, {z0\.b-z1\.b}, z0\.b, z28\[0\]
+ *[0-9a-f]+:	81609c00 	utmopa	za0\.s, {z0\.b-z1\.b}, z0\.b, z31\[0\]
+ *[0-9a-f]+:	81608030 	utmopa	za0\.s, {z0\.b-z1\.b}, z0\.b, z20\[3\]
--- a/gas/testsuite/gas/aarch64/sme-tmop.s
+++ b/gas/testsuite/gas/aarch64/sme-tmop.s
@@ -0,0 +1,80 @@
+	bftmopa	za0.s, { z0.h-z1.h }, z0.h, z20[0]
+	bftmopa	za3.s, { z0.h-z1.h }, z0.h, z20[0]
+	bftmopa	za0.s, { z30.h-z31.h }, z0.h, z20[0]
+	bftmopa	za0.s, { z0.h-z1.h }, z31.h, z20[0]
+	bftmopa	za0.s, { z0.h-z1.h }, z0.h, z23[0]
+	bftmopa	za0.s, { z0.h-z1.h }, z0.h, z28[0]
+	bftmopa	za0.s, { z0.h-z1.h }, z0.h, z31[0]
+	bftmopa	za0.s, { z0.h-z1.h }, z0.h, z20[3]
+
+	ftmopa	za0.s, { z0.s-z1.s }, z0.s, z20[0]
+	ftmopa	za3.s, { z0.s-z1.s }, z0.s, z20[0]
+	ftmopa	za0.s, { z30.s-z31.s }, z0.s, z20[0]
+	ftmopa	za0.s, { z0.s-z1.s }, z31.s, z20[0]
+	ftmopa	za0.s, { z0.s-z1.s }, z0.s, z23[0]
+	ftmopa	za0.s, { z0.s-z1.s }, z0.s, z28[0]
+	ftmopa	za0.s, { z0.s-z1.s }, z0.s, z31[0]
+	ftmopa	za0.s, { z0.s-z1.s }, z0.s, z20[3]
+
+	ftmopa	za0.s, { z0.h-z1.h }, z0.h, z20[0]
+	ftmopa	za3.s, { z0.h-z1.h }, z0.h, z20[0]
+	ftmopa	za0.s, { z30.h-z31.h }, z0.h, z20[0]
+	ftmopa	za0.s, { z0.h-z1.h }, z31.h, z20[0]
+	ftmopa	za0.s, { z0.h-z1.h }, z0.h, z23[0]
+	ftmopa	za0.s, { z0.h-z1.h }, z0.h, z28[0]
+	ftmopa	za0.s, { z0.h-z1.h }, z0.h, z31[0]
+	ftmopa	za0.s, { z0.h-z1.h }, z0.h, z20[3]
+
+	stmopa	za0.s, { z0.h-z1.h }, z0.h, z20[0]
+	stmopa	za3.s, { z0.h-z1.h }, z0.h, z20[0]
+	stmopa	za0.s, { z30.h-z31.h }, z0.h, z20[0]
+	stmopa	za0.s, { z0.h-z1.h }, z31.h, z20[0]
+	stmopa	za0.s, { z0.h-z1.h }, z0.h, z23[0]
+	stmopa	za0.s, { z0.h-z1.h }, z0.h, z28[0]
+	stmopa	za0.s, { z0.h-z1.h }, z0.h, z31[0]
+	stmopa	za0.s, { z0.h-z1.h }, z0.h, z20[3]
+
+	stmopa	za0.s, { z0.b-z1.b }, z0.b, z20[0]
+	stmopa	za3.s, { z0.b-z1.b }, z0.b, z20[0]
+	stmopa	za0.s, { z30.b-z31.b }, z0.b, z20[0]
+	stmopa	za0.s, { z0.b-z1.b }, z31.b, z20[0]
+	stmopa	za0.s, { z0.b-z1.b }, z0.b, z23[0]
+	stmopa	za0.s, { z0.b-z1.b }, z0.b, z28[0]
+	stmopa	za0.s, { z0.b-z1.b }, z0.b, z31[0]
+	stmopa	za0.s, { z0.b-z1.b }, z0.b, z20[3]
+
+	sutmopa	za0.s, { z0.b-z1.b }, z0.b, z20[0]
+	sutmopa	za3.s, { z0.b-z1.b }, z0.b, z20[0]
+	sutmopa	za0.s, { z30.b-z31.b }, z0.b, z20[0]
+	sutmopa	za0.s, { z0.b-z1.b }, z31.b, z20[0]
+	sutmopa	za0.s, { z0.b-z1.b }, z0.b, z23[0]
+	sutmopa	za0.s, { z0.b-z1.b }, z0.b, z28[0]
+	sutmopa	za0.s, { z0.b-z1.b }, z0.b, z31[0]
+	sutmopa	za0.s, { z0.b-z1.b }, z0.b, z20[3]
+
+	ustmopa	za0.s, { z0.b-z1.b }, z0.b, z20[0]
+	ustmopa	za3.s, { z0.b-z1.b }, z0.b, z20[0]
+	ustmopa	za0.s, { z30.b-z31.b }, z0.b, z20[0]
+	ustmopa	za0.s, { z0.b-z1.b }, z31.b, z20[0]
+	ustmopa	za0.s, { z0.b-z1.b }, z0.b, z23[0]
+	ustmopa	za0.s, { z0.b-z1.b }, z0.b, z28[0]
+	ustmopa	za0.s, { z0.b-z1.b }, z0.b, z31[0]
+	ustmopa	za0.s, { z0.b-z1.b }, z0.b, z20[3]
+
+	utmopa	za0.s, { z0.h-z1.h }, z0.h, z20[0]
+	utmopa	za3.s, { z0.h-z1.h }, z0.h, z20[0]
+	utmopa	za0.s, { z30.h-z31.h }, z0.h, z20[0]
+	utmopa	za0.s, { z0.h-z1.h }, z31.h, z20[0]
+	utmopa	za0.s, { z0.h-z1.h }, z0.h, z23[0]
+	utmopa	za0.s, { z0.h-z1.h }, z0.h, z28[0]
+	utmopa	za0.s, { z0.h-z1.h }, z0.h, z31[0]
+	utmopa	za0.s, { z0.h-z1.h }, z0.h, z20[3]
+
+	utmopa	za0.s, { z0.b-z1.b }, z0.b, z20[0]
+	utmopa	za3.s, { z0.b-z1.b }, z0.b, z20[0]
+	utmopa	za0.s, { z30.b-z31.b }, z0.b, z20[0]
+	utmopa	za0.s, { z0.b-z1.b }, z31.b, z20[0]
+	utmopa	za0.s, { z0.b-z1.b }, z0.b, z23[0]
+	utmopa	za0.s, { z0.b-z1.b }, z0.b, z28[0]
+	utmopa	za0.s, { z0.b-z1.b }, z0.b, z31[0]
+	utmopa	za0.s, { z0.b-z1.b }, z0.b, z20[3]