x86: reject further invalid AVX-512 masking constructs

For one the register type used for masking should be validated. And then
we shouldn't accept input producing encodings which will #UD when
executed, as is the case when EVEX.Z is set while EVEX.AAA is zero.

gas/ChangeLog

@@ -1,3 +1,14 @@
+2017-11-24  Jan Beulich  <jbeulich@suse.com>
+
+	* config/tc-i386.c (check_VecOperations): Check register type
+	for masking. Quote the actual register name in the respective
+	diagnostic. Check {z} wasn't specified on its own.
+	* testsuite/gas/i386/inval-avx512f.s,
+	testsuite/gas/i386/x86-64-inval-avx512f.s: Add further bad
+	masking tests.
+	* testsuite/gas/i386/inval-avx512f.l,
+	testsuite/gas/i386/x86-64-inval-avx512f.l: Adjust expectations.
+
 2017-11-24  Jan Beulich  <jbeulich@suse.com>
 
 	* testsuite/gas/i386/intel.d, testsuite/gas/i386/opcode.d,

gas/config/tc-i386.c

@@ -8138,10 +8138,10 @@ check_VecOperations (char *op_string, char *op_end)
 	  else if ((mask = parse_register (op_string, &end_op)) != NULL)
 	    {
 	      /* k0 can't be used for write mask.  */
-	      if (mask->reg_num == 0)
+	      if (!mask->reg_type.bitfield.regmask || mask->reg_num == 0)
 		{
-		  as_bad (_("`%s' can't be used for write mask"),
-			  op_string);
+		  as_bad (_("`%s%s' can't be used for write mask"),
+			  register_prefix, mask->reg_name);
 		  return NULL;
 		}
 
@@ -8220,6 +8220,12 @@ check_VecOperations (char *op_string, char *op_end)
       return NULL;
     }
 
+  if (i.mask && i.mask->zeroing && !i.mask->mask)
+    {
+      as_bad (_("zeroing-masking only allowed with write mask"));
+      return NULL;
+    }
+
   return op_string;
 }
 

gas/testsuite/gas/i386/inval-avx512f.l

@@ -36,6 +36,10 @@
 .*:48: Error: .*
 .*:49: Error: .*
 .*:50: Error: .*
+.*:53: Error: .*
+.*:54: Error: .*
+.*:57: Error: .*
+.*:58: Error: .*
 GAS LISTING .*
 
 
@@ -89,3 +93,13 @@ GAS LISTING .*
 [ 	]*48[ 	]+vaddps zmm2, zmm1, ZMMWORD PTR \[eax\]\{1to16\}
 [ 	]*49[ 	]+vaddps zmm2, zmm1, DWORD PTR \[eax\]
 [ 	]*50[ 	]+vaddpd zmm2, zmm1, QWORD PTR \[eax\]
+[ 	]*51[ 	]*
+[ 	]*52[ 	]+\.att_syntax prefix
+[ 	]*53[ 	]+vaddps %zmm0, %zmm1, %zmm2\{%ecx\}
+[ 	]*54[ 	]+vaddps %zmm0, %zmm1, %zmm2\{z\}
+[ 	]*55[ 	]*
+[ 	]*56[ 	]+\.intel_syntax noprefix
+[ 	]*57[ 	]+vaddps zmm2\{ecx\}, zmm1, zmm0
+GAS LISTING .*
+#...
+[ 	]*58[ 	]+vaddps zmm2\{z\}, zmm1, zmm0

gas/testsuite/gas/i386/inval-avx512f.s

@@ -48,3 +48,11 @@ _start:
 	vaddps zmm2, zmm1, ZMMWORD PTR [eax]{1to16}
 	vaddps zmm2, zmm1, DWORD PTR [eax]
 	vaddpd zmm2, zmm1, QWORD PTR [eax]
+
+	.att_syntax prefix
+	vaddps %zmm0, %zmm1, %zmm2{%ecx}
+	vaddps %zmm0, %zmm1, %zmm2{z}
+
+	.intel_syntax noprefix
+	vaddps zmm2{ecx}, zmm1, zmm0
+	vaddps zmm2{z}, zmm1, zmm0

gas/testsuite/gas/i386/x86-64-inval-avx512f.l

@@ -34,6 +34,10 @@
 .*:46: Error: .*
 .*:47: Error: .*
 .*:48: Error: .*
+.*:51: Error: .*
+.*:52: Error: .*
+.*:55: Error: .*
+.*:56: Error: .*
 GAS LISTING .*
 
 
@@ -85,3 +89,11 @@ GAS LISTING .*
 [ 	]*46[ 	]+vaddps zmm2, zmm1, ZMMWORD PTR \[rax\]\{1to16\}
 [ 	]*47[ 	]+vaddps zmm2, zmm1, DWORD PTR \[rax\]
 [ 	]*48[ 	]+vaddpd zmm2, zmm1, QWORD PTR \[rax\]
+[ 	]*49[ 	]*
+[ 	]*50[ 	]+\.att_syntax prefix
+[ 	]*51[ 	]+vaddps %zmm0, %zmm1, %zmm2\{%rcx\}
+[ 	]*52[ 	]+vaddps %zmm0, %zmm1, %zmm2\{z\}
+[ 	]*53[ 	]*
+[ 	]*54[ 	]+\.intel_syntax noprefix
+[ 	]*55[ 	]+vaddps zmm2\{rcx\}, zmm1, zmm0
+[ 	]*56[ 	]+vaddps zmm2\{z\}, zmm1, zmm0

gas/testsuite/gas/i386/x86-64-inval-avx512f.s

@@ -46,3 +46,11 @@ _start:
 	vaddps zmm2, zmm1, ZMMWORD PTR [rax]{1to16}
 	vaddps zmm2, zmm1, DWORD PTR [rax]
 	vaddpd zmm2, zmm1, QWORD PTR [rax]
+
+	.att_syntax prefix
+	vaddps %zmm0, %zmm1, %zmm2{%rcx}
+	vaddps %zmm0, %zmm1, %zmm2{z}
+
+	.intel_syntax noprefix
+	vaddps zmm2{rcx}, zmm1, zmm0
+	vaddps zmm2{z}, zmm1, zmm0