x86: SYSENTER/SYSEXIT are unavailable in 64-bit mode on AMD

The disassembler change is such that in default mode we'd disassemble
the insns (for there not ebing any conflicts), but when AMD64 mode was
explicitly requested, we'd show them as "(bad)".

gas/ChangeLog

@@ -1,3 +1,12 @@
+2020-01-09  Jan Beulich  <jbeulich@suse.com>
+
+	* testsuite/gas/i386/x86-64-sysenter-amd.s,
+	testsuite/gas/i386/x86-64-sysenter-amd.d,
+	testsuite/gas/i386/x86-64-sysenter-amd.l,
+	testsuite/gas/i386/x86-64-sysenter-intel.d,
+	testsuite/gas/i386/x86-64-sysenter-mixed.d: New.
+	* testsuite/gas/i386/i386.exp: Run new tests.
+
 2020-01-08  Nick Clifton  <nickc@redhat.com>
 
 	PR 25284

gas/testsuite/gas/i386/i386.exp

@@ -724,6 +724,10 @@ if [expr ([istarget "i*86-*-*"] || [istarget "x86_64-*-*"]) && [gas_64_check]] t
     run_dump_test "x86-64-nops-5"
     run_dump_test "x86-64-nops-5-k8"
     run_dump_test "x86-64-nops-7"
+    run_dump_test "x86-64-sysenter-intel"
+    run_dump_test "x86-64-sysenter-mixed"
+    run_dump_test "x86-64-sysenter-amd"
+    run_list_test "x86-64-sysenter-amd"
     run_dump_test "noreg64"
     run_list_test "cvtsi2sX"
     run_dump_test "x86-64-sse4_1"

gas/testsuite/gas/i386/x86-64-sysenter-amd.d

@@ -0,0 +1,14 @@
+#as: -mintel64
+#objdump: -dw -Mamd64
+#name: x86-64 sysenter (Intel64/AMD64)
+
+.*: +file format .*
+
+Disassembly of section .text:
+
+0+ <.text>:
+[ 	]*[a-f0-9]+:[ 	]+0f 34[ 	]+\(bad\)[ 	]*
+[ 	]*[a-f0-9]+:[ 	]+0f 35[ 	]+\(bad\)[ 	]*
+[ 	]*[a-f0-9]+:[ 	]+0f 34[ 	]+\(bad\)[ 	]*
+[ 	]*[a-f0-9]+:[ 	]+0f 35[ 	]+\(bad\)[ 	]*
+#pass

gas/testsuite/gas/i386/x86-64-sysenter-amd.l

@@ -0,0 +1,5 @@
+.*: Assembler messages:
+.*:2: Error: .*
+.*:3: Error: .*
+.*:6: Error: .*
+.*:7: Error: .*

gas/testsuite/gas/i386/x86-64-sysenter-amd.s

@@ -0,0 +1,7 @@
+	.text
+	sysenter
+	sysexit
+
+	.intel_syntax noprefix
+	sysenter
+	sysexit

gas/testsuite/gas/i386/x86-64-sysenter-intel.d

@@ -0,0 +1,15 @@
+#as: -mintel64
+#objdump: -dw -Mintel64
+#name: x86-64 sysenter (Intel64/Intel64)
+#source: x86-64-sysenter-amd.s
+
+.*: +file format .*
+
+Disassembly of section .text:
+
+0+ <.text>:
+[ 	]*[a-f0-9]+:	0f 34                	sysenter *
+[ 	]*[a-f0-9]+:	0f 35                	sysexit *
+[ 	]*[a-f0-9]+:	0f 34                	sysenter *
+[ 	]*[a-f0-9]+:	0f 35                	sysexit *
+#pass

gas/testsuite/gas/i386/x86-64-sysenter-mixed.d

@@ -0,0 +1,6 @@
+#as: -mintel64
+#objdump: -dw
+#name: x86-64 sysenter (Intel64/Default)
+#source: x86-64-sysenter-amd.s
+#dump: x86-64-sysenter-intel.d
+

opcodes/ChangeLog

@@ -1,3 +1,14 @@
+2020-01-09  Jan Beulich  <jbeulich@suse.com>
+
+	* i386-dis.c (SEP_Fixup): New.
+	(SEP): Define.
+	(dis386_twobyte): Use it for sysenter/sysexit.
+	(enum x86_64_isa): Change amd64 enumerator to value 1.
+	(OP_J): Compare isa64 against intel64 instead of amd64.
+	* i386-opc.tbl (sysenter, sysexit): Split into AMD64 and Intel64
+	forms.
+	* i386-tbl.h: Re-generate.
+
 2020-01-08  Alan Modra  <amodra@gmail.com>
 
 	* z8k-dis.c: Include libiberty.h

opcodes/i386-dis.c

@@ -107,6 +107,7 @@ static void OP_3DNowSuffix (int, int);
 static void CMP_Fixup (int, int);
 static void BadOp (void);
 static void REP_Fixup (int, int);
+static void SEP_Fixup (int, int);
 static void BND_Fixup (int, int);
 static void NOTRACK_Fixup (int, int);
 static void HLE_Fixup1 (int, int);
@@ -412,6 +413,7 @@ fetch_data (struct disassemble_info *info, bfd_byte *addr)
 #define EMCq { OP_EMC, q_mode }
 #define MXC { OP_MXC, 0 }
 #define OPSUF { OP_3DNowSuffix, 0 }
+#define SEP { SEP_Fixup, 0 }
 #define CMP { CMP_Fixup, 0 }
 #define XMM0 { XMM_Fixup, 0 }
 #define FXSAVE { FXSAVE_Fixup, 0 }
@@ -2713,8 +2715,8 @@ static const struct dis386 dis386_twobyte[] = {
   { "rdtsc",		{ XX }, 0 },
   { "rdmsr",		{ XX }, 0 },
   { "rdpmc",		{ XX }, 0 },
-  { "sysenter",		{ XX }, 0 },
+  { "sysenter",		{ SEP }, 0 },
-  { "sysexit",		{ XX }, 0 },
+  { "sysexit",		{ SEP }, 0 },
   { Bad_Opcode },
   { "getsec",		{ XX }, 0 },
   /* 38 */
@@ -11316,7 +11318,7 @@ static char scale_char;
 
 enum x86_64_isa
 {
-  amd64 = 0,
+  amd64 = 1,
   intel64
 };
 
@@ -14829,12 +14831,12 @@ OP_J (int bytemode, int sizeflag)
 	disp -= 0x100;
       break;
     case v_mode:
-      if (isa64 == amd64)
+      if (isa64 != intel64)
     case dqw_mode:
 	USED_REX (REX_W);
       if ((sizeflag & DFLAG)
 	  || (address_mode == mode_64bit
-	      && ((isa64 != amd64 && bytemode != dqw_mode)
+	      && ((isa64 == intel64 && bytemode != dqw_mode)
 		  || (rex & REX_W))))
 	disp = get32s ();
       else
@@ -14852,7 +14854,7 @@ OP_J (int bytemode, int sizeflag)
 		       & ~((bfd_vma) 0xffff));
 	}
       if (address_mode != mode_64bit
-	  || (isa64 == amd64 && !(rex & REX_W)))
+	  || (isa64 != intel64 && !(rex & REX_W)))
 	used_prefixes |= (prefixes & PREFIX_DATA);
       break;
     default:
@@ -15602,6 +15604,18 @@ REP_Fixup (int bytemode, int sizeflag)
     }
 }
 
+static void
+SEP_Fixup (int bytemode ATTRIBUTE_UNUSED, int sizeflag ATTRIBUTE_UNUSED)
+{
+  if ( isa64 != amd64 )
+    return;
+
+  obufp = obuf;
+  BadOp ();
+  mnemonicendp = obufp;
+  ++codep;
+}
+
 /* For BND-prefixed instructions 0xF2 prefix should be displayed as
    "bnd".  */
 

opcodes/i386-opc.tbl

@@ -904,8 +904,10 @@ rdmsr, 0, 0xf32, None, 2, Cpu586, No_bSuf|No_wSuf|No_lSuf|No_sSuf|No_qSuf|No_ldS
 cmpxchg8b, 1, 0xfc7, 0x1, 2, Cpu586, Modrm|No_bSuf|No_wSuf|No_lSuf|No_sSuf|No_ldSuf|IsLockable|NoRex64|HLEPrefixOk, { Qword|Unspecified|BaseIndex }
 
 // Pentium II/Pentium Pro extensions.
-sysenter, 0, 0xf34, None, 2, Cpu686, No_bSuf|No_wSuf|No_lSuf|No_sSuf|No_qSuf|No_ldSuf, { 0 }
+sysenter, 0, 0xf34, None, 2, Cpu686, Intel64|No_bSuf|No_wSuf|No_lSuf|No_sSuf|No_qSuf|No_ldSuf, { 0 }
-sysexit, 0, 0xf35, None, 2, Cpu686, No_bSuf|No_wSuf|No_lSuf|No_sSuf|No_qSuf|No_ldSuf, { 0 }
+sysenter, 0, 0xf34, None, 2, Cpu686|CpuNo64, AMD64|No_bSuf|No_wSuf|No_lSuf|No_sSuf|No_qSuf|No_ldSuf, { 0 }
+sysexit, 0, 0xf35, None, 2, Cpu686, Intel64|No_bSuf|No_wSuf|No_lSuf|No_sSuf|No_qSuf|No_ldSuf, { 0 }
+sysexit, 0, 0xf35, None, 2, Cpu686|CpuNo64, AMD64|No_bSuf|No_wSuf|No_lSuf|No_sSuf|No_qSuf|No_ldSuf, { 0 }
 fxsave, 1, 0xfae, 0x0, 2, CpuFXSR, Modrm|No_bSuf|No_wSuf|No_lSuf|No_sSuf|No_ldSuf, { Unspecified|BaseIndex }
 fxsave64, 1, 0xfae, 0x0, 2, CpuFXSR|Cpu64, Modrm|No_bSuf|No_wSuf|No_lSuf|No_sSuf|No_qSuf|No_ldSuf|Rex64, { Unspecified|BaseIndex }
 fxrstor, 1, 0xfae, 0x1, 2, CpuFXSR, Modrm|No_bSuf|No_wSuf|No_lSuf|No_sSuf|No_ldSuf, { Unspecified|BaseIndex }

opcodes/i386-tbl.h

@@ -7805,7 +7805,19 @@ const insn_template i386_optab[] =
         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } },
     { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0,
       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 },
+    { { { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	  0, 0, 0, 0, 0, 0 } } } },
+  { "sysenter", 0xf34, None, 2, 0,
+    { { 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0 } },
+    { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0 },
     { { { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 	  0, 0, 0, 0, 0, 0 } } } },
   { "sysexit", 0xf35, None, 2, 0,
@@ -7817,7 +7829,19 @@ const insn_template i386_optab[] =
         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } },
     { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0,
       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 },
+    { { { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	  0, 0, 0, 0, 0, 0 } } } },
+  { "sysexit", 0xf35, None, 2, 0,
+    { { 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0 } },
+    { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0 },
     { { { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 	  0, 0, 0, 0, 0, 0 } } } },
   { "fxsave", 0xfae, 0x0, 2, 1,