gdb: error if /r and /b are used with disassemble command

The disassembler gained a new /b flag in this commit:

  commit d4ce49b7ac
  Date:   Tue Jun 21 20:23:35 2022 +0100

      gdb: disassembler opcode display formatting

The /b and /r flags result in the instruction opcodes displayed in
different formats, so it's not possible to have both at the same
time.  Currently the /b flag overrides the /r flag.

We have a similar situation with the /m and /s flags, but here, if the
user tries to use both flags then they will get an error.

I think the error is clearer, so in this commit I propose that we add
an error if /r and /b are both used.

Obviously this change breaks backwards compatibility.  I don't have a
compelling argument for why we should make the change beyond my
feeling that it was a mistake not to add this error from the start,
and that the new behaviour is better.

Reviewed-By: Eli Zaretskii <eliz@gnu.org>

gdb/NEWS

@@ -9,6 +9,13 @@
 * GDB index now contains information about the main function.  This speeds up
   startup when it is being used for some large binaries.
 
+* Changed commands
+
+disassemble
+  Attempting to use both the 'r' and 'b' flags with the disassemble
+  command will now give an error.  Previously the 'b' flag would
+  always override the 'r' flag.
+
 * Python API
 
   ** New function gdb.notify_mi(NAME, DATA), that emits custom

gdb/cli/cli-cmds.c

@@ -1644,6 +1644,10 @@ disassemble_command (const char *arg, int from_tty)
       == (DISASSEMBLY_SOURCE_DEPRECATED | DISASSEMBLY_SOURCE))
     error (_("Cannot specify both /m and /s."));
 
+  if ((flags & (DISASSEMBLY_RAW_INSN | DISASSEMBLY_RAW_BYTES))
+      == (DISASSEMBLY_RAW_INSN | DISASSEMBLY_RAW_BYTES))
+    error (_("Cannot specify both /r and /b."));
+
   if (! p || ! *p)
     {
       flags |= DISASSEMBLY_OMIT_FNAME;

gdb/doc/gdb.texinfo

@@ -10060,12 +10060,20 @@ This specialized command dumps a range of memory as machine
 instructions.  It can also print mixed source+disassembly by specifying
 the @code{/m} or @code{/s} modifier and print the raw instructions in
 hex as well as in symbolic form by specifying the @code{/r} or @code{/b}
-modifier.  The default memory range is the function surrounding the
-program counter of the selected frame.  A single argument to this
-command is a program counter value; @value{GDBN} dumps the function
-surrounding this value.  When two arguments are given, they should be
-separated by a comma, possibly surrounded by whitespace.  The arguments
-specify a range of addresses to dump, in one of two forms:
+modifier.
+
+Only one of @code{/m} and @code{/s} can be used, attempting to use
+both flag will give an error.
+
+Only one of @code{/r} and @code{/b} can be used, attempting to use
+both flag will give an error.
+
+The default memory range is the function surrounding the program
+counter of the selected frame.  A single argument to this command is a
+program counter value; @value{GDBN} dumps the function surrounding
+this value.  When two arguments are given, they should be separated by
+a comma, possibly surrounded by whitespace.  The arguments specify a
+range of addresses to dump, in one of two forms:
 
 @table @code
 @item @var{start},@var{end}

gdb/testsuite/gdb.disasm/basics.c

@@ -0,0 +1,22 @@
+/* This testcase is part of GDB, the GNU debugger.
+
+   Copyright (C) 2023 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+int
+main ()
+{
+  return 0;
+}

gdb/testsuite/gdb.disasm/basics.exp

@@ -0,0 +1,39 @@
+# Copyright (C) 2023 Free Software Foundation, Inc.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Some basic tests of the disassemble command.  Tests in this script
+# should be architecture independent.
+
+standard_testfile
+
+if { [prepare_for_testing "failed to prepare" $testfile ${srcfile}] == -1 } {
+    return -1
+}
+
+if ![runto_main] {
+    return -1
+}
+
+# Check the '/s' and '/m' flags can't be used together.
+gdb_test "disassemble /sm main" \
+    "Cannot specify both /m and /s\\."
+gdb_test "disassemble /ms main" \
+    "Cannot specify both /m and /s\\."
+
+# Check the '/r' and '/b' flags can't be used together.
+gdb_test "disassemble /rb main" \
+    "Cannot specify both /r and /b\\."
+gdb_test "disassemble /br main" \
+    "Cannot specify both /r and /b\\."