Move DWARF index-related things to a separate file

I want to add a DWARF index-related feature (automatically produce index
files when loading objfiles in GDB), but I don't want to add many
hundred lines to the already too big dwarf2read.c.  I thought it would
be a logical split to move everything related to the DWARF index to its
own file.

I first tried to move everything that reads and writes DWARF indices to
a separate file, but found that the "read" part is a little bit
entangled with the rest of dwarf2read.c, so the line is hard to draw
about where to split.  The write part is quite isolated though, so I
moved this part to a new file, dwarf-index-write.c.  Some things are
necessary to both reading and writing indices, so I placed them in
dwarf-index-common.{c,h}.  The idea would be to have a
dwarf-index-read.c eventually that would use it too (for now that code
is still in dwarf2read.c).

This required moving some things to a new dwarf2read.h header, so they
can be read by the code that writes the index.

The patch is big in number of lines, but it's all existing code being
moved around.  The only changes are that some functions are not static
anymore, a declaration is added in a .h file, and therefore the comment
is moved there.

I built-tested it with a little and big endian target.

This patch is also available on the users/simark/split-dwarf2read
branch.

gdb/ChangeLog:

	* Makefile.in (COMMON_SFILES): Add dwarf-index-common.c and
	dwarf-index-write.c
	(HFILES_NO_SRCDIR): Add dwarf-index-common.h and dwarf2read.h.
	* dwarf-index-common.c: New file.
	* dwarf-index-common.h: New file.
	* dwarf-index-write.c: New file.
	* dwarf2read.c: Include dwarf2read.h and dwarf-index-common.h.
	(struct dwarf2_section_info): Move from here.
	(dwarf2_section_info_def): Likewise.
	(DEF_VEC_O (dwarf2_section_info_def)): Likewise.
	(offset_type): Likewise.
	(DW2_GDB_INDEX_SYMBOL_STATIC_SET_VALUE): Likewise.
	(DW2_GDB_INDEX_SYMBOL_KIND_SET_VALUE): Likewise.
	(DW2_GDB_INDEX_CU_SET_VALUE): Likewise.
	(byte_swap): Likewise.
	(MAYBE_SWAP): Likewise.
	(dwarf2_per_cu_ptr): Likewise.
	(DEF_VEC_P (dwarf2_per_cu_ptr)): Likewise.
	(struct tu_stats): Likewise.
	(struct dwarf2_per_objfile): Likewise.
	(struct dwarf2_per_cu_data): Likewise.
	(struct signatured_type): Likewise.
	(sig_type_ptr): Likewise.
	(DEF_VEC_P (sig_type_ptr)): Likewise.
	(INDEX4_SUFFIX): Likewise.
	(INDEX5_SUFFIX): Likewise.
	(DEBUG_STR_SUFFIX): Likewise.
	(dwarf2_read_section): Make non-static.
	(mapped_index_string_hash): Move from here.
	(dwarf5_djb_hash): Likewise.
	(file_write): Likewise.
	(class data_buf): Likewise.
	(struct symtab_index_entry): Likewise.
	(struct mapped_symtab): Likewise.
	(find_slot): Likewise.
	(hash_expand): Likewise.
	(add_index_entry): Likewise.
	(uniquify_cu_indices): Likewise.
	(class c_str_view): Likewise.
	(class c_str_view_hasher): Likewise.
	(class vector_hasher): Likewise.
	(write_hash_table): Likewise.
	(psym_index_map): Likewise.
	(struct addrmap_index_data): Likewise.
	(add_address_entry): Likewise.
	(add_address_entry_worker): Likewise.
	(write_address_map): Likewise.
	(symbol_kind): Likewise.
	(write_psymbols): Likewise.
	(struct signatured_type_index_data): Likewise.
	(write_one_signatured_type): Likewise.
	(recursively_count_psymbols): Likewise.
	(recursively_write_psymbols): Likewise.
	(class debug_names): Likewise.
	(check_dwarf64_offsets): Likewise.
	(psyms_seen_size): Likewise.
	(write_gdbindex): Likewise.
	(write_debug_names): Likewise.
	(assert_file_size): Likewise.
	(write_psymtabs_to_index): Likewise.
	(save_gdb_index_command): Likewise.
	(_initialize_dwarf2_read): Don't register the "save gdb-index"
	command.
	* dwarf2read.h: New file.

gdb/Makefile.in

@@ -992,6 +992,8 @@ COMMON_SFILES = \
 	disasm.c \
 	disasm-selftests.c \
 	dummy-frame.c \
+	dwarf-index-common.c \
+	dwarf-index-write.c \
 	dwarf2-frame.c \
 	dwarf2-frame-tailcall.c \
 	dwarf2expr.c \
@@ -1213,10 +1215,12 @@ HFILES_NO_SRCDIR = \
 	dictionary.h \
 	disasm.h \
 	dummy-frame.h \
+	dwarf-index-common.h \
 	dwarf2-frame.h \
 	dwarf2-frame-tailcall.h \
 	dwarf2expr.h \
 	dwarf2loc.h \
+	dwarf2read.h \
 	event-loop.h \
 	event-top.h \
 	exceptions.h \

gdb/dwarf-index-common.c

@@ -0,0 +1,56 @@
+/* Things needed for both reading and writing DWARF indices.
+
+   Copyright (C) 1994-2018 Free Software Foundation, Inc.
+
+   This file is part of GDB.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+#include "defs.h"
+#include "dwarf-index-common.h"
+
+/* See dwarf-index-common.h.  */
+
+hashval_t
+mapped_index_string_hash (int index_version, const void *p)
+{
+  const unsigned char *str = (const unsigned char *) p;
+  hashval_t r = 0;
+  unsigned char c;
+
+  while ((c = *str++) != 0)
+    {
+      if (index_version >= 5)
+	c = tolower (c);
+      r = r * 67 + c - 113;
+    }
+
+  return r;
+}
+
+/* See dwarf-index-common.h.  */
+
+uint32_t
+dwarf5_djb_hash (const char *str_)
+{
+  const unsigned char *str = (const unsigned char *) str_;
+
+  /* Note: tolower here ignores UTF-8, which isn't fully compliant.
+     See http://dwarfstd.org/ShowIssue.php?issue=161027.1.  */
+
+  uint32_t hash = 5381;
+  while (int c = *str++)
+    hash = hash * 33 + tolower (c);
+  return hash;
+}

gdb/dwarf-index-common.h

@@ -0,0 +1,63 @@
+/* Things needed for both reading and writing DWARF indices.
+
+   Copyright (C) 1994-2018 Free Software Foundation, Inc.
+
+   This file is part of GDB.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+#ifndef DWARF_INDEX_COMMON_H
+#define DWARF_INDEX_COMMON_H
+
+/* All offsets in the index are of this type.  It must be
+   architecture-independent.  */
+typedef uint32_t offset_type;
+
+#if WORDS_BIGENDIAN
+
+/* Convert VALUE between big- and little-endian.  */
+
+static inline offset_type
+byte_swap (offset_type value)
+{
+  offset_type result;
+
+  result = (value & 0xff) << 24;
+  result |= (value & 0xff00) << 8;
+  result |= (value & 0xff0000) >> 8;
+  result |= (value & 0xff000000) >> 24;
+  return result;
+}
+
+#define MAYBE_SWAP(V)  byte_swap (V)
+
+#else
+#define MAYBE_SWAP(V) static_cast<offset_type> (V)
+#endif /* WORDS_BIGENDIAN */
+
+/* The hash function for strings in the mapped index.  This is the same as
+   SYMBOL_HASH_NEXT, but we keep a separate copy to maintain control over the
+   implementation.  This is necessary because the hash function is tied to the
+   format of the mapped index file.  The hash values do not have to match with
+   SYMBOL_HASH_NEXT.
+
+   Use INT_MAX for INDEX_VERSION if you generate the current index format.  */
+
+hashval_t mapped_index_string_hash (int index_version, const void *p);
+
+/* Symbol name hashing function as specified by DWARF-5.  */
+
+uint32_t dwarf5_djb_hash (const char *str_);
+
+#endif /* DWARF_INDEX_COMMON_H */

gdb/dwarf2read.h

@@ -0,0 +1,375 @@
+/* DWARF 2 debugging format support for GDB.
+
+   Copyright (C) 1994-2018 Free Software Foundation, Inc.
+
+   This file is part of GDB.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+#ifndef DWARF2READ_H
+#define DWARF2READ_H
+
+#include "filename-seen-cache.h"
+#include "gdb_obstack.h"
+
+typedef struct dwarf2_per_cu_data *dwarf2_per_cu_ptr;
+DEF_VEC_P (dwarf2_per_cu_ptr);
+
+/* A descriptor for dwarf sections.
+
+   S.ASECTION, SIZE are typically initialized when the objfile is first
+   scanned.  BUFFER, READIN are filled in later when the section is read.
+   If the section contained compressed data then SIZE is updated to record
+   the uncompressed size of the section.
+
+   DWP file format V2 introduces a wrinkle that is easiest to handle by
+   creating the concept of virtual sections contained within a real section.
+   In DWP V2 the sections of the input DWO files are concatenated together
+   into one section, but section offsets are kept relative to the original
+   input section.
+   If this is a virtual dwp-v2 section, S.CONTAINING_SECTION is a backlink to
+   the real section this "virtual" section is contained in, and BUFFER,SIZE
+   describe the virtual section.  */
+
+struct dwarf2_section_info
+{
+  union
+  {
+    /* If this is a real section, the bfd section.  */
+    asection *section;
+    /* If this is a virtual section, pointer to the containing ("real")
+       section.  */
+    struct dwarf2_section_info *containing_section;
+  } s;
+  /* Pointer to section data, only valid if readin.  */
+  const gdb_byte *buffer;
+  /* The size of the section, real or virtual.  */
+  bfd_size_type size;
+  /* If this is a virtual section, the offset in the real section.
+     Only valid if is_virtual.  */
+  bfd_size_type virtual_offset;
+  /* True if we have tried to read this section.  */
+  char readin;
+  /* True if this is a virtual section, False otherwise.
+     This specifies which of s.section and s.containing_section to use.  */
+  char is_virtual;
+};
+
+typedef struct dwarf2_section_info dwarf2_section_info_def;
+DEF_VEC_O (dwarf2_section_info_def);
+
+/* Read the contents of the section INFO.
+   OBJFILE is the main object file, but not necessarily the file where
+   the section comes from.  E.g., for DWO files the bfd of INFO is the bfd
+   of the DWO file.
+   If the section is compressed, uncompress it before returning.  */
+
+void dwarf2_read_section (struct objfile *objfile, dwarf2_section_info *info);
+
+struct tu_stats
+{
+  int nr_uniq_abbrev_tables;
+  int nr_symtabs;
+  int nr_symtab_sharers;
+  int nr_stmt_less_type_units;
+  int nr_all_type_units_reallocs;
+};
+
+struct dwarf2_debug_sections;
+struct mapped_index;
+struct mapped_debug_names;
+
+/* Collection of data recorded per objfile.
+   This hangs off of dwarf2_objfile_data_key.  */
+
+struct dwarf2_per_objfile : public allocate_on_obstack
+{
+  /* Construct a dwarf2_per_objfile for OBJFILE.  NAMES points to the
+     dwarf2 section names, or is NULL if the standard ELF names are
+     used.  */
+  dwarf2_per_objfile (struct objfile *objfile,
+		      const dwarf2_debug_sections *names);
+
+  ~dwarf2_per_objfile ();
+
+  DISABLE_COPY_AND_ASSIGN (dwarf2_per_objfile);
+
+  /* Free all cached compilation units.  */
+  void free_cached_comp_units ();
+private:
+  /* This function is mapped across the sections and remembers the
+     offset and size of each of the debugging sections we are
+     interested in.  */
+  void locate_sections (bfd *abfd, asection *sectp,
+			const dwarf2_debug_sections &names);
+
+public:
+  dwarf2_section_info info {};
+  dwarf2_section_info abbrev {};
+  dwarf2_section_info line {};
+  dwarf2_section_info loc {};
+  dwarf2_section_info loclists {};
+  dwarf2_section_info macinfo {};
+  dwarf2_section_info macro {};
+  dwarf2_section_info str {};
+  dwarf2_section_info line_str {};
+  dwarf2_section_info ranges {};
+  dwarf2_section_info rnglists {};
+  dwarf2_section_info addr {};
+  dwarf2_section_info frame {};
+  dwarf2_section_info eh_frame {};
+  dwarf2_section_info gdb_index {};
+  dwarf2_section_info debug_names {};
+  dwarf2_section_info debug_aranges {};
+
+  VEC (dwarf2_section_info_def) *types = NULL;
+
+  /* Back link.  */
+  struct objfile *objfile = NULL;
+
+  /* Table of all the compilation units.  This is used to locate
+     the target compilation unit of a particular reference.  */
+  struct dwarf2_per_cu_data **all_comp_units = NULL;
+
+  /* The number of compilation units in ALL_COMP_UNITS.  */
+  int n_comp_units = 0;
+
+  /* The number of .debug_types-related CUs.  */
+  int n_type_units = 0;
+
+  /* The number of elements allocated in all_type_units.
+     If there are skeleton-less TUs, we add them to all_type_units lazily.  */
+  int n_allocated_type_units = 0;
+
+  /* The .debug_types-related CUs (TUs).
+     This is stored in malloc space because we may realloc it.  */
+  struct signatured_type **all_type_units = NULL;
+
+  /* Table of struct type_unit_group objects.
+     The hash key is the DW_AT_stmt_list value.  */
+  htab_t type_unit_groups {};
+
+  /* A table mapping .debug_types signatures to its signatured_type entry.
+     This is NULL if the .debug_types section hasn't been read in yet.  */
+  htab_t signatured_types {};
+
+  /* Type unit statistics, to see how well the scaling improvements
+     are doing.  */
+  struct tu_stats tu_stats {};
+
+  /* A chain of compilation units that are currently read in, so that
+     they can be freed later.  */
+  dwarf2_per_cu_data *read_in_chain = NULL;
+
+  /* A table mapping DW_AT_dwo_name values to struct dwo_file objects.
+     This is NULL if the table hasn't been allocated yet.  */
+  htab_t dwo_files {};
+
+  /* True if we've checked for whether there is a DWP file.  */
+  bool dwp_checked = false;
+
+  /* The DWP file if there is one, or NULL.  */
+  struct dwp_file *dwp_file = NULL;
+
+  /* The shared '.dwz' file, if one exists.  This is used when the
+     original data was compressed using 'dwz -m'.  */
+  struct dwz_file *dwz_file = NULL;
+
+  /* A flag indicating whether this objfile has a section loaded at a
+     VMA of 0.  */
+  bool has_section_at_zero = false;
+
+  /* True if we are using the mapped index,
+     or we are faking it for OBJF_READNOW's sake.  */
+  bool using_index = false;
+
+  /* The mapped index, or NULL if .gdb_index is missing or not being used.  */
+  mapped_index *index_table = NULL;
+
+  /* The mapped index, or NULL if .debug_names is missing or not being used.  */
+  std::unique_ptr<mapped_debug_names> debug_names_table;
+
+  /* When using index_table, this keeps track of all quick_file_names entries.
+     TUs typically share line table entries with a CU, so we maintain a
+     separate table of all line table entries to support the sharing.
+     Note that while there can be way more TUs than CUs, we've already
+     sorted all the TUs into "type unit groups", grouped by their
+     DW_AT_stmt_list value.  Therefore the only sharing done here is with a
+     CU and its associated TU group if there is one.  */
+  htab_t quick_file_names_table {};
+
+  /* Set during partial symbol reading, to prevent queueing of full
+     symbols.  */
+  bool reading_partial_symbols = false;
+
+  /* Table mapping type DIEs to their struct type *.
+     This is NULL if not allocated yet.
+     The mapping is done via (CU/TU + DIE offset) -> type.  */
+  htab_t die_type_hash {};
+
+  /* The CUs we recently read.  */
+  VEC (dwarf2_per_cu_ptr) *just_read_cus = NULL;
+
+  /* Table containing line_header indexed by offset and offset_in_dwz.  */
+  htab_t line_header_hash {};
+
+  /* Table containing all filenames.  This is an optional because the
+     table is lazily constructed on first access.  */
+  gdb::optional<filename_seen_cache> filenames_cache;
+};
+
+/* Get the dwarf2_per_objfile associated to OBJFILE.  */
+
+dwarf2_per_objfile *get_dwarf2_per_objfile (struct objfile *objfile);
+
+/* Persistent data held for a compilation unit, even when not
+   processing it.  We put a pointer to this structure in the
+   read_symtab_private field of the psymtab.  */
+
+struct dwarf2_per_cu_data
+{
+  /* The start offset and length of this compilation unit.
+     NOTE: Unlike comp_unit_head.length, this length includes
+     initial_length_size.
+     If the DIE refers to a DWO file, this is always of the original die,
+     not the DWO file.  */
+  sect_offset sect_off;
+  unsigned int length;
+
+  /* DWARF standard version this data has been read from (such as 4 or 5).  */
+  short dwarf_version;
+
+  /* Flag indicating this compilation unit will be read in before
+     any of the current compilation units are processed.  */
+  unsigned int queued : 1;
+
+  /* This flag will be set when reading partial DIEs if we need to load
+     absolutely all DIEs for this compilation unit, instead of just the ones
+     we think are interesting.  It gets set if we look for a DIE in the
+     hash table and don't find it.  */
+  unsigned int load_all_dies : 1;
+
+  /* Non-zero if this CU is from .debug_types.
+     Struct dwarf2_per_cu_data is contained in struct signatured_type iff
+     this is non-zero.  */
+  unsigned int is_debug_types : 1;
+
+  /* Non-zero if this CU is from the .dwz file.  */
+  unsigned int is_dwz : 1;
+
+  /* Non-zero if reading a TU directly from a DWO file, bypassing the stub.
+     This flag is only valid if is_debug_types is true.
+     We can't read a CU directly from a DWO file: There are required
+     attributes in the stub.  */
+  unsigned int reading_dwo_directly : 1;
+
+  /* Non-zero if the TU has been read.
+     This is used to assist the "Stay in DWO Optimization" for Fission:
+     When reading a DWO, it's faster to read TUs from the DWO instead of
+     fetching them from random other DWOs (due to comdat folding).
+     If the TU has already been read, the optimization is unnecessary
+     (and unwise - we don't want to change where gdb thinks the TU lives
+     "midflight").
+     This flag is only valid if is_debug_types is true.  */
+  unsigned int tu_read : 1;
+
+  /* The section this CU/TU lives in.
+     If the DIE refers to a DWO file, this is always the original die,
+     not the DWO file.  */
+  struct dwarf2_section_info *section;
+
+  /* Set to non-NULL iff this CU is currently loaded.  When it gets freed out
+     of the CU cache it gets reset to NULL again.  This is left as NULL for
+     dummy CUs (a CU header, but nothing else).  */
+  struct dwarf2_cu *cu;
+
+  /* The corresponding dwarf2_per_objfile.  */
+  struct dwarf2_per_objfile *dwarf2_per_objfile;
+
+  /* When dwarf2_per_objfile->using_index is true, the 'quick' field
+     is active.  Otherwise, the 'psymtab' field is active.  */
+  union
+  {
+    /* The partial symbol table associated with this compilation unit,
+       or NULL for unread partial units.  */
+    struct partial_symtab *psymtab;
+
+    /* Data needed by the "quick" functions.  */
+    struct dwarf2_per_cu_quick_data *quick;
+  } v;
+
+  /* The CUs we import using DW_TAG_imported_unit.  This is filled in
+     while reading psymtabs, used to compute the psymtab dependencies,
+     and then cleared.  Then it is filled in again while reading full
+     symbols, and only deleted when the objfile is destroyed.
+
+     This is also used to work around a difference between the way gold
+     generates .gdb_index version <=7 and the way gdb does.  Arguably this
+     is a gold bug.  For symbols coming from TUs, gold records in the index
+     the CU that includes the TU instead of the TU itself.  This breaks
+     dw2_lookup_symbol: It assumes that if the index says symbol X lives
+     in CU/TU Y, then one need only expand Y and a subsequent lookup in Y
+     will find X.  Alas TUs live in their own symtab, so after expanding CU Y
+     we need to look in TU Z to find X.  Fortunately, this is akin to
+     DW_TAG_imported_unit, so we just use the same mechanism: For
+     .gdb_index version <=7 this also records the TUs that the CU referred
+     to.  Concurrently with this change gdb was modified to emit version 8
+     indices so we only pay a price for gold generated indices.
+     http://sourceware.org/bugzilla/show_bug.cgi?id=15021.  */
+  VEC (dwarf2_per_cu_ptr) *imported_symtabs;
+};
+
+/* Entry in the signatured_types hash table.  */
+
+struct signatured_type
+{
+  /* The "per_cu" object of this type.
+     This struct is used iff per_cu.is_debug_types.
+     N.B.: This is the first member so that it's easy to convert pointers
+     between them.  */
+  struct dwarf2_per_cu_data per_cu;
+
+  /* The type's signature.  */
+  ULONGEST signature;
+
+  /* Offset in the TU of the type's DIE, as read from the TU header.
+     If this TU is a DWO stub and the definition lives in a DWO file
+     (specified by DW_AT_GNU_dwo_name), this value is unusable.  */
+  cu_offset type_offset_in_tu;
+
+  /* Offset in the section of the type's DIE.
+     If the definition lives in a DWO file, this is the offset in the
+     .debug_types.dwo section.
+     The value is zero until the actual value is known.
+     Zero is otherwise not a valid section offset.  */
+  sect_offset type_offset_in_section;
+
+  /* Type units are grouped by their DW_AT_stmt_list entry so that they
+     can share them.  This points to the containing symtab.  */
+  struct type_unit_group *type_unit_group;
+
+  /* The type.
+     The first time we encounter this type we fully read it in and install it
+     in the symbol tables.  Subsequent times we only need the type.  */
+  struct type *type;
+
+  /* Containing DWO unit.
+     This field is valid iff per_cu.reading_dwo_directly.  */
+  struct dwo_unit *dwo_unit;
+};
+
+typedef struct signatured_type *sig_type_ptr;
+DEF_VEC_P (sig_type_ptr);
+
+#endif /* DWARF2READ_H */