libctf: create, types: variables and datasecs (REVIEW NEEDED)

This is an area of significant difference from CTFv3.  The API changes
significantly, with quite a few additions to allow creation and querying of
these new datasec entities:

-typedef int ctf_variable_f (const char *name, ctf_id_t type, void *arg);
+typedef int ctf_variable_f (ctf_dict_t *, const char *name, ctf_id_t type,
+			    void *arg);
+typedef int ctf_datasec_var_f (ctf_dict_t *fp, ctf_id_t type, size_t offset,
+			       size_t datasec_size, void *arg);

+/* Search a datasec for a variable covering a given offset.
+
+   Errors with ECTF_NODATASEC if not found.  */
+
+ctf_id_t ctf_datasec_var_offset (ctf_dict_t *fp, ctf_id_t datasec,
+				 uint32_t offset);
+
+/* Return the datasec that a given variable appears in, or ECTF_NODATASEC if
+   none.  */
+
+ctf_id_t ctf_variable_datasec (ctf_dict_t *fp, ctf_id_t var);

+int ctf_datasec_var_iter (ctf_dict_t *, ctf_id_t, ctf_datasec_var_f *,
+			  void *);
+ctf_id_t ctf_datasec_var_next (ctf_dict_t *, ctf_id_t, ctf_next_t **,
+			       size_t *size, size_t *offset);

-int ctf_add_variable (ctf_dict_t *, const char *, ctf_id_t);
+/* ctf_add_variable adds variables to no datasec at all;
+   ctf_add_section_variable adds them to the given datasec, or to no datasec at
+   all if the datasec is NULL.  */
+
+ctf_id_t ctf_add_variable (ctf_dict_t *, const char *, int linkage, ctf_id_t);
+ctf_id_t ctf_add_section_variable (ctf_dict_t *, uint32_t,
+				   const char *datasec, const char *name,
+				   int linkage, ctf_id_t type,
+				   size_t size, size_t offset);

We tie datasecs quite closely to variables at addition (and, as should
become clear later, dedup) time: you never create datasecs, you only create
variables *in* datasecs, and the datasec springs into existence when you do
so: datasecs are always found in the same dict as the variables they contain
(the variables are never in the parent if the datasec is in a child or
anything).  We keep track of the variable->datasec mapping in
ctf_var_datasecs (populating it at addition and open time), to allow
ctf_variable_datasec to work at reasonable speed.  (But, as yet, there are
no tests of this function at all.)

The datasecs are created unsorted (to avoid variable addition becoming
O(n^2)) and sorted at serialization time, and when ctf_datasec_var_offset is
invoked.

We reuse the natural-alignment code from struct addition to get a plausible
offset in datasecs if an alignment of -1 is specified: maybe this is
unnecessary now (it was originally added when ctf_add_variable added
variables to a "default datasec", while now it just leaves them out of
all datasecs, like externs are).

One constraint of this is that we currently prohibit the addition of
nonrepresentable-typed variables, because we can't tell what their natural
alignment is: if we dropped the whole "align" and just required everyone
adding a variable to a datasec to specify an offset, we could drop that
restriction. WDYT?

One additional caveat: right now, ctf_lookup_variable() looks up the type of
a variable (because when it was invented, variables were not entities in
themselves that you could look up).  This name is confusing as hell as a
result.  It might be less confusing to make it return the CTF_K_VAR, but
that would be awful to adapt callers to, since both are represented with
ctf_id_t's, so the compiler wouldn't warn about the needed change at all...
I've vacillated on this three or four times now.

include/ctf-api.h

@@ -229,7 +229,7 @@ typedef struct ctf_snapshot_id
   _CTF_ITEM (ECTF_RDONLY, "CTF container is read-only.") \
   _CTF_ITEM (ECTF_DTFULL, "CTF type is full (no more members allowed).") \
   _CTF_ITEM (ECTF_FULL, "CTF container is full.") \
-  _CTF_ITEM (ECTF_DUPLICATE, "Duplicate member, enumerator, or variable name.") \
+  _CTF_ITEM (ECTF_DUPLICATE, "Duplicate member, enumerator, datasec, or variable name.") \
   _CTF_ITEM (ECTF_CONFLICT, "Conflicting type is already defined.") \
   _CTF_ITEM (ECTF_OVERROLLBACK, "Attempt to roll back past a ctf_update.") \
   _CTF_ITEM (ECTF_COMPRESS, "Failed to compress CTF data.") \
@@ -255,6 +255,10 @@ typedef struct ctf_snapshot_id
   _CTF_ITEM (ECTF_NOTSERIALIZED, "CTF dict must be serialized first.") \
   _CTF_ITEM (ECTF_NOTBITSOU, "Type is not a bitfield-capable struct or union.") \
   _CTF_ITEM (ECTF_DESCENDING, "Structure offsets may not descend.") \
+  _CTF_ITEM (ECTF_LINKAGE, "Invalid linkage.") \
+  _CTF_ITEM (ECTF_NOTDATASEC, "This function requires a datasec.") \
+  _CTF_ITEM (ECTF_NOTVAR, "This function requires a variable.") \
+  _CTF_ITEM (ECTF_NODATASEC, "Variable not found in datasec.") \
 
 #define	ECTF_BASE	1000	/* Base value for libctf errnos.  */
 
@@ -306,7 +310,6 @@ _CTF_ERRORS
 
 typedef int ctf_visit_f (const char *name, ctf_id_t type, unsigned long offset,
 			 int depth, void *arg);
-typedef int ctf_variable_f (const char *name, ctf_id_t type, void *arg);
 typedef int ctf_type_f (ctf_id_t type, void *arg);
 typedef int ctf_type_all_f (ctf_id_t type, int flag, void *arg);
 			 void *arg);
@@ -314,6 +317,10 @@ typedef int ctf_member_f (ctf_dict_t *, const char *name, ctf_id_t membtype,
 			  size_t offset, int bit_width, void *arg);
 typedef int ctf_enum_f (const char *name, int64_t val, void *arg);
 typedef int ctf_unsigned_enum_f (const char *name, uint64_t val, void *arg);
+typedef int ctf_variable_f (ctf_dict_t *, const char *name, ctf_id_t type,
+			    void *arg);
+typedef int ctf_datasec_var_f (ctf_dict_t *fp, ctf_id_t type, size_t offset,
+			       size_t datasec_size, void *arg);
 typedef int ctf_archive_member_f (ctf_dict_t *fp, const char *name, void *arg);
 typedef int ctf_archive_raw_member_f (const char *name, const void *content,
 				      size_t len, void *arg);
@@ -563,7 +570,10 @@ extern ctf_id_t ctf_lookup_by_name (ctf_dict_t *, const char *);
    relationship to a symbol table.  Before linking, everything with types in the
    symbol table will be in the variable table as well; after linking, only those
    typed functions and data objects that are not asssigned to symbols by the
-   linker are left in the variable table here.  */
+   linker are left in the variable table here.
+
+   Note: this looks up a variable's *type*, not the variable itself.
+   For that, use ctf_lookup_by_kind, below.  */
 
 extern ctf_id_t ctf_lookup_variable (ctf_dict_t *, const char *);
 
@@ -713,6 +723,18 @@ extern int ctf_member_info (ctf_dict_t *, ctf_id_t, const char *,
 			    ctf_membinfo_t *);
 extern ssize_t ctf_member_count (ctf_dict_t *, ctf_id_t);
 
+/* Search a datasec for a variable covering a given offset.
+
+   Errors with ECTF_NODATASEC if not found.  */
+
+extern ctf_id_t ctf_datasec_var_offset (ctf_dict_t *fp, ctf_id_t datasec,
+					uint32_t offset);
+
+/* Return the datasec that a given variable appears in, or ECTF_NODATASEC if
+   none.  */
+
+extern ctf_id_t ctf_variable_datasec (ctf_dict_t *fp, ctf_id_t var);
+
 /* Iterators.  */
 
 /* ctf_member_next is a _next-style iterator that can additionally traverse into
@@ -764,7 +786,10 @@ extern ctf_id_t ctf_arc_lookup_enumerator_next (ctf_archive_t *, const char *nam
    of CTF_ADD_ROOT for such types.  ctf_type_next allows you to choose whether
    to see non-root types or not with the want_hidden arg: if set, the flag (if
    passed) returns the non-root state of each type in turn.  Types in parent
-   dictionaries are not returned.  */
+   dictionaries are not returned.
+
+   These days, even variables are included in the things returned by ctf_type*()
+   (type kind CTF_K_VAR).  */
 
 extern int ctf_type_iter (ctf_dict_t *, ctf_type_f *, void *);
 extern int ctf_type_iter_all (ctf_dict_t *, ctf_type_all_f *, void *);
@@ -775,6 +800,11 @@ extern int ctf_variable_iter (ctf_dict_t *, ctf_variable_f *, void *);
 extern ctf_id_t ctf_variable_next (ctf_dict_t *, ctf_next_t **,
 				   const char **);
 
+extern int ctf_datasec_var_iter (ctf_dict_t *, ctf_id_t, ctf_datasec_var_f *,
+				 void *);
+extern ctf_id_t ctf_datasec_var_next (ctf_dict_t *, ctf_id_t, ctf_next_t **,
+				      size_t *size, size_t *offset);
+
 /* ctf_archive_iter and ctf_archive_next open each member dict for you,
    automatically importing any parent dict as usual: ctf_archive_iter closes the
    dict on return from ctf_archive_member_f, but for ctf_archive_next the caller
@@ -915,7 +945,16 @@ extern int ctf_add_member_bitfield (ctf_dict_t *, ctf_id_t souid,
                                     unsigned long bit_offset,
                                     int bit_width);
 
-extern int ctf_add_variable (ctf_dict_t *, const char *, ctf_id_t);
+/* ctf_add_variable adds variables to no datasec at all;
+   ctf_add_section_variable adds them to the given datasec, or to no datasec at
+   all if the datasec is NULL.  */
+
+extern ctf_id_t ctf_add_variable (ctf_dict_t *, const char *, int linkage,
+				  ctf_id_t);
+extern ctf_id_t ctf_add_section_variable (ctf_dict_t *, uint32_t,
+					  const char *datasec, const char *name,
+					  int linkage, ctf_id_t type,
+					  size_t size, size_t offset);
 
 /* Set the size and member and index types of an array.  */
 

libctf/ctf-create.c

@@ -272,6 +272,8 @@ ctf_name_table (ctf_dict_t *fp, int kind)
     case CTF_K_ENUM:
     case CTF_K_ENUM64:
       return fp->ctf_enums;
+    case CTF_K_DATASEC:
+      return fp->ctf_datasecs;
     default:
       return fp->ctf_names;
     }
@@ -1656,60 +1658,200 @@ ctf_add_member (ctf_dict_t *fp, ctf_id_t souid, const char *name,
   return ctf_add_member_offset (fp, souid, name, type, (unsigned long) - 1);
 }
 
-/* Add a variable regardless of whether or not it is already present.
+/* Add a DATASEC to hang variables off of.  */
 
-   Internal use only.  */
+static ctf_id_t
-int
+ctf_add_datasec (ctf_dict_t *fp, uint32_t flag, const char *datasec)
-ctf_add_variable_forced (ctf_dict_t *fp, const char *name, ctf_id_t ref)
 {
-  ctf_dvdef_t *dvd;
+  ctf_dtdef_t *dtd;
-  ctf_dict_t *tmp = fp;
+  size_t initial_vlen = sizeof (ctf_var_secinfo_t) * INITIAL_VLEN;
 
-  if (ctf_lookup_by_id (&tmp, ref) == NULL)
+  if ((dtd = ctf_add_generic (fp, flag, datasec, CTF_K_DATASEC,
-    return -1;			/* errno is set for us.  */
+			      0, 0, initial_vlen, NULL)) == NULL)
+    return CTF_ERR;		/* errno is set for us. */
 
-  /* Make sure this type is representable.  */
+  dtd->dtd_data->ctt_info = CTF_TYPE_INFO (CTF_K_DATASEC, 0, 0);
-  if ((ctf_type_resolve (fp, ref) == CTF_ERR)
+  dtd->dtd_data->ctt_size = 0;
-      && (ctf_errno (fp) == ECTF_NONREPRESENTABLE))
-    return -1;
 
-  if ((dvd = malloc (sizeof (ctf_dvdef_t))) == NULL)
+  return dtd->dtd_type;
-    return (ctf_set_errno (fp, EAGAIN));
-
-  if (name != NULL && (dvd->dvd_name = strdup (name)) == NULL)
-    {
-      free (dvd);
-      return (ctf_set_errno (fp, EAGAIN));
-    }
-  dvd->dvd_type = ref;
-  dvd->dvd_snapshots = fp->ctf_snapshots;
-
-  if (ctf_dvd_insert (fp, dvd) < 0)
-    {
-      free (dvd->dvd_name);
-      free (dvd);
-      return -1;			/* errno is set for us.  */
-    }
-
-  return 0;
 }
 
-int
+ctf_id_t
-ctf_add_variable (ctf_dict_t *fp, const char *name, ctf_id_t ref)
+ctf_add_variable (ctf_dict_t *fp, const char *name, int linkage, ctf_id_t ref)
 {
+  return ctf_add_section_variable (fp, CTF_ADD_ROOT, NULL, name, linkage, ref,
+				   0, (unsigned long) -1);
+}
+
+/* Add variable, interning it in the specified DATASEC (which must be in the
+   same dict, but which may be NULL, meaning "no datasec").  As with structs, an
+   offset of -1 means "next natural alignment".  A size of zero means "get it
+   from the type" and is the common case.  */
+ctf_id_t
+ctf_add_section_variable (ctf_dict_t *fp, uint32_t flag, const char *datasec,
+			  const char *name, int linkage, ctf_id_t type,
+			  size_t size, size_t offset)
+{
+  ctf_dtdef_t *sec_dtd = NULL;
+  ctf_dtdef_t *var_dtd = NULL;
+
+  uint32_t kind, kflag;
+  size_t vlen;
+
+  ctf_linkage_t *l;
+  ctf_var_secinfo_t *sec;
+
+  ctf_dict_t *tmp = fp;
+  ctf_id_t datasec_id = 0;
+  int is_incomplete = 0;
+  ctf_snapshot_id_t err_snap = ctf_snapshot (fp);
+
   if (fp->ctf_flags & LCTF_NO_STR)
-    return (ctf_set_errno (fp, ECTF_NOPARENT));
+    return (ctf_set_typed_errno (fp, ECTF_NOPARENT));
 
-  if (fp->ctf_flags & LCTF_NO_TYPE)
+  if (name == NULL || name[0] == '\0')
-    return (ctf_set_errno (fp, ECTF_NOTSERIALIZED));
+    return (ctf_set_typed_errno (fp, ECTF_NONAME));
 
-  if (ctf_lookup_variable_here (fp, name) != CTF_ERR)
+  if (linkage < 0 || linkage > 2)
-    return (ctf_set_errno (fp, ECTF_DUPLICATE));
+    return (ctf_set_typed_errno (fp, ECTF_LINKAGE));
 
-  if (ctf_errno (fp) != ECTF_NOTYPEDAT)
+  if (flag == CTF_ADD_ROOT && ctf_lookup_by_rawname (fp, CTF_K_VAR, name) != 0)
-    return -1;				/* errno is set for us.  */
+    return (ctf_set_typed_errno (fp, ECTF_DUPLICATE));
 
-  return ctf_add_variable_forced (fp, name, ref);
+  /* First, create the variable.  Make sure its type exists.  */
+
+  if (ctf_lookup_by_id (&tmp, type, NULL) == NULL)
+    return CTF_ERR;			/* errno is set for us.  */
+
+  /* Make sure this type is representable: if a variable is nonrepresentable
+     there's nothing the end-user can do with it even if they know it's there.
+     Allow type 0: this is used for const void variables in BTF input.  */
+
+  if ((ctf_type_resolve_nonrepresentable (fp, type, 1) == CTF_ERR)
+      && (ctf_errno (fp) == ECTF_NONREPRESENTABLE))
+    return CTF_ERR;
+
+  if ((var_dtd = ctf_add_generic (fp, flag, name, CTF_K_VAR, 0,
+				  sizeof (ctf_linkage_t), 0, NULL)) == NULL)
+    return CTF_ERR;			/* errno is set for us.  */
+
+  l = (ctf_linkage_t *) var_dtd->dtd_vlen;
+  var_dtd->dtd_data->ctt_info = CTF_TYPE_INFO (CTF_K_VAR, 0, 0);
+  var_dtd->dtd_data->ctt_type = type;
+  l->ctl_linkage = linkage;
+
+  /* Add it to the datasec, if requested, creating the datasec if need be.  */
+
+  if (!datasec)
+    return var_dtd->dtd_type;
+
+  if ((datasec_id = ctf_lookup_by_rawname (fp, CTF_K_DATASEC,
+					   datasec)) == 0)
+    {
+      if ((datasec_id = ctf_add_datasec (fp, CTF_ADD_ROOT,
+					 datasec)) == CTF_ERR)
+	goto err;				/* errno is set for us.  */
+    }
+
+  sec_dtd = ctf_dtd_lookup (fp, datasec_id);
+
+  kind = LCTF_KIND (fp, sec_dtd->dtd_buf);
+  kflag = CTF_INFO_KFLAG (sec_dtd->dtd_data->ctt_info);
+  vlen = LCTF_VLEN (fp, sec_dtd->dtd_buf);
+
+  if (vlen == CTF_MAX_RAW_VLEN)
+    {
+      ctf_set_typed_errno (fp, ECTF_DTFULL);
+      goto err;
+    }
+
+  /* DATASECs do not support CTF_K_BIG (yet).  */
+  if (vlen == CTF_MAX_RAW_VLEN)
+    {
+      ctf_set_typed_errno (fp, ECTF_DTFULL);
+      goto err;
+    }
+
+  /* Allow for variables of void-like types.  */
+  if (type == 0)
+    is_incomplete = 1;
+  else if (ctf_type_align (fp, type) < 0)
+    {
+      /* See the comment in ctf_add_member_bitfield.  We don't need to worry
+	 about norepresentable types, because we just added this one: we know
+	 it's representable.  We do not know it's complete.  */
+
+      if (ctf_errno (fp) == ECTF_INCOMPLETE)
+	is_incomplete = 1;
+      else
+	goto err;			/* errno is set for us.  */
+    }
+
+  /* Here we just add the var info to the end of the datasec, naturally aligning
+     the offset as with struct/union membership addition if no offset is
+     specified.  */
+
+  sec = (ctf_var_secinfo_t *) sec_dtd->dtd_vlen;
+
+  if (vlen != 0)
+    {
+      /* To avoid causing trouble with existing code promiscuously adding
+	 variables without caring about their types, if natural alignment fails
+	 due to incomplete types, just set the next offset to something aligned
+	 mod 8.  It might be a waste of space but it'll avoid an error and
+	 should suffice for a long time to come.  */
+
+      if (offset == (unsigned long) -1 && is_incomplete)
+	offset = roundup (offset, 8);
+      else if (offset == (unsigned long) -1)
+	{
+	  /* Natural alignment.  */
+
+	  ssize_t bit_offset = offset * 8;
+
+	  if ((bit_offset = ctf_type_align_natural (fp, sec[vlen - 1].cvs_type,
+						    type, sec[vlen -1].cvs_offset)) < 0)
+	    offset = roundup (offset, 8);
+	  else
+	    offset = bit_offset / CHAR_BIT;
+	}
+
+      /* This DTD may need sorting.  */
+
+      if (offset < sec[vlen - 1].cvs_offset)
+	sec_dtd->dtd_flags |= ~DTD_F_UNSORTED;
+
+    } else if (offset == (unsigned long) -1)
+	offset = 0;
+
+  /* Remember the variable -> datasec mapping.  */
+
+  if (ctf_dynhash_insert (fp->ctf_var_datasecs,
+			  (void *) (ptrdiff_t) var_dtd->dtd_type,
+			  (void *) (ptrdiff_t) datasec_id) != 0)
+    {
+      ctf_set_typed_errno (fp, ENOMEM);
+      goto err;
+    }
+
+  if (ctf_grow_vlen (fp, sec_dtd, sizeof (ctf_var_secinfo_t) * (vlen + 1)) < 0)
+    goto err;
+
+  sec_dtd->dtd_vlen_size += sizeof (ctf_var_secinfo_t);
+  sec = (ctf_var_secinfo_t *) sec_dtd->dtd_vlen;
+
+  sec[vlen].cvs_type = (uint32_t) var_dtd->dtd_type;
+  sec[vlen].cvs_offset = (uint32_t) offset;
+  sec[vlen].cvs_size = (uint32_t) size;
+  sec_dtd->dtd_data->ctt_info = CTF_TYPE_INFO (kind, kflag, vlen + 1);
+
+  return var_dtd->dtd_type;
+
+err:
+  ctf_dynhash_remove (fp->ctf_var_datasecs,
+		      (void *) (ptrdiff_t) var_dtd->dtd_type);
+  ctf_rollback (fp, err_snap);
+  return CTF_ERR;
 }
 
 /* Add a function or object symbol regardless of whether or not it is already
@@ -1767,6 +1909,39 @@ ctf_add_func_sym (ctf_dict_t *fp, const char *name, ctf_id_t id)
   return (ctf_add_funcobjt_sym (fp, 1, name, id));
 }
 
+/* Sort function used by ctf_datasec_sort.  */
+
+static int
+ctf_datasec_sort_ascending (const void *one_, const void *two_)
+{
+  ctf_var_secinfo_t *one = (ctf_var_secinfo_t *) one_;
+  ctf_var_secinfo_t *two = (ctf_var_secinfo_t *) two_;
+
+  if (one->cvs_offset < two->cvs_offset)
+    return -1;
+  else if (one->cvs_offset > two->cvs_offset)
+    return 1;
+  return 0;
+}
+
+/* Sort a datasec into order.  Needed before serialization or query
+   operations.  */
+
+void
+ctf_datasec_sort (ctf_dict_t *fp, ctf_dtdef_t *dtd)
+{
+  size_t vlen;
+
+  if (!(dtd->dtd_flags & DTD_F_UNSORTED))
+    return;
+
+  vlen = LCTF_VLEN (fp, dtd->dtd_buf);
+
+  qsort (dtd->dtd_vlen, vlen, sizeof (ctf_var_secinfo_t),
+	 ctf_datasec_sort_ascending);
+  dtd->dtd_flags &= ~DTD_F_UNSORTED;
+}
+
 /* Add an enumeration constant observed in a given enum type as an identifier.
    They appear as names that cite the enum type.
 

libctf/ctf-decl.c

@@ -122,6 +122,11 @@ ctf_decl_push (ctf_decl_t *cd, ctf_dict_t *fp, ctf_id_t type)
       prec = CTF_PREC_POINTER;
       break;
 
+    case CTF_K_VAR:
+      ctf_decl_push (cd, fp, suffix->ctt_type);
+      prec = CTF_PREC_BASE; /* UPTODO probably wrong */
+      break;
+
     case CTF_K_SLICE:
       /* Slices themselves have no print representation and should not appear in
 	 the decl stack.  */

libctf/ctf-impl.h

@@ -176,6 +176,8 @@ typedef struct ctf_decl
   int cd_enomem;		     /* Nonzero if OOM during printing.  */
 } ctf_decl_t;
 
+#define DTD_F_UNSORTED	0x0001	/* set on a datasec if it needs sorting.  */
+
 typedef struct ctf_dtdef
 {
   ctf_list_t dtd_list;		/* List forward/back pointers.  */
@@ -431,8 +433,6 @@ struct ctf_dict
   ctf_link_sym_t **ctf_dynsymidx;  /* Indexes ctf_dynsyms by symidx.  */
   uint32_t ctf_dynsymmax;	  /* Maximum ctf_dynsym index.  */
   ctf_list_t ctf_in_flight_dynsyms; /* Dynsyms during accumulation.  */
-  struct ctf_varent *ctf_vars;	  /* Sorted variable->type mapping.  */
-  unsigned long ctf_nvars;	  /* Number of variables in ctf_vars.  */
   uint32_t ctf_typemax;		  /* Maximum valid type index.  */
   uint32_t ctf_idmax;		  /* Maximum valid non-provisional type ID.  */
   uint32_t ctf_stypes;		  /* Number of static (non-dynamic) types.  */
@@ -630,7 +630,6 @@ extern ctf_id_t ctf_index_to_type (const ctf_dict_t *, uint32_t);
 #define LCTF_PRESERIALIZED	0x0020  /* Already serialized all but the strtab.  */
 
 extern ctf_dynhash_t *ctf_name_table (ctf_dict_t *, int);
-extern ctf_id_t ctf_lookup_variable_here (ctf_dict_t *fp, const char *name);
 extern const ctf_type_t *ctf_lookup_by_id (ctf_dict_t **, ctf_id_t,
 					   const ctf_type_t **suffix);
 extern const ctf_type_t *ctf_find_prefix (ctf_dict_t *, const ctf_type_t *,
@@ -743,7 +742,6 @@ extern ctf_id_t ctf_add_encoded (ctf_dict_t *, uint32_t, const char *,
 				 const ctf_encoding_t *, uint32_t kind);
 extern ctf_id_t ctf_add_reftype (ctf_dict_t *, uint32_t, ctf_id_t,
 				 uint32_t kind);
-extern int ctf_add_variable_forced (ctf_dict_t *, const char *, ctf_id_t);
 extern int ctf_add_funcobjt_sym_forced (ctf_dict_t *, int is_function,
 					const char *, ctf_id_t);
 
@@ -815,6 +813,9 @@ extern ctf_id_t ctf_type_resolve_nonrepresentable (ctf_dict_t *, ctf_id_t, int a
 extern int ctf_type_kind_unsliced (ctf_dict_t *, ctf_id_t);
 extern ssize_t ctf_type_align_natural (ctf_dict_t *fp, ctf_id_t prev_type,
 				       ctf_id_t type, ssize_t bit_offset);
+extern ctf_var_secinfo_t *ctf_datasec_entry (ctf_dict_t *, ctf_id_t,
+					     int component_idx);
+extern void ctf_datasec_sort (ctf_dict_t *, ctf_dtdef_t *);
 
 _libctf_printflike_ (1, 2)
 extern void ctf_dprintf (const char *, ...);

libctf/ctf-lookup.c

@@ -471,6 +471,11 @@ ctf_lookup_by_kind (ctf_dict_t *fp, int kind, const char *name)
   return ctf_set_typed_errno (fp, ECTF_NOTYPE);
 }
 
+/* Look up a variable by name, in this dict or the parent.  */
+ctf_id_t
+ctf_lookup_variable (ctf_dict_t *fp, const char *name)
+{
+  return ctf_lookup_by_kind (fp, CTF_K_VAR, name);
 }
 
 /* Look up a single enumerator by enumeration constant name.  Returns the ID of

libctf/ctf-types.c

@@ -608,7 +608,7 @@ ctf_variable_iter (ctf_dict_t *fp, ctf_variable_f *func, void *arg)
   while ((type = ctf_variable_next (fp, &i, &name)) != CTF_ERR)
     {
       int rc;
-      if ((rc = func (name, type, arg)) != 0)
+      if ((rc = func (fp, name, type, arg)) != 0)
 	{
 	  ctf_next_destroy (i);
 	  return rc;
@@ -643,7 +643,7 @@ ctf_variable_next (ctf_dict_t *fp, ctf_next_t **it, const char **name)
 
       i->cu.ctn_fp = fp;
       i->ctn_iter_fun = (void (*) (void)) ctf_variable_next;
-      i->u.ctn_dvd = ctf_list_next (&fp->ctf_dvdefs);
+      i->ctn_next = NULL;
       *it = i;
     }
 
@@ -653,25 +653,110 @@ ctf_variable_next (ctf_dict_t *fp, ctf_next_t **it, const char **name)
   if (fp != i->cu.ctn_fp)
     return (ctf_set_typed_errno (fp, ECTF_NEXT_WRONGFP));
 
-  if (i->ctn_n < fp->ctf_nvars)
+  if ((id = ctf_type_kind_next (fp, &i->ctn_next, CTF_K_VAR)) == CTF_ERR)
     {
-      *name = ctf_strptr (fp, fp->ctf_vars[i->ctn_n].ctv_name);
+      if (ctf_errno (fp) == ECTF_NEXT_END)
-      return fp->ctf_vars[i->ctn_n++].ctv_type;
+	ctf_next_destroy (i);
-
     }
 
-  if (i->u.ctn_dvd == NULL)
+  if (name)
+    *name = ctf_type_name_raw (fp, id);
+
+  return id;
+}
+
+/* Iterate over every variable in the given DATASEC, in arbitrary order.  We
+   pass the type ID, datasec-recorded size (usually 0), and offset of each
+   variable to the specified callback function.  */
+
+int
+ctf_datasec_var_iter (ctf_dict_t *fp, ctf_id_t datasec,
+		      ctf_datasec_var_f *func, void *arg)
+{
+  ctf_next_t *i = NULL;
+  ctf_id_t type;
+  size_t size, offset;
+
+  while ((type = ctf_datasec_var_next (fp, datasec, &i, &size, &offset)) != CTF_ERR)
+    {
+      int rc;
+      if ((rc = func (fp, type, offset, size, arg)) != 0)
+	{
+	  ctf_next_destroy (i);
+	  return rc;
+	}
+    }
+  if (ctf_errno (fp) != ECTF_NEXT_END)
+    return -1;					/* errno is set for us.  */
+
+  return 0;
+}
+
+/* Iterate over every variable in the given CTF datasec, in arbitrary order,
+   returning the name and type of each variable in turn.  Returns CTF_ERR on end
+   of iteration or error.
+
+   (The order is arbitrary so we don't need to worry about sorting unsorted
+   datasecs.)  */
+
+ctf_id_t
+ctf_datasec_var_next (ctf_dict_t *fp, ctf_id_t datasec, ctf_next_t **it,
+		      size_t *size, size_t *offset)
+{
+  ctf_next_t *i = *it;
+  ctf_id_t type;
+
+  if (!i)
+    {
+      const ctf_type_t *tp;
+      unsigned char *vlen;
+      ctf_dict_t *ofp = fp;
+
+      if ((datasec = ctf_type_resolve_unsliced (fp, datasec)) == CTF_ERR)
+	return CTF_ERR;			/* errno is set for us.  */
+
+      if (ctf_type_kind (fp, datasec) != CTF_K_DATASEC)
+	return (ctf_set_typed_errno (ofp, ECTF_NOTDATASEC));
+
+      if ((tp = ctf_lookup_by_id (&fp, datasec, NULL)) == NULL)
+	return CTF_ERR;			/* errno is set for us.  */
+
+      if ((i = ctf_next_create ()) == NULL)
+	return (ctf_set_typed_errno (ofp, ENOMEM));
+
+      i->cu.ctn_fp = ofp;
+      i->ctn_iter_fun = (void (*) (void)) ctf_datasec_var_next;
+      vlen = ctf_vlen (fp, datasec, tp, &i->ctn_n);
+
+      i->u.ctn_datasec = (const ctf_var_secinfo_t *) vlen;
+
+      *it = i;
+    }
+
+  if ((void (*) (void)) ctf_datasec_var_next != i->ctn_iter_fun)
+    return (ctf_set_typed_errno (fp, ECTF_NEXT_WRONGFUN));
+
+  if (fp != i->cu.ctn_fp)
+    return (ctf_set_typed_errno (fp, ECTF_NEXT_WRONGFP));
+
+  if (i->ctn_n == 0)
     goto end_iter;
 
-  *name = i->u.ctn_dvd->dvd_name;
+  if (size)
-  id = i->u.ctn_dvd->dvd_type;
+    *size = i->u.ctn_datasec->cvs_size;
-  i->u.ctn_dvd = ctf_list_next (i->u.ctn_dvd);
+  if (offset)
-  return id;
+    *offset = i->u.ctn_datasec->cvs_offset;
+  type = i->u.ctn_datasec->cvs_type;
+
+  i->u.ctn_datasec++;
+  i->ctn_n--;
+
+  return type;
 
  end_iter:
   ctf_next_destroy (i);
   *it = NULL;
-  return ctf_set_typed_errno (fp, ECTF_NEXT_END);
+  return (ctf_set_typed_errno (fp, ECTF_NEXT_END));
 }
 
 /* Follow a given type through the graph for TYPEDEF, VOLATILE, CONST, and
@@ -874,8 +959,10 @@ ctf_type_aname (ctf_dict_t *fp, ctf_id_t type)
 	    case CTF_K_INTEGER:
 	    case CTF_K_FLOAT:
 	    case CTF_K_TYPEDEF:
-	      /* Integers, floats, and typedefs must always be named types.  */
 	    case CTF_K_BTF_FLOAT:
+	    case CTF_K_DATASEC:
+	      /* Integers, floats, typedefs, and datasecs must always be named
+		 types.  */
 
 	      if (name[0] == '\0')
 		{
@@ -884,7 +971,11 @@ ctf_type_aname (ctf_dict_t *fp, ctf_id_t type)
 		  return NULL;
 		}
 
-	      ctf_decl_sprintf (&cd, "%s", name);
+	      if (cdp->cd_kind != CTF_K_DATASEC)
+		ctf_decl_sprintf (&cd, "%s", name);
+	      else
+		ctf_decl_sprintf (&cd, "DATASEC (\"%s\", %i)", name,
+				  LCTF_VLEN (rfp, tp));
 	      break;
 	    case CTF_K_POINTER:
 	      ctf_decl_sprintf (&cd, "*");
@@ -1967,6 +2058,110 @@ ctf_func_type_args (ctf_dict_t *fp, ctf_id_t type, uint32_t argc, ctf_id_t *argv
   return 0;
 }
 
+/* bsearch_r comparison function for datasec searches.  */
+static int
+search_datasec_by_offset (const void *key_, const void *arr_)
+{
+  uint32_t *key = (uint32_t *) key_;
+  ctf_var_secinfo_t *arr = (ctf_var_secinfo_t *) arr_;
+
+  if (*key < arr->cvs_offset)
+    return -1;
+  else if (*key > arr->cvs_offset)
+    return 1;
+
+  return 0;
+}
+
+/* Search a datasec for a variable covering a given offset.
+
+   Errors with ECTF_NODATASEC if not found.  */
+
+ctf_id_t
+ctf_datasec_var_offset (ctf_dict_t *fp, ctf_id_t datasec, uint32_t offset)
+{
+  ctf_dtdef_t *dtd;
+  const ctf_type_t *tp;
+  unsigned char *vlen;
+  size_t vlen_len;
+  ctf_var_secinfo_t *sec;
+  ctf_var_secinfo_t *el;
+  ssize_t size;
+
+  if ((tp = ctf_lookup_by_id (&fp, datasec, NULL)) == NULL)
+    return -1;			/* errno is set for us.  */
+
+  if (ctf_type_kind (fp, datasec) != CTF_K_DATASEC)
+    return ctf_set_typed_errno (fp, ECTF_NOTDATASEC);
+
+  if ((dtd = ctf_dynamic_type (fp, datasec)) != NULL)
+    {
+      if (dtd->dtd_flags & DTD_F_UNSORTED)
+	ctf_datasec_sort (fp, dtd);
+    }
+
+  vlen = ctf_vlen (fp, datasec, tp, &vlen_len);
+  sec = (ctf_var_secinfo_t *) vlen;
+
+  if ((el = bsearch (&offset, sec, vlen_len, sizeof (ctf_var_secinfo_t),
+		     search_datasec_by_offset)) == NULL)
+    return ctf_set_typed_errno (fp, ECTF_NODATASEC);
+
+  if (el->cvs_offset == offset)
+    return el->cvs_type;
+
+  if ((size = ctf_type_size (fp, el->cvs_type)) >= 0)
+    if (el->cvs_offset < offset && el->cvs_offset + size > offset)
+      return el->cvs_type;
+
+  return ctf_set_typed_errno (fp, ECTF_NODATASEC);
+}
+
+/* Return the entry corresponding to a given component_idx in a datasec.
+
+   Not currently public API.  */
+
+ctf_var_secinfo_t *
+ctf_datasec_entry (ctf_dict_t *fp, ctf_id_t datasec, int component_idx)
+{
+  const ctf_type_t *tp;
+  unsigned char *vlen;
+  size_t vlen_len;
+  ctf_var_secinfo_t *sec;
+
+  if ((tp = ctf_lookup_by_id (&fp, datasec, NULL)) == NULL)
+    return NULL;		/* errno is set for us.  */
+
+  /* No type kind check: internal function.  */
+  vlen = ctf_vlen (fp, datasec, tp, &vlen_len);
+  sec = (ctf_var_secinfo_t *) vlen;
+
+  if (component_idx < 0 || (size_t) component_idx > vlen_len)
+    {
+      ctf_set_errno (fp, EOVERFLOW);
+      return NULL;
+    }
+
+  return &sec[component_idx];
+}
+
+/* Return the datasec that a given variable appears in, or ECTF_NODATASEC if
+   none.  */
+
+ctf_id_t ctf_variable_datasec (ctf_dict_t *fp, ctf_id_t var)
+{
+  void *sec;
+
+  if (ctf_type_kind (fp, var) != CTF_K_VAR)
+    return (ctf_set_typed_errno (fp, ECTF_NOTVAR));
+
+  if (ctf_dynhash_lookup_kv (fp->ctf_var_datasecs, (void *) (ptrdiff_t) var,
+			     NULL, &sec))
+    return (ctf_id_t) sec;
+
+  return (ctf_set_typed_errno (fp, ECTF_NODATASEC));
+}
+
 /* Recursively visit the members of any type.  This function is used as the
    engine for ctf_type_visit, below.  We resolve the input type, recursively
    invoke ourself for each type member if the type is a struct or union, and

libctf/libctf.ver

@@ -104,6 +104,8 @@ LIBCTF_2.0 {
 	ctf_type_iter_all;
 	ctf_variable_iter;
 	ctf_variable_next;
+	ctf_datasec_iter;
+	ctf_datasec_next;
 
 	ctf_next_create;
 	ctf_next_destroy;
@@ -124,19 +126,20 @@ LIBCTF_2.0 {
 	ctf_add_type;
 	ctf_add_typedef;
 	ctf_add_restrict;
+	ctf_add_section_variable;
 	ctf_add_slice;
 	ctf_add_struct;
 	ctf_add_union;
 	ctf_add_struct_sized;
 	ctf_add_union_sized;
 	ctf_add_unknown;
+	ctf_add_variable;
 	ctf_add_volatile;
 
 	ctf_add_enumerator;
 	ctf_add_member;
 	ctf_add_member_offset;
 	ctf_add_member_encoded;
-	ctf_add_variable;
 	ctf_add_member_bitfield;
 
 	ctf_set_array;