platform_gen: only generate regions that are used

Right now the `platform_gen.json` and `platform_gen.yaml` files contain
regions that are not actually used by the kernel. This is fine for
`platform_gen.h` because they are guarded by #ifdefs with kernel config
defines but the same approach does not work for YAML and JSON.

So, this patch changes the Python scripts that generate this files so
that they only generate the regions that actually end up being used.

Signed-off-by: Ivan-Velickovic <i.velickovic@unsw.edu.au>

config.cmake

@@ -104,133 +104,6 @@ if(NOT "${KernelCustomDTS}" STREQUAL "")
     message(STATUS "Using custom ${KernelCustomDTS} device tree, ignoring default dts and overlays")
 endif()
 
-if(DEFINED KernelDTSList AND (NOT "${KernelDTSList}" STREQUAL ""))
-    set(KernelDTSIntermediate "${CMAKE_CURRENT_BINARY_DIR}/kernel.dts")
-    set(
-        KernelDTBPath "${CMAKE_CURRENT_BINARY_DIR}/kernel.dtb"
-        CACHE INTERNAL "Location of kernel DTB file"
-    )
-    set(compatibility_outfile "${CMAKE_CURRENT_BINARY_DIR}/kernel_compat.txt")
-    set(device_dest "${CMAKE_CURRENT_BINARY_DIR}/gen_headers/plat/machine/devices_gen.h")
-    set(
-        platform_yaml "${CMAKE_CURRENT_BINARY_DIR}/gen_headers/plat/machine/platform_gen.yaml"
-        CACHE INTERNAL "Location of platform YAML description"
-    )
-    set(
-        platform_json "${CMAKE_CURRENT_BINARY_DIR}/gen_headers/plat/machine/platform_gen.json"
-        CACHE INTERNAL "Location of platform JSON description"
-    )
-    set(config_file "${CMAKE_CURRENT_SOURCE_DIR}/tools/hardware.yml")
-    set(config_schema "${CMAKE_CURRENT_SOURCE_DIR}/tools/hardware_schema.yml")
-    set(
-        KernelCustomDTSOverlay ""
-        CACHE
-            STRING
-            "Provide an additional list of overlays to append to the selected KernelPlatform's \
-        device tree during build time"
-    )
-    if(NOT "${KernelCustomDTSOverlay}" STREQUAL "")
-        foreach(dts_entry IN ITEMS ${KernelCustomDTSOverlay})
-            if(NOT EXISTS ${dts_entry})
-                message(FATAL_ERROR "Can't open external overlay file '${dts_entry}'!")
-            endif()
-            list(APPEND KernelDTSList "${dts_entry}")
-            message(STATUS "Appending ${dts_entry} overlay")
-        endforeach()
-    endif()
-
-    find_program(DTC_TOOL dtc)
-    if("${DTC_TOOL}" STREQUAL "DTC_TOOL-NOTFOUND")
-        message(FATAL_ERROR "Cannot find 'dtc' program.")
-    endif()
-    find_program(STAT_TOOL stat)
-    if("${STAT_TOOL}" STREQUAL "STAT_TOOL-NOTFOUND")
-        message(FATAL_ERROR "Cannot find 'stat' program.")
-    endif()
-    mark_as_advanced(DTC_TOOL STAT_TOOL)
-    # Generate final DTS based on Linux DTS + seL4 overlay[s]
-    foreach(entry ${KernelDTSList})
-        get_absolute_source_or_binary(dts_tmp ${entry})
-        list(APPEND dts_list "${dts_tmp}")
-    endforeach()
-
-    check_outfile_stale(regen ${KernelDTBPath} dts_list ${CMAKE_CURRENT_BINARY_DIR}/dts.cmd)
-    if(regen)
-        file(REMOVE "${KernelDTSIntermediate}")
-        foreach(entry ${dts_list})
-            file(READ ${entry} CONTENTS)
-            file(APPEND "${KernelDTSIntermediate}" "${CONTENTS}")
-        endforeach()
-        # Compile DTS to DTB
-        execute_process(
-            COMMAND
-                ${DTC_TOOL} -q -I dts -O dtb -o ${KernelDTBPath} ${KernelDTSIntermediate}
-            RESULT_VARIABLE error
-        )
-        if(error)
-            message(FATAL_ERROR "Failed to compile DTS to DTB: ${KernelDTSIntermediate}")
-        endif()
-        # The macOS and GNU coreutils `stat` utilities have different interfaces.
-        # Check if we're using the macOS version, otherwise assume GNU coreutils.
-        # CMAKE_HOST_APPLE is a built-in CMake variable.
-        if(CMAKE_HOST_APPLE AND "${STAT_TOOL}" STREQUAL "/usr/bin/stat")
-            set(STAT_ARGS "-f%z")
-        else()
-            set(STAT_ARGS "-c '%s'")
-        endif()
-        # Track the size of the DTB for downstream tools
-        execute_process(
-            COMMAND ${STAT_TOOL} ${STAT_ARGS} ${KernelDTBPath}
-            OUTPUT_VARIABLE KernelDTBSize
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-            RESULT_VARIABLE error
-        )
-        if(error)
-            message(FATAL_ERROR "Failed to determine KernelDTBSize: ${KernelDTBPath}")
-        endif()
-        string(
-            REPLACE
-                "\'"
-                ""
-                KernelDTBSize
-                ${KernelDTBSize}
-        )
-        set(KernelDTBSize "${KernelDTBSize}" CACHE INTERNAL "Size of DTB blob, in bytes")
-    endif()
-
-    set(deps ${KernelDTBPath} ${config_file} ${config_schema} ${HARDWARE_GEN_PATH})
-    check_outfile_stale(regen ${device_dest} deps ${CMAKE_CURRENT_BINARY_DIR}/gen_header.cmd)
-    if(regen)
-        # Generate devices_gen header based on DTB
-        message(STATUS "${device_dest} is out of date. Regenerating from DTB...")
-        file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/gen_headers/plat/machine/")
-        execute_process(
-            COMMAND
-                ${PYTHON3} "${HARDWARE_GEN_PATH}" --dtb "${KernelDTBPath}" --compat-strings
-                --compat-strings-out "${compatibility_outfile}" --c-header --header-out
-                "${device_dest}" --hardware-config "${config_file}" --hardware-schema
-                "${config_schema}" --yaml --yaml-out "${platform_yaml}" --sel4arch
-                "${KernelSel4Arch}" --addrspace-max "${KernelPaddrUserTop}" --json --json-out
-                "${platform_json}"
-            RESULT_VARIABLE error
-        )
-        if(error)
-            message(FATAL_ERROR "Failed to generate from DTB: ${device_dest}")
-        endif()
-    endif()
-    file(READ "${compatibility_outfile}" compatibility_strings)
-
-    # Mark all file dependencies as CMake rerun dependencies.
-    set(cmake_deps ${deps} ${KernelDTSIntermediate} ${KernelDTSList} ${compatibility_outfile})
-    set_property(
-        DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
-        APPEND
-        PROPERTY CMAKE_CONFIGURE_DEPENDS ${cmake_deps}
-    )
-
-    include(src/drivers/config.cmake)
-endif()
-
 # Enshrine common variables in the config
 config_set(KernelHaveFPU HAVE_FPU "${KernelHaveFPU}")
 config_set(KernelPaddrUserTop PADDR_USER_DEVICE_TOP "${KernelPaddrUserTop}")
@@ -598,4 +471,133 @@ config_option(
     DEFAULT OFF
 )
 
+if(DEFINED KernelDTSList AND (NOT "${KernelDTSList}" STREQUAL ""))
+    set(KernelDTSIntermediate "${CMAKE_CURRENT_BINARY_DIR}/kernel.dts")
+    set(
+        KernelDTBPath "${CMAKE_CURRENT_BINARY_DIR}/kernel.dtb"
+        CACHE INTERNAL "Location of kernel DTB file"
+    )
+    set(compatibility_outfile "${CMAKE_CURRENT_BINARY_DIR}/kernel_compat.txt")
+    set(device_dest "${CMAKE_CURRENT_BINARY_DIR}/gen_headers/plat/machine/devices_gen.h")
+    set(
+        platform_yaml "${CMAKE_CURRENT_BINARY_DIR}/gen_headers/plat/machine/platform_gen.yaml"
+        CACHE INTERNAL "Location of platform YAML description"
+    )
+    set(
+        platform_json "${CMAKE_CURRENT_BINARY_DIR}/gen_headers/plat/machine/platform_gen.json"
+        CACHE INTERNAL "Location of platform JSON description"
+    )
+    set(config_file "${CMAKE_CURRENT_SOURCE_DIR}/tools/hardware.yml")
+    set(config_schema "${CMAKE_CURRENT_SOURCE_DIR}/tools/hardware_schema.yml")
+    set(
+        KernelCustomDTSOverlay ""
+        CACHE
+            STRING
+            "Provide an additional list of overlays to append to the selected KernelPlatform's \
+        device tree during build time"
+    )
+    if(NOT "${KernelCustomDTSOverlay}" STREQUAL "")
+        foreach(dts_entry IN ITEMS ${KernelCustomDTSOverlay})
+            if(NOT EXISTS ${dts_entry})
+                message(FATAL_ERROR "Can't open external overlay file '${dts_entry}'!")
+            endif()
+            list(APPEND KernelDTSList "${dts_entry}")
+            message(STATUS "Appending ${dts_entry} overlay")
+        endforeach()
+    endif()
+
+    find_program(DTC_TOOL dtc)
+    if("${DTC_TOOL}" STREQUAL "DTC_TOOL-NOTFOUND")
+        message(FATAL_ERROR "Cannot find 'dtc' program.")
+    endif()
+    find_program(STAT_TOOL stat)
+    if("${STAT_TOOL}" STREQUAL "STAT_TOOL-NOTFOUND")
+        message(FATAL_ERROR "Cannot find 'stat' program.")
+    endif()
+    mark_as_advanced(DTC_TOOL STAT_TOOL)
+    # Generate final DTS based on Linux DTS + seL4 overlay[s]
+    foreach(entry ${KernelDTSList})
+        get_absolute_source_or_binary(dts_tmp ${entry})
+        list(APPEND dts_list "${dts_tmp}")
+    endforeach()
+
+    check_outfile_stale(regen ${KernelDTBPath} dts_list ${CMAKE_CURRENT_BINARY_DIR}/dts.cmd)
+    if(regen)
+        file(REMOVE "${KernelDTSIntermediate}")
+        foreach(entry ${dts_list})
+            file(READ ${entry} CONTENTS)
+            file(APPEND "${KernelDTSIntermediate}" "${CONTENTS}")
+        endforeach()
+        # Compile DTS to DTB
+        execute_process(
+            COMMAND
+                ${DTC_TOOL} -q -I dts -O dtb -o ${KernelDTBPath} ${KernelDTSIntermediate}
+            RESULT_VARIABLE error
+        )
+        if(error)
+            message(FATAL_ERROR "Failed to compile DTS to DTB: ${KernelDTSIntermediate}")
+        endif()
+        # The macOS and GNU coreutils `stat` utilities have different interfaces.
+        # Check if we're using the macOS version, otherwise assume GNU coreutils.
+        # CMAKE_HOST_APPLE is a built-in CMake variable.
+        if(CMAKE_HOST_APPLE AND "${STAT_TOOL}" STREQUAL "/usr/bin/stat")
+            set(STAT_ARGS "-f%z")
+        else()
+            set(STAT_ARGS "-c '%s'")
+        endif()
+        # Track the size of the DTB for downstream tools
+        execute_process(
+            COMMAND ${STAT_TOOL} ${STAT_ARGS} ${KernelDTBPath}
+            OUTPUT_VARIABLE KernelDTBSize
+            OUTPUT_STRIP_TRAILING_WHITESPACE
+            RESULT_VARIABLE error
+        )
+        if(error)
+            message(FATAL_ERROR "Failed to determine KernelDTBSize: ${KernelDTBPath}")
+        endif()
+        string(
+            REPLACE
+                "\'"
+                ""
+                KernelDTBSize
+                ${KernelDTBSize}
+        )
+        set(KernelDTBSize "${KernelDTBSize}" CACHE INTERNAL "Size of DTB blob, in bytes")
+    endif()
+
+    set(deps ${KernelDTBPath} ${config_file} ${config_schema} ${HARDWARE_GEN_PATH})
+    check_outfile_stale(regen ${device_dest} deps ${CMAKE_CURRENT_BINARY_DIR}/gen_header.cmd)
+    if(regen)
+        # Generate devices_gen header based on DTB
+        message(STATUS "${device_dest} is out of date. Regenerating from DTB...")
+        file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/gen_headers/plat/machine/")
+        execute_process(
+            COMMAND
+                ${PYTHON3} "${HARDWARE_GEN_PATH}" --dtb "${KernelDTBPath}" --compat-strings
+                --compat-strings-out "${compatibility_outfile}" --c-header --header-out
+                "${device_dest}" --hardware-config "${config_file}" --hardware-schema
+                "${config_schema}" --yaml --yaml-out "${platform_yaml}" --sel4arch
+                "${KernelSel4Arch}" --addrspace-max "${KernelPaddrUserTop}" --json --json-out
+                "${platform_json}" --kernel-config-flags "CONFIG_PRINTING=${KernelPrinting}"
+                "CONFIG_ARM_HYPERVISOR_SUPPORT=${KernelArmHypervisorSupport}"
+                "CONFIG_ARM_SMMU=${KernelArmSMMU}" "CONFIG_TK1_SMMU=${KernelTk1SMMU}"
+            RESULT_VARIABLE error
+        )
+        if(error)
+            message(FATAL_ERROR "Failed to generate from DTB: ${device_dest}")
+        endif()
+    endif()
+    file(READ "${compatibility_outfile}" compatibility_strings)
+
+    # Mark all file dependencies as CMake rerun dependencies.
+    set(cmake_deps ${deps} ${KernelDTSIntermediate} ${KernelDTSList} ${compatibility_outfile})
+    set_property(
+        DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+        APPEND
+        PROPERTY CMAKE_CONFIGURE_DEPENDS ${cmake_deps}
+    )
+
+    include(src/drivers/config.cmake)
+endif()
+
 add_config_library(kernel "${configure_string}")

tools/hardware/outputs/c_header.py

@@ -131,7 +131,7 @@ static const p_region_t BOOT_RODATA avail_p_regs[] = {
 '''
 
 
-def get_kernel_devices(tree: FdtParser, hw_yaml: HardwareYaml) -> (List, Dict):
+def get_kernel_devices(tree: FdtParser, hw_yaml: HardwareYaml, kernel_config_dict: Dict[str, str]) -> (List, Dict):
     '''
     Given a device tree and a set of rules, returns a tuple (groups, offsets).
 
@@ -208,12 +208,12 @@ def create_c_header_file(config, kernel_irqs: List, kernel_macros: Dict,
         outputStream.write(data)
 
 
-def run(tree: FdtParser, hw_yaml: HardwareYaml, config: Config, args: argparse.Namespace):
+def run(tree: FdtParser, hw_yaml: HardwareYaml, config: Config, kernel_config_dict, args: argparse.Namespace):
     if not args.header_out:
         raise ValueError('You need to specify a header-out to use c header output')
 
     physical_memory, reserved, physBase = hardware.utils.memory.get_physical_memory(tree, config)
-    kernel_regions, kernel_macros = get_kernel_devices(tree, hw_yaml)
+    kernel_regions, kernel_macros = get_kernel_devices(tree, hw_yaml, kernel_config_dict)
 
     create_c_header_file(
         config,

tools/hardware/outputs/compat_strings.py

@@ -9,10 +9,11 @@ import argparse
 from hardware.config import Config
 from hardware.fdt import FdtParser
 from hardware.utils.rule import HardwareYaml
+from typing import Dict
 
 
 def run(tree: FdtParser, hw_yaml: HardwareYaml, config: Config,
-        args: argparse.Namespace):
+        kernel_config_dict: Dict[str, str], args: argparse.Namespace):
     if not args.compat_strings_out:
         raise ValueError('You need to specify a compat-strings-out to use compat strings output')
     chosen = tree.get_kernel_devices()

tools/hardware/outputs/elfloader.py

@@ -12,7 +12,7 @@ import logging
 import pyfdt.pyfdt
 
 from jinja2 import Environment, BaseLoader
-from typing import List
+from typing import Dict, List
 
 from hardware import config, device, fdt
 from hardware.utils import cpu, memory, rule
@@ -140,7 +140,7 @@ def get_elfloader_cpus(tree: fdt.FdtParser, devices: List[device.WrappedNode]) -
     return sorted(cpu_info, key=lambda a: a['cpuid'])
 
 
-def run(tree: fdt.FdtParser, hardware: rule.HardwareYaml, config: config.Config, args: argparse.Namespace):
+def run(tree: fdt.FdtParser, hardware: rule.HardwareYaml, config: config.Config, kernel_config_dict: Dict[str, str], args: argparse.Namespace):
     devices = tree.get_elfloader_devices()
     cpu_info = get_elfloader_cpus(tree, devices)
 

tools/hardware/outputs/json.py

@@ -8,7 +8,7 @@
 
 import argparse
 import json
-from typing import List
+from typing import Dict, List
 import hardware
 from hardware.config import Config
 from hardware.fdt import FdtParser
@@ -35,24 +35,29 @@ def create_json_file(dev_mem, phys_mem, output_stream):
         json.dump(json_obj, output_stream)
 
 
-def get_kernel_devices(tree: FdtParser, hw_yaml: HardwareYaml):
+def get_kernel_devices(tree: FdtParser, hw_yaml: HardwareYaml, kernel_config_dict: Dict[str, str]):
     kernel_devices = tree.get_kernel_devices()
 
     groups = []
     for dev in kernel_devices:
         rule = hw_yaml.get_rule(dev)
-        groups += rule.get_regions(dev)
+        new_regions = rule.get_regions(dev)
+        for reg in new_regions:
+            if reg.macro in kernel_config_dict:
+                if kernel_config_dict[reg.macro] != "ON":
+                    continue
+            groups.append(reg)
 
     return groups
 
 
 def run(tree: FdtParser, hw_yaml: HardwareYaml, config: Config,
-        args: argparse.Namespace):
+        kernel_config_dict, args: argparse.Namespace):
     if not args.json_out:
         raise ValueError('you need to provide a json-out to use the JSON output method')
 
     phys_mem, reserved, _ = hardware.utils.memory.get_physical_memory(tree, config)
-    kernel_devs = get_kernel_devices(tree, hw_yaml)
+    kernel_devs = get_kernel_devices(tree, hw_yaml, kernel_config_dict)
     dev_mem = hardware.utils.memory.get_addrspace_exclude(
         list(reserved) + phys_mem + kernel_devs, config)
 

tools/hardware/outputs/yaml.py

@@ -8,7 +8,7 @@
 
 import argparse
 import yaml
-from typing import List
+from typing import Dict, List
 import hardware
 from hardware.config import Config
 from hardware.fdt import FdtParser
@@ -40,24 +40,29 @@ def create_yaml_file(dev_mem, phys_mem, outputStream):
         yaml.dump(yaml_obj, outputStream)
 
 
-def get_kernel_devices(tree: FdtParser, hw_yaml: HardwareYaml):
+def get_kernel_devices(tree: FdtParser, hw_yaml: HardwareYaml, kernel_config_dict: Dict[str, str]):
     kernel_devices = tree.get_kernel_devices()
 
     groups = []
     for dev in kernel_devices:
         rule = hw_yaml.get_rule(dev)
-        groups += rule.get_regions(dev)
+        new_regions = rule.get_regions(dev)
+        for reg in new_regions:
+            if reg.macro in kernel_config_dict:
+                if kernel_config_dict[reg.macro] != "ON":
+                    continue
+            groups.append(reg)
 
     return groups
 
 
 def run(tree: FdtParser, hw_yaml: HardwareYaml, config: Config,
-        args: argparse.Namespace):
+        kernel_config_dict, args: argparse.Namespace):
     if not args.yaml_out:
         raise ValueError('you need to provide a yaml-out to use the yaml output method')
 
     phys_mem, reserved, _ = hardware.utils.memory.get_physical_memory(tree, config)
-    kernel_devs = get_kernel_devices(tree, hw_yaml)
+    kernel_devs = get_kernel_devices(tree, hw_yaml, kernel_config_dict)
     dev_mem = hardware.utils.memory.get_addrspace_exclude(
         list(reserved) + phys_mem + kernel_devs, config)
 

tools/hardware_gen.py

@@ -59,7 +59,7 @@ def main(args: argparse.Namespace):
     arg_dict = vars(args)
     for t in sorted(OUTPUTS.keys()):
         if arg_dict[t]:
-            OUTPUTS[t].run(parsed_dt, hw_yaml, cfg, args)
+            OUTPUTS[t].run(parsed_dt, hw_yaml, cfg, kernel_config_dict, args)
 
 
 if __name__ == '__main__':
@@ -75,6 +75,8 @@ if __name__ == '__main__':
                         required=True, type=argparse.FileType('r'))
     parser.add_argument('--sel4arch', help='seL4 architecture to generate for',
                         required=True)
+    parser.add_argument('--kernel-config-flags',
+                        help='List of kernel config params', action='append', nargs='+')
     parser.add_argument('--addrspace-max',
                         help='maximum address that is available as device untyped', type=int, default=32)
 
@@ -85,6 +87,12 @@ if __name__ == '__main__':
 
     args = parser.parse_args()
 
+    kernel_config_dict = dict()
+    if args.kernel_config_flags:
+        for option in sum(args.kernel_config_flags, []):
+            name, val = option.split('=')
+            kernel_config_dict[name] = val
+
     if args.enable_profiling:
         import cProfile
         cProfile.run('main(args)', sort='cumtime')