From 1cbaeb97f0ab2238640ef8eb27598b7845bc42b2 Mon Sep 17 00:00:00 2001 From: Ivan-Velickovic Date: Mon, 10 Feb 2025 11:07:51 +1100 Subject: [PATCH] platform_gen: only generate regions that are used Right now the `platform_gen.json` and `platform_gen.yaml` files contain regions that are not actually used by the kernel. This is fine for `platform_gen.h` because they are guarded by #ifdefs with kernel config defines but the same approach does not work for YAML and JSON. So, this patch changes the Python scripts that generate this files so that they only generate the regions that actually end up being used. Signed-off-by: Ivan-Velickovic --- config.cmake | 256 ++++++++++++----------- tools/hardware/outputs/c_header.py | 6 +- tools/hardware/outputs/compat_strings.py | 3 +- tools/hardware/outputs/elfloader.py | 4 +- tools/hardware/outputs/json.py | 15 +- tools/hardware/outputs/yaml.py | 15 +- tools/hardware_gen.py | 10 +- 7 files changed, 165 insertions(+), 144 deletions(-) diff --git a/config.cmake b/config.cmake index 8a15d4f80..57eadaf3e 100644 --- a/config.cmake +++ b/config.cmake @@ -104,133 +104,6 @@ if(NOT "${KernelCustomDTS}" STREQUAL "") message(STATUS "Using custom ${KernelCustomDTS} device tree, ignoring default dts and overlays") endif() -if(DEFINED KernelDTSList AND (NOT "${KernelDTSList}" STREQUAL "")) - set(KernelDTSIntermediate "${CMAKE_CURRENT_BINARY_DIR}/kernel.dts") - set( - KernelDTBPath "${CMAKE_CURRENT_BINARY_DIR}/kernel.dtb" - CACHE INTERNAL "Location of kernel DTB file" - ) - set(compatibility_outfile "${CMAKE_CURRENT_BINARY_DIR}/kernel_compat.txt") - set(device_dest "${CMAKE_CURRENT_BINARY_DIR}/gen_headers/plat/machine/devices_gen.h") - set( - platform_yaml "${CMAKE_CURRENT_BINARY_DIR}/gen_headers/plat/machine/platform_gen.yaml" - CACHE INTERNAL "Location of platform YAML description" - ) - set( - platform_json "${CMAKE_CURRENT_BINARY_DIR}/gen_headers/plat/machine/platform_gen.json" - CACHE INTERNAL "Location of platform JSON description" - ) - set(config_file "${CMAKE_CURRENT_SOURCE_DIR}/tools/hardware.yml") - set(config_schema "${CMAKE_CURRENT_SOURCE_DIR}/tools/hardware_schema.yml") - set( - KernelCustomDTSOverlay "" - CACHE - STRING - "Provide an additional list of overlays to append to the selected KernelPlatform's \ - device tree during build time" - ) - if(NOT "${KernelCustomDTSOverlay}" STREQUAL "") - foreach(dts_entry IN ITEMS ${KernelCustomDTSOverlay}) - if(NOT EXISTS ${dts_entry}) - message(FATAL_ERROR "Can't open external overlay file '${dts_entry}'!") - endif() - list(APPEND KernelDTSList "${dts_entry}") - message(STATUS "Appending ${dts_entry} overlay") - endforeach() - endif() - - find_program(DTC_TOOL dtc) - if("${DTC_TOOL}" STREQUAL "DTC_TOOL-NOTFOUND") - message(FATAL_ERROR "Cannot find 'dtc' program.") - endif() - find_program(STAT_TOOL stat) - if("${STAT_TOOL}" STREQUAL "STAT_TOOL-NOTFOUND") - message(FATAL_ERROR "Cannot find 'stat' program.") - endif() - mark_as_advanced(DTC_TOOL STAT_TOOL) - # Generate final DTS based on Linux DTS + seL4 overlay[s] - foreach(entry ${KernelDTSList}) - get_absolute_source_or_binary(dts_tmp ${entry}) - list(APPEND dts_list "${dts_tmp}") - endforeach() - - check_outfile_stale(regen ${KernelDTBPath} dts_list ${CMAKE_CURRENT_BINARY_DIR}/dts.cmd) - if(regen) - file(REMOVE "${KernelDTSIntermediate}") - foreach(entry ${dts_list}) - file(READ ${entry} CONTENTS) - file(APPEND "${KernelDTSIntermediate}" "${CONTENTS}") - endforeach() - # Compile DTS to DTB - execute_process( - COMMAND - ${DTC_TOOL} -q -I dts -O dtb -o ${KernelDTBPath} ${KernelDTSIntermediate} - RESULT_VARIABLE error - ) - if(error) - message(FATAL_ERROR "Failed to compile DTS to DTB: ${KernelDTSIntermediate}") - endif() - # The macOS and GNU coreutils `stat` utilities have different interfaces. - # Check if we're using the macOS version, otherwise assume GNU coreutils. - # CMAKE_HOST_APPLE is a built-in CMake variable. - if(CMAKE_HOST_APPLE AND "${STAT_TOOL}" STREQUAL "/usr/bin/stat") - set(STAT_ARGS "-f%z") - else() - set(STAT_ARGS "-c '%s'") - endif() - # Track the size of the DTB for downstream tools - execute_process( - COMMAND ${STAT_TOOL} ${STAT_ARGS} ${KernelDTBPath} - OUTPUT_VARIABLE KernelDTBSize - OUTPUT_STRIP_TRAILING_WHITESPACE - RESULT_VARIABLE error - ) - if(error) - message(FATAL_ERROR "Failed to determine KernelDTBSize: ${KernelDTBPath}") - endif() - string( - REPLACE - "\'" - "" - KernelDTBSize - ${KernelDTBSize} - ) - set(KernelDTBSize "${KernelDTBSize}" CACHE INTERNAL "Size of DTB blob, in bytes") - endif() - - set(deps ${KernelDTBPath} ${config_file} ${config_schema} ${HARDWARE_GEN_PATH}) - check_outfile_stale(regen ${device_dest} deps ${CMAKE_CURRENT_BINARY_DIR}/gen_header.cmd) - if(regen) - # Generate devices_gen header based on DTB - message(STATUS "${device_dest} is out of date. Regenerating from DTB...") - file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/gen_headers/plat/machine/") - execute_process( - COMMAND - ${PYTHON3} "${HARDWARE_GEN_PATH}" --dtb "${KernelDTBPath}" --compat-strings - --compat-strings-out "${compatibility_outfile}" --c-header --header-out - "${device_dest}" --hardware-config "${config_file}" --hardware-schema - "${config_schema}" --yaml --yaml-out "${platform_yaml}" --sel4arch - "${KernelSel4Arch}" --addrspace-max "${KernelPaddrUserTop}" --json --json-out - "${platform_json}" - RESULT_VARIABLE error - ) - if(error) - message(FATAL_ERROR "Failed to generate from DTB: ${device_dest}") - endif() - endif() - file(READ "${compatibility_outfile}" compatibility_strings) - - # Mark all file dependencies as CMake rerun dependencies. - set(cmake_deps ${deps} ${KernelDTSIntermediate} ${KernelDTSList} ${compatibility_outfile}) - set_property( - DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" - APPEND - PROPERTY CMAKE_CONFIGURE_DEPENDS ${cmake_deps} - ) - - include(src/drivers/config.cmake) -endif() - # Enshrine common variables in the config config_set(KernelHaveFPU HAVE_FPU "${KernelHaveFPU}") config_set(KernelPaddrUserTop PADDR_USER_DEVICE_TOP "${KernelPaddrUserTop}") @@ -598,4 +471,133 @@ config_option( DEFAULT OFF ) +if(DEFINED KernelDTSList AND (NOT "${KernelDTSList}" STREQUAL "")) + set(KernelDTSIntermediate "${CMAKE_CURRENT_BINARY_DIR}/kernel.dts") + set( + KernelDTBPath "${CMAKE_CURRENT_BINARY_DIR}/kernel.dtb" + CACHE INTERNAL "Location of kernel DTB file" + ) + set(compatibility_outfile "${CMAKE_CURRENT_BINARY_DIR}/kernel_compat.txt") + set(device_dest "${CMAKE_CURRENT_BINARY_DIR}/gen_headers/plat/machine/devices_gen.h") + set( + platform_yaml "${CMAKE_CURRENT_BINARY_DIR}/gen_headers/plat/machine/platform_gen.yaml" + CACHE INTERNAL "Location of platform YAML description" + ) + set( + platform_json "${CMAKE_CURRENT_BINARY_DIR}/gen_headers/plat/machine/platform_gen.json" + CACHE INTERNAL "Location of platform JSON description" + ) + set(config_file "${CMAKE_CURRENT_SOURCE_DIR}/tools/hardware.yml") + set(config_schema "${CMAKE_CURRENT_SOURCE_DIR}/tools/hardware_schema.yml") + set( + KernelCustomDTSOverlay "" + CACHE + STRING + "Provide an additional list of overlays to append to the selected KernelPlatform's \ + device tree during build time" + ) + if(NOT "${KernelCustomDTSOverlay}" STREQUAL "") + foreach(dts_entry IN ITEMS ${KernelCustomDTSOverlay}) + if(NOT EXISTS ${dts_entry}) + message(FATAL_ERROR "Can't open external overlay file '${dts_entry}'!") + endif() + list(APPEND KernelDTSList "${dts_entry}") + message(STATUS "Appending ${dts_entry} overlay") + endforeach() + endif() + + find_program(DTC_TOOL dtc) + if("${DTC_TOOL}" STREQUAL "DTC_TOOL-NOTFOUND") + message(FATAL_ERROR "Cannot find 'dtc' program.") + endif() + find_program(STAT_TOOL stat) + if("${STAT_TOOL}" STREQUAL "STAT_TOOL-NOTFOUND") + message(FATAL_ERROR "Cannot find 'stat' program.") + endif() + mark_as_advanced(DTC_TOOL STAT_TOOL) + # Generate final DTS based on Linux DTS + seL4 overlay[s] + foreach(entry ${KernelDTSList}) + get_absolute_source_or_binary(dts_tmp ${entry}) + list(APPEND dts_list "${dts_tmp}") + endforeach() + + check_outfile_stale(regen ${KernelDTBPath} dts_list ${CMAKE_CURRENT_BINARY_DIR}/dts.cmd) + if(regen) + file(REMOVE "${KernelDTSIntermediate}") + foreach(entry ${dts_list}) + file(READ ${entry} CONTENTS) + file(APPEND "${KernelDTSIntermediate}" "${CONTENTS}") + endforeach() + # Compile DTS to DTB + execute_process( + COMMAND + ${DTC_TOOL} -q -I dts -O dtb -o ${KernelDTBPath} ${KernelDTSIntermediate} + RESULT_VARIABLE error + ) + if(error) + message(FATAL_ERROR "Failed to compile DTS to DTB: ${KernelDTSIntermediate}") + endif() + # The macOS and GNU coreutils `stat` utilities have different interfaces. + # Check if we're using the macOS version, otherwise assume GNU coreutils. + # CMAKE_HOST_APPLE is a built-in CMake variable. + if(CMAKE_HOST_APPLE AND "${STAT_TOOL}" STREQUAL "/usr/bin/stat") + set(STAT_ARGS "-f%z") + else() + set(STAT_ARGS "-c '%s'") + endif() + # Track the size of the DTB for downstream tools + execute_process( + COMMAND ${STAT_TOOL} ${STAT_ARGS} ${KernelDTBPath} + OUTPUT_VARIABLE KernelDTBSize + OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE error + ) + if(error) + message(FATAL_ERROR "Failed to determine KernelDTBSize: ${KernelDTBPath}") + endif() + string( + REPLACE + "\'" + "" + KernelDTBSize + ${KernelDTBSize} + ) + set(KernelDTBSize "${KernelDTBSize}" CACHE INTERNAL "Size of DTB blob, in bytes") + endif() + + set(deps ${KernelDTBPath} ${config_file} ${config_schema} ${HARDWARE_GEN_PATH}) + check_outfile_stale(regen ${device_dest} deps ${CMAKE_CURRENT_BINARY_DIR}/gen_header.cmd) + if(regen) + # Generate devices_gen header based on DTB + message(STATUS "${device_dest} is out of date. Regenerating from DTB...") + file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/gen_headers/plat/machine/") + execute_process( + COMMAND + ${PYTHON3} "${HARDWARE_GEN_PATH}" --dtb "${KernelDTBPath}" --compat-strings + --compat-strings-out "${compatibility_outfile}" --c-header --header-out + "${device_dest}" --hardware-config "${config_file}" --hardware-schema + "${config_schema}" --yaml --yaml-out "${platform_yaml}" --sel4arch + "${KernelSel4Arch}" --addrspace-max "${KernelPaddrUserTop}" --json --json-out + "${platform_json}" --kernel-config-flags "CONFIG_PRINTING=${KernelPrinting}" + "CONFIG_ARM_HYPERVISOR_SUPPORT=${KernelArmHypervisorSupport}" + "CONFIG_ARM_SMMU=${KernelArmSMMU}" "CONFIG_TK1_SMMU=${KernelTk1SMMU}" + RESULT_VARIABLE error + ) + if(error) + message(FATAL_ERROR "Failed to generate from DTB: ${device_dest}") + endif() + endif() + file(READ "${compatibility_outfile}" compatibility_strings) + + # Mark all file dependencies as CMake rerun dependencies. + set(cmake_deps ${deps} ${KernelDTSIntermediate} ${KernelDTSList} ${compatibility_outfile}) + set_property( + DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + APPEND + PROPERTY CMAKE_CONFIGURE_DEPENDS ${cmake_deps} + ) + + include(src/drivers/config.cmake) +endif() + add_config_library(kernel "${configure_string}") diff --git a/tools/hardware/outputs/c_header.py b/tools/hardware/outputs/c_header.py index 9cab57ef0..fdc7a9705 100644 --- a/tools/hardware/outputs/c_header.py +++ b/tools/hardware/outputs/c_header.py @@ -131,7 +131,7 @@ static const p_region_t BOOT_RODATA avail_p_regs[] = { ''' -def get_kernel_devices(tree: FdtParser, hw_yaml: HardwareYaml) -> (List, Dict): +def get_kernel_devices(tree: FdtParser, hw_yaml: HardwareYaml, kernel_config_dict: Dict[str, str]) -> (List, Dict): ''' Given a device tree and a set of rules, returns a tuple (groups, offsets). @@ -208,12 +208,12 @@ def create_c_header_file(config, kernel_irqs: List, kernel_macros: Dict, outputStream.write(data) -def run(tree: FdtParser, hw_yaml: HardwareYaml, config: Config, args: argparse.Namespace): +def run(tree: FdtParser, hw_yaml: HardwareYaml, config: Config, kernel_config_dict, args: argparse.Namespace): if not args.header_out: raise ValueError('You need to specify a header-out to use c header output') physical_memory, reserved, physBase = hardware.utils.memory.get_physical_memory(tree, config) - kernel_regions, kernel_macros = get_kernel_devices(tree, hw_yaml) + kernel_regions, kernel_macros = get_kernel_devices(tree, hw_yaml, kernel_config_dict) create_c_header_file( config, diff --git a/tools/hardware/outputs/compat_strings.py b/tools/hardware/outputs/compat_strings.py index 3fe67ffa8..7b14a0ab1 100644 --- a/tools/hardware/outputs/compat_strings.py +++ b/tools/hardware/outputs/compat_strings.py @@ -9,10 +9,11 @@ import argparse from hardware.config import Config from hardware.fdt import FdtParser from hardware.utils.rule import HardwareYaml +from typing import Dict def run(tree: FdtParser, hw_yaml: HardwareYaml, config: Config, - args: argparse.Namespace): + kernel_config_dict: Dict[str, str], args: argparse.Namespace): if not args.compat_strings_out: raise ValueError('You need to specify a compat-strings-out to use compat strings output') chosen = tree.get_kernel_devices() diff --git a/tools/hardware/outputs/elfloader.py b/tools/hardware/outputs/elfloader.py index 2d00712be..24af655d4 100644 --- a/tools/hardware/outputs/elfloader.py +++ b/tools/hardware/outputs/elfloader.py @@ -12,7 +12,7 @@ import logging import pyfdt.pyfdt from jinja2 import Environment, BaseLoader -from typing import List +from typing import Dict, List from hardware import config, device, fdt from hardware.utils import cpu, memory, rule @@ -140,7 +140,7 @@ def get_elfloader_cpus(tree: fdt.FdtParser, devices: List[device.WrappedNode]) - return sorted(cpu_info, key=lambda a: a['cpuid']) -def run(tree: fdt.FdtParser, hardware: rule.HardwareYaml, config: config.Config, args: argparse.Namespace): +def run(tree: fdt.FdtParser, hardware: rule.HardwareYaml, config: config.Config, kernel_config_dict: Dict[str, str], args: argparse.Namespace): devices = tree.get_elfloader_devices() cpu_info = get_elfloader_cpus(tree, devices) diff --git a/tools/hardware/outputs/json.py b/tools/hardware/outputs/json.py index d10fcae88..99044f3e1 100644 --- a/tools/hardware/outputs/json.py +++ b/tools/hardware/outputs/json.py @@ -8,7 +8,7 @@ import argparse import json -from typing import List +from typing import Dict, List import hardware from hardware.config import Config from hardware.fdt import FdtParser @@ -35,24 +35,29 @@ def create_json_file(dev_mem, phys_mem, output_stream): json.dump(json_obj, output_stream) -def get_kernel_devices(tree: FdtParser, hw_yaml: HardwareYaml): +def get_kernel_devices(tree: FdtParser, hw_yaml: HardwareYaml, kernel_config_dict: Dict[str, str]): kernel_devices = tree.get_kernel_devices() groups = [] for dev in kernel_devices: rule = hw_yaml.get_rule(dev) - groups += rule.get_regions(dev) + new_regions = rule.get_regions(dev) + for reg in new_regions: + if reg.macro in kernel_config_dict: + if kernel_config_dict[reg.macro] != "ON": + continue + groups.append(reg) return groups def run(tree: FdtParser, hw_yaml: HardwareYaml, config: Config, - args: argparse.Namespace): + kernel_config_dict, args: argparse.Namespace): if not args.json_out: raise ValueError('you need to provide a json-out to use the JSON output method') phys_mem, reserved, _ = hardware.utils.memory.get_physical_memory(tree, config) - kernel_devs = get_kernel_devices(tree, hw_yaml) + kernel_devs = get_kernel_devices(tree, hw_yaml, kernel_config_dict) dev_mem = hardware.utils.memory.get_addrspace_exclude( list(reserved) + phys_mem + kernel_devs, config) diff --git a/tools/hardware/outputs/yaml.py b/tools/hardware/outputs/yaml.py index 3a35fd8bd..23ab54103 100644 --- a/tools/hardware/outputs/yaml.py +++ b/tools/hardware/outputs/yaml.py @@ -8,7 +8,7 @@ import argparse import yaml -from typing import List +from typing import Dict, List import hardware from hardware.config import Config from hardware.fdt import FdtParser @@ -40,24 +40,29 @@ def create_yaml_file(dev_mem, phys_mem, outputStream): yaml.dump(yaml_obj, outputStream) -def get_kernel_devices(tree: FdtParser, hw_yaml: HardwareYaml): +def get_kernel_devices(tree: FdtParser, hw_yaml: HardwareYaml, kernel_config_dict: Dict[str, str]): kernel_devices = tree.get_kernel_devices() groups = [] for dev in kernel_devices: rule = hw_yaml.get_rule(dev) - groups += rule.get_regions(dev) + new_regions = rule.get_regions(dev) + for reg in new_regions: + if reg.macro in kernel_config_dict: + if kernel_config_dict[reg.macro] != "ON": + continue + groups.append(reg) return groups def run(tree: FdtParser, hw_yaml: HardwareYaml, config: Config, - args: argparse.Namespace): + kernel_config_dict, args: argparse.Namespace): if not args.yaml_out: raise ValueError('you need to provide a yaml-out to use the yaml output method') phys_mem, reserved, _ = hardware.utils.memory.get_physical_memory(tree, config) - kernel_devs = get_kernel_devices(tree, hw_yaml) + kernel_devs = get_kernel_devices(tree, hw_yaml, kernel_config_dict) dev_mem = hardware.utils.memory.get_addrspace_exclude( list(reserved) + phys_mem + kernel_devs, config) diff --git a/tools/hardware_gen.py b/tools/hardware_gen.py index 481fbe2b3..1b95986ce 100644 --- a/tools/hardware_gen.py +++ b/tools/hardware_gen.py @@ -59,7 +59,7 @@ def main(args: argparse.Namespace): arg_dict = vars(args) for t in sorted(OUTPUTS.keys()): if arg_dict[t]: - OUTPUTS[t].run(parsed_dt, hw_yaml, cfg, args) + OUTPUTS[t].run(parsed_dt, hw_yaml, cfg, kernel_config_dict, args) if __name__ == '__main__': @@ -75,6 +75,8 @@ if __name__ == '__main__': required=True, type=argparse.FileType('r')) parser.add_argument('--sel4arch', help='seL4 architecture to generate for', required=True) + parser.add_argument('--kernel-config-flags', + help='List of kernel config params', action='append', nargs='+') parser.add_argument('--addrspace-max', help='maximum address that is available as device untyped', type=int, default=32) @@ -85,6 +87,12 @@ if __name__ == '__main__': args = parser.parse_args() + kernel_config_dict = dict() + if args.kernel_config_flags: + for option in sum(args.kernel_config_flags, []): + name, val = option.split('=') + kernel_config_dict[name] = val + if args.enable_profiling: import cProfile cProfile.run('main(args)', sort='cumtime')