rtems/bsps/aarch64/shared/cache/cache.c

/* SPDX-License-Identifier: BSD-2-Clause */

/**
 * @file
 *
 * @ingroup RTEMSBSPsAArch64Shared
 *
 * @brief AArch64 cache defines and implementation.
 */

/*
 * Copyright (C) 2020 On-Line Applications Research Corporation (OAR)
 * Written by Kinsey Moore <kinsey.moore@oarcorp.com>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <rtems.h>
#include <bsp.h>
#include <rtems/score/aarch64-system-registers.h>

#define CPU_DATA_CACHE_ALIGNMENT 64

#define CPU_INSTRUCTION_CACHE_ALIGNMENT 64

#define CPU_CACHE_SUPPORT_PROVIDES_RANGE_FUNCTIONS

#define CPU_CACHE_SUPPORT_PROVIDES_CACHE_SIZE_FUNCTIONS

#define AARCH64_CACHE_L1_CPU_DATA_ALIGNMENT ( (size_t) 64 )
#define AARCH64_CACHE_PREPARE_MVA(mva) (const void *) \
  RTEMS_ALIGN_DOWN ( (size_t) mva, AARCH64_CACHE_L1_CPU_DATA_ALIGNMENT )

static inline
void AArch64_data_cache_clean_and_invalidate_line(const void *d_addr)
{
  d_addr = AARCH64_CACHE_PREPARE_MVA(d_addr);

  __asm__ volatile (
    "dc civac, %[d_addr]"
    :
    : [d_addr] "r" (d_addr)
    : "memory"
  );
}

static inline void
_CPU_cache_flush_data_range(
  const void *d_addr,
  size_t      n_bytes
)
{
  _AARCH64_Data_synchronization_barrier();
  if ( n_bytes != 0 ) {
    size_t adx = (size_t) AARCH64_CACHE_PREPARE_MVA ( d_addr );
    const size_t ADDR_LAST = (size_t) d_addr + n_bytes - 1;

    for (; adx <= ADDR_LAST; adx += AARCH64_CACHE_L1_CPU_DATA_ALIGNMENT ) {
      /* Store and invalidate the Data cache line */
      AArch64_data_cache_clean_and_invalidate_line( (void*)adx );
    }
    /* Wait for L1 store to complete */
    _AARCH64_Data_synchronization_barrier();
  }
  _AARCH64_Data_synchronization_barrier();
}

static inline void AArch64_data_cache_invalidate_line(const void *d_addr)
{
  d_addr = AARCH64_CACHE_PREPARE_MVA(d_addr);

  __asm__ volatile (
    "dc ivac, %[d_addr]"
    :
    : [d_addr] "r" (d_addr)
    : "memory"
  );
}

static inline void
_CPU_cache_invalidate_data_range(
  const void *d_addr,
  size_t     n_bytes
)
{
  if ( n_bytes != 0 ) {
    size_t adx = (size_t) AARCH64_CACHE_PREPARE_MVA ( d_addr );
    const size_t end = (size_t)d_addr + n_bytes -1;

    /* Back starting address up to start of a line and invalidate until end */
    for (;
         adx <= end;
         adx += AARCH64_CACHE_L1_CPU_DATA_ALIGNMENT ) {
        /* Invalidate the Instruction cache line */
        AArch64_data_cache_invalidate_line( (void*)adx );
    }
    /* Wait for L1 invalidate to complete */
    _AARCH64_Data_synchronization_barrier();
  }
}

static inline void _CPU_cache_freeze_data(void)
{
  /* TODO */
}

static inline void _CPU_cache_unfreeze_data(void)
{
  /* TODO */
}

static inline void AArch64_instruction_cache_invalidate_line(const void *i_addr)
{
  /* __builtin___clear_cache is explicitly only for instruction cacche */
  __builtin___clear_cache((void *)i_addr, ((char *)i_addr) + sizeof(void*) - 1);
}

static inline void
_CPU_cache_invalidate_instruction_range( const void *i_addr, size_t n_bytes)
{
  if ( n_bytes != 0 ) {
    __builtin___clear_cache((void *)i_addr, ((char *)i_addr) + n_bytes - 1);
  }
  _AARCH64_Instruction_synchronization_barrier();
}

static inline void _CPU_cache_freeze_instruction(void)
{
  /* TODO */
}

static inline void _CPU_cache_unfreeze_instruction(void)
{
  /* TODO */
}

static inline uint64_t AArch64_get_ccsidr_for_level(
  uint64_t level, bool instruction
)
{
  uint64_t csselr = AARCH64_CSSELR_EL1_LEVEL(level - 1);

  csselr |= instruction ? AARCH64_CSSELR_EL1_IND : 0;

  _AArch64_Write_csselr_el1(csselr);
  _AARCH64_Instruction_synchronization_barrier();
  return _AArch64_Read_ccsidr_el1();
}

static inline uint64_t
AArch64_ccsidr_get_line_power(uint64_t ccsidr)
{
  return AARCH64_CCSIDR_EL1_LINESIZE_GET(ccsidr) + 4;
}

static inline uint64_t
AArch64_ccsidr_get_associativity(uint64_t ccsidr)
{
  return AARCH64_CCSIDR_EL1_ASSOCIATIVITY_GET_0(ccsidr) + 1;
}

static inline uint64_t
AArch64_ccsidr_get_num_sets(uint64_t ccsidr)
{
  return AARCH64_CCSIDR_EL1_NUMSETS_GET_0(ccsidr) + 1;
}


static inline void
AArch64_data_cache_clean_and_invalidate_level(uint64_t level)
{
  uint64_t ccsidr;
  uint64_t line_power;
  uint64_t associativity;
  uint64_t way;
  uint64_t way_shift;

  ccsidr = AArch64_get_ccsidr_for_level(level, false);

  line_power = AArch64_ccsidr_get_line_power(ccsidr);
  associativity = AArch64_ccsidr_get_associativity(ccsidr);
  way_shift = __builtin_clz(associativity - 1);

  for (way = 0; way < associativity; ++way) {
    uint64_t num_sets = AArch64_ccsidr_get_num_sets(ccsidr);
    uint64_t set;

    for (set = 0; set < num_sets; ++set) {
      uint64_t set_and_way = (way << way_shift)
        | (set << line_power)
        | ((level - 1) << 1);

      __asm__ volatile (
        "dc cisw, %[set_and_way]"
        :
        : [set_and_way] "r" (set_and_way)
        : "memory"
      );
    }
  }
}

static inline
uint64_t AArch64_clidr_get_cache_type(uint64_t clidr, uint64_t level)
{
  return (clidr >> (3 * level)) & 0x7;
}

static inline uint64_t AArch64_clidr_get_level_of_coherency(uint64_t clidr)
{
  return AARCH64_CLIDR_EL1_LOC_GET(clidr);
}

static inline void AArch64_data_cache_clean_and_invalidate_all_levels(void)
{
  uint64_t clidr = _AArch64_Read_clidr_el1();
  uint64_t loc = AArch64_clidr_get_level_of_coherency(clidr);
  uint64_t level = 0;

  for (level = 1; level <= loc; ++level) {
    /* Assume that all levels have a data cache */
    AArch64_data_cache_clean_and_invalidate_level(level);
  }
}

static inline void _CPU_cache_flush_entire_data(void)
{
  rtems_interrupt_level isr_level;

  rtems_interrupt_local_disable(isr_level);
  _AARCH64_Data_synchronization_barrier();
  AArch64_data_cache_clean_and_invalidate_all_levels();
  _AARCH64_Data_synchronization_barrier();
  rtems_interrupt_local_enable(isr_level);
}

static inline void AArch64_cache_invalidate_level(uint64_t level)
{
  uint64_t ccsidr;
  uint64_t line_power;
  uint64_t associativity;
  uint64_t way;
  uint64_t way_shift;

  ccsidr = AArch64_get_ccsidr_for_level(level, false);

  line_power = AArch64_ccsidr_get_line_power(ccsidr);
  associativity = AArch64_ccsidr_get_associativity(ccsidr);
  way_shift = __builtin_clz(associativity - 1);

  for (way = 0; way < associativity; ++way) {
    uint64_t num_sets = AArch64_ccsidr_get_num_sets(ccsidr);
    uint64_t set;

    for (set = 0; set < num_sets; ++set) {
      uint64_t set_and_way = (way << way_shift)
        | (set << line_power)
        | ((level - 1) << 1);

      __asm__ volatile (
        "dc isw, %[set_and_way]"
        :
        : [set_and_way] "r" (set_and_way)
        : "memory"
      );
    }
  }
}

static inline void AArch64_data_cache_invalidate_all_levels(void)
{
  uint64_t clidr = _AArch64_Read_clidr_el1();
  uint64_t loc = AArch64_clidr_get_level_of_coherency(clidr);
  uint64_t level = 0;

  for (level = 1; level <= loc; ++level) {
    /* Assume that all levels have a data cache */
    AArch64_cache_invalidate_level(level);
  }
}

static inline void _CPU_cache_invalidate_entire_data(void)
{
  rtems_interrupt_level isr_level;

  rtems_interrupt_local_disable(isr_level);
  _AARCH64_Data_synchronization_barrier();
  AArch64_data_cache_invalidate_all_levels();
  _AARCH64_Data_synchronization_barrier();
  rtems_interrupt_local_enable(isr_level);
}

static inline void _CPU_cache_enable_data(void)
{
  rtems_interrupt_level isr_level;
  uint64_t sctlr;

  rtems_interrupt_local_disable(isr_level);
  sctlr = _AArch64_Read_sctlr_el1();
  sctlr |= AARCH64_SCTLR_EL1_C;
  _AArch64_Write_sctlr_el1(sctlr);
  rtems_interrupt_local_enable(isr_level);
}

static RTEMS_NO_RETURN inline void _CPU_cache_disable_data(void)
{
  _Internal_error( INTERNAL_ERROR_CANNOT_DISABLE_DATA_CACHE );
}

static inline void _CPU_cache_invalidate_entire_instruction(void)
{
  /*
   * There is no way to manage branch prediction in AArch64.  See D4.4.12 in
   * the ARM Architecture Reference Manual, ARMv8, for ARMv8-A architecture
   * profile (ARM DDI 0487D.a).
   */

  __asm__ volatile (
#ifdef RTEMS_SMP
    /*
     * Invalidate all instruction caches up to
     * Point of Unification, Inner Shareable.
     */
    "ic ialluis\n"
#else
    /* Invalidate all instruction caches up to Point of Unification */
    "ic iallu\n"
#endif
    "isb"
    :
    :
    : "memory"
  );
}

static inline void _CPU_cache_enable_instruction(void)
{
  rtems_interrupt_level isr_level;
  uint64_t sctlr;

  rtems_interrupt_local_disable(isr_level);
  sctlr = _AArch64_Read_sctlr_el1();
  sctlr |= AARCH64_SCTLR_EL1_I;
  _AArch64_Write_sctlr_el1(sctlr);
  rtems_interrupt_local_enable(isr_level);
}

static inline void _CPU_cache_disable_instruction(void)
{
  rtems_interrupt_level isr_level;
  uint64_t sctlr;

  rtems_interrupt_local_disable(isr_level);
  sctlr = _AArch64_Read_sctlr_el1();
  sctlr &= ~AARCH64_SCTLR_EL1_I;
  _AArch64_Write_sctlr_el1(sctlr);
  rtems_interrupt_local_enable(isr_level);
}

static inline size_t AArch64_get_cache_size(
  uint64_t level,
  bool instruction
)
{
  rtems_interrupt_level isr_level;
  uint64_t clidr;
  uint64_t loc;
  uint64_t ccsidr;

  clidr = _AArch64_Read_clidr_el1();
  loc = AArch64_clidr_get_level_of_coherency(clidr);

  if (level > loc) {
    return 0;
  }

  rtems_interrupt_local_disable(isr_level);
  ccsidr = AArch64_get_ccsidr_for_level(level, instruction);
  rtems_interrupt_local_enable(isr_level);

  return (1U << (AArch64_ccsidr_get_line_power(ccsidr)+4))
    * AArch64_ccsidr_get_associativity(ccsidr)
    * AArch64_ccsidr_get_num_sets(ccsidr);
}

static inline size_t _CPU_cache_get_data_cache_size(uint64_t level)
{
  return AArch64_get_cache_size(level, false);
}

static inline size_t _CPU_cache_get_instruction_cache_size(uint64_t level)
{
  return AArch64_get_cache_size(level, true);
}

#include "../../shared/cache/cacheimpl.h"