1. 概述

在linux啟動過程中會打印出如下信息，這些信息為我們呈現出系統下的保留內存空間情況。

Reserved memory: created DMA memory pool at 0x4c000000, size 8 MiB
OF: reserved mem: initialized node vram@4c000000, compatible id shared-dma-pool

本文只介紹基本的保留內存，不涉及CMA部分內容

保留內存的初始化流程如下圖所示：

本文所說的保留內存的作用，概況來講包括如下四方面：

1. 驅動程序特定使用
2. 加載固件到指定內存
3. DDR中某段內存區域存放特定數據，如多核處理相關代碼
4. 調試驅動

2. 保留內存初始化流程

從setup_arch()函數中我們可以發現，保留內存初始化是在設備樹釋放之前，通過解析FDT，獲取保留內存的參數來進行初始化。

void __init setup_arch(char **cmdline_p)
{
...
 arm_memblock_init(mdesc);
...
 unflatten_device_tree();
...

2.1 解析內核中的保留內存空間

在各平臺初始化過程中調用early_init_fdt_scan_reserved_mem()進行保留內存的初始化。

setup_arch_memory in init.c (arch\\arc\\mm) :  early_init_fdt_scan_reserved_mem();
arm64_memblock_init in init.c (arch\\arm64\\mm) :  early_init_fdt_scan_reserved_mem();
arm_memblock_init in init.c (arch\\arm\\mm) :  early_init_fdt_scan_reserved_mem();
setup_bootmem in init.c (arch\\riscv\\mm) :  early_init_fdt_scan_reserved_mem();
bootmem_init in init.c (arch\\xtensa\\mm) :  early_init_fdt_scan_reserved_mem();
sh_of_mem_reserve in of-generic.c (arch\\sh\\boards) :  early_init_fdt_scan_reserved_mem();
of_fdt.h (include\\linux) line 66 : extern void early_init_fdt_scan_reserved_mem(void);
of_fdt.h (include\\linux) line 94 : static inline void early_init_fdt_scan_reserved_mem(void) {}
early_reserve_mem_dt in prom.c (arch\\powerpc\\kernel) :  early_init_fdt_scan_reserved_mem();
csky_memblock_init in setup.c (arch\\csky\\kernel) :  early_init_fdt_scan_reserved_mem();
bootmem_init in setup.c (arch\\h8300\\kernel) :  early_init_fdt_scan_reserved_mem();
arch_mem_init in setup.c (arch\\mips\\kernel) :  early_init_fdt_scan_reserved_mem();
setup_memory in setup.c (arch\\nds32\\kernel) :  early_init_fdt_scan_reserved_mem();
setup_arch in setup.c (arch\\nios2\\kernel) :  early_init_fdt_scan_reserved_mem();
setup_memory in setup.c (arch\\openrisc\\kernel) :  early_init_fdt_scan_reserved_mem();

它的定義位于drivers/of/fdt.c中，需要內核配置打開CONFIG_OF_EARLY_FLATTREE宏。

主體函數如下：

void __init early_init_fdt_scan_reserved_mem(void)
{
 int n;
 u64 base, size;

 if (!initial_boot_params)
  return;

 for (n = 0; ; n++) {
  fdt_get_mem_rsv(initial_boot_params, n, &base, &size);
  if (!size)
   break;
  early_init_dt_reserve_memory_arch(base, size, false);
 }

 of_scan_flat_dt(__fdt_scan_reserved_mem, NULL);
 fdt_init_reserved_mem();
}

2.1.1 解析memreserve

從early_init_fdt_scan_reserved_mem()函數中的initial_boot_params可以再次確定這一點。initial_boot_params代表的是fdt的地址，如下：

## Flattened Device Tree blob at 41000000
   Booting using the fdt blob at 0x41000000
   Loading Kernel Image ... OK
   Loading Device Tree to 4ffef000, end 4ffffff2 ... OK

Starting kernel ...

Uncompressing Linux... done, booting the kernel.
[    0.000000] Booting Linux on physical CPU 0xa00
...
[    0.000000] --- initial_boot_params(fdt addr 0x4ffef000)

通過fdt_get_mem_rsv()解析設備樹中的/memreserve/fields，例如樹莓派處理器的設備樹中定義了該屬性，通常來講，這部分內存區域是存放和rom或者多核啟動相關的程序，需要注意的是內核無法使用這部分內存。這是和reserver memory的區別。

/memreserve/ 0x00000000 0x00001000;
...
/ {
        compatible = "brcm,bcm2835";
...

若在設備樹中查找到了memreserve且未進行映射，則通過memblock_reserve()將這部分內存區域加入到memblock.reserved，當進行memblock到buddy轉換時，釋放掉memblock.reserved所標記的內存區域。

int __init_memblock memblock_reserve(phys_addr_t base, phys_addr_t size)
{
 phys_addr_t end = base + size - 1;

 memblock_dbg("memblock_reserve: [%pa-%pa] %pS\\n",
       &base, &end, (void *)_RET_IP_);

 return memblock_add_range(&memblock.reserved, base, size, MAX_NUMNODES, 0);
}

2.1.2 解析reserve memory

通過__fdt_scan_reserved_mem()解析設備樹中保留內存相關的結點信息。

static int __init __fdt_scan_reserved_mem(unsigned long node, const char *uname,
       int depth, void *data)
{
 static int found;
 int err;

 if (!found && depth == 1 && strcmp(uname, "reserved-memory") == 0) {
  ...
 }

 if (!of_fdt_device_is_available(initial_boot_params, node))
  return 0;

 err = __reserved_mem_reserve_reg(node, uname);
 if (err == -ENOENT && of_get_flat_dt_prop(node, "size", NULL))
  fdt_reserved_mem_save_node(node, uname, 0, 0);
...
}

該函數首先解析設備樹中reserved-memory結點并確認是否有效。若有效，繼續檢查reg或size屬性定義的內存區域，通過fdt_reserved_mem_save_node()將內存信息更新到數據結構struct reserved_mem。

void __init fdt_reserved_mem_save_node(unsigned long node, const char *uname,phys_addr_t base, phys_addr_t size)
{
 struct reserved_mem *rmem = &reserved_mem[reserved_mem_count];

 if (reserved_mem_count == ARRAY_SIZE(reserved_mem)) {
  pr_err("not enough space all defined regions.\\n");
  return;
 }

 rmem- >fdt_node = node;
 rmem- >name = uname;
 rmem- >base = base;
 rmem- >size = size;

 reserved_mem_count++;
 return;
}

2.2 保留內存初始化

保留內存初始化的主體函數是fdt_init_reserved_mem()，其首先解析設備樹結點no-map、phandle等信息，最后通過關鍵函數__reserved_mem_init_node()完成保留內存子節點的初始化。

放開解析保留內存解析相關的打印，再回頭再看kernel的啟動信息，啟動信息中保留內存相關內容正是此處打印出來的。

OF: fdt: Reserved memory: reserved region for node 'vram@4c000000': base 0x4c000000, size 8 MiB
Reserved memory: created DMA memory pool at 0x4c000000, size 8 MiB

created DMA memory pool...來自函數rmem_dma_setup()，這個函數從數據結構struct reserved_mem獲取保留內存的信息。

static int __init rmem_dma_setup(struct reserved_mem *rmem)
{
 unsigned long node = rmem- >fdt_node;

 if (of_get_flat_dt_prop(node, "reusable", NULL))
  return -EINVAL;

#ifdef CONFIG_ARM
 if (!of_get_flat_dt_prop(node, "no-map", NULL)) {
  pr_err("Reserved memory: regions without no-map are not yet supported\\n");
  return -EINVAL;
 }

 if (of_get_flat_dt_prop(node, "linux,dma-default", NULL)) {
  WARN(dma_reserved_default_memory,
       "Reserved memory: region for default DMA coherent area is redefined\\n");
  dma_reserved_default_memory = rmem;
 }
#endif

 rmem- >ops = &rmem_dma_ops;
 pr_info("Reserved memory: created DMA memory pool at %pa, size %ld MiB\\n",
  &rmem- >base, (unsigned long)rmem- >size / SZ_1M);
 return 0;
}

在函數rmem_dma_setup()中還會例化reserved_mem.ops，如下：

static const struct reserved_mem_ops rmem_dma_ops = {
 .device_init = rmem_dma_device_init,
 .device_release = rmem_dma_device_release,
};

3. 設備樹中保留內存的定義方式

以vexpress-v2p-ca9.dts中保留內存的定義方式為例，說明dts文件中如何定義保留內存。

reserved-memory {
        #address-cells = < 1 >;
        #size-cells = < 1 >;
        ranges;

        /* Chipselect 3 is physically at 0x4c000000 */
        vram: vram@4c000000 {
                /* 8 MB of designated video RAM */
                compatible = "shared-dma-pool";
                reg = < 0x4c000000 0x00800000 >;
                no-map;
        };
};

保留內存由根節點和1個或多個子結點組成。

根節點包括如下信息：

• #address-cells、#size-cells

  必須項，需要同dts根節點中相關屬性保持一致。

/dts-v1/;
#include "vexpress-v2m.dtsi"

/ {
        model = "V2P-CA9";
        arm,hbi = < 0x191 >;
        arm,vexpress,site = < 0xf >;
        compatible = "arm,vexpress,v2p-ca9", "arm,vexpress";
        interrupt-parent = < &gic >;
        #address-cells = < 1 >;
        #size-cells = < 1 >;
...

• ranges

  必須項，且定義為空

子結點包括如下信息：

• 空間大小

  可以通過reg或size來指定保留內存空間大小，若二者同時存在，以reg屬性為準。通過size的方式如下：

reserved-memory {
        #address-cells = < 1 >;
        #size-cells = < 1 >;
        ranges;

        mfc_left: region_mfc_left {
                compatible = "shared-dma-pool";
                no-map;
                size = < 0x2400000 >;
...

• alignment

  可選項

• alloc-ranges

  可選項，通常可以和size同時使用。

reserved-memory {
        #address-cells = < 1 >;
        #size-cells = < 1 >;
        ranges;
        default-pool {
                compatible = "shared-dma-pool";
                size = < 0x6000000 >;
                alloc-ranges = < 0x40000000 0x10000000 >;
...

• compatible

  可能包括shared-dma-pool或者shared-dma-pool。

  主要關注shared-dma-pool，當驅動程序需要申請DMA空間時，可以從這里進行申請內存空間。

• no-map

  該屬性意為不會為這段內存創建地址映射，在使用之前，需要調用者通過ioremap創建頁表映射關系才可以正常訪問。這個屬性與reusable是互斥的。

• no-map-fixup

  保持內存映射。

• reusable

  當驅動程序不使用這些內存的時候，OS可以使用這些內存。

• linux,cma-default

  定義該段保留內存空間是默認的CMA內存池。

reserved-memory {
        #address-cells = < 1 >;
        #size-cells = < 1 >;
        ranges;
        default-pool {
                compatible = "shared-dma-pool";
                size = < 0x6000000 >;
                alloc-ranges = < 0x40000000 0x10000000 >;
                reusable;
                linux,cma-default;
        };
};

4. 保留內存的使用

4.1 設備樹編碼

定義保留內存：

memory@60000000 {
        device_type = "memory";
        reg = < 0x60000000 0x40000000 >;
};

reserved-memory {
        #address-cells = < 1 >;
        #size-cells = < 1 >;
        ranges;

        /* test reserve memory */
        test_reserve: test_reserve@90000000 {
                /* 1 MB reserve memory */
                compatible = "shared-dma-pool";
                reg = < 0x90000000 0x00100000 >;
                no-map;
        };
};

定義memory-region，將保留內存指定給特定設備，如下：

driver-test@8000 {
        /* compatible = "test_driver_0", "simple_bus"; */
        compatible = "test_driver_0";
        reg = < 0x80008000 0x1000 >;

        interrupt-parent = < &gic >;
        interrupts= < 0 89 4 >, < 0 90 4444 >;
        interrupt-names = "first_irq", "second_irq";

        clocks = < &oscclk2 >;
        clock-names = "apb_pclk";

        memory-region = < &test_reserve >;

        status = "okay";

        simple_bus_test{
            compatile = "simple_bus_test";
        };
};

加載kernel后，保留內存相關的打印信息如下：

Reserved memory: created DMA memory pool at 0x90000000, size 1 MiB
OF: reserved mem: initialized node test_reserve@90000000, compatible id shared-dma-pool

4.2 驅動程序編碼

驅動程序中調用of_reserved_mem_device_init()申請保留內存空間。

static inline int of_reserved_mem_device_init(struct device *dev)
{
 return of_reserved_mem_device_init_by_idx(dev, dev- >of_node, 0);
}

主體函數是of_reserved_mem_device_init_by_idx()

int of_reserved_mem_device_init_by_idx(struct device *dev,
           struct device_node *np, int idx)
{
 struct rmem_assigned_device *rd;
 struct device_node *target;
 struct reserved_mem *rmem;
 int ret;

 if (!np || !dev)
  return -EINVAL;

 target = of_parse_phandle(np, "memory-region", idx);
 if (!target)
  return -ENODEV;

 if (!of_device_is_available(target)) {
  of_node_put(target);
  return 0;
 }

 rmem = __find_rmem(target);
 of_node_put(target);

 if (!rmem || !rmem- >ops || !rmem- >ops- >device_init)
  return -EINVAL;

 rd = kmalloc(sizeof(struct rmem_assigned_device), GFP_KERNEL);
 if (!rd)
  return -ENOMEM;

 ret = rmem- >ops- >device_init(rmem, dev);
 if (ret == 0) {
  rd- >dev = dev;
  rd- >rmem = rmem;

  mutex_lock(&of_rmem_assigned_device_mutex);
  list_add(&rd- >list, &of_rmem_assigned_device_list);
  mutex_unlock(&of_rmem_assigned_device_mutex);

  dev_info(dev, "assigned reserved memory node %s\\n", rmem- >name);
 } else {
  kfree(rd);
 }

 return ret;
}

然后可以通過dma_alloc_coherent()在保留內存空間申請DMA空間。

void *dma_vaddr;
dma_addr_t dma_handler;

/* Start: test reserve memory */
ret = of_reserved_mem_device_init(&pdev- >dev);
if (!ret) {
        dev_info(&pdev- >dev, "using device-specific reserved memory\\n");
}

dma_set_coherent_mask(&pdev- >dev, 0xFFFFFFFF);
dma_vaddr = dma_alloc_coherent(&pdev- >dev, 64*1024, &dma_handler, GFP_KERNEL);
if (!dma_vaddr) {
        pr_notice("DMA allocation failed\\n");
        return false;
}
dev_info(&pdev- >dev, "DMA alloc phy addr 0x%X\\n", (u32)dma_handler);
/* End: test reserve memory */

執行結果：

test_driver ahb:driver-test@8000: assigned reserved memory node test_reserve@90000000
test_driver ahb:driver-test@8000: using device-specific reserved memory
test_driver ahb:driver-test@8000: DMA alloc phy addr 0x90000000

結果表明，DMA申請的內存落于保留內存空間0x90000000-0x90100000。