SbsaQemu can configure with numa-related arguments, but OS cannot identify the numa architecture without SRAT tables. We add supporting for generating SRAT tables at runtime to solve this issue.
Changes in v2: - get the information of numa via SMC calls rather than parse DT.
Xiong Yining (1): SbsaQemu: AcpiDxe: Create SRAT table at runtime
.../Drivers/SbsaQemuAcpiDxe/SbsaQemuAcpiDxe.c | 91 +++++++++++++++++++ .../Drivers/SbsaQemuAcpiDxe/SbsaQemuAcpiDxe.h | 28 ++++++ .../SbsaQemu/Include/Library/QemuSbsaSmc.h | 10 ++ .../Library/SbsaQemuSmc/SbsaQemuSmc.c | 22 +++++ .../Library/SbsaQemuSmc/SbsaQemuSmc.inf | 3 + 5 files changed, 154 insertions(+)
Add support to create SRAT(System resource affinity table) for sbsa platform at runtime.
Signed-off-by: Xiong Yining xiongyining1480@phytium.com.cn Signed-off-by: Chen Baozi chenbaozi@phytium.com.cn --- .../Drivers/SbsaQemuAcpiDxe/SbsaQemuAcpiDxe.c | 91 +++++++++++++++++++ .../Drivers/SbsaQemuAcpiDxe/SbsaQemuAcpiDxe.h | 28 ++++++ .../SbsaQemu/Include/Library/QemuSbsaSmc.h | 10 ++ .../Library/SbsaQemuSmc/SbsaQemuSmc.c | 22 +++++ .../Library/SbsaQemuSmc/SbsaQemuSmc.inf | 3 + 5 files changed, 154 insertions(+)
diff --git a/Silicon/Qemu/SbsaQemu/Drivers/SbsaQemuAcpiDxe/SbsaQemuAcpiDxe.c b/Silicon/Qemu/SbsaQemu/Drivers/SbsaQemuAcpiDxe/SbsaQemuAcpiDxe.c index 89c4b797d2..abbf891514 100644 --- a/Silicon/Qemu/SbsaQemu/Drivers/SbsaQemuAcpiDxe/SbsaQemuAcpiDxe.c +++ b/Silicon/Qemu/SbsaQemu/Drivers/SbsaQemuAcpiDxe/SbsaQemuAcpiDxe.c @@ -748,6 +748,90 @@ DisableXhciOnOlderPlatVer ( return Status; }
+/* + * A function that adds the SRAT ACPI table. + */ +EFI_STATUS +AddSratTable ( + IN EFI_ACPI_TABLE_PROTOCOL *AcpiTable + ) +{ + EFI_STATUS Status; + UINT8 *New; + EFI_PHYSICAL_ADDRESS PageAddress; + UINTN TableHandle; + UINT32 TableSize; + UINT32 Index; + UINT32 NodeId; + UINT32 NumMemNode; + MemoryInfo MemInfo; + UINT32 NumCores = PcdGet32 (PcdCoreCount); + + // Initialize MADT ACPI Header + EFI_ACPI_6_4_SYSTEM_RESOURCE_AFFINITY_TABLE_HEADER Header = { + SBSAQEMU_ACPI_HEADER (EFI_ACPI_6_4_SYSTEM_RESOURCE_AFFINITY_TABLE_SIGNATURE, + EFI_ACPI_6_4_SYSTEM_RESOURCE_AFFINITY_TABLE_HEADER, + EFI_ACPI_6_4_SYSTEM_RESOURCE_AFFINITY_TABLE_REVISION), + 1, 0 }; + + NumMemNode = SbsaQemuGetMemNode(); + + // Calculate the new table size based on the number of cores + TableSize = sizeof (EFI_ACPI_6_4_SYSTEM_RESOURCE_AFFINITY_TABLE_HEADER) + + (sizeof (EFI_ACPI_6_4_MEMORY_AFFINITY_STRUCTURE) * NumMemNode) + + (sizeof (EFI_ACPI_6_4_GICC_AFFINITY_STRUCTURE) * NumCores); + + Status = gBS->AllocatePages ( + AllocateAnyPages, + EfiACPIReclaimMemory, + EFI_SIZE_TO_PAGES (TableSize), + &PageAddress + ); + + if (EFI_ERROR(Status)) { + DEBUG ((DEBUG_ERROR, "Failed to allocate pages for SRAT table\n")); + return EFI_OUT_OF_RESOURCES; + } + + New = (UINT8 *)(UINTN) PageAddress; + ZeroMem (New, TableSize); + + // Add the ACPI Description table header + CopyMem (New, &Header, sizeof (EFI_ACPI_6_4_SYSTEM_RESOURCE_AFFINITY_TABLE_HEADER)); + ((EFI_ACPI_DESCRIPTION_HEADER*) New)->Length = TableSize; + New += sizeof (EFI_ACPI_6_4_SYSTEM_RESOURCE_AFFINITY_TABLE_HEADER); + + // Add new memory structures + for (Index = 0; Index < NumMemNode; Index++) { + MemInfo = SbsaQemuGetMemInfo (Index); + EFI_ACPI_6_4_MEMORY_AFFINITY_STRUCTURE memory = SBSAQEMU_ACPI_MEMORY_AFFINITY_STRUCTURE_INIT (MemInfo.NodeId, MemInfo.AddressBase, MemInfo.AddressSize, 1); + CopyMem (New, &memory, sizeof (EFI_ACPI_6_4_MEMORY_AFFINITY_STRUCTURE)); + New += sizeof (EFI_ACPI_6_4_MEMORY_AFFINITY_STRUCTURE); + } + + // Add new professor structures for the Cores + for (Index = 0; Index < NumCores; Index++) { + NodeId = SbsaQemuGetCpuNumaNode(Index); + EFI_ACPI_6_4_GICC_AFFINITY_STRUCTURE gicc = SBSAQEMU_ACPI_GICC_AFFINITY_STRUCTURE_INIT(NodeId, Index, 1, 0); + CopyMem (New, &gicc, sizeof (EFI_ACPI_6_4_GICC_AFFINITY_STRUCTURE)); + New += sizeof (EFI_ACPI_6_4_GICC_AFFINITY_STRUCTURE); + } + + // Perform Checksum + AcpiPlatformChecksum ((UINT8*) PageAddress, TableSize); + + Status = AcpiTable->InstallAcpiTable ( + AcpiTable, + (EFI_ACPI_COMMON_HEADER *)PageAddress, + TableSize, + &TableHandle + ); + if (EFI_ERROR(Status)) { + DEBUG ((DEBUG_ERROR, "Failed to install SRAT table\n")); + } + + return Status; +}
EFI_STATUS EFIAPI @@ -799,6 +883,13 @@ InitializeSbsaQemuAcpiDxe ( DEBUG ((DEBUG_ERROR, "Failed to add GTDT table\n")); }
+ if (SbsaQemuGetNumNumaNode() > 1){ + Status = AddSratTable (AcpiTable); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "Failed to add SRAT table\n")); + } + } + Status = DisableXhciOnOlderPlatVer (); if (EFI_ERROR (Status)) { DEBUG ((DEBUG_ERROR, "Failed to handle XHCI enablement\n")); diff --git a/Silicon/Qemu/SbsaQemu/Drivers/SbsaQemuAcpiDxe/SbsaQemuAcpiDxe.h b/Silicon/Qemu/SbsaQemu/Drivers/SbsaQemuAcpiDxe/SbsaQemuAcpiDxe.h index 7595df4c8a..1cf14952e5 100644 --- a/Silicon/Qemu/SbsaQemu/Drivers/SbsaQemuAcpiDxe/SbsaQemuAcpiDxe.h +++ b/Silicon/Qemu/SbsaQemu/Drivers/SbsaQemuAcpiDxe/SbsaQemuAcpiDxe.h @@ -39,6 +39,34 @@ typedef struct { EFI_ACPI_6_3_GTDT_SBSA_GENERIC_WATCHDOG_STRUCTURE mGwdt; } GENERIC_TIMER_DESCRIPTION_TABLES;
+#define SBSAQEMU_ACPI_MEMORY_AFFINITY_STRUCTURE_INIT( \ + ProximityDomain, Base, Length, Flags) \ + { \ + 1, /* Type */ \ + sizeof (EFI_ACPI_6_4_MEMORY_AFFINITY_STRUCTURE), /* Length */ \ + ProximityDomain, /* Proximity Domain */ \ + 0, /* Reserved */ \ + (Base) & 0xffffffff, /* Base Address Low */ \ + ((Base) >> 32) & 0xffffffff , /* Base Address High */ \ + (Length) & 0xffffffff, /* Length Low */ \ + ((Length) >> 32) & 0xffffffff, /* Length High */ \ + 0, /* Reserved */ \ + Flags, /* Flags */ \ + 0 /* Reserved */ \ + } + + +#define SBSAQEMU_ACPI_GICC_AFFINITY_STRUCTURE_INIT( \ + ProximityDomain, ACPIProcessorUID, Flags, ClockDomain) \ + { \ + 3, /* Type */ \ + sizeof (EFI_ACPI_6_4_GICC_AFFINITY_STRUCTURE), /* Length */ \ + ProximityDomain, /* Proximity Domain */ \ + ACPIProcessorUID, /* ACPI Processor UID */ \ + Flags, /* Flags */ \ + ClockDomain /* Clock Domain */ \ + } + #ifndef SYSTEM_TIMER_BASE_ADDRESS #define SYSTEM_TIMER_BASE_ADDRESS MAX_ADDRESS #endif diff --git a/Silicon/Qemu/SbsaQemu/Include/Library/QemuSbsaSmc.h b/Silicon/Qemu/SbsaQemu/Include/Library/QemuSbsaSmc.h index 4d9b8b95e7..db55d7e407 100644 --- a/Silicon/Qemu/SbsaQemu/Include/Library/QemuSbsaSmc.h +++ b/Silicon/Qemu/SbsaQemu/Include/Library/QemuSbsaSmc.h @@ -61,4 +61,14 @@ SbsaQemuGetMemInfo ( IN UINTN MemNode );
+/** + Get the number of numa node from device tree passed by Qemu. + + @retval the number of numa node. +**/ +UINT64 +SbsaQemuGetNumNumaNode ( + VOID + ); + #endif /* QEMU_SBSA_SMC_ */ diff --git a/Silicon/Qemu/SbsaQemu/Library/SbsaQemuSmc/SbsaQemuSmc.c b/Silicon/Qemu/SbsaQemu/Library/SbsaQemuSmc/SbsaQemuSmc.c index f7c5308a63..4c0a0af1e4 100644 --- a/Silicon/Qemu/SbsaQemu/Library/SbsaQemuSmc/SbsaQemuSmc.c +++ b/Silicon/Qemu/SbsaQemu/Library/SbsaQemuSmc/SbsaQemuSmc.c @@ -97,4 +97,26 @@ SbsaQemuGetMemInfo ( }
return MemInfo; +} + +UINT64 +SbsaQemuGetNumNumaNode ( + VOID +) +{ + UINT64 Arg; + UINT32 Index; + UINT32 NumNumaNodes; + UINT32 NumCores = PcdGet32 (PcdCoreCount); + + NumNumaNodes = 1; + for (Index = 0; Index < NumCores; Index ++){ + Arg = SbsaQemuGetCpuNumaNode(Index); + + if (NumNumaNodes == 1 || NumNumaNodes < (Arg + 1)){ + NumNumaNodes = Arg + 1; + } + } + + return NumNumaNodes; } \ No newline at end of file diff --git a/Silicon/Qemu/SbsaQemu/Library/SbsaQemuSmc/SbsaQemuSmc.inf b/Silicon/Qemu/SbsaQemu/Library/SbsaQemuSmc/SbsaQemuSmc.inf index 3eae73650e..95302b6895 100644 --- a/Silicon/Qemu/SbsaQemu/Library/SbsaQemuSmc/SbsaQemuSmc.inf +++ b/Silicon/Qemu/SbsaQemu/Library/SbsaQemuSmc/SbsaQemuSmc.inf @@ -26,3 +26,6 @@ ArmSmcLib BaseMemoryLib DebugLib + +[Pcd] + gArmVirtSbsaQemuPlatformTokenSpaceGuid.PcdCoreCount
W dniu 13.12.2023 o 10:20, Xiong Yining pisze:
Add support to create SRAT(System resource affinity table) for sbsa platform at runtime.
Signed-off-by: Xiong Yining xiongyining1480@phytium.com.cn Signed-off-by: Chen Baozi chenbaozi@phytium.com.cn
.../Drivers/SbsaQemuAcpiDxe/SbsaQemuAcpiDxe.c | 91 +++++++++++++++++++ .../Drivers/SbsaQemuAcpiDxe/SbsaQemuAcpiDxe.h | 28 ++++++ .../SbsaQemu/Include/Library/QemuSbsaSmc.h | 10 ++ .../Library/SbsaQemuSmc/SbsaQemuSmc.c | 22 +++++ .../Library/SbsaQemuSmc/SbsaQemuSmc.inf | 3 + 5 files changed, 154 insertions(+)
diff --git a/Silicon/Qemu/SbsaQemu/Drivers/SbsaQemuAcpiDxe/SbsaQemuAcpiDxe.c b/Silicon/Qemu/SbsaQemu/Drivers/SbsaQemuAcpiDxe/SbsaQemuAcpiDxe.c index 89c4b797d2..abbf891514 100644 --- a/Silicon/Qemu/SbsaQemu/Drivers/SbsaQemuAcpiDxe/SbsaQemuAcpiDxe.c +++ b/Silicon/Qemu/SbsaQemu/Drivers/SbsaQemuAcpiDxe/SbsaQemuAcpiDxe.c @@ -748,6 +748,90 @@ DisableXhciOnOlderPlatVer ( return Status; } +/*
- A function that adds the SRAT ACPI table.
- */
+EFI_STATUS +AddSratTable (
- IN EFI_ACPI_TABLE_PROTOCOL *AcpiTable
- )
+{
- EFI_STATUS Status;
- UINT8 *New;
- EFI_PHYSICAL_ADDRESS PageAddress;
- UINTN TableHandle;
- UINT32 TableSize;
- UINT32 Index;
- UINT32 NodeId;
- UINT32 NumMemNode;
NumMemNodeCount or NumMemNodes
- MemoryInfo MemInfo;
- UINT32 NumCores = PcdGet32 (PcdCoreCount);
- // Initialize MADT ACPI Header
s/MADT/SRAT/
- EFI_ACPI_6_4_SYSTEM_RESOURCE_AFFINITY_TABLE_HEADER Header = {
SBSAQEMU_ACPI_HEADER (EFI_ACPI_6_4_SYSTEM_RESOURCE_AFFINITY_TABLE_SIGNATURE,EFI_ACPI_6_4_SYSTEM_RESOURCE_AFFINITY_TABLE_HEADER,EFI_ACPI_6_4_SYSTEM_RESOURCE_AFFINITY_TABLE_REVISION),1, 0 };- NumMemNode = SbsaQemuGetMemNode();
- // Calculate the new table size based on the number of cores
- TableSize = sizeof (EFI_ACPI_6_4_SYSTEM_RESOURCE_AFFINITY_TABLE_HEADER) +
(sizeof (EFI_ACPI_6_4_MEMORY_AFFINITY_STRUCTURE) * NumMemNode) +(sizeof (EFI_ACPI_6_4_GICC_AFFINITY_STRUCTURE) * NumCores);- Status = gBS->AllocatePages (
AllocateAnyPages,EfiACPIReclaimMemory,EFI_SIZE_TO_PAGES (TableSize),&PageAddress);- if (EFI_ERROR(Status)) {
- DEBUG ((DEBUG_ERROR, "Failed to allocate pages for SRAT table\n"));
- return EFI_OUT_OF_RESOURCES;
- }
- New = (UINT8 *)(UINTN) PageAddress;
- ZeroMem (New, TableSize);
- // Add the ACPI Description table header
- CopyMem (New, &Header, sizeof (EFI_ACPI_6_4_SYSTEM_RESOURCE_AFFINITY_TABLE_HEADER));
- ((EFI_ACPI_DESCRIPTION_HEADER*) New)->Length = TableSize;
- New += sizeof (EFI_ACPI_6_4_SYSTEM_RESOURCE_AFFINITY_TABLE_HEADER);
- // Add new memory structures
s/new//
- for (Index = 0; Index < NumMemNode; Index++) {
- MemInfo = SbsaQemuGetMemInfo (Index);
- EFI_ACPI_6_4_MEMORY_AFFINITY_STRUCTURE memory = SBSAQEMU_ACPI_MEMORY_AFFINITY_STRUCTURE_INIT (MemInfo.NodeId, MemInfo.AddressBase, MemInfo.AddressSize, 1);
- CopyMem (New, &memory, sizeof (EFI_ACPI_6_4_MEMORY_AFFINITY_STRUCTURE));
- New += sizeof (EFI_ACPI_6_4_MEMORY_AFFINITY_STRUCTURE);
- }
- // Add new professor structures for the Cores
s/professor/processor/ s/Cores/cores
- for (Index = 0; Index < NumCores; Index++) {
- NodeId = SbsaQemuGetCpuNumaNode(Index);
- EFI_ACPI_6_4_GICC_AFFINITY_STRUCTURE gicc = SBSAQEMU_ACPI_GICC_AFFINITY_STRUCTURE_INIT(NodeId, Index, 1, 0);
- CopyMem (New, &gicc, sizeof (EFI_ACPI_6_4_GICC_AFFINITY_STRUCTURE));
- New += sizeof (EFI_ACPI_6_4_GICC_AFFINITY_STRUCTURE);
- }
- // Perform Checksum
- AcpiPlatformChecksum ((UINT8*) PageAddress, TableSize);
- Status = AcpiTable->InstallAcpiTable (
AcpiTable,(EFI_ACPI_COMMON_HEADER *)PageAddress,TableSize,&TableHandle);- if (EFI_ERROR(Status)) {
- DEBUG ((DEBUG_ERROR, "Failed to install SRAT table\n"));
- }
- return Status;
+} EFI_STATUS EFIAPI @@ -799,6 +883,13 @@ InitializeSbsaQemuAcpiDxe ( DEBUG ((DEBUG_ERROR, "Failed to add GTDT table\n")); }
- if (SbsaQemuGetNumNumaNode() > 1){
Can we put SRAT for 1 node system or is it against rules?
- Status = AddSratTable (AcpiTable);
- if (EFI_ERROR (Status)) {
DEBUG ((DEBUG_ERROR, "Failed to add SRAT table\n"));- }
- }
- Status = DisableXhciOnOlderPlatVer (); if (EFI_ERROR (Status)) { DEBUG ((DEBUG_ERROR, "Failed to handle XHCI enablement\n"));
diff --git a/Silicon/Qemu/SbsaQemu/Drivers/SbsaQemuAcpiDxe/SbsaQemuAcpiDxe.h b/Silicon/Qemu/SbsaQemu/Drivers/SbsaQemuAcpiDxe/SbsaQemuAcpiDxe.h index 7595df4c8a..1cf14952e5 100644 --- a/Silicon/Qemu/SbsaQemu/Drivers/SbsaQemuAcpiDxe/SbsaQemuAcpiDxe.h +++ b/Silicon/Qemu/SbsaQemu/Drivers/SbsaQemuAcpiDxe/SbsaQemuAcpiDxe.h @@ -39,6 +39,34 @@ typedef struct { EFI_ACPI_6_3_GTDT_SBSA_GENERIC_WATCHDOG_STRUCTURE mGwdt; } GENERIC_TIMER_DESCRIPTION_TABLES; +#define SBSAQEMU_ACPI_MEMORY_AFFINITY_STRUCTURE_INIT( \
ProximityDomain, Base, Length, Flags) \- { \
- 1, /* Type */ \
- sizeof (EFI_ACPI_6_4_MEMORY_AFFINITY_STRUCTURE), /* Length */ \
- ProximityDomain, /* Proximity Domain */ \
- 0, /* Reserved */ \
- (Base) & 0xffffffff, /* Base Address Low */ \
- ((Base) >> 32) & 0xffffffff , /* Base Address High */ \
- (Length) & 0xffffffff, /* Length Low */ \
- ((Length) >> 32) & 0xffffffff, /* Length High */ \
- 0, /* Reserved */ \
- Flags, /* Flags */ \
- 0 /* Reserved */ \
- }
+#define SBSAQEMU_ACPI_GICC_AFFINITY_STRUCTURE_INIT( \
ProximityDomain, ACPIProcessorUID, Flags, ClockDomain) \- { \
- 3, /* Type */ \
- sizeof (EFI_ACPI_6_4_GICC_AFFINITY_STRUCTURE), /* Length */ \
- ProximityDomain, /* Proximity Domain */ \
- ACPIProcessorUID, /* ACPI Processor UID */ \
- Flags, /* Flags */ \
- ClockDomain /* Clock Domain */ \
- }
- #ifndef SYSTEM_TIMER_BASE_ADDRESS #define SYSTEM_TIMER_BASE_ADDRESS MAX_ADDRESS #endif
diff --git a/Silicon/Qemu/SbsaQemu/Include/Library/QemuSbsaSmc.h b/Silicon/Qemu/SbsaQemu/Include/Library/QemuSbsaSmc.h index 4d9b8b95e7..db55d7e407 100644 --- a/Silicon/Qemu/SbsaQemu/Include/Library/QemuSbsaSmc.h +++ b/Silicon/Qemu/SbsaQemu/Include/Library/QemuSbsaSmc.h @@ -61,4 +61,14 @@ SbsaQemuGetMemInfo ( IN UINTN MemNode ); +/**
- Get the number of numa node from device tree passed by Qemu.
- @retval the number of numa node.
+**/ +UINT64 +SbsaQemuGetNumNumaNode (
- VOID
- );
- #endif /* QEMU_SBSA_SMC_ */
diff --git a/Silicon/Qemu/SbsaQemu/Library/SbsaQemuSmc/SbsaQemuSmc.c b/Silicon/Qemu/SbsaQemu/Library/SbsaQemuSmc/SbsaQemuSmc.c index f7c5308a63..4c0a0af1e4 100644 --- a/Silicon/Qemu/SbsaQemu/Library/SbsaQemuSmc/SbsaQemuSmc.c +++ b/Silicon/Qemu/SbsaQemu/Library/SbsaQemuSmc/SbsaQemuSmc.c @@ -97,4 +97,26 @@ SbsaQemuGetMemInfo ( } return MemInfo; +}
+UINT64 +SbsaQemuGetNumNumaNode (
- VOID
+) +{
- UINT64 Arg;
- UINT32 Index;
- UINT32 NumNumaNodes;
- UINT32 NumCores = PcdGet32 (PcdCoreCount);
- NumNumaNodes = 1;
- for (Index = 0; Index < NumCores; Index ++){
For multiple numa nodes I use those qemu arguments:
-smp 4,sockets=4,maxcpus=4 -m 4G,slots=2,maxmem=5G -object memory-backend-ram,size=3G,id=m0 -object memory-backend-ram,size=1G,id=m1 -numa node,nodeid=0,cpus=0-1,memdev=m0 -numa node,nodeid=1,memdev=m1,initiator=0 -numa node,nodeid=2,cpus=2-3
("--numa" argument to my "boot-sbsa-ref.sh" script [1])
1. https://github.com/hrw/sbsa-ref-status/blob/main/boot-sbsa-ref.sh
Node 1 has no cpus. Will your code report 2 numa nodes or 3?
Or if I move memory only node to be last:
-numa node,nodeid=0,cpus=0-1,memdev=m0 -numa node,nodeid=1,cpus=2-3 -numa node,nodeid=2,memdev=m1,initiator=0
Then function will see node 0, node 1 and finish.
Maybe it should walk through cpu nodes and memory nodes and check numa node ids? (do we ignore no-cpu, no-memory numa nodes here?)
- Arg = SbsaQemuGetCpuNumaNode(Index);
- if (NumNumaNodes == 1 || NumNumaNodes < (Arg + 1)){
NumNumaNodes = Arg + 1;- }
- }
- return NumNumaNodes; }
\ No newline at end of file diff --git a/Silicon/Qemu/SbsaQemu/Library/SbsaQemuSmc/SbsaQemuSmc.inf b/Silicon/Qemu/SbsaQemu/Library/SbsaQemuSmc/SbsaQemuSmc.inf index 3eae73650e..95302b6895 100644 --- a/Silicon/Qemu/SbsaQemu/Library/SbsaQemuSmc/SbsaQemuSmc.inf +++ b/Silicon/Qemu/SbsaQemu/Library/SbsaQemuSmc/SbsaQemuSmc.inf @@ -26,3 +26,6 @@ ArmSmcLib BaseMemoryLib DebugLib
+[Pcd]
- gArmVirtSbsaQemuPlatformTokenSpaceGuid.PcdCoreCount
-
Marcin Juszkiewicz -
Xiong Yining