649 lines
16 KiB
C
649 lines
16 KiB
C
/*++
|
|
|
|
Copyright (c) 2000 Microsoft Corporation
|
|
|
|
Module Name:
|
|
|
|
allproc.c
|
|
|
|
Abstract:
|
|
|
|
This module allocates and initializes kernel resources required to
|
|
start a new processor, and passes a complete process state structure
|
|
to the hal to obtain a new processor.
|
|
|
|
Author:
|
|
|
|
David N. Cutler (davec) 5-May-2000
|
|
|
|
Environment:
|
|
|
|
Kernel mode only.
|
|
|
|
Revision History:
|
|
|
|
--*/
|
|
|
|
#include "ki.h"
|
|
#include "pool.h"
|
|
|
|
//
|
|
// Define local macros.
|
|
//
|
|
|
|
#define ROUNDUP16(x) (((x) + 15) & ~15)
|
|
|
|
//
|
|
// Define prototypes for forward referenced functions.
|
|
//
|
|
|
|
#if !defined(NT_UP)
|
|
|
|
VOID
|
|
KiCopyDescriptorMemory (
|
|
IN PKDESCRIPTOR Source,
|
|
IN PKDESCRIPTOR Destination,
|
|
IN PVOID Base
|
|
);
|
|
|
|
NTSTATUS
|
|
KiNotNumaQueryProcessorNode (
|
|
IN ULONG ProcessorNumber,
|
|
OUT PUSHORT Identifier,
|
|
OUT PUCHAR Node
|
|
);
|
|
|
|
VOID
|
|
KiSetDescriptorBase (
|
|
IN USHORT Selector,
|
|
IN PKGDTENTRY64 GdtBase,
|
|
IN PVOID Base
|
|
);
|
|
|
|
PHALNUMAQUERYPROCESSORNODE KiQueryProcessorNode = KiNotNumaQueryProcessorNode;
|
|
|
|
//
|
|
// Statically allocate enough KNODE structures to allow memory management
|
|
// to allocate pages by node during system initialization. As processors
|
|
// are brought online, real KNODE structures are allocated in the correct
|
|
// memory for the node.
|
|
//
|
|
|
|
#pragma data_seg("INITDATA")
|
|
|
|
KNODE KiNodeInit[MAXIMUM_CCNUMA_NODES];
|
|
|
|
#pragma data_seg()
|
|
|
|
#pragma alloc_text(INIT, KiAllProcessorsStarted)
|
|
#pragma alloc_text(INIT, KiCopyDescriptorMemory)
|
|
#pragma alloc_text(INIT, KiNotNumaQueryProcessorNode)
|
|
#pragma alloc_text(INIT, KiSetDescriptorBase)
|
|
|
|
#endif // !defined(NT_UP)
|
|
|
|
#pragma alloc_text(INIT, KeStartAllProcessors)
|
|
|
|
ULONG KiBarrierWait = 0;
|
|
|
|
VOID
|
|
KeStartAllProcessors (
|
|
VOID
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This function is called during phase 1 initialization on the master boot
|
|
processor to start all of the other registered processors.
|
|
|
|
Arguments:
|
|
|
|
None.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
--*/
|
|
|
|
{
|
|
|
|
#if !defined(NT_UP)
|
|
|
|
ULONG AllocationSize;
|
|
PUCHAR Base;
|
|
PKPCR CurrentPcr = KeGetPcr();
|
|
PVOID DataBlock;
|
|
PVOID DpcStack;
|
|
PKGDTENTRY64 GdtBase;
|
|
ULONG GdtOffset;
|
|
ULONG IdtOffset;
|
|
PVOID KernelStack;
|
|
PKNODE Node;
|
|
UCHAR NodeNumber;
|
|
UCHAR Number;
|
|
PKPCR PcrBase;
|
|
USHORT ProcessorId;
|
|
KPROCESSOR_STATE ProcessorState;
|
|
NTSTATUS Status;
|
|
PKTSS64 SysTssBase;
|
|
PETHREAD Thread;
|
|
|
|
//
|
|
// Do not start additional processors if the RELOCATEPHYSICAL loader
|
|
// switch has been specified.
|
|
//
|
|
|
|
if (KeLoaderBlock->LoadOptions != NULL) {
|
|
if (strstr(KeLoaderBlock->LoadOptions, "RELOCATEPHYSICAL") != NULL) {
|
|
return;
|
|
}
|
|
}
|
|
|
|
//
|
|
// If processor zero is not on node zero, then move it to the appropriate
|
|
// node.
|
|
//
|
|
|
|
if (KeNumberNodes > 1) {
|
|
Status = KiQueryProcessorNode(0, &ProcessorId, &NodeNumber);
|
|
if (NT_SUCCESS(Status)) {
|
|
|
|
//
|
|
// Adjust the data structures to reflect that P0 is not on Node 0.
|
|
//
|
|
|
|
if (NodeNumber != 0) {
|
|
|
|
ASSERT(KeNodeBlock[0] == &KiNode0);
|
|
|
|
KeNodeBlock[0]->ProcessorMask &= ~1;
|
|
KiNodeInit[0] = *KeNodeBlock[0];
|
|
KeNodeBlock[0] = &KiNodeInit[0];
|
|
KiNode0 = *KeNodeBlock[NodeNumber];
|
|
KeNodeBlock[NodeNumber] = &KiNode0;
|
|
KeNodeBlock[NodeNumber]->ProcessorMask |= 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
//
|
|
// Calculate the size of the per processor data structures.
|
|
//
|
|
// This includes:
|
|
//
|
|
// PCR (including the PRCB)
|
|
// System TSS
|
|
// Idle Thread Object
|
|
// Double Fault/NMI Panic Stack
|
|
// Machine Check Stack
|
|
// GDT
|
|
// IDT
|
|
//
|
|
// If this is a multinode system, the KNODE structure is also allocated.
|
|
//
|
|
// A DPC and Idle stack are also allocated, but they are done separately.
|
|
//
|
|
|
|
AllocationSize = ROUNDUP16(sizeof(KPCR)) +
|
|
ROUNDUP16(sizeof(KTSS64)) +
|
|
ROUNDUP16(sizeof(ETHREAD)) +
|
|
ROUNDUP16(DOUBLE_FAULT_STACK_SIZE) +
|
|
ROUNDUP16(KERNEL_MCA_EXCEPTION_STACK_SIZE);
|
|
|
|
AllocationSize += ROUNDUP16(sizeof(KNODE));
|
|
|
|
//
|
|
// Save the offset of the GDT in the allocation structure and add in
|
|
// the size of the GDT.
|
|
//
|
|
|
|
GdtOffset = AllocationSize;
|
|
AllocationSize +=
|
|
CurrentPcr->Prcb.ProcessorState.SpecialRegisters.Gdtr.Limit + 1;
|
|
|
|
//
|
|
// Save the offset of the IDT in the allocation structure and add in
|
|
// the size of the IDT.
|
|
//
|
|
|
|
IdtOffset = AllocationSize;
|
|
AllocationSize +=
|
|
CurrentPcr->Prcb.ProcessorState.SpecialRegisters.Idtr.Limit + 1;
|
|
|
|
//
|
|
// If the registered number of processors is greater than the maximum
|
|
// number of processors supported, then only allow the maximum number
|
|
// of supported processors.
|
|
//
|
|
|
|
if (KeRegisteredProcessors > MAXIMUM_PROCESSORS) {
|
|
KeRegisteredProcessors = MAXIMUM_PROCESSORS;
|
|
}
|
|
|
|
//
|
|
// Set barrier that will prevent any other processor from entering the
|
|
// idle loop until all processors have been started.
|
|
//
|
|
|
|
KiBarrierWait = 1;
|
|
|
|
//
|
|
// Initialize the fixed part of the processor state that will be used to
|
|
// start processors. Each processor starts in the system initialization
|
|
// code with address of the loader parameter block as an argument.
|
|
//
|
|
|
|
RtlZeroMemory(&ProcessorState, sizeof(KPROCESSOR_STATE));
|
|
ProcessorState.ContextFrame.Rcx = (ULONG64)KeLoaderBlock;
|
|
ProcessorState.ContextFrame.Rip = (ULONG64)KiSystemStartup;
|
|
ProcessorState.ContextFrame.SegCs = KGDT64_R0_CODE;
|
|
ProcessorState.ContextFrame.SegDs = KGDT64_R3_DATA | RPL_MASK;
|
|
ProcessorState.ContextFrame.SegEs = KGDT64_R3_DATA | RPL_MASK;
|
|
ProcessorState.ContextFrame.SegFs = KGDT64_R3_CMTEB | RPL_MASK;
|
|
ProcessorState.ContextFrame.SegGs = KGDT64_R3_DATA | RPL_MASK;
|
|
ProcessorState.ContextFrame.SegSs = KGDT64_NULL;
|
|
|
|
//
|
|
// Loop trying to start a new processors until a new processor can't be
|
|
// started or an allocation failure occurs.
|
|
//
|
|
|
|
Number = 0;
|
|
while ((ULONG)KeNumberProcessors < KeRegisteredProcessors) {
|
|
Number += 1;
|
|
Status = KiQueryProcessorNode(Number, &ProcessorId, &NodeNumber);
|
|
if (!NT_SUCCESS(Status)) {
|
|
|
|
//
|
|
// No such processor, advance to next.
|
|
//
|
|
|
|
continue;
|
|
}
|
|
|
|
Node = KeNodeBlock[NodeNumber];
|
|
|
|
//
|
|
// Allocate memory for the new processor specific data. If the
|
|
// allocation fails, then stop starting processors.
|
|
//
|
|
|
|
DataBlock = MmAllocateIndependentPages(AllocationSize, NodeNumber);
|
|
if (DataBlock == NULL) {
|
|
break;
|
|
}
|
|
|
|
//
|
|
// Allocate a pool tag table for the new processor.
|
|
//
|
|
|
|
if (ExCreatePoolTagTable(Number, NodeNumber) == NULL) {
|
|
MmFreeIndependentPages(DataBlock, AllocationSize);
|
|
break;
|
|
}
|
|
|
|
//
|
|
// Zero the allocated memory.
|
|
//
|
|
|
|
Base = (PUCHAR)DataBlock;
|
|
RtlZeroMemory(DataBlock, AllocationSize);
|
|
|
|
//
|
|
// Copy and initialize the GDT for the next processor.
|
|
//
|
|
|
|
KiCopyDescriptorMemory(&CurrentPcr->Prcb.ProcessorState.SpecialRegisters.Gdtr,
|
|
&ProcessorState.SpecialRegisters.Gdtr,
|
|
Base + GdtOffset);
|
|
|
|
GdtBase = (PKGDTENTRY64)ProcessorState.SpecialRegisters.Gdtr.Base;
|
|
|
|
//
|
|
// Copy and initialize the IDT for the next processor.
|
|
//
|
|
|
|
KiCopyDescriptorMemory(&CurrentPcr->Prcb.ProcessorState.SpecialRegisters.Idtr,
|
|
&ProcessorState.SpecialRegisters.Idtr,
|
|
Base + IdtOffset);
|
|
|
|
//
|
|
// Set the PCR base address for the next processor and set the
|
|
// processor number.
|
|
//
|
|
// N.B. The PCR address is passed to the next processor by computing
|
|
// the containing address with respect to the PRCB.
|
|
//
|
|
|
|
PcrBase = (PKPCR)Base;
|
|
PcrBase->Number = Number;
|
|
PcrBase->Prcb.Number = Number;
|
|
Base += ROUNDUP16(sizeof(KPCR));
|
|
|
|
//
|
|
// Set the system TSS descriptor base for the next processor.
|
|
//
|
|
|
|
SysTssBase = (PKTSS64)Base;
|
|
KiSetDescriptorBase(KGDT64_SYS_TSS / 16, GdtBase, SysTssBase);
|
|
Base += ROUNDUP16(sizeof(KTSS64));
|
|
|
|
//
|
|
// Initialize the panic stack address for double fault and NMI.
|
|
//
|
|
|
|
Base += DOUBLE_FAULT_STACK_SIZE;
|
|
SysTssBase->Ist[TSS_IST_PANIC] = (ULONG64)Base;
|
|
|
|
//
|
|
// Initialize the machine check stack address.
|
|
//
|
|
|
|
Base += KERNEL_MCA_EXCEPTION_STACK_SIZE;
|
|
SysTssBase->Ist[TSS_IST_MCA] = (ULONG64)Base;
|
|
|
|
//
|
|
// Idle Thread thread object.
|
|
//
|
|
|
|
Thread = (PETHREAD)Base;
|
|
Base += ROUNDUP16(sizeof(ETHREAD));
|
|
|
|
//
|
|
// Set other special registers in the processor state.
|
|
//
|
|
|
|
ProcessorState.SpecialRegisters.Cr0 = ReadCR0();
|
|
ProcessorState.SpecialRegisters.Cr3 = ReadCR3();
|
|
ProcessorState.ContextFrame.EFlags = 0;
|
|
ProcessorState.SpecialRegisters.Tr = KGDT64_SYS_TSS;
|
|
GdtBase[KGDT64_SYS_TSS / 16].Bytes.Flags1 = 0x89;
|
|
ProcessorState.SpecialRegisters.Cr4 = ReadCR4();
|
|
|
|
//
|
|
// Allocate a kernel stack and a DPC stack for the next processor.
|
|
//
|
|
|
|
KernelStack = MmCreateKernelStack(FALSE, NodeNumber);
|
|
if (KernelStack == NULL) {
|
|
MmFreeIndependentPages(DataBlock, AllocationSize);
|
|
break;
|
|
}
|
|
|
|
DpcStack = MmCreateKernelStack(FALSE, NodeNumber);
|
|
if (DpcStack == NULL) {
|
|
MmDeleteKernelStack(KernelStack, FALSE);
|
|
MmFreeIndependentPages(DataBlock, AllocationSize);
|
|
break;
|
|
}
|
|
|
|
//
|
|
// Initialize the kernel stack for the system TSS.
|
|
//
|
|
|
|
SysTssBase->Rsp0 = (ULONG64)KernelStack - sizeof(PVOID) * 4;
|
|
ProcessorState.ContextFrame.Rsp = (ULONG64)KernelStack;
|
|
|
|
//
|
|
// If this is the first processor on this node, then use the space
|
|
// allocated for KNODE as the KNODE.
|
|
//
|
|
|
|
if (KeNodeBlock[NodeNumber] == &KiNodeInit[NodeNumber]) {
|
|
Node = (PKNODE)Base;
|
|
*Node = KiNodeInit[NodeNumber];
|
|
KeNodeBlock[NodeNumber] = Node;
|
|
}
|
|
|
|
Base += ROUNDUP16(sizeof(KNODE));
|
|
PcrBase->Prcb.ParentNode = Node;
|
|
|
|
//
|
|
// Adjust the loader block so it has the next processor state. Ensure
|
|
// that the KernelStack has space for home registers for up to four
|
|
// parameters.
|
|
//
|
|
|
|
KeLoaderBlock->KernelStack = (ULONG64)DpcStack - (sizeof(PVOID) * 4);
|
|
KeLoaderBlock->Thread = (ULONG64)Thread;
|
|
KeLoaderBlock->Prcb = (ULONG64)(&PcrBase->Prcb);
|
|
|
|
//
|
|
// Attempt to start the next processor. If a processor cannot be
|
|
// started, then deallocate memory and stop starting processors.
|
|
//
|
|
|
|
if (HalStartNextProcessor(KeLoaderBlock, &ProcessorState) == 0) {
|
|
ExDeletePoolTagTable (Number);
|
|
MmFreeIndependentPages(DataBlock, AllocationSize);
|
|
MmDeleteKernelStack(KernelStack, FALSE);
|
|
MmDeleteKernelStack(DpcStack, FALSE);
|
|
break;
|
|
}
|
|
|
|
Node->ProcessorMask |= AFFINITY_MASK(Number);
|
|
|
|
//
|
|
// Wait for processor to initialize.
|
|
//
|
|
|
|
while (*((volatile ULONG64 *)&KeLoaderBlock->Prcb) != 0) {
|
|
KeYieldProcessor();
|
|
}
|
|
}
|
|
|
|
//
|
|
// All processors have been stated.
|
|
//
|
|
|
|
KiAllProcessorsStarted();
|
|
|
|
//
|
|
// Reset and synchronize the performance counters of all processors, by
|
|
// applying a null adjustment to the interrupt time
|
|
//
|
|
|
|
KeAdjustInterruptTime(0);
|
|
|
|
//
|
|
// Allow all processors that were started to enter the idle loop and
|
|
// begin execution.
|
|
//
|
|
|
|
KiBarrierWait = 0;
|
|
|
|
#endif // !defined(NT_UP)
|
|
|
|
return;
|
|
}
|
|
|
|
#if !defined(NT_UP)
|
|
|
|
VOID
|
|
KiSetDescriptorBase (
|
|
IN USHORT Selector,
|
|
IN PKGDTENTRY64 GdtBase,
|
|
IN PVOID Base
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This function sets the base address of a descriptor to the specified
|
|
base address.
|
|
|
|
Arguments:
|
|
|
|
Selector - Supplies the selector for the descriptor.
|
|
|
|
GdtBase - Supplies a pointer to the GDT.
|
|
|
|
Base - Supplies a pointer to the base address.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
--*/
|
|
|
|
{
|
|
|
|
GdtBase = &GdtBase[Selector];
|
|
GdtBase->BaseLow = (USHORT)((ULONG64)Base);
|
|
GdtBase->Bytes.BaseMiddle = (UCHAR)((ULONG64)Base >> 16);
|
|
GdtBase->Bytes.BaseHigh = (UCHAR)((ULONG64)Base >> 24);
|
|
GdtBase->BaseUpper = (ULONG)((ULONG64)Base >> 32);
|
|
return;
|
|
}
|
|
|
|
VOID
|
|
KiCopyDescriptorMemory (
|
|
IN PKDESCRIPTOR Source,
|
|
IN PKDESCRIPTOR Destination,
|
|
IN PVOID Base
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This function copies the specified descriptor memory to the new memory
|
|
and initializes a descriptor for the new memory.
|
|
|
|
Arguments:
|
|
|
|
Source - Supplies a pointer to the source descriptor that describes
|
|
the memory to copy.
|
|
|
|
Destination - Supplies a pointer to the destination descriptor to be
|
|
initialized.
|
|
|
|
Base - Supplies a pointer to the new memory.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
--*/
|
|
|
|
{
|
|
|
|
Destination->Limit = Source->Limit;
|
|
Destination->Base = Base;
|
|
RtlCopyMemory(Base, Source->Base, Source->Limit + 1);
|
|
return;
|
|
}
|
|
|
|
VOID
|
|
KiAllProcessorsStarted (
|
|
VOID
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine is called once all processors in the system have been started.
|
|
|
|
Arguments:
|
|
|
|
None.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
--*/
|
|
|
|
{
|
|
|
|
ULONG i;
|
|
|
|
//
|
|
// Make sure there are no references to the temporary nodes used during
|
|
// initialization.
|
|
//
|
|
|
|
for (i = 0; i < KeNumberNodes; i += 1) {
|
|
if (KeNodeBlock[i] == &KiNodeInit[i]) {
|
|
|
|
//
|
|
// No processor started on this node so no new node structure has
|
|
// been allocated. This is possible if the node contains memory
|
|
// only or IO busses. At this time we need to allocate a permanent
|
|
// node structure for the node.
|
|
//
|
|
|
|
KeNodeBlock[i] = ExAllocatePoolWithTag(NonPagedPool,
|
|
sizeof(KNODE),
|
|
' eK');
|
|
|
|
if (KeNodeBlock[i]) {
|
|
*KeNodeBlock[i] = KiNodeInit[i];
|
|
}
|
|
}
|
|
|
|
//
|
|
// Set the node number.
|
|
//
|
|
|
|
KeNodeBlock[i]->NodeNumber = (UCHAR)i;
|
|
}
|
|
|
|
for (i = KeNumberNodes; i < MAXIMUM_CCNUMA_NODES; i += 1) {
|
|
KeNodeBlock[i] = NULL;
|
|
}
|
|
|
|
if (KeNumberNodes == 1) {
|
|
|
|
//
|
|
// For Non NUMA machines, Node 0 gets all processors.
|
|
//
|
|
|
|
KeNodeBlock[0]->ProcessorMask = KeActiveProcessors;
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
NTSTATUS
|
|
KiNotNumaQueryProcessorNode (
|
|
IN ULONG ProcessorNumber,
|
|
OUT PUSHORT Identifier,
|
|
OUT PUCHAR Node
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
This routine is a stub used on non NUMA systems to provide a
|
|
consistent method of determining the NUMA configuration rather
|
|
than checking for the presense of multiple nodes inline.
|
|
|
|
Arguments:
|
|
|
|
ProcessorNumber supplies the system logical processor number.
|
|
Identifier supplies the address of a variable to receive
|
|
the unique identifier for this processor.
|
|
NodeNumber supplies the address of a variable to receive
|
|
the number of the node this processor resides on.
|
|
|
|
Return Value:
|
|
|
|
Returns success.
|
|
|
|
--*/
|
|
|
|
{
|
|
*Identifier = (USHORT)ProcessorNumber;
|
|
*Node = 0;
|
|
return STATUS_SUCCESS;
|
|
}
|
|
|
|
#endif // !defined(NT_UP)
|