diff --git a/README.md b/README.md index 2999d9671c..c0910cbbab 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ for the original description of the approach. | RTX 3090 | Pairwise NVLink where available, PCIe BAR1 otherwise | | RTX 4090 | PCIe BAR1 | | RTX 5090 | PCIe BAR1 | +| RTX 5060 Ti / 5060 (GB206) | PCIe BAR1, including with a display attached | P2P also works between different devices of the same generation, for example RTX 5090 to RTX PRO 6000 Blackwell. @@ -23,6 +24,10 @@ This enables BAR1 P2P on consumer GPUs where NVLink isn't available, and falls b NVLink where it is. For PCIe pairs, transfers write directly to the other GPU's physical address over DMA. +On GB206 cards (RTX 5060 Ti / 5060) BAR1 P2P works even when a display is attached: the +static BAR1 window is placed above the console reservation instead of requiring all of +BAR1, and allocations that don't fit in the window fall back to dynamic mappings. + > [!WARNING] > IOMMU must be in passthrough mode (`iommu=pt`), not translating, or DMA will go through > IOMMU page tables and transfers will fail. This is very dangerous if you run untrusted diff --git a/src/nvidia/src/kernel/gpu/bus/arch/maxwell/kern_bus_gm200.c b/src/nvidia/src/kernel/gpu/bus/arch/maxwell/kern_bus_gm200.c index 1d56500465..6711be5e07 100644 --- a/src/nvidia/src/kernel/gpu/bus/arch/maxwell/kern_bus_gm200.c +++ b/src/nvidia/src/kernel/gpu/bus/arch/maxwell/kern_bus_gm200.c @@ -37,6 +37,59 @@ ((PCIE_P2P_WRITE_MAILBOX_SIZE << DRF_SIZE(NV_P2P_WMBOX_ADDR_ADDR)) - \ PCIE_P2P_WRITE_MAILBOX_SIZE) +static NV_STATUS +_kbusSetupMailboxes_GM200 +( + OBJGPU *pGpu0, + KernelBus *pKernelBus0, + OBJGPU *pGpu1, + KernelBus *pKernelBus1, + NvU32 local2Remote, + NvU32 remote2Local, + NvBool *pbLocalMailboxTeardownAttempted, + NvBool *pbRemoteMailboxTeardownAttempted +); + +static NV_STATUS +_kbusProgramPciePeerMask_GM200 +( + OBJGPU *pGpu, + NvU32 peerMask +) +{ + RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu); + NV2080_CTRL_INTERNAL_HSHUB_PEER_CONN_CONFIG_PARAMS params = {0}; + + params.programPciePeerMask = peerMask; + + return pRmApi->Control(pRmApi, + pGpu->hInternalClient, + pGpu->hInternalSubdevice, + NV2080_CTRL_CMD_INTERNAL_HSHUB_PEER_CONN_CONFIG, + ¶ms, + sizeof(params)); +} + +static NV_STATUS +_kbusInvalidatePeerMask_GM200 +( + OBJGPU *pGpu, + NvU32 peerMask +) +{ + RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu); + NV2080_CTRL_INTERNAL_HSHUB_PEER_CONN_CONFIG_PARAMS params = {0}; + + params.invalidatePeerMask = peerMask; + + return pRmApi->Control(pRmApi, + pGpu->hInternalClient, + pGpu->hInternalSubdevice, + NV2080_CTRL_CMD_INTERNAL_HSHUB_PEER_CONN_CONFIG, + ¶ms, + sizeof(params)); +} + /*! * @brief Setup the mailboxes of 2 GPUs so that the local GPU can access remote GPU. * @@ -59,6 +112,27 @@ kbusSetupMailboxes_GM200 NvU32 local2Remote, NvU32 remote2Local ) +{ + NV_STATUS status = _kbusSetupMailboxes_GM200(pGpu0, pKernelBus0, + pGpu1, pKernelBus1, + local2Remote, remote2Local, + NULL, NULL); + + NV_ASSERT_OK(status); +} + +static NV_STATUS +_kbusSetupMailboxes_GM200 +( + OBJGPU *pGpu0, + KernelBus *pKernelBus0, + OBJGPU *pGpu1, + KernelBus *pKernelBus1, + NvU32 local2Remote, + NvU32 remote2Local, + NvBool *pbLocalMailboxTeardownAttempted, + NvBool *pbRemoteMailboxTeardownAttempted +) { PMEMORY_DESCRIPTOR *ppMemDesc = NULL; RmPhysAddr localP2PDomainRemoteAddr; @@ -72,35 +146,65 @@ kbusSetupMailboxes_GM200 NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_LOCAL_PARAMS params0 = {0}; NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_REMOTE_PARAMS params1 = {0}; NV_STATUS status; + NvBool bRemoteWMBoxMapped = NV_FALSE; + NvBool bLocalP2PDomainMapped = NV_FALSE; + NvBool bRemoteP2PDomainMapped = NV_FALSE; + NvBool bLocalMailboxControl = NV_FALSE; + NvBool bRemoteMailboxControl = NV_FALSE; + NvBool bMailboxTagWritten = NV_FALSE; + + if (pbLocalMailboxTeardownAttempted != NULL) + { + *pbLocalMailboxTeardownAttempted = NV_FALSE; + } + if (pbRemoteMailboxTeardownAttempted != NULL) + { + *pbRemoteMailboxTeardownAttempted = NV_FALSE; + } - NV_ASSERT_OR_RETURN_VOID(local2Remote < P2P_MAX_NUM_PEERS); - NV_ASSERT_OR_RETURN_VOID(remote2Local < P2P_MAX_NUM_PEERS); + NV_ASSERT_OR_RETURN(local2Remote < P2P_MAX_NUM_PEERS, NV_ERR_INVALID_ARGUMENT); + NV_ASSERT_OR_RETURN(remote2Local < P2P_MAX_NUM_PEERS, NV_ERR_INVALID_ARGUMENT); // Ensure we have the correct bidirectional peer mapping - NV_ASSERT_OR_RETURN_VOID(pKernelBus1->p2pPcie.busPeer[remote2Local].remotePeerId == - local2Remote); - NV_ASSERT_OR_RETURN_VOID(pKernelBus0->p2pPcie.busPeer[local2Remote].remotePeerId == - remote2Local); + NV_ASSERT_OR_RETURN(pKernelBus1->p2pPcie.busPeer[remote2Local].remotePeerId == + local2Remote, NV_ERR_INVALID_STATE); + NV_ASSERT_OR_RETURN(pKernelBus0->p2pPcie.busPeer[local2Remote].remotePeerId == + remote2Local, NV_ERR_INVALID_STATE); ppMemDesc = &pKernelBus0->p2pPcie.busPeer[local2Remote].pRemoteWMBoxMemDesc; remoteWMBoxLocalAddr = kbusSetupMailboxAccess_HAL(pGpu1, pKernelBus1, pGpu0, remote2Local, ppMemDesc); - NV_ASSERT_OR_RETURN_VOID(remoteWMBoxLocalAddr != ~0ULL); + if (remoteWMBoxLocalAddr == ~0ULL) + { + status = NV_ERR_INVALID_ADDRESS; + goto kbusSetupMailboxes_cleanup; + } + bRemoteWMBoxMapped = NV_TRUE; ppMemDesc = &pKernelBus1->p2pPcie.busPeer[remote2Local].pRemoteP2PDomMemDesc; localP2PDomainRemoteAddr = kbusSetupP2PDomainAccess_HAL(pGpu0, pKernelBus0, pGpu1, ppMemDesc); - NV_ASSERT_OR_RETURN_VOID(localP2PDomainRemoteAddr != ~0ULL); + if (localP2PDomainRemoteAddr == ~0ULL) + { + status = NV_ERR_INVALID_ADDRESS; + goto kbusSetupMailboxes_cleanup; + } + bLocalP2PDomainMapped = NV_TRUE; ppMemDesc = &pKernelBus0->p2pPcie.busPeer[local2Remote].pRemoteP2PDomMemDesc; remoteP2PDomainLocalAddr = kbusSetupP2PDomainAccess_HAL(pGpu1, pKernelBus1, pGpu0, ppMemDesc); - NV_ASSERT_OR_RETURN_VOID(remoteP2PDomainLocalAddr != ~0ULL); + if (remoteP2PDomainLocalAddr == ~0ULL) + { + status = NV_ERR_INVALID_ADDRESS; + goto kbusSetupMailboxes_cleanup; + } + bRemoteP2PDomainMapped = NV_TRUE; // Setup the local GPU to access remote GPU's FB. @@ -110,7 +214,11 @@ kbusSetupMailboxes_GM200 PCIE_P2P_WRITE_MAILBOX_SIZE * remote2Local; // Write mailbox data window needs to be 64KB aligned. - NV_ASSERT((remoteWMBoxAddrU64 & 0xFFFF) == 0); + if ((remoteWMBoxAddrU64 & 0xFFFF) != 0) + { + status = NV_ERR_INVALID_ADDRESS; + goto kbusSetupMailboxes_cleanup; + } // Setup PCIE P2P Mailbox on local GPU params0.local2Remote = local2Remote; @@ -127,7 +235,11 @@ kbusSetupMailboxes_GM200 NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_LOCAL, ¶ms0, sizeof(NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_LOCAL_PARAMS)); - NV_ASSERT(status == NV_OK); + if (status != NV_OK) + { + goto kbusSetupMailboxes_cleanup; + } + bLocalMailboxControl = NV_TRUE; // Setup PCIE P2P Mailbox on remote GPU params1.local2Remote = local2Remote; @@ -143,9 +255,62 @@ kbusSetupMailboxes_GM200 NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_REMOTE, ¶ms1, sizeof(NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_REMOTE_PARAMS)); - NV_ASSERT(status == NV_OK); + if (status != NV_OK) + { + goto kbusSetupMailboxes_cleanup; + } + bRemoteMailboxControl = NV_TRUE; kbusWriteP2PWmbTag_HAL(pGpu1, pKernelBus1, remote2Local, params0.p2pWmbTag); + bMailboxTagWritten = NV_TRUE; + + return NV_OK; + +kbusSetupMailboxes_cleanup: + NV_PRINTF(LEVEL_ERROR, + "P2P_MAILBOX_SETUP_FAIL localGpu=%u remoteGpu=%u localPeer=%u remotePeer=%u " + "status=0x%x wmboxMapped=%u localDomainMapped=%u remoteDomainMapped=%u " + "localCtrl=%u remoteCtrl=%u tagWritten=%u tag=0x%llx\n", + gpuGetInstance(pGpu0), + gpuGetInstance(pGpu1), + local2Remote, + remote2Local, + status, + bRemoteWMBoxMapped, + bLocalP2PDomainMapped, + bRemoteP2PDomainMapped, + bLocalMailboxControl, + bRemoteMailboxControl, + bMailboxTagWritten, + (NvU64)params0.p2pWmbTag); + + if (bLocalMailboxControl) + { + kbusDestroyMailbox(pGpu0, pKernelBus0, pGpu1, local2Remote); + if (pbLocalMailboxTeardownAttempted != NULL) + { + *pbLocalMailboxTeardownAttempted = NV_TRUE; + } + } + else if (bRemoteWMBoxMapped || bRemoteP2PDomainMapped) + { + kbusDestroyPeerAccess_HAL(pGpu0, pKernelBus0, local2Remote); + } + + if (bRemoteMailboxControl || bMailboxTagWritten) + { + kbusDestroyMailbox(pGpu1, pKernelBus1, pGpu0, remote2Local); + if (pbRemoteMailboxTeardownAttempted != NULL) + { + *pbRemoteMailboxTeardownAttempted = NV_TRUE; + } + } + else if (bLocalP2PDomainMapped) + { + kbusDestroyPeerAccess_HAL(pGpu1, pKernelBus1, remote2Local); + } + + return status; } void @@ -198,11 +363,28 @@ kbusSetupMailboxAccess_GM200 PMEMORY_DESCRIPTOR *ppWMBoxMemDesc ) { - return kbusSetupPeerBarAccess(pGpu0, pGpu1, - gpumgrGetGpuPhysFbAddr(pGpu0) + - pKernelBus0->p2pPcie.writeMailboxBar1Addr + - PCIE_P2P_WRITE_MAILBOX_SIZE * local2Remote, - PCIE_P2P_WRITE_MAILBOX_SIZE, ppWMBoxMemDesc); + RmPhysAddr fbBase = gpumgrGetGpuPhysFbAddr(pGpu0); + NvU64 mailboxOffset = pKernelBus0->p2pPcie.writeMailboxBar1Addr; + NvU64 peerOffset = PCIE_P2P_WRITE_MAILBOX_SIZE * local2Remote; + RmPhysAddr base; + + if (pKernelBus0->p2pPcie.writeMailboxBar1Addr == + PCIE_P2P_INVALID_WRITE_MAILBOX_ADDR) + { + NV_PRINTF(LEVEL_ERROR, + "PCIe mailbox P2P requested without an allocated mailbox area " + "ownerGpu=%u accessorGpu=%u peer=%u writeMailboxBar1Addr=0x%llx\n", + gpuGetInstance(pGpu0), + gpuGetInstance(pGpu1), + local2Remote, + pKernelBus0->p2pPcie.writeMailboxBar1Addr); + return ~0ULL; + } + + base = fbBase + mailboxOffset + peerOffset; + + return kbusSetupPeerBarAccess(pGpu0, pGpu1, base, + PCIE_P2P_WRITE_MAILBOX_SIZE, ppWMBoxMemDesc); } void @@ -353,9 +535,20 @@ kbusCreateP2PMappingForMailbox_GM200 NvU32 attributes ) { - RM_API *pRmApi; - NV2080_CTRL_INTERNAL_HSHUB_PEER_CONN_CONFIG_PARAMS params; NvU32 gpuInst0, gpuInst1; + NvBool bPeer0HshubProgrammed = NV_FALSE; + NvBool bPeer1HshubProgrammed = NV_FALSE; + NvBool bPeer0MailboxTeardownNeeded = NV_FALSE; + NvBool bPeer1MailboxTeardownNeeded = NV_FALSE; + NvBool bPeer0MailboxTeardownAttempted = NV_FALSE; + NvBool bPeer1MailboxTeardownAttempted = NV_FALSE; + NvU32 oldPeer0RemotePeerId; + NvU32 oldPeer1RemotePeerId; + NvU32 oldPeer0RefCount; + NvU32 oldPeer1RefCount; + NvU32 oldPeerMask0; + NvU32 oldPeerMask1; + NV_STATUS status; if (IS_VIRTUAL(pGpu0) || IS_VIRTUAL(pGpu1)) { @@ -397,25 +590,26 @@ kbusCreateP2PMappingForMailbox_GM200 NV_ASSERT(pKernelBus0->p2pPcie.busPeer[*peer0].remotePeerId == *peer1); NV_ASSERT(pKernelBus1->p2pPcie.busPeer[*peer1].remotePeerId == *peer0); - pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu0); - portMemSet(¶ms, 0, sizeof(params)); - params.programPciePeerMask = NVBIT32(*peer0); - NV_ASSERT_OK_OR_RETURN(pRmApi->Control(pRmApi, - pGpu0->hInternalClient, - pGpu0->hInternalSubdevice, - NV2080_CTRL_CMD_INTERNAL_HSHUB_PEER_CONN_CONFIG, - ¶ms, - sizeof(params))); - - pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu1); - portMemSet(¶ms, 0, sizeof(params)); - params.programPciePeerMask = NVBIT32(*peer1); - NV_ASSERT_OK_OR_RETURN(pRmApi->Control(pRmApi, - pGpu1->hInternalClient, - pGpu1->hInternalSubdevice, - NV2080_CTRL_CMD_INTERNAL_HSHUB_PEER_CONN_CONFIG, - ¶ms, - sizeof(params))); + status = _kbusProgramPciePeerMask_GM200(pGpu0, NVBIT32(*peer0)); + if (status != NV_OK) + { + pKernelBus0->p2pPcie.busPeer[*peer0].refCount--; + pKernelBus1->p2pPcie.busPeer[*peer1].refCount--; + return status; + } + + status = _kbusProgramPciePeerMask_GM200(pGpu1, NVBIT32(*peer1)); + if (status != NV_OK) + { + // + // The mapping pre-exists and its HSHUB peer masks are + // still needed by the existing references, so only drop + // the references taken above. + // + pKernelBus0->p2pPcie.busPeer[*peer0].refCount--; + pKernelBus1->p2pPcie.busPeer[*peer1].refCount--; + return status; + } return NV_OK; } @@ -449,25 +643,26 @@ kbusCreateP2PMappingForMailbox_GM200 NV_ASSERT(!pKernelBus0->p2pPcie.busPeer[*peer0].bReserved); NV_ASSERT(!pKernelBus1->p2pPcie.busPeer[*peer1].bReserved); - pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu0); - portMemSet(¶ms, 0, sizeof(params)); - params.programPciePeerMask = NVBIT32(*peer0); - NV_ASSERT_OK_OR_RETURN(pRmApi->Control(pRmApi, - pGpu0->hInternalClient, - pGpu0->hInternalSubdevice, - NV2080_CTRL_CMD_INTERNAL_HSHUB_PEER_CONN_CONFIG, - ¶ms, - sizeof(params))); - - pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu1); - portMemSet(¶ms, 0, sizeof(params)); - params.programPciePeerMask = NVBIT32(*peer1); - NV_ASSERT_OK_OR_RETURN(pRmApi->Control(pRmApi, - pGpu1->hInternalClient, - pGpu1->hInternalSubdevice, - NV2080_CTRL_CMD_INTERNAL_HSHUB_PEER_CONN_CONFIG, - ¶ms, - sizeof(params))); + status = _kbusProgramPciePeerMask_GM200(pGpu0, NVBIT32(*peer0)); + if (status != NV_OK) + { + pKernelBus0->p2pPcie.busPeer[*peer0].refCount--; + pKernelBus1->p2pPcie.busPeer[*peer1].refCount--; + return status; + } + + status = _kbusProgramPciePeerMask_GM200(pGpu1, NVBIT32(*peer1)); + if (status != NV_OK) + { + // + // The mapping pre-exists and its HSHUB peer masks are still + // needed by the existing references, so only drop the + // references taken above. + // + pKernelBus0->p2pPcie.busPeer[*peer0].refCount--; + pKernelBus1->p2pPcie.busPeer[*peer1].refCount--; + return status; + } return NV_OK; } @@ -521,6 +716,13 @@ kbusCreateP2PMappingForMailbox_GM200 } busCreateP2PMapping_setupMapping: + oldPeer0RemotePeerId = pKernelBus0->p2pPcie.busPeer[*peer0].remotePeerId; + oldPeer1RemotePeerId = pKernelBus1->p2pPcie.busPeer[*peer1].remotePeerId; + oldPeer0RefCount = pKernelBus0->p2pPcie.busPeer[*peer0].refCount; + oldPeer1RefCount = pKernelBus1->p2pPcie.busPeer[*peer1].refCount; + oldPeerMask0 = pKernelBus0->p2pPcie.peerNumberMask[gpuInst1]; + oldPeerMask1 = pKernelBus1->p2pPcie.peerNumberMask[gpuInst0]; + pKernelBus0->p2pPcie.busPeer[*peer0].remotePeerId = *peer1; pKernelBus0->p2pPcie.peerNumberMask[gpuInst1] |= NVBIT(*peer0); pKernelBus1->p2pPcie.busPeer[*peer1].remotePeerId = *peer0; @@ -538,34 +740,80 @@ kbusCreateP2PMappingForMailbox_GM200 pKernelBus0->p2pPcie.busPeer[*peer0].refCount++; pKernelBus1->p2pPcie.busPeer[*peer1].refCount++; - pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu0); - portMemSet(¶ms, 0, sizeof(params)); - params.programPciePeerMask = NVBIT32(*peer0); - NV_ASSERT_OK_OR_RETURN(pRmApi->Control(pRmApi, - pGpu0->hInternalClient, - pGpu0->hInternalSubdevice, - NV2080_CTRL_CMD_INTERNAL_HSHUB_PEER_CONN_CONFIG, - ¶ms, - sizeof(params))); - - pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu1); - portMemSet(¶ms, 0, sizeof(params)); - params.programPciePeerMask = NVBIT32(*peer1); - NV_ASSERT_OK_OR_RETURN(pRmApi->Control(pRmApi, - pGpu1->hInternalClient, - pGpu1->hInternalSubdevice, - NV2080_CTRL_CMD_INTERNAL_HSHUB_PEER_CONN_CONFIG, - ¶ms, - sizeof(params))); + status = _kbusProgramPciePeerMask_GM200(pGpu0, NVBIT32(*peer0)); + if (status != NV_OK) + { + goto busCreateP2PMapping_rollback; + } + bPeer0HshubProgrammed = NV_TRUE; + + status = _kbusProgramPciePeerMask_GM200(pGpu1, NVBIT32(*peer1)); + if (status != NV_OK) + { + goto busCreateP2PMapping_rollback; + } + bPeer1HshubProgrammed = NV_TRUE; + + status = _kbusSetupMailboxes_GM200(pGpu0, pKernelBus0, pGpu1, pKernelBus1, + *peer0, *peer1, + &bPeer0MailboxTeardownAttempted, + &bPeer1MailboxTeardownAttempted); + if (status != NV_OK) + { + goto busCreateP2PMapping_rollback; + } + bPeer0MailboxTeardownNeeded = NV_TRUE; + bPeer1MailboxTeardownNeeded = NV_TRUE; + + status = _kbusSetupMailboxes_GM200(pGpu1, pKernelBus1, pGpu0, pKernelBus0, + *peer1, *peer0, + &bPeer1MailboxTeardownAttempted, + &bPeer0MailboxTeardownAttempted); + if (status != NV_OK) + { + goto busCreateP2PMapping_rollback; + } NV_PRINTF(LEVEL_INFO, "added PCIe P2P mapping between GPU%u (peer %u) and GPU%u (peer %u)\n", gpuInst0, *peer0, gpuInst1, *peer1); - kbusSetupMailboxes_HAL(pGpu0, pKernelBus0, pGpu1, pKernelBus1, *peer0, *peer1); - kbusSetupMailboxes_HAL(pGpu1, pKernelBus1, pGpu0, pKernelBus0, *peer1, *peer0); - return NV_OK; + +busCreateP2PMapping_rollback: + if (bPeer0MailboxTeardownNeeded && !bPeer0MailboxTeardownAttempted) + { + kbusDestroyMailbox(pGpu0, pKernelBus0, pGpu1, *peer0); + bPeer0MailboxTeardownAttempted = NV_TRUE; + } + + if (bPeer1MailboxTeardownNeeded && !bPeer1MailboxTeardownAttempted) + { + kbusDestroyMailbox(pGpu1, pKernelBus1, pGpu0, *peer1); + bPeer1MailboxTeardownAttempted = NV_TRUE; + } + + if (bPeer0HshubProgrammed && !bPeer0MailboxTeardownAttempted) + { + NV_ASSERT_OK(_kbusInvalidatePeerMask_GM200(pGpu0, NVBIT32(*peer0))); + } + + if (bPeer1HshubProgrammed && !bPeer1MailboxTeardownAttempted) + { + NV_ASSERT_OK(_kbusInvalidatePeerMask_GM200(pGpu1, NVBIT32(*peer1))); + } + + pKernelBus0->p2pPcie.busPeer[*peer0].remotePeerId = oldPeer0RemotePeerId; + pKernelBus1->p2pPcie.busPeer[*peer1].remotePeerId = oldPeer1RemotePeerId; + pKernelBus0->p2pPcie.busPeer[*peer0].refCount = oldPeer0RefCount; + pKernelBus1->p2pPcie.busPeer[*peer1].refCount = oldPeer1RefCount; + pKernelBus0->p2pPcie.peerNumberMask[gpuInst1] = oldPeerMask0; + pKernelBus1->p2pPcie.peerNumberMask[gpuInst0] = oldPeerMask1; + + *peer0 = BUS_INVALID_PEER; + *peer1 = BUS_INVALID_PEER; + + return status; } /*! @@ -797,6 +1045,15 @@ kbusSetP2PMailboxBar1Area_GM200 if (!kbusIsP2pMailboxClientAllocated(pKernelBus)) { + if (pKernelBus->p2pPcie.writeMailboxBar1Addr == + PCIE_P2P_INVALID_WRITE_MAILBOX_ADDR) + { + NV_PRINTF(LEVEL_ERROR, + "P2P mailbox area expected from RM but no valid address is installed gpu=%u\n", + gpuGetInstance(pGpu)); + return NV_ERR_INVALID_STATE; + } + // P2P mailbox area already allocated by RM. Nothing to do. return NV_OK; } diff --git a/src/nvidia/src/kernel/gpu/bus/arch/turing/kern_bus_tu102.c b/src/nvidia/src/kernel/gpu/bus/arch/turing/kern_bus_tu102.c index f06fb0953f..544ad065c6 100644 --- a/src/nvidia/src/kernel/gpu/bus/arch/turing/kern_bus_tu102.c +++ b/src/nvidia/src/kernel/gpu/bus/arch/turing/kern_bus_tu102.c @@ -385,8 +385,26 @@ kbusIsStaticBar1Supported_TU102 // NvU64 fbSize = pMemoryManager->Ram.fbAddrSpaceSizeMb << 20; NvU64 fbSizeAligned = RM_ALIGN_UP(fbSize, RM_PAGE_SIZE_2M); + NvU64 clientFbSize = memmgrGetClientFbAddrSpaceSize(pGpu, pMemoryManager); + NvU64 clientFbSizeAligned = RM_ALIGN_DOWN(clientFbSize, RM_PAGE_SIZE_2M); NvU64 bar1VASize = pKernelBus->bar1[gfid].mappableLength; NvU64 bar1VASizeAligned = RM_ALIGN_DOWN(bar1VASize, RM_PAGE_SIZE_2M); + NvU64 staticBar1Offset = NV_ALIGN_UP(consoleSize + mailboxSize, RM_PAGE_SIZE_512M); + NvBool bGb206Bar1P2PDefault = + ((gpuGetChipImpl(pGpu) == GPU_IMPLEMENTATION_GB206) && + pKernelBus->getProperty(pKernelBus, PDB_PROP_KBUS_SUPPORT_BAR1_P2P_BY_DEFAULT)); + NvU64 gb206MaxStaticMapSize = + (bar1VASizeAligned > staticBar1Offset) ? + RM_ALIGN_DOWN(bar1VASizeAligned - staticBar1Offset, RM_PAGE_SIZE_2M) : 0; + // + // GB206 enables BAR1 P2P by default. A display-attached GPU may already + // consume low BAR1 VA for console mappings, so allow the static mapping to + // cover the post-console BAR1 extent instead of requiring all client FB. + // + NvU64 autoStaticMapSize = bGb206Bar1P2PDefault ? + ((clientFbSizeAligned < gb206MaxStaticMapSize) ? + clientFbSizeAligned : gb206MaxStaticMapSize) : + fbSizeAligned; if (gfid != 0) { @@ -427,14 +445,13 @@ kbusIsStaticBar1Supported_TU102 // really wants to enable static BAR1 regardless of the auto checks // NvU64 bar1MapSize = - RM_ALIGN_DOWN(memmgrGetClientFbAddrSpaceSize(pGpu, pMemoryManager), - RM_PAGE_SIZE_2M); + clientFbSizeAligned; - if (bar1VASizeAligned < bar1MapSize) + if (bar1VASizeAligned < (staticBar1Offset + bar1MapSize)) { NV_PRINTF(LEVEL_ERROR, "BAR1 size %lld is not large enough to map FB size" - "%lld to force static BAR1\n", - bar1VASizeAligned, bar1MapSize); + "%lld at offset %lld to force static BAR1\n", + bar1VASizeAligned, bar1MapSize, staticBar1Offset); DBG_BREAKPOINT(); return NV_ERR_INVALID_REGISTRY_KEY; @@ -469,19 +486,17 @@ kbusIsStaticBar1Supported_TU102 // NvU32 userdSize = 0; NvU32 numChannels = kfifoGetMaxChannelsInSystem(pGpu, pKernelFifo); - NvU64 requiredAutoBar1Size = fbSizeAligned; + NvU64 requiredAutoBar1Size = autoStaticMapSize; NvU64 mmioPrivSize = 16 * RM_PAGE_SIZE; NvU64 doorbellSize = 16 * RM_PAGE_SIZE; + NvU64 alignmentPadding = staticBar1Offset - (consoleSize + mailboxSize); + NvU64 dynamicBar1Size; kfifoGetUserdSizeAlign_HAL(pKernelFifo, &userdSize, NULL); userdSize *= numChannels; - requiredAutoBar1Size += userdSize; - requiredAutoBar1Size += mmioPrivSize; - requiredAutoBar1Size += doorbellSize; - requiredAutoBar1Size += consoleSize; - requiredAutoBar1Size += mailboxSize; + dynamicBar1Size = userdSize + mmioPrivSize + doorbellSize; // // Console mappings are already mapped from the bottom of the BAR1 VASpace, @@ -493,10 +508,30 @@ kbusIsStaticBar1Supported_TU102 // if ((consoleSize != 0) || (mailboxSize != 0)) { - requiredAutoBar1Size += RM_PAGE_SIZE_512M - ((consoleSize + mailboxSize) % RM_PAGE_SIZE_512M); + if (bGb206Bar1P2PDefault) + { + requiredAutoBar1Size += staticBar1Offset; + + if (dynamicBar1Size > alignmentPadding) + { + requiredAutoBar1Size += dynamicBar1Size - alignmentPadding; + } + } + else + { + requiredAutoBar1Size += dynamicBar1Size; + requiredAutoBar1Size += consoleSize; + requiredAutoBar1Size += mailboxSize; + requiredAutoBar1Size += alignmentPadding; + } + } + else + { + requiredAutoBar1Size += dynamicBar1Size; } - if (bar1VASizeAligned >= requiredAutoBar1Size) + if ((autoStaticMapSize != 0) && + (bar1VASizeAligned >= requiredAutoBar1Size)) { NV_PRINTF(LEVEL_INFO, "Enabling static BAR1 automatically!\n"); return NV_OK; @@ -535,8 +570,13 @@ kbusEnableStaticBar1Mapping_TU102 MEMORY_DESCRIPTOR *pDmaMemDesc = NULL; NV_STATUS status = NV_OK; NvU64 bar1MapSize; + NvU64 clientFbSizeAligned; + NvU64 bar1VASizeAligned; NvU64 bar1BusAddr; NvU32 mapFlags = BUS_MAP_FB_FLAGS_MAP_UNICAST | BUS_MAP_FB_FLAGS_MAP_OFFSET_FIXED; + NvBool bGb206Bar1P2PDefault = + ((gpuGetChipImpl(pGpu) == GPU_IMPLEMENTATION_GB206) && + pKernelBus->getProperty(pKernelBus, PDB_PROP_KBUS_SUPPORT_BAR1_P2P_BY_DEFAULT)); // // But use memmgrGetClientFbAddrSpaceSize @@ -548,8 +588,28 @@ kbusEnableStaticBar1Mapping_TU102 // The last client FB addresses not aligned to 2MB will // not be mappable to a 2MB mapping. // - bar1MapSize = RM_ALIGN_DOWN(memmgrGetClientFbAddrSpaceSize(pGpu, pMemoryManager), - RM_PAGE_SIZE_2M); + clientFbSizeAligned = RM_ALIGN_DOWN(memmgrGetClientFbAddrSpaceSize(pGpu, pMemoryManager), + RM_PAGE_SIZE_2M); + bar1VASizeAligned = RM_ALIGN_DOWN(pKernelBus->bar1[gfid].mappableLength, + RM_PAGE_SIZE_2M); + bar1MapSize = clientFbSizeAligned; + + if (bGb206Bar1P2PDefault && (bar1Offset < bar1VASizeAligned)) + { + // + // Mirror the eligibility logic above: for display-attached GB206, map + // only the BAR1 VA that remains after fixed console/mailbox mappings. + // + NvU64 maxStaticMapSize = + RM_ALIGN_DOWN(bar1VASizeAligned - bar1Offset, RM_PAGE_SIZE_2M); + + if (bar1MapSize > maxStaticMapSize) + { + bar1MapSize = maxStaticMapSize; + } + } + + NV_ASSERT_OR_RETURN(bar1MapSize != 0, NV_ERR_NOT_SUPPORTED); // // The static mapping is not backed by an allocated physical FB. @@ -1043,7 +1103,6 @@ kbusGetStaticFbAperture_TU102 NvBool bDiscontigAllowed = !!(busMapFlags & BUS_MAP_FB_FLAGS_ALLOW_DISCONTIG); NvBool bInStaticRegion = NV_FALSE; NvBool bInDynamicRegion = NV_FALSE; - NvBool bInLastPage = NV_TRUE; NV_CHECK_OR_RETURN(LEVEL_SILENT, kbusIsStaticBar1Enabled(pGpu, pKernelBus), NV_ERR_NOT_SUPPORTED); @@ -1076,7 +1135,6 @@ kbusGetStaticFbAperture_TU102 if (curLimit > staticBar1Size) { bInDynamicRegion = NV_TRUE; - bInLastPage = bInLastPage && ((curLimit - staticBar1Size) < RM_PAGE_SIZE_2M); } else { @@ -1090,25 +1148,12 @@ kbusGetStaticFbAperture_TU102 if (bInDynamicRegion && bInStaticRegion) { // - // With rounding down the static region to 2MB, - // we can allocate the last non-2MB aligned region - // but not have a mapping for it + // The static region may not cover all of client FB: it is rounded + // down to 2MB, and on display-attached GB206 it is clipped to the + // BAR1 VA left after the console/mailbox reservation. A range that + // spans the static/dynamic boundary falls back to a dynamic mapping. // - if (bInLastPage) - { - return NV_ERR_NOT_SUPPORTED; - } - - NV_PRINTF(LEVEL_ERROR, "MemDesc spans both static and dynamic region," - "which is unsupported.\n"); - NV_PRINTF(LEVEL_ERROR, "static Bar1 map [0, 0x%llx]\n", - pKernelBus->bar1[gfid].staticBar1.size); - NV_PRINTF(LEVEL_ERROR, "Requested map range 0x%llx to 0x%llx, mapGranularity 0x%llx\n", - mapRange.start, mrangeLimit(mapRange) - 1llu, mapRange.size); - - memdescPrintMemdesc(pMemDesc, NV_TRUE, MAKE_NV_PRINTF_STR("Dumping memdesc:")); - - return NV_ERR_INVALID_ARGUMENT; + return NV_ERR_NOT_SUPPORTED; } if (bInDynamicRegion) diff --git a/src/nvidia/src/kernel/rmapi/nv_gpu_ops.c b/src/nvidia/src/kernel/rmapi/nv_gpu_ops.c index 3de77d045a..9e8b7ceac0 100644 --- a/src/nvidia/src/kernel/rmapi/nv_gpu_ops.c +++ b/src/nvidia/src/kernel/rmapi/nv_gpu_ops.c @@ -3934,22 +3934,42 @@ nvGpuOpsMemGetPageSize * * @param[in] pAddresses : Array of physical addresses to be encoded. * @param[in] dmaBaseAddress : IOVA base address. + * @param[in] dmaSize : IOVA window size. + * @param[in] pageSize : Size covered by each physical address. * @param[in] count : Count of physical addresses. */ -static void +static NV_STATUS _nvGpuOpsEncodeBar1P2PAddrs ( NvU64 *pAddresses, NvU64 dmaBaseAddress, + NvU64 dmaSize, + NvU64 pageSize, NvU64 count ) { - NvU32 i; + NvU64 i; for (i = 0; i < count; i++) { - pAddresses[i] = dmaBaseAddress + pAddresses[i]; + NvU64 offset = pAddresses[i]; + NvU64 encodedAddress; + + if ((offset >= dmaSize) || + (pageSize > (dmaSize - offset)) || + !portSafeAddU64(dmaBaseAddress, offset, &encodedAddress)) + { + NV_PRINTF(LEVEL_ERROR, + "BAR1 P2P address range exceeds DMA window: " + "offset=0x%llx pageSize=0x%llx dmaBase=0x%llx dmaSize=0x%llx\n", + offset, pageSize, dmaBaseAddress, dmaSize); + return NV_ERR_INVALID_ADDRESS; + } + + pAddresses[i] = encodedAddress; } + + return NV_OK; } static @@ -4309,7 +4329,13 @@ nvGpuOpsBuildExternalAllocPtes status = NV_ERR_INVALID_STATE; goto done; } - _nvGpuOpsEncodeBar1P2PAddrs(physicalAddresses, dmaBaseAddress, pteCount); + NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, + _nvGpuOpsEncodeBar1P2PAddrs(physicalAddresses, + dmaBaseAddress, + dmaSize, + mappingPageSize, + pteCount), + done); } else { @@ -4649,7 +4675,13 @@ nvGpuOpsBuildExternalAllocPhysAddrs status = NV_ERR_INVALID_STATE; goto done; } - _nvGpuOpsEncodeBar1P2PAddrs(physicalAddresses, dmaBaseAddress, physAddrCount); + NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, + _nvGpuOpsEncodeBar1P2PAddrs(physicalAddresses, + dmaBaseAddress, + dmaSize, + mappingPageSize, + physAddrCount), + done); } else {