//===--- AMDGPUIGroupLP.cpp - AMDGPU IGroupLP ------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// \file This file defines a set of schedule DAG mutations that can be used to
// override default scheduler behavior to enforce specific scheduling patterns.
// They should be used in cases where runtime performance considerations such as
// inter-wavefront interactions, mean that compile-time heuristics cannot
// predict the optimal instruction ordering, or in kernels where optimum
// instruction scheduling is important enough to warrant manual intervention.
//
//===----------------------------------------------------------------------===//
#include "AMDGPUIGroupLP.h"
#include "AMDGPUTargetMachine.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIInstrInfo.h"
#include "SIMachineFunctionInfo.h"
#include "llvm/ADT/BitmaskEnum.h"
#include "llvm/CodeGen/MachineScheduler.h"
#include "llvm/CodeGen/TargetOpcodes.h"
using namespace llvm;
#define DEBUG_TYPE "machine-scheduler"
namespace {
static cl::opt<bool>
EnableIGroupLP("amdgpu-igrouplp",
cl::desc("Enable construction of Instruction Groups and "
"their ordering for scheduling"),
cl::init(false));
static cl::opt<Optional<unsigned>>
VMEMGroupMaxSize("amdgpu-igrouplp-vmem-group-size", cl::init(None),
cl::Hidden,
cl::desc("The maximum number of instructions to include "
"in VMEM group."));
static cl::opt<Optional<unsigned>>
MFMAGroupMaxSize("amdgpu-igrouplp-mfma-group-size", cl::init(None),
cl::Hidden,
cl::desc("The maximum number of instructions to include "
"in MFMA group."));
static cl::opt<Optional<unsigned>>
LDRGroupMaxSize("amdgpu-igrouplp-ldr-group-size", cl::init(None),
cl::Hidden,
cl::desc("The maximum number of instructions to include "
"in lds/gds read group."));
static cl::opt<Optional<unsigned>>
LDWGroupMaxSize("amdgpu-igrouplp-ldw-group-size", cl::init(None),
cl::Hidden,
cl::desc("The maximum number of instructions to include "
"in lds/gds write group."));
typedef function_ref<bool(const MachineInstr &, const SIInstrInfo *)>
CanAddMIFn;
// Classify instructions into groups to enable fine tuned control over the
// scheduler. These groups may be more specific than current SchedModel
// instruction classes.
class SchedGroup {
private:
// Function that returns true if a non-bundle MI may be inserted into this
// group.
const CanAddMIFn canAddMI;
// Maximum number of SUnits that can be added to this group.
Optional<unsigned> MaxSize;
// Collection of SUnits that are classified as members of this group.
SmallVector<SUnit *, 32> Collection;
ScheduleDAGInstrs *DAG;
void tryAddEdge(SUnit *A, SUnit *B) {
if (A != B && DAG->canAddEdge(B, A)) {
DAG->addEdge(B, SDep(A, SDep::Artificial));
LLVM_DEBUG(dbgs() << "Adding edge...\n"
<< "from: SU(" << A->NodeNum << ") " << *A->getInstr()
<< "to: SU(" << B->NodeNum << ") " << *B->getInstr());
}
}
public:
// Add DAG dependencies from all SUnits in this SchedGroup and this SU. If
// MakePred is true, SU will be a predecessor of the SUnits in this
// SchedGroup, otherwise SU will be a successor.
void link(SUnit &SU, bool MakePred = false) {
for (auto A : Collection) {
SUnit *B = &SU;
if (MakePred)
std::swap(A, B);
tryAddEdge(A, B);
}
}
// Add DAG dependencies from all SUnits in this SchedGroup and this SU. Use
// the predicate to determine whether SU should be a predecessor (P = true)
// or a successor (P = false) of this SchedGroup.
void link(SUnit &SU, function_ref<bool(const SUnit *A, const SUnit *B)> P) {
for (auto A : Collection) {
SUnit *B = &SU;
if (P(A, B))
std::swap(A, B);
tryAddEdge(A, B);
}
}
// Add DAG dependencies such that SUnits in this group shall be ordered
// before SUnits in OtherGroup.
void link(SchedGroup &OtherGroup) {
for (auto B : OtherGroup.Collection)
link(*B);
}
// Returns true if no more instructions may be added to this group.
bool isFull() { return MaxSize && Collection.size() >= *MaxSize; }
// Returns true if SU can be added to this SchedGroup.
bool canAddSU(SUnit &SU, const SIInstrInfo *TII) {
if (isFull())
return false;
MachineInstr &MI = *SU.getInstr();
if (MI.getOpcode() != TargetOpcode::BUNDLE)
return canAddMI(MI, TII);
// Special case for bundled MIs.
const MachineBasicBlock *MBB = MI.getParent();
MachineBasicBlock::instr_iterator B = MI.getIterator(), E = ++B;
while (E != MBB->end() && E->isBundledWithPred())
++E;
// Return true if all of the bundled MIs can be added to this group.
return std::all_of(
B, E, [this, TII](MachineInstr &MI) { return canAddMI(MI, TII); });
}
void add(SUnit &SU) { Collection.push_back(&SU); }
SchedGroup(CanAddMIFn canAddMI, Optional<unsigned> MaxSize,
ScheduleDAGInstrs *DAG)
: canAddMI(canAddMI), MaxSize(MaxSize), DAG(DAG) {}
};
bool isMFMASGMember(const MachineInstr &MI, const SIInstrInfo *TII) {
return TII->isMFMA(MI);
}
bool isVALUSGMember(const MachineInstr &MI, const SIInstrInfo *TII) {
return TII->isVALU(MI) && !TII->isMFMA(MI);
}
bool isSALUSGMember(const MachineInstr &MI, const SIInstrInfo *TII) {
return TII->isSALU(MI);
}
bool isVMEMSGMember(const MachineInstr &MI, const SIInstrInfo *TII) {
return TII->isVMEM(MI) || (TII->isFLAT(MI) && !TII->isDS(MI));
}
bool isVMEMReadSGMember(const MachineInstr &MI, const SIInstrInfo *TII) {
return MI.mayLoad() &&
(TII->isVMEM(MI) || (TII->isFLAT(MI) && !TII->isDS(MI)));
}
bool isVMEMWriteSGMember(const MachineInstr &MI, const SIInstrInfo *TII) {
return MI.mayStore() &&
(TII->isVMEM(MI) || (TII->isFLAT(MI) && !TII->isDS(MI)));
}
bool isDSWriteSGMember(const MachineInstr &MI, const SIInstrInfo *TII) {
return MI.mayStore() && TII->isDS(MI);
}
bool isDSReadSGMember(const MachineInstr &MI, const SIInstrInfo *TII) {
return MI.mayLoad() && TII->isDS(MI);
}
class IGroupLPDAGMutation : public ScheduleDAGMutation {
public:
const SIInstrInfo *TII;
ScheduleDAGMI *DAG;
IGroupLPDAGMutation() = default;
void apply(ScheduleDAGInstrs *DAGInstrs) override;
};
// DAG mutation that coordinates with the SCHED_BARRIER instruction and
// corresponding builtin. The mutation adds edges from specific instruction
// classes determined by the SCHED_BARRIER mask so that they cannot be
// scheduled around the SCHED_BARRIER.
class SchedBarrierDAGMutation : public ScheduleDAGMutation {
private:
const SIInstrInfo *TII;
ScheduleDAGMI *DAG;
// Components of the mask that determines which instructions may not be
// scheduled across the SCHED_BARRIER.
enum class SchedBarrierMasks {
NONE = 0u,
ALU = 1u << 0,
VALU = 1u << 1,
SALU = 1u << 2,
MFMA = 1u << 3,
VMEM = 1u << 4,
VMEM_READ = 1u << 5,
VMEM_WRITE = 1u << 6,
DS = 1u << 7,
DS_READ = 1u << 8,
DS_WRITE = 1u << 9,
LLVM_MARK_AS_BITMASK_ENUM(/* LargestFlag = */ DS_WRITE)
};
// Cache SchedGroups of each type if we have multiple SCHED_BARRIERs in a
// region.
//
std::unique_ptr<SchedGroup> MFMASchedGroup = nullptr;
std::unique_ptr<SchedGroup> VALUSchedGroup = nullptr;
std::unique_ptr<SchedGroup> SALUSchedGroup = nullptr;
std::unique_ptr<SchedGroup> VMEMReadSchedGroup = nullptr;
std::unique_ptr<SchedGroup> VMEMWriteSchedGroup = nullptr;
std::unique_ptr<SchedGroup> DSWriteSchedGroup = nullptr;
std::unique_ptr<SchedGroup> DSReadSchedGroup = nullptr;
// Use a SCHED_BARRIER's mask to identify instruction SchedGroups that should
// not be reordered accross the SCHED_BARRIER.
void getSchedGroupsFromMask(int32_t Mask,
SmallVectorImpl<SchedGroup *> &SchedGroups);
// Add DAG edges that enforce SCHED_BARRIER ordering.
void addSchedBarrierEdges(SUnit &SU);
// Classify instructions and add them to the SchedGroup.
void initSchedGroup(SchedGroup *SG);
// Remove all existing edges from a SCHED_BARRIER.
void resetSchedBarrierEdges(SUnit &SU);
public:
void apply(ScheduleDAGInstrs *DAGInstrs) override;
SchedBarrierDAGMutation() = default;
};
void IGroupLPDAGMutation::apply(ScheduleDAGInstrs *DAGInstrs) {
const GCNSubtarget &ST = DAGInstrs->MF.getSubtarget<GCNSubtarget>();
TII = ST.getInstrInfo();
DAG = static_cast<ScheduleDAGMI *>(DAGInstrs);
const TargetSchedModel *TSchedModel = DAGInstrs->getSchedModel();
if (!TSchedModel || DAG->SUnits.empty())
return;
LLVM_DEBUG(dbgs() << "Applying IGroupLPDAGMutation...\n");
// The order of InstructionGroups in this vector defines the
// order in which edges will be added. In other words, given the
// present ordering, we will try to make each VMEMRead instruction
// a predecessor of each DSRead instruction, and so on.
SmallVector<SchedGroup, 4> PipelineOrderGroups = {
SchedGroup(isVMEMSGMember, VMEMGroupMaxSize, DAG),
SchedGroup(isDSReadSGMember, LDRGroupMaxSize, DAG),
SchedGroup(isMFMASGMember, MFMAGroupMaxSize, DAG),
SchedGroup(isDSWriteSGMember, LDWGroupMaxSize, DAG)};
for (SUnit &SU : DAG->SUnits) {
LLVM_DEBUG(dbgs() << "Checking Node"; DAG->dumpNode(SU));
for (auto &SG : PipelineOrderGroups)
if (SG.canAddSU(SU, TII))
SG.add(SU);
}
for (unsigned i = 0; i < PipelineOrderGroups.size() - 1; i++) {
auto &GroupA = PipelineOrderGroups[i];
for (unsigned j = i + 1; j < PipelineOrderGroups.size(); j++) {
auto &GroupB = PipelineOrderGroups[j];
GroupA.link(GroupB);
}
}
}
void SchedBarrierDAGMutation::apply(ScheduleDAGInstrs *DAGInstrs) {
const TargetSchedModel *TSchedModel = DAGInstrs->getSchedModel();
if (!TSchedModel || DAGInstrs->SUnits.empty())
return;
LLVM_DEBUG(dbgs() << "Applying SchedBarrierDAGMutation...\n");
const GCNSubtarget &ST = DAGInstrs->MF.getSubtarget<GCNSubtarget>();
TII = ST.getInstrInfo();
DAG = static_cast<ScheduleDAGMI *>(DAGInstrs);
for (auto &SU : DAG->SUnits)
if (SU.getInstr()->getOpcode() == AMDGPU::SCHED_BARRIER)
addSchedBarrierEdges(SU);
}
void SchedBarrierDAGMutation::addSchedBarrierEdges(SUnit &SchedBarrier) {
MachineInstr &MI = *SchedBarrier.getInstr();
assert(MI.getOpcode() == AMDGPU::SCHED_BARRIER);
// Remove all existing edges from the SCHED_BARRIER that were added due to the
// instruction having side effects.
resetSchedBarrierEdges(SchedBarrier);
SmallVector<SchedGroup *, 4> SchedGroups;
int32_t Mask = MI.getOperand(0).getImm();
getSchedGroupsFromMask(Mask, SchedGroups);
for (auto SG : SchedGroups)
SG->link(
SchedBarrier, (function_ref<bool(const SUnit *A, const SUnit *B)>)[](
const SUnit *A, const SUnit *B) {
return A->NodeNum > B->NodeNum;
});
}
void SchedBarrierDAGMutation::getSchedGroupsFromMask(
int32_t Mask, SmallVectorImpl<SchedGroup *> &SchedGroups) {
SchedBarrierMasks SBMask = (SchedBarrierMasks)Mask;
// See IntrinsicsAMDGPU.td for an explanation of these masks and their
// mappings.
//
if ((SBMask & SchedBarrierMasks::VALU) == SchedBarrierMasks::NONE &&
(SBMask & SchedBarrierMasks::ALU) == SchedBarrierMasks::NONE) {
if (!VALUSchedGroup) {
VALUSchedGroup = std::make_unique<SchedGroup>(isVALUSGMember, None, DAG);
initSchedGroup(VALUSchedGroup.get());
}
SchedGroups.push_back(VALUSchedGroup.get());
}
if ((SBMask & SchedBarrierMasks::SALU) == SchedBarrierMasks::NONE &&
(SBMask & SchedBarrierMasks::ALU) == SchedBarrierMasks::NONE) {
if (!SALUSchedGroup) {
SALUSchedGroup = std::make_unique<SchedGroup>(isSALUSGMember, None, DAG);
initSchedGroup(SALUSchedGroup.get());
}
SchedGroups.push_back(SALUSchedGroup.get());
}
if ((SBMask & SchedBarrierMasks::MFMA) == SchedBarrierMasks::NONE &&
(SBMask & SchedBarrierMasks::ALU) == SchedBarrierMasks::NONE) {
if (!MFMASchedGroup) {
MFMASchedGroup = std::make_unique<SchedGroup>(isMFMASGMember, None, DAG);
initSchedGroup(MFMASchedGroup.get());
}
SchedGroups.push_back(MFMASchedGroup.get());
}
if ((SBMask & SchedBarrierMasks::VMEM_READ) == SchedBarrierMasks::NONE &&
(SBMask & SchedBarrierMasks::VMEM) == SchedBarrierMasks::NONE) {
if (!VMEMReadSchedGroup) {
VMEMReadSchedGroup =
std::make_unique<SchedGroup>(isVMEMReadSGMember, None, DAG);
initSchedGroup(VMEMReadSchedGroup.get());
}
SchedGroups.push_back(VMEMReadSchedGroup.get());
}
if ((SBMask & SchedBarrierMasks::VMEM_WRITE) == SchedBarrierMasks::NONE &&
(SBMask & SchedBarrierMasks::VMEM) == SchedBarrierMasks::NONE) {
if (!VMEMWriteSchedGroup) {
VMEMWriteSchedGroup =
std::make_unique<SchedGroup>(isVMEMWriteSGMember, None, DAG);
initSchedGroup(VMEMWriteSchedGroup.get());
}
SchedGroups.push_back(VMEMWriteSchedGroup.get());
}
if ((SBMask & SchedBarrierMasks::DS_READ) == SchedBarrierMasks::NONE &&
(SBMask & SchedBarrierMasks::DS) == SchedBarrierMasks::NONE) {
if (!DSReadSchedGroup) {
DSReadSchedGroup =
std::make_unique<SchedGroup>(isDSReadSGMember, None, DAG);
initSchedGroup(DSReadSchedGroup.get());
}
SchedGroups.push_back(DSReadSchedGroup.get());
}
if ((SBMask & SchedBarrierMasks::DS_WRITE) == SchedBarrierMasks::NONE &&
(SBMask & SchedBarrierMasks::DS) == SchedBarrierMasks::NONE) {
if (!DSWriteSchedGroup) {
DSWriteSchedGroup =
std::make_unique<SchedGroup>(isDSWriteSGMember, None, DAG);
initSchedGroup(DSWriteSchedGroup.get());
}
SchedGroups.push_back(DSWriteSchedGroup.get());
}
}
void SchedBarrierDAGMutation::initSchedGroup(SchedGroup *SG) {
assert(SG);
for (auto &SU : DAG->SUnits)
if (SG->canAddSU(SU, TII))
SG->add(SU);
}
void SchedBarrierDAGMutation::resetSchedBarrierEdges(SUnit &SU) {
assert(SU.getInstr()->getOpcode() == AMDGPU::SCHED_BARRIER);
for (auto &P : SU.Preds)
SU.removePred(P);
for (auto &S : SU.Succs) {
for (auto &SP : S.getSUnit()->Preds) {
if (SP.getSUnit() == &SU) {
S.getSUnit()->removePred(SP);
}
}
}
}
} // namespace
namespace llvm {
std::unique_ptr<ScheduleDAGMutation> createIGroupLPDAGMutation() {
return EnableIGroupLP ? std::make_unique<IGroupLPDAGMutation>() : nullptr;
}
std::unique_ptr<ScheduleDAGMutation> createSchedBarrierDAGMutation() {
return std::make_unique<SchedBarrierDAGMutation>();
}
} // end namespace llvm
