| //===-- InstrProfCorrelator.cpp -------------------------------------------===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include "llvm/ProfileData/InstrProfCorrelator.h" |
| #include "llvm/DebugInfo/DIContext.h" |
| #include "llvm/DebugInfo/DWARF/DWARFContext.h" |
| #include "llvm/DebugInfo/DWARF/DWARFDie.h" |
| #include "llvm/DebugInfo/DWARF/DWARFExpression.h" |
| #include "llvm/DebugInfo/DWARF/DWARFFormValue.h" |
| #include "llvm/DebugInfo/DWARF/DWARFLocationExpression.h" |
| #include "llvm/DebugInfo/DWARF/DWARFUnit.h" |
| #include "llvm/Object/MachO.h" |
| #include "llvm/Support/Debug.h" |
| #include <optional> |
| |
| #define DEBUG_TYPE "correlator" |
| |
| using namespace llvm; |
| |
| /// Get the __llvm_prf_cnts section. |
| Expected<object::SectionRef> getCountersSection(const object::ObjectFile &Obj) { |
| for (auto &Section : Obj.sections()) |
| if (auto SectionName = Section.getName()) |
| if (SectionName.get() == INSTR_PROF_CNTS_SECT_NAME) |
| return Section; |
| return make_error<InstrProfError>( |
| instrprof_error::unable_to_correlate_profile, |
| "could not find counter section (" INSTR_PROF_CNTS_SECT_NAME ")"); |
| } |
| |
| const char *InstrProfCorrelator::FunctionNameAttributeName = "Function Name"; |
| const char *InstrProfCorrelator::CFGHashAttributeName = "CFG Hash"; |
| const char *InstrProfCorrelator::NumCountersAttributeName = "Num Counters"; |
| |
| llvm::Expected<std::unique_ptr<InstrProfCorrelator::Context>> |
| InstrProfCorrelator::Context::get(std::unique_ptr<MemoryBuffer> Buffer, |
| const object::ObjectFile &Obj) { |
| auto CountersSection = getCountersSection(Obj); |
| if (auto Err = CountersSection.takeError()) |
| return std::move(Err); |
| auto C = std::make_unique<Context>(); |
| C->Buffer = std::move(Buffer); |
| C->CountersSectionStart = CountersSection->getAddress(); |
| C->CountersSectionEnd = C->CountersSectionStart + CountersSection->getSize(); |
| C->ShouldSwapBytes = Obj.isLittleEndian() != sys::IsLittleEndianHost; |
| return Expected<std::unique_ptr<Context>>(std::move(C)); |
| } |
| |
| llvm::Expected<std::unique_ptr<InstrProfCorrelator>> |
| InstrProfCorrelator::get(StringRef DebugInfoFilename) { |
| auto DsymObjectsOrErr = |
| object::MachOObjectFile::findDsymObjectMembers(DebugInfoFilename); |
| if (auto Err = DsymObjectsOrErr.takeError()) |
| return std::move(Err); |
| if (!DsymObjectsOrErr->empty()) { |
| // TODO: Enable profile correlation when there are multiple objects in a |
| // dSYM bundle. |
| if (DsymObjectsOrErr->size() > 1) |
| return make_error<InstrProfError>( |
| instrprof_error::unable_to_correlate_profile, |
| "using multiple objects is not yet supported"); |
| DebugInfoFilename = *DsymObjectsOrErr->begin(); |
| } |
| auto BufferOrErr = |
| errorOrToExpected(MemoryBuffer::getFile(DebugInfoFilename)); |
| if (auto Err = BufferOrErr.takeError()) |
| return std::move(Err); |
| |
| return get(std::move(*BufferOrErr)); |
| } |
| |
| llvm::Expected<std::unique_ptr<InstrProfCorrelator>> |
| InstrProfCorrelator::get(std::unique_ptr<MemoryBuffer> Buffer) { |
| auto BinOrErr = object::createBinary(*Buffer); |
| if (auto Err = BinOrErr.takeError()) |
| return std::move(Err); |
| |
| if (auto *Obj = dyn_cast<object::ObjectFile>(BinOrErr->get())) { |
| auto CtxOrErr = Context::get(std::move(Buffer), *Obj); |
| if (auto Err = CtxOrErr.takeError()) |
| return std::move(Err); |
| auto T = Obj->makeTriple(); |
| if (T.isArch64Bit()) |
| return InstrProfCorrelatorImpl<uint64_t>::get(std::move(*CtxOrErr), *Obj); |
| if (T.isArch32Bit()) |
| return InstrProfCorrelatorImpl<uint32_t>::get(std::move(*CtxOrErr), *Obj); |
| } |
| return make_error<InstrProfError>( |
| instrprof_error::unable_to_correlate_profile, "not an object file"); |
| } |
| |
| std::optional<size_t> InstrProfCorrelator::getDataSize() const { |
| if (auto *C = dyn_cast<InstrProfCorrelatorImpl<uint32_t>>(this)) { |
| return C->getDataSize(); |
| } else if (auto *C = dyn_cast<InstrProfCorrelatorImpl<uint64_t>>(this)) { |
| return C->getDataSize(); |
| } |
| return {}; |
| } |
| |
| namespace llvm { |
| |
| template <> |
| InstrProfCorrelatorImpl<uint32_t>::InstrProfCorrelatorImpl( |
| std::unique_ptr<InstrProfCorrelator::Context> Ctx) |
| : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_32Bit, |
| std::move(Ctx)) {} |
| template <> |
| InstrProfCorrelatorImpl<uint64_t>::InstrProfCorrelatorImpl( |
| std::unique_ptr<InstrProfCorrelator::Context> Ctx) |
| : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_64Bit, |
| std::move(Ctx)) {} |
| template <> |
| bool InstrProfCorrelatorImpl<uint32_t>::classof(const InstrProfCorrelator *C) { |
| return C->getKind() == InstrProfCorrelatorKind::CK_32Bit; |
| } |
| template <> |
| bool InstrProfCorrelatorImpl<uint64_t>::classof(const InstrProfCorrelator *C) { |
| return C->getKind() == InstrProfCorrelatorKind::CK_64Bit; |
| } |
| |
| } // end namespace llvm |
| |
| template <class IntPtrT> |
| llvm::Expected<std::unique_ptr<InstrProfCorrelatorImpl<IntPtrT>>> |
| InstrProfCorrelatorImpl<IntPtrT>::get( |
| std::unique_ptr<InstrProfCorrelator::Context> Ctx, |
| const object::ObjectFile &Obj) { |
| if (Obj.isELF() || Obj.isMachO()) { |
| auto DICtx = DWARFContext::create(Obj); |
| return std::make_unique<DwarfInstrProfCorrelator<IntPtrT>>(std::move(DICtx), |
| std::move(Ctx)); |
| } |
| return make_error<InstrProfError>( |
| instrprof_error::unable_to_correlate_profile, |
| "unsupported debug info format (only DWARF is supported)"); |
| } |
| |
| template <class IntPtrT> |
| Error InstrProfCorrelatorImpl<IntPtrT>::correlateProfileData() { |
| assert(Data.empty() && Names.empty() && NamesVec.empty()); |
| correlateProfileDataImpl(); |
| if (Data.empty() || NamesVec.empty()) |
| return make_error<InstrProfError>( |
| instrprof_error::unable_to_correlate_profile, |
| "could not find any profile metadata in debug info"); |
| auto Result = |
| collectPGOFuncNameStrings(NamesVec, /*doCompression=*/false, Names); |
| CounterOffsets.clear(); |
| NamesVec.clear(); |
| return Result; |
| } |
| |
| template <> struct yaml::MappingTraits<InstrProfCorrelator::CorrelationData> { |
| static void mapping(yaml::IO &io, |
| InstrProfCorrelator::CorrelationData &Data) { |
| io.mapRequired("Probes", Data.Probes); |
| } |
| }; |
| |
| template <> struct yaml::MappingTraits<InstrProfCorrelator::Probe> { |
| static void mapping(yaml::IO &io, InstrProfCorrelator::Probe &P) { |
| io.mapRequired("Function Name", P.FunctionName); |
| io.mapOptional("Linkage Name", P.LinkageName); |
| io.mapRequired("CFG Hash", P.CFGHash); |
| io.mapRequired("Counter Offset", P.CounterOffset); |
| io.mapRequired("Num Counters", P.NumCounters); |
| io.mapOptional("File", P.FilePath); |
| io.mapOptional("Line", P.LineNumber); |
| } |
| }; |
| |
| template <> struct yaml::SequenceElementTraits<InstrProfCorrelator::Probe> { |
| static const bool flow = false; |
| }; |
| |
| template <class IntPtrT> |
| Error InstrProfCorrelatorImpl<IntPtrT>::dumpYaml(raw_ostream &OS) { |
| InstrProfCorrelator::CorrelationData Data; |
| correlateProfileDataImpl(&Data); |
| if (Data.Probes.empty()) |
| return make_error<InstrProfError>( |
| instrprof_error::unable_to_correlate_profile, |
| "could not find any profile metadata in debug info"); |
| yaml::Output YamlOS(OS); |
| YamlOS << Data; |
| return Error::success(); |
| } |
| |
| template <class IntPtrT> |
| void InstrProfCorrelatorImpl<IntPtrT>::addProbe(StringRef FunctionName, |
| uint64_t CFGHash, |
| IntPtrT CounterOffset, |
| IntPtrT FunctionPtr, |
| uint32_t NumCounters) { |
| // Check if a probe was already added for this counter offset. |
| if (!CounterOffsets.insert(CounterOffset).second) |
| return; |
| Data.push_back({ |
| maybeSwap<uint64_t>(IndexedInstrProf::ComputeHash(FunctionName)), |
| maybeSwap<uint64_t>(CFGHash), |
| // In this mode, CounterPtr actually stores the section relative address |
| // of the counter. |
| maybeSwap<IntPtrT>(CounterOffset), |
| maybeSwap<IntPtrT>(FunctionPtr), |
| // TODO: Value profiling is not yet supported. |
| /*ValuesPtr=*/maybeSwap<IntPtrT>(0), |
| maybeSwap<uint32_t>(NumCounters), |
| /*NumValueSites=*/{maybeSwap<uint16_t>(0), maybeSwap<uint16_t>(0)}, |
| }); |
| NamesVec.push_back(FunctionName.str()); |
| } |
| |
| template <class IntPtrT> |
| std::optional<uint64_t> |
| DwarfInstrProfCorrelator<IntPtrT>::getLocation(const DWARFDie &Die) const { |
| auto Locations = Die.getLocations(dwarf::DW_AT_location); |
| if (!Locations) { |
| consumeError(Locations.takeError()); |
| return {}; |
| } |
| auto &DU = *Die.getDwarfUnit(); |
| auto AddressSize = DU.getAddressByteSize(); |
| for (auto &Location : *Locations) { |
| DataExtractor Data(Location.Expr, DICtx->isLittleEndian(), AddressSize); |
| DWARFExpression Expr(Data, AddressSize); |
| for (auto &Op : Expr) { |
| if (Op.getCode() == dwarf::DW_OP_addr) { |
| return Op.getRawOperand(0); |
| } else if (Op.getCode() == dwarf::DW_OP_addrx) { |
| uint64_t Index = Op.getRawOperand(0); |
| if (auto SA = DU.getAddrOffsetSectionItem(Index)) |
| return SA->Address; |
| } |
| } |
| } |
| return {}; |
| } |
| |
| template <class IntPtrT> |
| bool DwarfInstrProfCorrelator<IntPtrT>::isDIEOfProbe(const DWARFDie &Die) { |
| const auto &ParentDie = Die.getParent(); |
| if (!Die.isValid() || !ParentDie.isValid() || Die.isNULL()) |
| return false; |
| if (Die.getTag() != dwarf::DW_TAG_variable) |
| return false; |
| if (!ParentDie.isSubprogramDIE()) |
| return false; |
| if (!Die.hasChildren()) |
| return false; |
| if (const char *Name = Die.getName(DINameKind::ShortName)) |
| return StringRef(Name).startswith(getInstrProfCountersVarPrefix()); |
| return false; |
| } |
| |
| template <class IntPtrT> |
| void DwarfInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl( |
| InstrProfCorrelator::CorrelationData *Data) { |
| auto maybeAddProbe = [&](DWARFDie Die) { |
| if (!isDIEOfProbe(Die)) |
| return; |
| std::optional<const char *> FunctionName; |
| std::optional<uint64_t> CFGHash; |
| std::optional<uint64_t> CounterPtr = getLocation(Die); |
| auto FnDie = Die.getParent(); |
| auto FunctionPtr = dwarf::toAddress(FnDie.find(dwarf::DW_AT_low_pc)); |
| std::optional<uint64_t> NumCounters; |
| for (const DWARFDie &Child : Die.children()) { |
| if (Child.getTag() != dwarf::DW_TAG_LLVM_annotation) |
| continue; |
| auto AnnotationFormName = Child.find(dwarf::DW_AT_name); |
| auto AnnotationFormValue = Child.find(dwarf::DW_AT_const_value); |
| if (!AnnotationFormName || !AnnotationFormValue) |
| continue; |
| auto AnnotationNameOrErr = AnnotationFormName->getAsCString(); |
| if (auto Err = AnnotationNameOrErr.takeError()) { |
| consumeError(std::move(Err)); |
| continue; |
| } |
| StringRef AnnotationName = *AnnotationNameOrErr; |
| if (AnnotationName.compare( |
| InstrProfCorrelator::FunctionNameAttributeName) == 0) { |
| if (auto EC = |
| AnnotationFormValue->getAsCString().moveInto(FunctionName)) |
| consumeError(std::move(EC)); |
| } else if (AnnotationName.compare( |
| InstrProfCorrelator::CFGHashAttributeName) == 0) { |
| CFGHash = AnnotationFormValue->getAsUnsignedConstant(); |
| } else if (AnnotationName.compare( |
| InstrProfCorrelator::NumCountersAttributeName) == 0) { |
| NumCounters = AnnotationFormValue->getAsUnsignedConstant(); |
| } |
| } |
| if (!FunctionName || !CFGHash || !CounterPtr || !NumCounters) { |
| LLVM_DEBUG(dbgs() << "Incomplete DIE for probe\n\tFunctionName: " |
| << FunctionName << "\n\tCFGHash: " << CFGHash |
| << "\n\tCounterPtr: " << CounterPtr |
| << "\n\tNumCounters: " << NumCounters); |
| LLVM_DEBUG(Die.dump(dbgs())); |
| return; |
| } |
| uint64_t CountersStart = this->Ctx->CountersSectionStart; |
| uint64_t CountersEnd = this->Ctx->CountersSectionEnd; |
| if (*CounterPtr < CountersStart || *CounterPtr >= CountersEnd) { |
| LLVM_DEBUG( |
| dbgs() << "CounterPtr out of range for probe\n\tFunction Name: " |
| << FunctionName << "\n\tExpected: [0x" |
| << Twine::utohexstr(CountersStart) << ", 0x" |
| << Twine::utohexstr(CountersEnd) << ")\n\tActual: 0x" |
| << Twine::utohexstr(*CounterPtr)); |
| LLVM_DEBUG(Die.dump(dbgs())); |
| return; |
| } |
| if (!FunctionPtr) { |
| LLVM_DEBUG(dbgs() << "Could not find address of " << *FunctionName |
| << "\n"); |
| LLVM_DEBUG(Die.dump(dbgs())); |
| } |
| IntPtrT CounterOffset = *CounterPtr - CountersStart; |
| if (Data) { |
| InstrProfCorrelator::Probe P; |
| P.FunctionName = *FunctionName; |
| if (auto Name = FnDie.getName(DINameKind::LinkageName)) |
| P.LinkageName = Name; |
| P.CFGHash = *CFGHash; |
| P.CounterOffset = CounterOffset; |
| P.NumCounters = *NumCounters; |
| auto FilePath = FnDie.getDeclFile( |
| DILineInfoSpecifier::FileLineInfoKind::RelativeFilePath); |
| if (!FilePath.empty()) |
| P.FilePath = FilePath; |
| if (auto LineNumber = FnDie.getDeclLine()) |
| P.LineNumber = LineNumber; |
| Data->Probes.push_back(P); |
| } else { |
| this->addProbe(*FunctionName, *CFGHash, CounterOffset, |
| FunctionPtr.value_or(0), *NumCounters); |
| } |
| }; |
| for (auto &CU : DICtx->normal_units()) |
| for (const auto &Entry : CU->dies()) |
| maybeAddProbe(DWARFDie(CU.get(), &Entry)); |
| for (auto &CU : DICtx->dwo_units()) |
| for (const auto &Entry : CU->dies()) |
| maybeAddProbe(DWARFDie(CU.get(), &Entry)); |
| } |
