third_party/llvm-16.0/llvm/lib/Transforms/Utils/SampleProfileLoaderBaseUtil.cpp

//===- SampleProfileLoaderBaseUtil.cpp - Profile loader Util func ---------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file implements the SampleProfileLoader base utility functions.
//
//===----------------------------------------------------------------------===//

#include "llvm/Transforms/Utils/SampleProfileLoaderBaseUtil.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Module.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"

namespace llvm {

cl::opt<unsigned> SampleProfileMaxPropagateIterations(
    "sample-profile-max-propagate-iterations", cl::init(100),
    cl::desc("Maximum number of iterations to go through when propagating "
             "sample block/edge weights through the CFG."));

cl::opt<unsigned> SampleProfileRecordCoverage(
    "sample-profile-check-record-coverage", cl::init(0), cl::value_desc("N"),
    cl::desc("Emit a warning if less than N% of records in the input profile "
             "are matched to the IR."));

cl::opt<unsigned> SampleProfileSampleCoverage(
    "sample-profile-check-sample-coverage", cl::init(0), cl::value_desc("N"),
    cl::desc("Emit a warning if less than N% of samples in the input profile "
             "are matched to the IR."));

cl::opt<bool> NoWarnSampleUnused(
    "no-warn-sample-unused", cl::init(false), cl::Hidden,
    cl::desc("Use this option to turn off/on warnings about function with "
             "samples but without debug information to use those samples. "));

cl::opt<bool> SampleProfileUseProfi(
    "sample-profile-use-profi", cl::Hidden,
    cl::desc("Use profi to infer block and edge counts."));

namespace sampleprofutil {

/// Return true if the given callsite is hot wrt to hot cutoff threshold.
///
/// Functions that were inlined in the original binary will be represented
/// in the inline stack in the sample profile. If the profile shows that
/// the original inline decision was "good" (i.e., the callsite is executed
/// frequently), then we will recreate the inline decision and apply the
/// profile from the inlined callsite.
///
/// To decide whether an inlined callsite is hot, we compare the callsite
/// sample count with the hot cutoff computed by ProfileSummaryInfo, it is
/// regarded as hot if the count is above the cutoff value.
///
/// When ProfileAccurateForSymsInList is enabled and profile symbol list
/// is present, functions in the profile symbol list but without profile will
/// be regarded as cold and much less inlining will happen in CGSCC inlining
/// pass, so we tend to lower the hot criteria here to allow more early
/// inlining to happen for warm callsites and it is helpful for performance.
bool callsiteIsHot(const FunctionSamples *CallsiteFS, ProfileSummaryInfo *PSI,
                   bool ProfAccForSymsInList) {
  if (!CallsiteFS)
    return false; // The callsite was not inlined in the original binary.

  assert(PSI && "PSI is expected to be non null");
  uint64_t CallsiteTotalSamples = CallsiteFS->getTotalSamples();
  if (ProfAccForSymsInList)
    return !PSI->isColdCount(CallsiteTotalSamples);
  else
    return PSI->isHotCount(CallsiteTotalSamples);
}

/// Mark as used the sample record for the given function samples at
/// (LineOffset, Discriminator).
///
/// \returns true if this is the first time we mark the given record.
bool SampleCoverageTracker::markSamplesUsed(const FunctionSamples *FS,
                                            uint32_t LineOffset,
                                            uint32_t Discriminator,
                                            uint64_t Samples) {
  LineLocation Loc(LineOffset, Discriminator);
  unsigned &Count = SampleCoverage[FS][Loc];
  bool FirstTime = (++Count == 1);
  if (FirstTime)
    TotalUsedSamples += Samples;
  return FirstTime;
}

/// Return the number of sample records that were applied from this profile.
///
/// This count does not include records from cold inlined callsites.
unsigned
SampleCoverageTracker::countUsedRecords(const FunctionSamples *FS,
                                        ProfileSummaryInfo *PSI) const {
  auto I = SampleCoverage.find(FS);

  // The size of the coverage map for FS represents the number of records
  // that were marked used at least once.
  unsigned Count = (I != SampleCoverage.end()) ? I->second.size() : 0;

  // If there are inlined callsites in this function, count the samples found
  // in the respective bodies. However, do not bother counting callees with 0
  // total samples, these are callees that were never invoked at runtime.
  for (const auto &I : FS->getCallsiteSamples())
    for (const auto &J : I.second) {
      const FunctionSamples *CalleeSamples = &J.second;
      if (callsiteIsHot(CalleeSamples, PSI, ProfAccForSymsInList))
        Count += countUsedRecords(CalleeSamples, PSI);
    }

  return Count;
}

/// Return the number of sample records in the body of this profile.
///
/// This count does not include records from cold inlined callsites.
unsigned
SampleCoverageTracker::countBodyRecords(const FunctionSamples *FS,
                                        ProfileSummaryInfo *PSI) const {
  unsigned Count = FS->getBodySamples().size();

  // Only count records in hot callsites.
  for (const auto &I : FS->getCallsiteSamples())
    for (const auto &J : I.second) {
      const FunctionSamples *CalleeSamples = &J.second;
      if (callsiteIsHot(CalleeSamples, PSI, ProfAccForSymsInList))
        Count += countBodyRecords(CalleeSamples, PSI);
    }

  return Count;
}

/// Return the number of samples collected in the body of this profile.
///
/// This count does not include samples from cold inlined callsites.
uint64_t
SampleCoverageTracker::countBodySamples(const FunctionSamples *FS,
                                        ProfileSummaryInfo *PSI) const {
  uint64_t Total = 0;
  for (const auto &I : FS->getBodySamples())
    Total += I.second.getSamples();

  // Only count samples in hot callsites.
  for (const auto &I : FS->getCallsiteSamples())
    for (const auto &J : I.second) {
      const FunctionSamples *CalleeSamples = &J.second;
      if (callsiteIsHot(CalleeSamples, PSI, ProfAccForSymsInList))
        Total += countBodySamples(CalleeSamples, PSI);
    }

  return Total;
}

/// Return the fraction of sample records used in this profile.
///
/// The returned value is an unsigned integer in the range 0-100 indicating
/// the percentage of sample records that were used while applying this
/// profile to the associated function.
unsigned SampleCoverageTracker::computeCoverage(unsigned Used,
                                                unsigned Total) const {
  assert(Used <= Total &&
         "number of used records cannot exceed the total number of records");
  return Total > 0 ? Used * 100 / Total : 100;
}

/// Create a global variable to flag FSDiscriminators are used.
void createFSDiscriminatorVariable(Module *M) {
  const char *FSDiscriminatorVar = "__llvm_fs_discriminator__";
  if (M->getGlobalVariable(FSDiscriminatorVar))
    return;

  auto &Context = M->getContext();
  // Place this variable to llvm.used so it won't be GC'ed.
  appendToUsed(*M, {new GlobalVariable(*M, Type::getInt1Ty(Context), true,
                                       GlobalValue::WeakODRLinkage,
                                       ConstantInt::getTrue(Context),
                                       FSDiscriminatorVar)});
}

} // end of namespace sampleprofutil
} // end of namespace llvm