Adding support for llvm.canonicalize intrinsic.

Change-Id: Ieed4a6beaf9089ef616071a05110ea88cc4719d0

Author: krystian-andrzejewski

IGC/Compiler/CISACodeGen/CISABuilder.cpp

@@ -4046,7 +4046,8 @@ namespace IGC
         {
             SaveOption(vISA_BXMLEncoder, false);
         }
-        if (IGC_IS_FLAG_ENABLED(DisableMixMode))
+        if (IGC_IS_FLAG_ENABLED(DisableMixMode) ||
+            context->getModuleMetaData()->disableMixMode)
         {
             SaveOption(vISA_DisableMixMode, true);
         }

IGC/Compiler/CISACodeGen/EmitVISAPass.cpp

@@ -8004,6 +8004,10 @@ void EmitPass::EmitIntrinsicMessage(llvm::IntrinsicInst* inst)
         emitSqrt(inst);
         break;
 
+    case Intrinsic::canonicalize:
+        emitCanonicalize(inst);
+        break;
+
     default:
         inst->print(IGC::Debug::ods());
         assert(0 && "unknown intrinsic");
@@ -15655,6 +15659,26 @@ void EmitPass::emitSqrt(Instruction* inst)
     m_encoder->Sqrt(m_destination, src0);
 }
 
+void IGC::EmitPass::emitCanonicalize(llvm::Instruction* inst)
+{
+    // Force to flush denormal fp value to zero. Select one of two possible solutions:
+    // 1. add inputVal, -0.0
+    // 2. mul inputVal, 1.0
+    // A normalized fp value isn't changed.
+    // The operation is done only if particular flags are set.
+    // If the instruction should be emitted anyway, flushing a subnormal to zero has to implemented in other way.
+    CodeGenContext* pCodeGenContext = getAnalysis<CodeGenContextWrapper>().getCodeGenContext();
+    bool flushVal = pCodeGenContext->m_floatDenormMode16 == ::IGC::FLOAT_DENORM_FLUSH_TO_ZERO && inst->getType()->isHalfTy();
+    flushVal = flushVal || (pCodeGenContext->m_floatDenormMode32 == ::IGC::FLOAT_DENORM_FLUSH_TO_ZERO && inst->getType()->isFloatTy());
+    flushVal = flushVal || (pCodeGenContext->m_floatDenormMode64 == ::IGC::FLOAT_DENORM_FLUSH_TO_ZERO && inst->getType()->isDoubleTy());
+    if (flushVal)
+    {
+        CVariable* inputVal = GetSymbol(inst->getOperand(0));
+        CVariable* negativeZero = m_currShader->GetScalarConstant(llvm::ConstantFP::get(inst->getType(), -0.0));
+        m_encoder->Add(m_destination, inputVal, negativeZero);
+    }
+}
+
 // emit llvm.bswap
 void EmitPass::emitLLVMbswap(IntrinsicInst* inst)
 {

IGC/Compiler/CISACodeGen/EmitVISAPass.hpp

@@ -407,6 +407,7 @@ namespace IGC
         void emitAddPairWithImm(CVariable* Dst, CVariable* Src, CVariable* Imm);
 
         void emitSqrt(llvm::Instruction* inst);
+        void emitCanonicalize(llvm::Instruction* inst);
         void emitRsq(llvm::Instruction* inst);
 
         void emitLLVMbswap(llvm::IntrinsicInst* inst);

IGC/Compiler/CISACodeGen/PatternMatchPass.cpp

@@ -2993,6 +2993,52 @@ namespace IGC
         return true;
     }
 
+    bool CodeGenPatternMatch::MatchCanonicalizeInstruction(llvm::Instruction& I)
+    {
+        struct CanonicalizeInstPattern : Pattern
+        {
+            CanonicalizeInstPattern(llvm::Instruction* pInst, bool isNeeded) : m_pInst(pInst), m_IsNeeded(isNeeded) {}
+
+            llvm::Instruction* m_pInst;
+            bool m_IsNeeded;
+
+            virtual void Emit(EmitPass* pass, const DstModifier& modifier)
+            {
+                assert(modifier.sat == false && modifier.flag == nullptr);
+                if (m_IsNeeded)
+                {
+                    pass->emitCanonicalize(m_pInst);
+                }
+            }
+        };
+
+        assert(I.getNumOperands() == 1);
+        bool isNeeded = true;
+
+        // FAdd, FSub, FMul, FDiv instructions flush subnormals to zero.
+        // However, mix mode and math instructions preserve subnormals.
+        // Other instructions also preserve subnormals.
+        if (llvm::BinaryOperator * pBianaryOperator = llvm::dyn_cast<llvm::BinaryOperator>(I.getOperand(0)))
+        {
+            switch (pBianaryOperator->getOpcode())
+            {
+            case llvm::BinaryOperator::BinaryOps::FAdd:
+            case llvm::BinaryOperator::BinaryOps::FMul:
+            case llvm::BinaryOperator::BinaryOps::FSub:
+            case llvm::BinaryOperator::BinaryOps::FDiv:
+                isNeeded = false;
+            default:
+                break;
+            }
+        }
+
+        CanonicalizeInstPattern* pattern = new (m_allocator) CanonicalizeInstPattern(&I, isNeeded);
+        MarkAsSource(I.getOperand(0));
+
+        AddPattern(pattern);
+        return true;
+    }
+
     bool CodeGenPatternMatch::MatchBranch(llvm::BranchInst& I)
     {
         struct CondBrInstPattern : Pattern

IGC/Compiler/CISACodeGen/PatternMatchPass.hpp

@@ -196,6 +196,7 @@ namespace IGC
         bool MatchCmpSext(llvm::Instruction& I);
         bool MatchModifier(llvm::Instruction& I, bool SupportSrc0Mod = true);
         bool MatchSingleInstruction(llvm::Instruction& I);
+        bool MatchCanonicalizeInstruction(llvm::Instruction& I);
         bool MatchBranch(llvm::BranchInst& I);
         bool MatchShuffleBroadCast(llvm::GenIntrinsicInst& I);
         bool MatchWaveShuffleIndex(llvm::GenIntrinsicInst& I);

IGC/Compiler/CISACodeGen/opCode.h

@@ -129,6 +129,7 @@ DECLARE_OPCODE(FPExt, Instruction, llvm_fpext, true, false, false, false, false,
 DECLARE_OPCODE(GenISA_f32tof16_rtz, GenISAIntrinsic, llvm_f32tof16_rtz, true, false, false, false, true, false, false)
 DECLARE_OPCODE(fabs, Intrinsic, llvm_fabs, false, false, true, false, true, false, false)
 DECLARE_OPCODE(GenISA_fsat, GenISAIntrinsic, llvm_fsat, true, false, true, true, true, false, false)
+DECLARE_OPCODE(canonicalize, Intrinsic, llvm_canonicalize, true, true, true, true, false, false, false)
 
 // GS Intrinsics
 DECLARE_OPCODE(GenISA_OUTPUTGS, GenISAIntrinsic, llvm_output_gs, false, false, false, false, false, false, false)

IGC/Compiler/CustomSafeOptPass.cpp

@@ -2351,6 +2351,17 @@ Constant* IGCConstProp::replaceShaderConstant(LoadInst* inst)
     return nullptr;
 }
 
+llvm::Constant* IGC::IGCConstantFolder::CreateCanonicalize(llvm::Constant* C0, bool flushDenorms /*= true*/) const
+{
+    assert(llvm::isa<ConstantFP>(C0));
+    auto APF = llvm::cast<ConstantFP>(C0)->getValueAPF();
+    if (flushDenorms && APF.isDenormal())
+    {
+        APF = APFloat::getZero(APF.getSemantics(), APF.isNegative());
+    }
+    return ConstantFP::get(C0->getContext(), APF);
+}
+
 Constant* IGCConstProp::ConstantFoldCallInstruction(CallInst* inst)
 {
     Constant* C = nullptr;
@@ -2453,6 +2464,22 @@ Constant* IGCConstProp::ConstantFoldCallInstruction(CallInst* inst)
                 C = ConstantFP::get(inst->getContext(), minnum(One, maxnum(zero, A)));
             }
         }
+        break;
+        case llvm_canonicalize:
+        {
+            // If the instruction should be emitted anyway, then remove the condition.
+            // Please, be aware of the fact that clients can understand the term canonical FP value in other way.
+            if (C0)
+            {
+                IGCConstantFolder constantFolder;
+                CodeGenContext* pCodeGenContext = getAnalysis<CodeGenContextWrapper>().getCodeGenContext();
+                bool flushVal = pCodeGenContext->m_floatDenormMode16 == ::IGC::FLOAT_DENORM_FLUSH_TO_ZERO && inst->getType()->isHalfTy();
+                flushVal = flushVal || (pCodeGenContext->m_floatDenormMode32 == ::IGC::FLOAT_DENORM_FLUSH_TO_ZERO && inst->getType()->isFloatTy());
+                flushVal = flushVal || (pCodeGenContext->m_floatDenormMode64 == ::IGC::FLOAT_DENORM_FLUSH_TO_ZERO && inst->getType()->isDoubleTy());
+                C = constantFolder.CreateCanonicalize(C0, flushVal);
+            }
+        }
+        break;
         default:
             break;
         }
@@ -2934,7 +2961,6 @@ bool IGCConstProp::runOnFunction(Function& F)
     return Changed;
 }
 
-
 namespace {
 
     class IGCIndirectICBPropagaion : public FunctionPass
@@ -2945,7 +2971,7 @@ namespace {
         {
             initializeIGCIndirectICBPropagaionPass(*PassRegistry::getPassRegistry());
         }
-        virtual llvm::StringRef getPassName() const { return "Indirect ICB Propagaion"; }
+        virtual llvm::StringRef getPassName() const { return "Indirect ICB Propagation"; }
         virtual bool runOnFunction(Function& F);
         virtual void getAnalysisUsage(llvm::AnalysisUsage& AU) const
         {
@@ -4186,5 +4212,4 @@ bool LogicalAndToBranch::runOnFunction(Function& F)
     }
 
     return changed;
-}
-
+}

IGC/Compiler/CustomSafeOptPass.hpp

@@ -33,6 +33,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #include <llvm/Analysis/TargetLibraryInfo.h>
 #include <llvm/Analysis/LoopPass.h>
 #include <llvm/IR/Instructions.h>
+#include <llvm/IR/ConstantFolder.h>
 #include "common/LLVMWarningsPop.hpp"
 
 namespace llvm
@@ -201,6 +202,16 @@ namespace IGC
         void visitSelectInst(llvm::SelectInst& I);
     };
 
+    class IGCConstantFolder : public llvm::ConstantFolder
+    {
+    public:
+        IGCConstantFolder() :
+            ConstantFolder()
+        {}
+
+        llvm::Constant* CreateCanonicalize(llvm::Constant* C0, bool flushDenorms = true) const;
+    };
+
     class IGCConstProp : public llvm::FunctionPass
     {
     public:
@@ -225,13 +236,14 @@ namespace IGC
             // specialized const-prop with shader-const replacement
             return "const-prop with shader-const replacement";
         }
+
     private:
         llvm::Module* module;
         llvm::Constant* ReplaceFromDynConstants(unsigned bufId, unsigned eltId, unsigned int size_in_bytes, llvm::LoadInst* inst);
         llvm::Constant* replaceShaderConstant(llvm::LoadInst* inst);
-        llvm::Constant* ConstantFoldCallInstruction(llvm::CallInst* inst);
         llvm::Constant* ConstantFoldCmpInst(llvm::CmpInst* inst);
         llvm::Constant* ConstantFoldExtractElement(llvm::ExtractElementInst* inst);
+        llvm::Constant* ConstantFoldCallInstruction(llvm::CallInst* inst);
         bool simplifyAdd(llvm::BinaryOperator* BO);
         bool simplifyGEP(llvm::GetElementPtrInst* GEP);
         bool m_enableMathConstProp;

IGC/Compiler/IGC_IR_spec.hpp

@@ -97,6 +97,7 @@ IGC_IR_LLVM_INTRINSIC(assume)
 IGC_IR_LLVM_INTRINSIC(bswap)
 IGC_IR_LLVM_INTRINSIC(maxnum)
 IGC_IR_LLVM_INTRINSIC(minnum)
+IGC_IR_LLVM_INTRINSIC(canonicalize)
 #if LLVM_VERSION_MAJOR >= 8
 IGC_IR_LLVM_INTRINSIC(fshl)
 IGC_IR_LLVM_INTRINSIC(fshr)

IGC/common/MDFrameWork.h

@@ -361,6 +361,7 @@ namespace IGC
         // When true compiler can assume that resources bound to two different
         // bindings do not alias.
         bool statefullResourcesNotAliased = false;
+        bool disableMixMode = false;
 
         unsigned int privateMemoryPerWI = 0;
         std::array<uint64_t, NUM_SHADER_RESOURCE_VIEW_SIZE> m_ShaderResourceViewMcsMask{};