Store3D doesn't allow to emit store wider than 128 bits. The LoadStorePointer pattern match doesn't consider the restrictions

krystian-andrzejewski · gfxbot · commit b878de87c694 · 2019-04-25T08:30:43.000-07:00
Change-Id: Ic39b791812e41ff496a64f1d13b9c65300f9ed5e
diff --git a/IGC/Compiler/CISACodeGen/PatternMatchPass.cpp b/IGC/Compiler/CISACodeGen/PatternMatchPass.cpp
@@ -96,7 +96,7 @@ void CodeGenPatternMatch::CodeGenNode( llvm::DomTreeNode* node )
     }
     llvm::BasicBlock* bb = node->getBlock();
     CodeGenBlock(bb);
-} 
+}
 
 
 bool CodeGenPatternMatch::runOnFunction(llvm::Function &F)
@@ -108,7 +108,7 @@ bool CodeGenPatternMatch::runOnFunction(llvm::Function &F)
 
     delete[] m_blocks;
     m_blocks = nullptr;
-    
+
     m_ctx = getAnalysis<CodeGenContextWrapper>().getCodeGenContext();
 
     MetaDataUtils *pMdUtils = getAnalysis<MetaDataUtilsWrapper>().getMetaDataUtils();
@@ -121,7 +121,7 @@ bool CodeGenPatternMatch::runOnFunction(llvm::Function &F)
     m_AllowContractions = true;
     if(m_ctx->m_DriverInfo.NeedCheckContractionAllowed())
     {
-        m_AllowContractions = 
+        m_AllowContractions =
             modMD->compOpt.FastRelaxedMath ||
             modMD->compOpt.MadEnable;
     }
@@ -152,7 +152,7 @@ inline bool HasSideEffect(llvm::Instruction& inst)
 
 inline bool HasPhiUse(llvm::Value& inst)
 {
-    for (auto UI = inst.user_begin(), E = inst.user_end(); UI != E; ++UI) 
+    for (auto UI = inst.user_begin(), E = inst.user_end(); UI != E; ++UI)
     {
         llvm::User *U = *UI;
         if( llvm::isa<llvm::PHINode>(U))
@@ -725,7 +725,7 @@ CodeGenPatternMatch::isIntegerSatTrunc(llvm::SelectInst *SI) {
         // since src is a source of a trunc instruction, and dst
         // have the same width as its destination.
         return std::make_tuple(nullptr, false, false);
-    }  
+    }
 
     if (CI->getValue() != UMax && CI->getValue() != SMax)
         return std::make_tuple(nullptr, false, false);
@@ -836,7 +836,7 @@ void CodeGenPatternMatch::visitCastInst(llvm::CastInst &I)
     }
     else if (I.getOpcode() == Instruction::Trunc)
     {
-        match = 
+        match =
             MatchModifier(I);
     }
     else
@@ -1004,12 +1004,12 @@ void CodeGenPatternMatch::visitBinaryOperator(llvm::BinaryOperator &I)
                 MatchModifier(I);
         break;
     case Instruction::And:
-        match = 
+        match =
                 MatchBoolOp(I) ||
                 MatchLogicAlu(I);
         break;
     case Instruction::Or:
-        match = 
+        match =
                 MatchBoolOp(I) ||
                 MatchLogicAlu(I);
         break;
@@ -1026,7 +1026,7 @@ void CodeGenPatternMatch::visitBinaryOperator(llvm::BinaryOperator &I)
 
 void CodeGenPatternMatch::visitCmpInst(llvm::CmpInst &I)
 {
-    bool match = MatchCondModifier(I) || 
+    bool match = MatchCondModifier(I) ||
         MatchModifier(I);
     assert(match);
 }
@@ -1109,7 +1109,7 @@ void CodeGenPatternMatch::visitCallInst(CallInst &I)
             break;
         case GenISAIntrinsic::GenISA_simdBlockRead:
         case GenISAIntrinsic::GenISA_simdBlockWrite:
-            match = MatchBlockReadWritePointer(*CI) || 
+            match = MatchBlockReadWritePointer(*CI) ||
                 MatchSingleInstruction(*CI);
             break;
         default:
@@ -1203,7 +1203,7 @@ void CodeGenPatternMatch::visitIntrinsicInst(llvm::IntrinsicInst &I)
         break;
     }
     assert(match && "no pattern found");
-}  
+}
 
 void CodeGenPatternMatch::visitStoreInst(StoreInst &I)
 {
@@ -1556,7 +1556,7 @@ bool CodeGenPatternMatch::MatchFrc(llvm::BinaryOperator& I)
     return found;
 }
 
-SSource CodeGenPatternMatch::GetSource(llvm::Value* value, bool modifier, bool regioning) 
+SSource CodeGenPatternMatch::GetSource(llvm::Value* value, bool modifier, bool regioning)
 {
     llvm::Value* sourceValue = value;
     e_modifier mod = EMOD_NONE;
@@ -1645,7 +1645,7 @@ bool CodeGenPatternMatch::MatchMad( llvm::BinaryOperator& I )
             pass->Mad(sources, modifier);
         }
     };
-    
+
     if (m_ctx->getModuleMetaData()->isPrecise)
     {
         return false;
@@ -1724,9 +1724,9 @@ bool CodeGenPatternMatch::MatchMad( llvm::BinaryOperator& I )
     {
         MadPattern *pattern = new (m_allocator) MadPattern();
         for(int i=0; i<3; i++)
-        {           
+        {
             pattern->sources[i] = GetSource(sources[i], src_mod[i], false);
-            if (isa<Constant>(sources[i]) && 
+            if (isa<Constant>(sources[i]) &&
                 (!m_Platform.support16BitImmSrcForMad() ||
                 (sources[i]->getType()->getTypeID() != llvm::Type::HalfTyID) || i == 1))
             {
@@ -1740,7 +1740,7 @@ bool CodeGenPatternMatch::MatchMad( llvm::BinaryOperator& I )
     return found;
 }
 
-// match simdblockRead/Write with preceding inttoptr if possible 
+// match simdblockRead/Write with preceding inttoptr if possible
 // to save a copy move
 bool CodeGenPatternMatch::MatchBlockReadWritePointer(llvm::GenIntrinsicInst& I)
 {
@@ -1818,6 +1818,14 @@ bool CodeGenPatternMatch::MatchLoadStorePointer(llvm::Instruction& I, llvm::Valu
     {
         return false;
     }
+
+    // Store3d supports only types equal or less than 128 bits.
+    StoreInst* storeInst = dyn_cast<StoreInst>(&I);
+    if (storeInst && storeInst->getValueOperand()->getType()->getPrimitiveSizeInBits() > 128)
+    {
+        return false;
+    }
+
     if (i2p || (ptr && ptr->getIntrinsicID() == GenISAIntrinsic::GenISA_OWordPtr))
     {
         LoadStorePointerPattern *pattern = new (m_allocator) LoadStorePointerPattern();
@@ -1996,9 +2004,9 @@ bool CodeGenPatternMatch::MatchLrp(llvm::BinaryOperator& I)
 
     if (!found)
     {
-        // match the case: dst = src2 - (src0 * src2) + (src0 * src1); 
-        // match the case: dst = (src0 * src1) + src2 - (src0 * src2); 
-        // match the case: dst = src2 + (src0 * src1) - (src0 * src2); 
+        // match the case: dst = src2 - (src0 * src2) + (src0 * src1);
+        // match the case: dst = (src0 * src1) + src2 - (src0 * src2);
+        // match the case: dst = src2 + (src0 * src1) - (src0 * src2);
         if (I.getOpcode() == Instruction::FAdd || I.getOpcode() == Instruction::FSub)
         {
             // dst = op[0] +/- op[1] +/- op[2]
@@ -2068,7 +2076,7 @@ bool CodeGenPatternMatch::MatchLrp(llvm::BinaryOperator& I)
                                                 // abort the cases marked as "skip" in the comment above
                                                 break;
                                             }
-                                            
+
                                             sources[0] = op[i]->getOperand(srci);
                                             sources[1] = op[k] == op[i]->getOperand(1 - srci) ? op[j]->getOperand(1 - srcj) : op[i]->getOperand(1 - srci);
                                             sources[2] = op[k];
@@ -2141,20 +2149,20 @@ bool CodeGenPatternMatch::MatchCmpSext(llvm::Instruction& I)
             pass->Cmp( inst->getPredicate(), sources, modifier );
         }
     };
-    bool match = false; 
+    bool match = false;
 
     if( CmpInst* cmpInst = dyn_cast<CmpInst>(I.getOperand(0)) )
     {
         if( cmpInst->getOperand(0)->getType()->getPrimitiveSizeInBits() == I.getType()->getPrimitiveSizeInBits() )
         {
             CmpSextPattern *pattern = new (m_allocator) CmpSextPattern();
             bool supportModifer = SupportsModifier(cmpInst);
-            
+
             pattern->inst = cmpInst;
             pattern->sources[0] = GetSource(cmpInst->getOperand(0), supportModifer, false);
             pattern->sources[1] = GetSource(cmpInst->getOperand(1), supportModifer, false);
             AddPattern(pattern);
-            match = true; 
+            match = true;
         }
     }
 
@@ -2420,16 +2428,16 @@ bool CodeGenPatternMatch::BitcastSearch(SSource& source, llvm::Value*& value, bo
         if (auto bTInst = dyn_cast<BitCastInst>(elemInst->getOperand(0)))
         {
             // Pattern Matching (Instruction) + ExtractElem + (Vector)Bitcast
-            // 
+            //
             // In order to set the regioning for the ALU operand
             // I require three things:
             //      -The first is the source number of elements
             //      -The second is the destination number of elements
             //      -The third is the index from the extract element
-            //      
+            //
             // For example if I have <4 x i32> to <16 x i8> all I need is
-            // the 4 (vstride) and the i8 (b) in this case the operand would look 
-            // like this -> r22.x <4;1,0>:b 
+            // the 4 (vstride) and the i8 (b) in this case the operand would look
+            // like this -> r22.x <4;1,0>:b
             // x is calculated below and later on using the simdsize
 
             uint32_t index, srcNElts, dstNElts, nEltsRatio;
@@ -2466,7 +2474,7 @@ bool CodeGenPatternMatch::BitcastSearch(SSource& source, llvm::Value*& value, bo
     }
     return false;
 }
-    
+
 
 bool CodeGenPatternMatch::MatchModifier(llvm::Instruction& I, bool SupportSrc0Mod)
 {
@@ -2539,7 +2547,7 @@ bool CodeGenPatternMatch::MatchSingleInstruction(llvm::Instruction& I)
         }
     }
     AddPattern(pattern);
-    return true;   
+    return true;
 }
 
 bool CodeGenPatternMatch::MatchBranch(llvm::BranchInst& I)
@@ -2596,18 +2604,18 @@ bool CodeGenPatternMatch::MatchSatModifier(llvm::Instruction& I)
             else
             {
                 pass->Mov(source, mod);
-            }            
+            }
         }
     };
-    bool match = false;    
+    bool match = false;
     llvm::Value* source = nullptr;
     if(isSat(&I, source))
     {
         SatPattern *satPattern = new (m_allocator) SatPattern();
         if(llvm::Instruction* inst = llvm::dyn_cast<Instruction>(source))
         {
-            // As an heuristic we only match saturate if the instruction has one use 
-            // to avoid duplicating expensive instructions and increasing reg pressure 
+            // As an heuristic we only match saturate if the instruction has one use
+            // to avoid duplicating expensive instructions and increasing reg pressure
             // without improve code quality this may be refined in the future
             if(inst->hasOneUse() &&  SupportsSaturate(inst))
             {
@@ -2772,7 +2780,7 @@ bool CodeGenPatternMatch::MatchPow(llvm::IntrinsicInst& I)
             pass->Pow(sources, modifier);
         }
     };
-    bool found = false;   
+    bool found = false;
     llvm::Value* source0 = NULL;
     llvm::Value* source1 = NULL;
     if(I.getIntrinsicID() == Intrinsic::exp2)
@@ -2794,7 +2802,7 @@ bool CodeGenPatternMatch::MatchPow(llvm::IntrinsicInst& I)
                     }
                 }
             }
-        }    
+        }
     }
     if(found)
     {
@@ -2889,7 +2897,7 @@ bool CodeGenPatternMatch::MatchBoolOp(llvm::BinaryOperator& I)
     return found;
 }
 
-//  
+//
 //  Assume that V is of type T (integer) with N bits;
 //  and amt is of integer type too.
 //
@@ -2917,7 +2925,7 @@ bool CodeGenPatternMatch::MatchBoolOp(llvm::BinaryOperator& I)
 //  ror can be handled similarly. Note that
 //    ror (x, amt) = ((unsigned)x >> amt) | ( x << (N - amt))
 //                 = rol (x, N - amt);
-//    
+//
 bool CodeGenPatternMatch::MatchRotate(llvm::Instruction& I)
 {
     using namespace llvm::PatternMatch;
@@ -2967,7 +2975,7 @@ bool CodeGenPatternMatch::MatchRotate(llvm::Instruction& I)
     {
         assert(false && "Should be invoked with Or/Trunc instruction");
     }
- 
+
     // Do rotate only if
     //   1) type is W/DW (HW only supports W/DW); and
     //   2) both operands are instructions.
@@ -3088,7 +3096,7 @@ bool CodeGenPatternMatch::MatchRotate(llvm::Instruction& I)
                  match(X1, m_And(m_Value(X2), m_SpecificInt(typeMask))) &&
                  (match(X2, m_Sub(m_SpecificInt(typeWidth), m_Value(X0))) ||
                   match(X2, m_Sub(m_Zero(), m_Value(X0)))));
-            
+
             if (isReverse)
             {
                 Amt = X0;
@@ -3101,7 +3109,7 @@ bool CodeGenPatternMatch::MatchRotate(llvm::Instruction& I)
     {
         return false;
     }
-  
+
     // Found the pattern.
     RotatePattern *pattern = new (m_allocator) RotatePattern();
     pattern->instruction = &I;
@@ -3246,7 +3254,7 @@ bool CodeGenPatternMatch::MatchDbgInstruction(llvm::DbgInfoIntrinsic& I)
         assert(false && "Unhandled Dbg intrinsic");
     }
     AddPattern(pattern);
-    return true;   
+    return true;
 }
 
 bool CodeGenPatternMatch::MatchAvg(llvm::Instruction& I)
@@ -3417,7 +3425,7 @@ bool CodeGenPatternMatch::MatchRegisterRegion(llvm::GenIntrinsicInst& I)
             {
                 uint shiftFactor = int_cast<uint>(simDOffSetInst->getZExtValue());
                 //Check to make sure we dont end up with an invalid Vertical Stride.
-                //Only 1, 2, 4, 8, 16 are supported. 
+                //Only 1, 2, 4, 8, 16 are supported.
                 if (shiftFactor <= 4)
                 {
                     verticalStride = (int)pow(2, shiftFactor);
@@ -3635,7 +3643,7 @@ bool isAbs(llvm::Value* abs, e_modifier& mod, llvm::Value*& source)
             return true;
         }
     }
-    
+
     llvm::SelectInst* select = llvm::dyn_cast<llvm::SelectInst>(abs);
     if (!select)
         return false;
@@ -3656,7 +3664,7 @@ bool isAbs(llvm::Value* abs, e_modifier& mod, llvm::Value*& source)
                     {
                         if(cmpSource == select->getOperand(1+sourceIndex))
                         {
-                            llvm::BinaryOperator* negate = 
+                            llvm::BinaryOperator* negate =
                                 llvm::dyn_cast<llvm::BinaryOperator>(select->getOperand(1+(1-sourceIndex)));
                             llvm::Value* negateSource = NULL;
                             if(negate && IsNegate(*negate, negateSource) && negateSource == cmpSource)
@@ -3669,7 +3677,7 @@ bool isAbs(llvm::Value* abs, e_modifier& mod, llvm::Value*& source)
                             }
                             break;
                         }
-                    }                          
+                    }
                     break;
                 }
             }
@@ -3904,7 +3912,7 @@ bool isSat(llvm::Instruction* sat, llvm::Value*& source)
                             found = true;
                             source = maxSources[1-j];
                             break;
-                        }                       
+                        }
                     }
                 }
                 break;
@@ -3927,7 +3935,7 @@ bool isSat(llvm::Instruction* sat, llvm::Value*& source)
                             found = true;
                             source = maxSources[1-j];
                             break;
-                        }                       
+                        }
                     }
                 }
                 break;
@@ -3941,7 +3949,7 @@ uint CodeGenPatternMatch::GetBlockId(llvm::BasicBlock* block)
 {
     auto it = m_blockMap.find(block);
     assert(it!=m_blockMap.end());
-    
+
     uint blockID = it->second->id;
     return blockID;
 }

Original file line number	Diff line number	Diff line change
`@@ -96,7 +96,7 @@ void CodeGenPatternMatch::CodeGenNode( llvm::DomTreeNode* node )`
`96`	`96`	`}`
`97`	`97`	`llvm::BasicBlock* bb = node->getBlock();`
`98`	`98`	`CodeGenBlock(bb);`
`99`		`-}`
	`99`	`+}`
`100`	`100`
`101`	`101`
`102`	`102`	`bool CodeGenPatternMatch::runOnFunction(llvm::Function &F)`
`@@ -108,7 +108,7 @@ bool CodeGenPatternMatch::runOnFunction(llvm::Function &F)`
`108`	`108`
`109`	`109`	`delete[] m_blocks;`
`110`	`110`	`m_blocks = nullptr;`
`111`		`-`
	`111`	`+`
`112`	`112`	`m_ctx = getAnalysis<CodeGenContextWrapper>().getCodeGenContext();`
`113`	`113`
`114`	`114`	`MetaDataUtils *pMdUtils = getAnalysis<MetaDataUtilsWrapper>().getMetaDataUtils();`
`@@ -121,7 +121,7 @@ bool CodeGenPatternMatch::runOnFunction(llvm::Function &F)`
`121`	`121`	`m_AllowContractions = true;`
`122`	`122`	`if(m_ctx->m_DriverInfo.NeedCheckContractionAllowed())`
`123`	`123`	`{`
`124`		`- m_AllowContractions =`
	`124`	`+ m_AllowContractions =`
`125`	`125`	`modMD->compOpt.FastRelaxedMath \|\|`
`126`	`126`	`modMD->compOpt.MadEnable;`
`127`	`127`	`}`
`@@ -152,7 +152,7 @@ inline bool HasSideEffect(llvm::Instruction& inst)`
`152`	`152`
`153`	`153`	`inline bool HasPhiUse(llvm::Value& inst)`
`154`	`154`	`{`
`155`		`- for (auto UI = inst.user_begin(), E = inst.user_end(); UI != E; ++UI)`
	`155`	`+ for (auto UI = inst.user_begin(), E = inst.user_end(); UI != E; ++UI)`
`156`	`156`	`{`
`157`	`157`	`llvm::User U = UI;`
`158`	`158`	`if( llvm::isa<llvm::PHINode>(U))`
`@@ -725,7 +725,7 @@ CodeGenPatternMatch::isIntegerSatTrunc(llvm::SelectInst *SI) {`
`725`	`725`	`// since src is a source of a trunc instruction, and dst`
`726`	`726`	`// have the same width as its destination.`
`727`	`727`	`return std::make_tuple(nullptr, false, false);`
`728`		`- }`
	`728`	`+ }`
`729`	`729`
`730`	`730`	`if (CI->getValue() != UMax && CI->getValue() != SMax)`
`731`	`731`	`return std::make_tuple(nullptr, false, false);`
`@@ -836,7 +836,7 @@ void CodeGenPatternMatch::visitCastInst(llvm::CastInst &I)`
`836`	`836`	`}`
`837`	`837`	`else if (I.getOpcode() == Instruction::Trunc)`
`838`	`838`	`{`
`839`		`- match =`
	`839`	`+ match =`
`840`	`840`	`MatchModifier(I);`
`841`	`841`	`}`
`842`	`842`	`else`
`@@ -1004,12 +1004,12 @@ void CodeGenPatternMatch::visitBinaryOperator(llvm::BinaryOperator &I)`
`1004`	`1004`	`MatchModifier(I);`
`1005`	`1005`	`break;`
`1006`	`1006`	`case Instruction::And:`
`1007`		`- match =`
	`1007`	`+ match =`
`1008`	`1008`	`MatchBoolOp(I) \|\|`
`1009`	`1009`	`MatchLogicAlu(I);`
`1010`	`1010`	`break;`
`1011`	`1011`	`case Instruction::Or:`
`1012`		`- match =`
	`1012`	`+ match =`
`1013`	`1013`	`MatchBoolOp(I) \|\|`
`1014`	`1014`	`MatchLogicAlu(I);`
`1015`	`1015`	`break;`
`@@ -1026,7 +1026,7 @@ void CodeGenPatternMatch::visitBinaryOperator(llvm::BinaryOperator &I)`
`1026`	`1026`
`1027`	`1027`	`void CodeGenPatternMatch::visitCmpInst(llvm::CmpInst &I)`
`1028`	`1028`	`{`
`1029`		`- bool match = MatchCondModifier(I) \|\|`
	`1029`	`+ bool match = MatchCondModifier(I) \|\|`
`1030`	`1030`	`MatchModifier(I);`
`1031`	`1031`	`assert(match);`
`1032`	`1032`	`}`
`@@ -1109,7 +1109,7 @@ void CodeGenPatternMatch::visitCallInst(CallInst &I)`
`1109`	`1109`	`break;`
`1110`	`1110`	`case GenISAIntrinsic::GenISA_simdBlockRead:`
`1111`	`1111`	`case GenISAIntrinsic::GenISA_simdBlockWrite:`
`1112`		`- match = MatchBlockReadWritePointer(*CI) \|\|`
	`1112`	`+ match = MatchBlockReadWritePointer(*CI) \|\|`
`1113`	`1113`	`MatchSingleInstruction(*CI);`
`1114`	`1114`	`break;`
`1115`	`1115`	`default:`
`@@ -1203,7 +1203,7 @@ void CodeGenPatternMatch::visitIntrinsicInst(llvm::IntrinsicInst &I)`
`1203`	`1203`	`break;`
`1204`	`1204`	`}`
`1205`	`1205`	`assert(match && "no pattern found");`
`1206`		`-}`
	`1206`	`+}`
`1207`	`1207`
`1208`	`1208`	`void CodeGenPatternMatch::visitStoreInst(StoreInst &I)`
`1209`	`1209`	`{`
`@@ -1556,7 +1556,7 @@ bool CodeGenPatternMatch::MatchFrc(llvm::BinaryOperator& I)`
`1556`	`1556`	`return found;`
`1557`	`1557`	`}`
`1558`	`1558`
`1559`		`-SSource CodeGenPatternMatch::GetSource(llvm::Value* value, bool modifier, bool regioning)`
	`1559`	`+SSource CodeGenPatternMatch::GetSource(llvm::Value* value, bool modifier, bool regioning)`
`1560`	`1560`	`{`
`1561`	`1561`	`llvm::Value* sourceValue = value;`
`1562`	`1562`	`e_modifier mod = EMOD_NONE;`
`@@ -1645,7 +1645,7 @@ bool CodeGenPatternMatch::MatchMad( llvm::BinaryOperator& I )`
`1645`	`1645`	`pass->Mad(sources, modifier);`
`1646`	`1646`	`}`
`1647`	`1647`	`};`
`1648`		`-`
	`1648`	`+`
`1649`	`1649`	`if (m_ctx->getModuleMetaData()->isPrecise)`
`1650`	`1650`	`{`
`1651`	`1651`	`return false;`
`@@ -1724,9 +1724,9 @@ bool CodeGenPatternMatch::MatchMad( llvm::BinaryOperator& I )`
`1724`	`1724`	`{`
`1725`	`1725`	`MadPattern *pattern = new (m_allocator) MadPattern();`
`1726`	`1726`	`for(int i=0; i<3; i++)`
`1727`		`- {`
	`1727`	`+ {`
`1728`	`1728`	`pattern->sources[i] = GetSource(sources[i], src_mod[i], false);`
`1729`		`- if (isa<Constant>(sources[i]) &&`
	`1729`	`+ if (isa<Constant>(sources[i]) &&`
`1730`	`1730`	`(!m_Platform.support16BitImmSrcForMad() \|\|`
`1731`	`1731`	`(sources[i]->getType()->getTypeID() != llvm::Type::HalfTyID) \|\| i == 1))`
`1732`	`1732`	`{`
`@@ -1740,7 +1740,7 @@ bool CodeGenPatternMatch::MatchMad( llvm::BinaryOperator& I )`
`1740`	`1740`	`return found;`
`1741`	`1741`	`}`
`1742`	`1742`
`1743`		`-// match simdblockRead/Write with preceding inttoptr if possible`
	`1743`	`+// match simdblockRead/Write with preceding inttoptr if possible`
`1744`	`1744`	`// to save a copy move`
`1745`	`1745`	`bool CodeGenPatternMatch::MatchBlockReadWritePointer(llvm::GenIntrinsicInst& I)`
`1746`	`1746`	`{`
`@@ -1818,6 +1818,14 @@ bool CodeGenPatternMatch::MatchLoadStorePointer(llvm::Instruction& I, llvm::Valu`
`1818`	`1818`	`{`
`1819`	`1819`	`return false;`
`1820`	`1820`	`}`
	`1821`	`+`
	`1822`	`+ // Store3d supports only types equal or less than 128 bits.`
	`1823`	`+ StoreInst* storeInst = dyn_cast<StoreInst>(&I);`
	`1824`	`+ if (storeInst && storeInst->getValueOperand()->getType()->getPrimitiveSizeInBits() > 128)`
	`1825`	`+ {`
	`1826`	`+ return false;`
	`1827`	`+ }`
	`1828`	`+`
`1821`	`1829`	`if (i2p \|\| (ptr && ptr->getIntrinsicID() == GenISAIntrinsic::GenISA_OWordPtr))`
`1822`	`1830`	`{`
`1823`	`1831`	`LoadStorePointerPattern *pattern = new (m_allocator) LoadStorePointerPattern();`
`@@ -1996,9 +2004,9 @@ bool CodeGenPatternMatch::MatchLrp(llvm::BinaryOperator& I)`
`1996`	`2004`
`1997`	`2005`	`if (!found)`
`1998`	`2006`	`{`
`1999`		`- // match the case: dst = src2 - (src0 * src2) + (src0 * src1);`
`2000`		`- // match the case: dst = (src0 * src1) + src2 - (src0 * src2);`
`2001`		`- // match the case: dst = src2 + (src0 * src1) - (src0 * src2);`
	`2007`	`+ // match the case: dst = src2 - (src0 * src2) + (src0 * src1);`
	`2008`	`+ // match the case: dst = (src0 * src1) + src2 - (src0 * src2);`
	`2009`	`+ // match the case: dst = src2 + (src0 * src1) - (src0 * src2);`
`2002`	`2010`	`if (I.getOpcode() == Instruction::FAdd \|\| I.getOpcode() == Instruction::FSub)`
`2003`	`2011`	`{`
`2004`	`2012`	`// dst = op[0] +/- op[1] +/- op[2]`
`@@ -2068,7 +2076,7 @@ bool CodeGenPatternMatch::MatchLrp(llvm::BinaryOperator& I)`
`2068`	`2076`	`// abort the cases marked as "skip" in the comment above`
`2069`	`2077`	`break;`
`2070`	`2078`	`}`
`2071`		`-`
	`2079`	`+`
`2072`	`2080`	`sources[0] = op[i]->getOperand(srci);`
`2073`	`2081`	`sources[1] = op[k] == op[i]->getOperand(1 - srci) ? op[j]->getOperand(1 - srcj) : op[i]->getOperand(1 - srci);`
`2074`	`2082`	`sources[2] = op[k];`
`@@ -2141,20 +2149,20 @@ bool CodeGenPatternMatch::MatchCmpSext(llvm::Instruction& I)`
`2141`	`2149`	`pass->Cmp( inst->getPredicate(), sources, modifier );`
`2142`	`2150`	`}`
`2143`	`2151`	`};`
`2144`		`- bool match = false;`
	`2152`	`+ bool match = false;`
`2145`	`2153`
`2146`	`2154`	`if( CmpInst* cmpInst = dyn_cast<CmpInst>(I.getOperand(0)) )`
`2147`	`2155`	`{`
`2148`	`2156`	`if( cmpInst->getOperand(0)->getType()->getPrimitiveSizeInBits() == I.getType()->getPrimitiveSizeInBits() )`
`2149`	`2157`	`{`
`2150`	`2158`	`CmpSextPattern *pattern = new (m_allocator) CmpSextPattern();`
`2151`	`2159`	`bool supportModifer = SupportsModifier(cmpInst);`
`2152`		`-`
	`2160`	`+`
`2153`	`2161`	`pattern->inst = cmpInst;`
`2154`	`2162`	`pattern->sources[0] = GetSource(cmpInst->getOperand(0), supportModifer, false);`
`2155`	`2163`	`pattern->sources[1] = GetSource(cmpInst->getOperand(1), supportModifer, false);`
`2156`	`2164`	`AddPattern(pattern);`
`2157`		`- match = true;`
	`2165`	`+ match = true;`
`2158`	`2166`	`}`
`2159`	`2167`	`}`
`2160`	`2168`
`@@ -2420,16 +2428,16 @@ bool CodeGenPatternMatch::BitcastSearch(SSource& source, llvm::Value*& value, bo`
`2420`	`2428`	`if (auto bTInst = dyn_cast<BitCastInst>(elemInst->getOperand(0)))`
`2421`	`2429`	`{`
`2422`	`2430`	`// Pattern Matching (Instruction) + ExtractElem + (Vector)Bitcast`
`2423`		`- //`
	`2431`	`+ //`
`2424`	`2432`	`// In order to set the regioning for the ALU operand`
`2425`	`2433`	`// I require three things:`
`2426`	`2434`	`// -The first is the source number of elements`
`2427`	`2435`	`// -The second is the destination number of elements`
`2428`	`2436`	`// -The third is the index from the extract element`
`2429`		`- //`
	`2437`	`+ //`
`2430`	`2438`	`// For example if I have <4 x i32> to <16 x i8> all I need is`
`2431`		`- // the 4 (vstride) and the i8 (b) in this case the operand would look`
`2432`		`- // like this -> r22.x <4;1,0>:b`
	`2439`	`+ // the 4 (vstride) and the i8 (b) in this case the operand would look`
	`2440`	`+ // like this -> r22.x <4;1,0>:b`
`2433`	`2441`	`// x is calculated below and later on using the simdsize`
`2434`	`2442`
`2435`	`2443`	`uint32_t index, srcNElts, dstNElts, nEltsRatio;`
`@@ -2466,7 +2474,7 @@ bool CodeGenPatternMatch::BitcastSearch(SSource& source, llvm::Value*& value, bo`
`2466`	`2474`	`}`
`2467`	`2475`	`return false;`
`2468`	`2476`	`}`
`2469`		`-`
	`2477`	`+`
`2470`	`2478`
`2471`	`2479`	`bool CodeGenPatternMatch::MatchModifier(llvm::Instruction& I, bool SupportSrc0Mod)`
`2472`	`2480`	`{`
`@@ -2539,7 +2547,7 @@ bool CodeGenPatternMatch::MatchSingleInstruction(llvm::Instruction& I)`
`2539`	`2547`	`}`
`2540`	`2548`	`}`
`2541`	`2549`	`AddPattern(pattern);`
`2542`		`- return true;`
	`2550`	`+ return true;`
`2543`	`2551`	`}`
`2544`	`2552`
`2545`	`2553`	`bool CodeGenPatternMatch::MatchBranch(llvm::BranchInst& I)`
`@@ -2596,18 +2604,18 @@ bool CodeGenPatternMatch::MatchSatModifier(llvm::Instruction& I)`
`2596`	`2604`	`else`
`2597`	`2605`	`{`
`2598`	`2606`	`pass->Mov(source, mod);`
`2599`		`- }`
	`2607`	`+ }`
`2600`	`2608`	`}`
`2601`	`2609`	`};`
`2602`		`- bool match = false;`
	`2610`	`+ bool match = false;`
`2603`	`2611`	`llvm::Value* source = nullptr;`
`2604`	`2612`	`if(isSat(&I, source))`
`2605`	`2613`	`{`
`2606`	`2614`	`SatPattern *satPattern = new (m_allocator) SatPattern();`
`2607`	`2615`	`if(llvm::Instruction* inst = llvm::dyn_cast<Instruction>(source))`
`2608`	`2616`	`{`
`2609`		`- // As an heuristic we only match saturate if the instruction has one use`
`2610`		`- // to avoid duplicating expensive instructions and increasing reg pressure`
	`2617`	`+ // As an heuristic we only match saturate if the instruction has one use`
	`2618`	`+ // to avoid duplicating expensive instructions and increasing reg pressure`
`2611`	`2619`	`// without improve code quality this may be refined in the future`
`2612`	`2620`	`if(inst->hasOneUse() && SupportsSaturate(inst))`
`2613`	`2621`	`{`
`@@ -2772,7 +2780,7 @@ bool CodeGenPatternMatch::MatchPow(llvm::IntrinsicInst& I)`
`2772`	`2780`	`pass->Pow(sources, modifier);`
`2773`	`2781`	`}`
`2774`	`2782`	`};`
`2775`		`- bool found = false;`
	`2783`	`+ bool found = false;`
`2776`	`2784`	`llvm::Value* source0 = NULL;`
`2777`	`2785`	`llvm::Value* source1 = NULL;`
`2778`	`2786`	`if(I.getIntrinsicID() == Intrinsic::exp2)`
`@@ -2794,7 +2802,7 @@ bool CodeGenPatternMatch::MatchPow(llvm::IntrinsicInst& I)`
`2794`	`2802`	`}`
`2795`	`2803`	`}`
`2796`	`2804`	`}`
`2797`		`- }`
	`2805`	`+ }`
`2798`	`2806`	`}`
`2799`	`2807`	`if(found)`
`2800`	`2808`	`{`
`@@ -2889,7 +2897,7 @@ bool CodeGenPatternMatch::MatchBoolOp(llvm::BinaryOperator& I)`
`2889`	`2897`	`return found;`
`2890`	`2898`	`}`
`2891`	`2899`
`2892`		`-//`
	`2900`	`+//`
`2893`	`2901`	`// Assume that V is of type T (integer) with N bits;`
`2894`	`2902`	`// and amt is of integer type too.`
`2895`	`2903`	`//`
`@@ -2917,7 +2925,7 @@ bool CodeGenPatternMatch::MatchBoolOp(llvm::BinaryOperator& I)`
`2917`	`2925`	`// ror can be handled similarly. Note that`
`2918`	`2926`	`// ror (x, amt) = ((unsigned)x >> amt) \| ( x << (N - amt))`
`2919`	`2927`	`// = rol (x, N - amt);`
`2920`		`-//`
	`2928`	`+//`
`2921`	`2929`	`bool CodeGenPatternMatch::MatchRotate(llvm::Instruction& I)`
`2922`	`2930`	`{`
`2923`	`2931`	`using namespace llvm::PatternMatch;`
`@@ -2967,7 +2975,7 @@ bool CodeGenPatternMatch::MatchRotate(llvm::Instruction& I)`
`2967`	`2975`	`{`
`2968`	`2976`	`assert(false && "Should be invoked with Or/Trunc instruction");`
`2969`	`2977`	`}`
`2970`		`-`
	`2978`	`+`
`2971`	`2979`	`// Do rotate only if`
`2972`	`2980`	`// 1) type is W/DW (HW only supports W/DW); and`
`2973`	`2981`	`// 2) both operands are instructions.`
`@@ -3088,7 +3096,7 @@ bool CodeGenPatternMatch::MatchRotate(llvm::Instruction& I)`
`3088`	`3096`	`match(X1, m_And(m_Value(X2), m_SpecificInt(typeMask))) &&`
`3089`	`3097`	`(match(X2, m_Sub(m_SpecificInt(typeWidth), m_Value(X0))) \|\|`
`3090`	`3098`	`match(X2, m_Sub(m_Zero(), m_Value(X0)))));`
`3091`		`-`
	`3099`	`+`
`3092`	`3100`	`if (isReverse)`
`3093`	`3101`	`{`
`3094`	`3102`	`Amt = X0;`
`@@ -3101,7 +3109,7 @@ bool CodeGenPatternMatch::MatchRotate(llvm::Instruction& I)`
`3101`	`3109`	`{`
`3102`	`3110`	`return false;`
`3103`	`3111`	`}`
`3104`		`-`
	`3112`	`+`
`3105`	`3113`	`// Found the pattern.`
`3106`	`3114`	`RotatePattern *pattern = new (m_allocator) RotatePattern();`
`3107`	`3115`	`pattern->instruction = &I;`
`@@ -3246,7 +3254,7 @@ bool CodeGenPatternMatch::MatchDbgInstruction(llvm::DbgInfoIntrinsic& I)`
`3246`	`3254`	`assert(false && "Unhandled Dbg intrinsic");`
`3247`	`3255`	`}`
`3248`	`3256`	`AddPattern(pattern);`
`3249`		`- return true;`
	`3257`	`+ return true;`
`3250`	`3258`	`}`
`3251`	`3259`
`3252`	`3260`	`bool CodeGenPatternMatch::MatchAvg(llvm::Instruction& I)`
`@@ -3417,7 +3425,7 @@ bool CodeGenPatternMatch::MatchRegisterRegion(llvm::GenIntrinsicInst& I)`
`3417`	`3425`	`{`
`3418`	`3426`	`uint shiftFactor = int_cast<uint>(simDOffSetInst->getZExtValue());`
`3419`	`3427`	`//Check to make sure we dont end up with an invalid Vertical Stride.`
`3420`		`- //Only 1, 2, 4, 8, 16 are supported.`
	`3428`	`+ //Only 1, 2, 4, 8, 16 are supported.`
`3421`	`3429`	`if (shiftFactor <= 4)`
`3422`	`3430`	`{`
`3423`	`3431`	`verticalStride = (int)pow(2, shiftFactor);`
`@@ -3635,7 +3643,7 @@ bool isAbs(llvm::Value* abs, e_modifier& mod, llvm::Value*& source)`
`3635`	`3643`	`return true;`
`3636`	`3644`	`}`
`3637`	`3645`	`}`
`3638`		`-`
	`3646`	`+`
`3639`	`3647`	`llvm::SelectInst* select = llvm::dyn_cast<llvm::SelectInst>(abs);`
`3640`	`3648`	`if (!select)`
`3641`	`3649`	`return false;`
`@@ -3656,7 +3664,7 @@ bool isAbs(llvm::Value* abs, e_modifier& mod, llvm::Value*& source)`
`3656`	`3664`	`{`
`3657`	`3665`	`if(cmpSource == select->getOperand(1+sourceIndex))`
`3658`	`3666`	`{`
`3659`		`- llvm::BinaryOperator* negate =`
	`3667`	`+ llvm::BinaryOperator* negate =`
`3660`	`3668`	`llvm::dyn_cast<llvm::BinaryOperator>(select->getOperand(1+(1-sourceIndex)));`
`3661`	`3669`	`llvm::Value* negateSource = NULL;`
`3662`	`3670`	`if(negate && IsNegate(*negate, negateSource) && negateSource == cmpSource)`
`@@ -3669,7 +3677,7 @@ bool isAbs(llvm::Value* abs, e_modifier& mod, llvm::Value*& source)`
`3669`	`3677`	`}`
`3670`	`3678`	`break;`
`3671`	`3679`	`}`
`3672`		`- }`
	`3680`	`+ }`
`3673`	`3681`	`break;`
`3674`	`3682`	`}`
`3675`	`3683`	`}`
`@@ -3904,7 +3912,7 @@ bool isSat(llvm::Instruction* sat, llvm::Value*& source)`
`3904`	`3912`	`found = true;`
`3905`	`3913`	`source = maxSources[1-j];`
`3906`	`3914`	`break;`
`3907`		`- }`
	`3915`	`+ }`
`3908`	`3916`	`}`
`3909`	`3917`	`}`
`3910`	`3918`	`break;`
`@@ -3927,7 +3935,7 @@ bool isSat(llvm::Instruction* sat, llvm::Value*& source)`
`3927`	`3935`	`found = true;`
`3928`	`3936`	`source = maxSources[1-j];`
`3929`	`3937`	`break;`
`3930`		`- }`
	`3938`	`+ }`
`3931`	`3939`	`}`
`3932`	`3940`	`}`
`3933`	`3941`	`break;`
`@@ -3941,7 +3949,7 @@ uint CodeGenPatternMatch::GetBlockId(llvm::BasicBlock* block)`
`3941`	`3949`	`{`
`3942`	`3950`	`auto it = m_blockMap.find(block);`
`3943`	`3951`	`assert(it!=m_blockMap.end());`
`3944`		`-`
	`3952`	`+`
`3945`	`3953`	`uint blockID = it->second->id;`
`3946`	`3954`	`return blockID;`
`3947`	`3955`	`}`