Skip to content

Commit 16c0f7b

Browse files
committed
[LV] Set correct costs for interleave group members.
This ensures each scalarized member has an accurate cost, matching the cost it would have if it would not have been considered for an interleave group. (cherry picked from commit 49605a4)
1 parent 88b3703 commit 16c0f7b

File tree

3 files changed

+73
-299
lines changed

3 files changed

+73
-299
lines changed

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5695,9 +5695,18 @@ void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
56955695
// If the instructions belongs to an interleave group, the whole group
56965696
// receives the same decision. The whole group receives the cost, but
56975697
// the cost will actually be assigned to one instruction.
5698-
if (const auto *Group = getInterleavedAccessGroup(&I))
5699-
setWideningDecision(Group, VF, Decision, Cost);
5700-
else
5698+
if (const auto *Group = getInterleavedAccessGroup(&I)) {
5699+
if (Decision == CM_Scalarize) {
5700+
for (unsigned Idx = 0; Idx < Group->getFactor(); ++Idx) {
5701+
if (auto *I = Group->getMember(Idx)) {
5702+
setWideningDecision(I, VF, Decision,
5703+
getMemInstScalarizationCost(I, VF));
5704+
}
5705+
}
5706+
} else {
5707+
setWideningDecision(Group, VF, Decision, Cost);
5708+
}
5709+
} else
57015710
setWideningDecision(&I, VF, Decision, Cost);
57025711
}
57035712
}

0 commit comments

Comments
 (0)