diff --git a/client-spark/common/src/main/java/org/apache/spark/shuffle/writer/WriteBufferManager.java b/client-spark/common/src/main/java/org/apache/spark/shuffle/writer/WriteBufferManager.java index 7e4e893e31..cf4b4bc51c 100644 --- a/client-spark/common/src/main/java/org/apache/spark/shuffle/writer/WriteBufferManager.java +++ b/client-spark/common/src/main/java/org/apache/spark/shuffle/writer/WriteBufferManager.java @@ -92,6 +92,7 @@ public class WriteBufferManager extends MemoryConsumer { private SerializationStream serializeStream; private WrappedByteArrayOutputStream arrayOutputStream; private long uncompressedDataLen = 0; + private long compressedDataLen = 0; private long requireMemoryInterval; private int requireMemoryRetryMax; private Optional codec; @@ -430,6 +431,7 @@ protected ShuffleBlockInfo createShuffleBlock(int partitionId, WriterBuffer wb) blockIdLayout.getBlockId(getNextSeqNo(partitionId), partitionId, taskAttemptId); blockCounter.incrementAndGet(); uncompressedDataLen += data.length; + compressedDataLen += compressed.length; shuffleWriteMetrics.incBytesWritten(compressed.length); // add memory to indicate bytes which will be sent to shuffle server inSendListBytes.addAndGet(wb.getMemoryUsed()); @@ -648,14 +650,18 @@ public String getManagerCostInfo() { + serializeTime + "], sortTime[" + sortTime - + "], compressTime[" - + compressTime + "], estimateTime[" + estimateTime + "], requireMemoryTime[" + requireMemoryTime + "], uncompressedDataLen[" + uncompressedDataLen + + "], compressedDataLen[" + + compressedDataLen + + "], compressTime[" + + compressTime + + "], compressRatio[" + + (compressedDataLen == 0 ? 0 : (float) uncompressedDataLen / compressedDataLen) + "]"; }