Skip to content

Commit b031bf2

Browse files
princepereiraprincepereira
committed
Network health script.
1 parent 7045e21 commit b031bf2

File tree

7 files changed

+2199
-0
lines changed

7 files changed

+2199
-0
lines changed

‎scripts/networkhealth/README.md‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,4 +35,30 @@ If the `-OutputMode` is set to `Event` or `All`, the script will register a new
3535
Example:
3636
kubectl logs -l name=networkhealth --all-containers=true >> networkhealth.txt
3737
Provide the generated networkhealth.txt
38+
```
39+
## Command to run startNetworkDiagnostics
40+
```
41+
Normal Execution: .\startNetworkDiagnostics.ps1 -TimeIntervalInSeconds 30 -PrintMatchedRules $true -PodNamePrefixes tcp-server,tcp-client
42+
43+
Execution with DNS Packet Capture: .\startNetworkDiagnostics.ps1 -DnsPktCap $true
44+
45+
Execution with DualStack Test: .\startNetworkDiagnostics.ps1 -DualStack $true
46+
47+
Execution with Vfp Rule Counter Dump for Pods : .\startNetworkDiagnostics.ps1 -PodNamePrefixes tcp-client,tcp-server
48+
49+
Execution with printing matched rule counter : .\startNetworkDiagnostics.ps1 -PodNamePrefixes tcp-client,tcp-server -PrintMatchedRules $true
50+
51+
Execution with validate loadbalancer rules for Service IPS : .\startNetworkDiagnostics.ps1 -ServiceIPS "10.0.0.1,10.0.0.2"
52+
53+
```
54+
55+
## Command to run vfpDropCounterMetrics
56+
```
57+
.\vfpDropCounterMetrics.ps1 -TimeIntervalInSeconds 30 -PrintMatchedRules $true -PodNamePrefixes tcp-server,tcp-client
58+
```
59+
60+
## DNS Health Check
61+
```
62+
Invoke-WebRequest https://raw.githubusercontent.com/microsoft/wcnscripts/2ea829ebaaf523cf58ef8e64120e54849eb4bd51/scripts/networkhealth/startNetworkDiagnostics.ps1 -OutFile startNetworkDiagnostics.ps1
63+
.\startNetworkDiagnostics.ps1
3864
```

‎scripts/networkhealth/RuleDump.ps1‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
2+
param (
3+
[Parameter(Mandatory=$false, HelpMessage="Rule for Pod")][string]$podIp = "10.224.0.53",
4+
[Parameter(Mandatory=$false, HelpMessage="Rule for Pod")][string]$ipFamily = "IPV4",
5+
[Parameter(Mandatory=$false, HelpMessage="Rule for Pod")][string]$protoFamily = "TCP",
6+
[Parameter(Mandatory=$false, HelpMessage="Rule for Pod")][bool]$PrintMatchedRules = $true
7+
)
8+
9+
10+
$podMac = (Get-HnsEndpoint | Where-Object IPAddress -Eq $podIp).MacAddress
11+
$podMacShortened = $podMac.Replace("-", "")
12+
$PodPortId = ((vfpctrl /list-vmswitch-port /format 1 | ConvertFrom-Json).Ports | Where-Object MacAddress -EQ $podMacShortened).Name
13+
$ExtPortId = ((vfpctrl /list-vmswitch-port /format 1 | ConvertFrom-Json).Ports | Where-Object Id -EQ "ExternalPort").Name
14+
$HostPortId = ((vfpctrl /list-vmswitch-port /format 1 | ConvertFrom-Json).Ports | Where-Object Id -Like "Container NIC*").Name
15+
16+
$podLayers = ((vfpctrl /port $PodPortId /list-layer /format 1 | ConvertFrom-Json).Layers | Sort-Object -Property Priority).Name
17+
$extPortLayers = ((vfpctrl /port $ExtPortId /list-layer /format 1 | ConvertFrom-Json).Layers | Sort-Object -Property Priority).Name
18+
$hostPortLayers = ((vfpctrl /port $HostPortId /list-layer /format 1 | ConvertFrom-Json).Layers | Sort-Object -Property Priority).Name
19+
20+
function RemoveNoise() {
21+
param (
22+
[Parameter(Mandatory=$true)][System.Object]$Rule
23+
)
24+
$Rule.PSObject.Properties.Remove('$type')
25+
$Rule.PSObject.Properties.Remove('Type')
26+
$Rule.PSObject.Properties.Remove('SubType')
27+
$Rule.PSObject.Properties.Remove('MssDelta')
28+
$Rule.PSObject.Properties.Remove('ReverseMssDelta')
29+
$Rule.PSObject.Properties.Remove('RuleFlags')
30+
$Rule.PSObject.Properties.Remove('PaRouteRuleFlags')
31+
$Rule.PSObject.Properties.Remove('CachePruningThreshold')
32+
$Rule.PSObject.Properties.Remove('InformationArray')
33+
$Rule.PSObject.Properties.Remove('NumHeaders')
34+
$Rule.PSObject.Properties.Remove('PartialRewriteTypes')
35+
return $Rule
36+
}
37+
38+
function LogVfpCounter {
39+
param (
40+
[parameter(Mandatory=$false)][string] $value = "",
41+
[parameter(Mandatory=$false)][bool] $error = $false
42+
)
43+
# Add-Content -Path $VfpRuleFile -Value $value
44+
if ($error -eq $true) {
45+
Write-Host $value -ForegroundColor Red
46+
}
47+
else {
48+
Write-Host $value
49+
}
50+
51+
}
52+
53+
function NewLine {
54+
param (
55+
[parameter(Mandatory=$false)][int] $NoOfLines = 1
56+
)
57+
for ($i = 1; $i -le $NoOfLines; $i++) {
58+
LogVfpCounter ""
59+
}
60+
}
61+
62+
function PrintRules {
63+
param (
64+
[Parameter(Mandatory=$true)][string[]]$Layers,
65+
[Parameter(Mandatory=$true)][string]$PortId,
66+
[Parameter(Mandatory=$false)][string]$Dir="OUT"
67+
)
68+
69+
$ruleCounterMap = @{}
70+
71+
foreach($layer in $Layers) {
72+
$groups = ((vfpctrl /port $PortId /layer $layer /list-group /format 1 | ConvertFrom-Json).Groups | Sort-Object -Property Priority).Name
73+
foreach($group in $groups) {
74+
if($group.Contains("_$DIR") -ne $true) {
75+
continue
76+
}
77+
if(($group.Contains("IPV4") -eq $true) -or ($group.Contains("IPV6") -eq $true)) {
78+
if($group.Contains($ipFamily) -ne $true) {
79+
continue
80+
}
81+
}
82+
if(($group.Contains("TCP") -eq $true) -or ($group.Contains("UDP") -eq $true) -or ($group.Contains("ICMP") -eq $true)) {
83+
if($group.Contains($protoFamily) -ne $true) {
84+
continue
85+
}
86+
}
87+
$rules = (vfpctrl /port $PortId /layer $layer /group $group /get-rule-counter /format 1 | ConvertFrom-Json | Sort-Object -Property Priority).Rules
88+
foreach ($rule in $rules) {
89+
90+
$ruleId = $rule.Name
91+
if (($rule.Id).Length -gt 0) {
92+
$ruleId = $rule.Id
93+
}
94+
95+
$ruleKey = "$portId-$layer-$group-$ruleId"
96+
97+
$informationArray = $rule.InformationArray
98+
99+
$rule = RemoveNoise -Rule $rule
100+
101+
if ($informationArray.Count -gt 0) {
102+
$rule | Add-Member -MemberType NoteProperty -Name RuleCounters -Value $informationArray[0].RuleCounters
103+
}
104+
105+
$ruleJson = $rule | ConvertTo-Json -Depth 10
106+
107+
if ($informationArray.Count -gt 0) {
108+
109+
$ruleCounters = $informationArray[0].RuleCounters
110+
$matchedPackets = $ruleCounters.MatchedPackets
111+
$droppedPackets = $ruleCounters.DroppedPackets
112+
$pendingPackets = $ruleCounters.PendingPackets
113+
$droppedFlows = $ruleCounters.DroppedFlows
114+
115+
if (($droppedPackets -gt 0) -or ($pendingPackets -gt 0) -or ($droppedFlows -gt 0)) {
116+
LogVfpCounter " Dropped Rule : " -error $true
117+
LogVfpCounter " ================ " -error $true
118+
NewLine 1
119+
LogVfpCounter " Layer : $layer , Group : $group , Id : $ruleId " -error $true
120+
NewLine 1
121+
LogVfpCounter " $ruleJson " -error $true
122+
NewLine 2
123+
}
124+
elseif (($PrintMatchedRules -eq $true) -and ($matchedPackets -gt 0)) {
125+
LogVfpCounter " Matched Rule : "
126+
LogVfpCounter " ================ "
127+
NewLine 1
128+
LogVfpCounter " Layer : $layer , Group : $group , Id : $ruleId "
129+
NewLine 1
130+
LogVfpCounter " $ruleJson "
131+
132+
NewLine 2
133+
}
134+
135+
$ruleCounterMap[$ruleKey] = $rule
136+
}
137+
}
138+
}
139+
}
140+
141+
return $ruleCounterMap
142+
}
143+
144+
Write-Host "#===================== Pod VFP Port Rules in Outbound Direction ================#"
145+
NewLine 2
146+
$podPortRulesOutbound = PrintRules -Layers $podLayers -portId $PodPortId -Dir "OUT"
147+
NewLine 2
148+
Write-Host "#===================== External VFP Port Rules in Outbound Direction ================#"
149+
NewLine 2
150+
$extPortRulesOutbound = PrintRules -Layers $extPortLayers -portId $ExtPortId -Dir "OUT"
151+
NewLine 2
152+
Write-Host "#===================== External VFP Port Rules in Inbound Direction ================#"
153+
NewLine 2
154+
$extPortRulesIntbound = PrintRules -Layers $extPortLayers -portId $ExtPortId -Dir "IN"
155+
NewLine 2
156+
Write-Host "#===================== Pod VFP Port Rules in Inbound Direction ================#"
157+
NewLine 2
158+
$podPortRulesIutbound = PrintRules -Layers $podLayers -portId $PodPortId -Dir "IN"

‎scripts/networkhealth/checkLbDsrMissing.ps1‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
param (
2+
[Parameter(Mandatory=$true)][string]$ServiceIPS = "",
3+
[Parameter(Mandatory=$false)][string]$RunFromOutside = $false,
4+
[Parameter(Mandatory=$false)][string]$CopyScriptToNode = $false,
5+
[Parameter(Mandatory=$false)][string]$HpcPodPrefix = "hpc",
6+
[Parameter(Mandatory=$false)][string]$HpcNamepsace = "demo",
7+
[Parameter(Mandatory=$false)][string]$Layer = "LB_DSR",
8+
[Parameter(Mandatory=$false)][string]$Group = "LB_DSR_IPv4_OUT"
9+
)
10+
11+
# Eg: .\checkLbDsrMissing.ps1 -ServiceIPS "20.51.25.211,10.0.10.189,10.0.92.4"
12+
13+
$Logfile = "health.log"
14+
15+
function LogError {
16+
param (
17+
[parameter(Mandatory=$true)][string] $message
18+
)
19+
Write-Host $message -ForegroundColor Red
20+
Add-content $Logfile -value "[FAILED] $message"
21+
}
22+
23+
function LogSuccess {
24+
param (
25+
[parameter(Mandatory=$true)][string] $message
26+
)
27+
Write-Host $message -ForegroundColor Green
28+
Add-content $Logfile -value "[SUCCESS] $message"
29+
}
30+
31+
function CheckLbDsrRuleMissing {
32+
33+
Write-Host "Checking $Layer Rule missing"
34+
$lbDsrRuleMissing = $false
35+
36+
$serviceIPList = $ServiceIPS.Split(",")
37+
38+
$portNameList = vfpctrl /list-vmswitch-port | sls "Port name"
39+
40+
foreach($portName in $portNameList) {
41+
$portId = $portName.ToString().Split(":")[1].Trim()
42+
$entries = vfpctrl /port $portId /layer $Layer /list-group
43+
if($entries.Count -le 8) {
44+
continue
45+
}
46+
foreach($serviceIP in $serviceIPList) {
47+
$entries = vfpctrl /port $portId /layer $Layer /group $Group /list-rule | sls $serviceIP
48+
if($entries.Count -le 1) {
49+
LogError "VFP $Layer Rule missing for Service IP : $serviceIP in VFP Port : $portName"
50+
$lbDsrRuleMissing = $true
51+
} else {
52+
LogSuccess "VFP $Layer Rule present for Service IP : $serviceIP in VFP Port : $portName"
53+
}
54+
}
55+
}
56+
57+
if($lbDsrRuleMissing){
58+
LogError "Mitigation : Restart-Service -f kubeproxy "
59+
return
60+
}
61+
62+
LogSuccess "No issues identified with $Layer Rule Missing for Service IPS : $ServiceIPS."
63+
}
64+
65+
function ExecFromOutside {
66+
$hpcPods = kubectl get pods -n $HpcNamepsace | sls $HpcPodPrefix
67+
if($CopyScriptToNode -eq $true) {
68+
foreach($hpcPod in $hpcPods) {
69+
$podId = $hpcPod.ToString().Split(" ")[0].Trim()
70+
Write-Host "Copying script to $podId"
71+
kubectl cp .\checkLbDsrMissing.ps1 "$podId`:`checkLbDsrMissing.ps1" -n $HpcNamepsace
72+
}
73+
}
74+
foreach($hpcPod in $hpcPods) {
75+
$podId = $hpcPod.ToString().Split(" ")[0].Trim()
76+
kubectl exec -it $podId -n $HpcNamepsace -- powershell ".\checkLbDsrMissing.ps1 -ServiceIPS '$ServiceIPS' -Layer $Layer -Group $Group"
77+
}
78+
}
79+
80+
if($RunFromOutside -eq $true) {
81+
ExecFromOutside
82+
}
83+
84+
CheckLbDsrRuleMissing

‎scripts/networkhealth/checkPortExhaustion.ps1‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
$Logfile = ".\health.log"
2+
$minThreshold = 50
3+
4+
function LogMessage {
5+
param (
6+
[parameter(Mandatory=$true)][string] $message
7+
)
8+
Add-content $Logfile -value "$message"
9+
}
10+
11+
function CountAvailableEphemeralPorts([string]$protocol = "TCP") {
12+
13+
[uint32]$portRangeSize = 64
14+
# First, remove all the text bells and whistle (plain text, table headers, dashes, empty lines, ...) from netsh output
15+
$tcpRanges = (netsh int ipv4 sh excludedportrange $protocol) -replace "[^0-9,\ ]", '' | ? { $_.trim() -ne "" }
16+
17+
# Then, remove any extra space characters. Only capture the numbers representing the beginning and end of range
18+
$tcpRangesArray = $tcpRanges -replace "\s+(\d+)\s+(\d+)\s+", '$1,$2' | ConvertFrom-String -Delimiter ","
19+
#Convert from PSCustomObject to Object[] type
20+
$tcpRangesArray = @($tcpRangesArray)
21+
22+
# Extract the ephemeral ports ranges
23+
$EphemeralPortRange = (netsh int ipv4 sh dynamicportrange $protocol) -replace "[^0-9]", '' | ? { $_.trim() -ne "" }
24+
$EphemeralPortStart = [Convert]::ToUInt32($EphemeralPortRange[0])
25+
$EphemeralPortEnd = $EphemeralPortStart + [Convert]::ToUInt32($EphemeralPortRange[1]) - 1
26+
27+
# Find the external interface
28+
$externalInterfaceIdx = (Get-NetRoute -DestinationPrefix "0.0.0.0/0")[0].InterfaceIndex
29+
$hostIP = (Get-NetIPConfiguration -ifIndex $externalInterfaceIdx).IPv4Address.IPAddress
30+
31+
# Extract the used TCP ports from the external interface
32+
$usedTcpPorts = (Get-NetTCPConnection -LocalAddress $hostIP -ErrorAction Ignore).LocalPort
33+
$usedTcpPorts | % { $tcpRangesArray += [pscustomobject]@{P1 = $_; P2 = $_ } }
34+
35+
# Extract the used TCP ports from the 0.0.0.0 interface
36+
$usedTcpGlobalPorts = (Get-NetTCPConnection -LocalAddress "0.0.0.0" -ErrorAction Ignore).LocalPort
37+
$usedTcpGlobalPorts | % { $tcpRangesArray += [pscustomobject]@{P1 = $_; P2 = $_ } }
38+
# Sort the list and remove duplicates
39+
$tcpRangesArray = ($tcpRangesArray | Sort-Object { $_.P1 } -Unique)
40+
41+
$tcpRangesList = New-Object System.Collections.ArrayList($null)
42+
$tcpRangesList.AddRange($tcpRangesArray)
43+
44+
# Remove overlapping ranges
45+
for ($i = $tcpRangesList.P1.Length - 2; $i -gt 0 ; $i--) {
46+
if ($tcpRangesList[$i].P2 -gt $tcpRangesList[$i + 1].P1 ) {
47+
$tcpRangesList.Remove($tcpRangesList[$i + 1])
48+
$i++
49+
}
50+
}
51+
52+
# Remove the non-ephemeral port reservations from the list
53+
$filteredTcpRangeArray = $tcpRangesList | ? { $_.P1 -ge $EphemeralPortStart }
54+
$filteredTcpRangeArray = $filteredTcpRangeArray | ? { $_.P2 -le $EphemeralPortEnd }
55+
56+
if ($null -eq $filteredTcpRangeArray) {
57+
$freeRanges = @($EphemeralPortRange[1])
58+
}
59+
else {
60+
$freeRanges = @()
61+
# The first free range goes from $EphemeralPortStart to the beginning of the first reserved range
62+
$freeRanges += ([Convert]::ToUInt32($filteredTcpRangeArray[0].P1) - $EphemeralPortStart)
63+
64+
for ($i = 1; $i -lt $filteredTcpRangeArray.length; $i++) {
65+
# Subsequent free ranges go from the end of the previous reserved range to the beginning of the current reserved range
66+
$freeRanges += ([Convert]::ToUInt32($filteredTcpRangeArray[$i].P1) - [Convert]::ToUInt32($filteredTcpRangeArray[$i - 1].P2) - 1)
67+
}
68+
69+
# The last free range goes from the end of the last reserved range to $EphemeralPortEnd
70+
$freeRanges += ($EphemeralPortEnd - [Convert]::ToUInt32($filteredTcpRangeArray[$filteredTcpRangeArray.length - 1].P2))
71+
}
72+
73+
# Count the number of available free ranges
74+
[uint32]$freeRangesCount = 0
75+
($freeRanges | % { $freeRangesCount += [Math]::Floor($_ / $portRangeSize) } )
76+
77+
return $freeRangesCount
78+
}
79+
80+
function CheckPortExhaustion {
81+
Write-Host "Checking Port Exhaustion"
82+
$avTcpPorts = CountAvailableEphemeralPorts -protocol TCP
83+
if($avTcpPorts -lt $minThreshold) {
84+
$message = "Available TCP ports are $avTcpPorts. Port exhaustion suspected."
85+
Write-Host "$message" -ForegroundColor Red
86+
LogMessage -message $message
87+
return $true
88+
}
89+
$avUdpPorts = CountAvailableEphemeralPorts -protocol UDP
90+
if($avTcpPorts -lt $minThreshold) {
91+
$message = "Available UDP ports are $avUdpPorts. Port exhaustion suspected."
92+
Write-Host "$message" -ForegroundColor Red
93+
LogMessage -message $message
94+
return $true
95+
}
96+
Write-Host "Available TCP Ports : $avTcpPorts , UDP Ports : $avUdpPorts . No port exhaustion suspected." -ForegroundColor Green
97+
return $false
98+
}
99+
100+
$i = 0
101+
Remove-Item $Logfile -ErrorAction Ignore
102+
103+
While($true) {
104+
$i++
105+
Write-Host "#============== Iteration : $i"
106+
if(CheckPortExhaustion) {
107+
Write-Host "DNS Issue Found." -ForegroundColor Red
108+
}
109+
Start-Sleep -Seconds 10
110+
}

0 commit comments

Comments
 (0)