operator: move pool metrics to controller

Author: djoshy

pkg/controller/common/metrics.go

@@ -32,18 +32,56 @@ var (
 			Name: "mcc_drain_err",
 			Help: "logs failed drain",
 		}, []string{"node"})
-	// MCCPoolAlert logs when the pool configuration changes in a way the user should know.
+
+	// MCCPoolAlert logs when the pool configuration changes in a way the user should know
 	MCCPoolAlert = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name: "mcc_pool_alert",
 			Help: "pool status alert",
 		}, []string{"node"})
+
 	// MCCSubControllerState logs the state of the subcontrollers of the MCC
 	MCCSubControllerState = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name: "mcc_sub_controller_state",
 			Help: "state of sub-controllers in the MCC",
 		}, []string{"subcontroller", "state", "object"})
+
+	// MCCState is the state of the machine config controller
+	// pause, updated, updating, degraded
+	MCCState = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "mco_state",
+			Help: "state of a specified pool",
+		}, []string{"node", "pool", "state", "reason"})
+
+	// MCCMachineCount is the total number of nodes in the pool
+	MCCMachineCount = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "mco_machine_count",
+			Help: "total number of machines in a specified pool",
+		}, []string{"pool"})
+
+	// MCCUpdatedMachineCount is the updated machines in the pool
+	MCCUpdatedMachineCount = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "mco_updated_machine_count",
+			Help: "total number of updated machines in specified pool",
+		}, []string{"pool"})
+
+	// MCCDegradedMachineCount is the degraded machines in the pool
+	MCCDegradedMachineCount = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "mco_degraded_machine_count",
+			Help: "total number of degraded machines in specified pool",
+		}, []string{"pool"})
+
+	// MCCUnavailableMachineCount is the unavailable machines in the pool
+	MCCUnavailableMachineCount = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "mco_unavailable_machine_count",
+			Help: "total number of unavailable machines in specified pool",
+		}, []string{"pool"})
 )
 
 func RegisterMCCMetrics() error {
@@ -52,13 +90,18 @@ func RegisterMCCMetrics() error {
 		MCCDrainErr,
 		MCCPoolAlert,
 		MCCSubControllerState,
+		MCCState,
+		MCCMachineCount,
+		MCCUpdatedMachineCount,
+		MCCDegradedMachineCount,
+		MCCUnavailableMachineCount,
 	})
 
 	if err != nil {
 		return fmt.Errorf("could not register machine-config-controller metrics: %w", err)
 	}
 
-	// Initilize GuageVecs to ensure that metrics of type GuageVec are accessible from the dashboard even if without a logged value
+	// Initialize GaugeVecs to ensure that metrics of type GaugeVec are accessible from the dashboard even if without a logged value
 	// Solution to OCPBUGS-20427: https://issues.redhat.com/browse/OCPBUGS-20427
 	OSImageURLOverride.WithLabelValues("initialize").Set(0)
 	MCCDrainErr.WithLabelValues("initialize").Set(0)

pkg/controller/node/node_controller.go

@@ -1319,7 +1319,13 @@ func (ctrl *Controller) syncMachineConfigPool(key string) error {
 		}
 		ctrlcommon.UpdateStateMetric(ctrlcommon.MCCSubControllerState, "machine-config-controller-node", "Sync Machine Config Pool", pool.Name)
 	}
-	return ctrl.syncStatusOnly(pool)
+
+	if err := ctrl.syncStatusOnly(pool); err != nil {
+		return err
+	}
+
+	// Update metrics after syncing the pool status
+	return ctrl.syncMetrics()
 }
 
 // checkIfNodeHasInProgressTaint checks if the given node has in progress taint
@@ -1708,3 +1714,33 @@ func (ctrl *Controller) isConfigOrBuildPresent(mosc *mcfgv1.MachineOSConfig, mos
 func (ctrl *Controller) isConfigAndBuildPresent(mosc *mcfgv1.MachineOSConfig, mosb *mcfgv1.MachineOSBuild) bool {
 	return (mosc != nil && mosb != nil)
 }
+
+// syncMetrics updates the metrics for all pools
+func (ctrl *Controller) syncMetrics() error {
+	pools, err := ctrl.mcpLister.List(labels.Everything())
+	if err != nil {
+		return err
+	}
+	// set metrics per pool, we need to get the latest condition to log for the state
+	var latestTime metav1.Time
+	latestTime.Time = time.Time{}
+	var cond mcfgv1.MachineConfigPoolCondition
+	for _, pool := range pools {
+		for _, condition := range pool.Status.Conditions {
+			if condition.Status == corev1.ConditionTrue && condition.LastTransitionTime.After(latestTime.Time) {
+				cond = condition
+				latestTime = cond.LastTransitionTime
+			}
+		}
+
+		nodes, _ := helpers.GetNodesForPool(ctrl.mcpLister, ctrl.nodeLister, pool)
+		for _, node := range nodes {
+			ctrlcommon.MCCState.WithLabelValues(node.Name, pool.Name, string(cond.Type), cond.Reason).SetToCurrentTime()
+		}
+		ctrlcommon.MCCMachineCount.WithLabelValues(pool.Name).Set(float64(pool.Status.MachineCount))
+		ctrlcommon.MCCUpdatedMachineCount.WithLabelValues(pool.Name).Set(float64(pool.Status.UpdatedMachineCount))
+		ctrlcommon.MCCDegradedMachineCount.WithLabelValues(pool.Name).Set(float64(pool.Status.DegradedMachineCount))
+		ctrlcommon.MCCUnavailableMachineCount.WithLabelValues(pool.Name).Set(float64(pool.Status.UnavailableMachineCount))
+	}
+	return nil
+}

pkg/operator/metrics.go

@@ -7,7 +7,6 @@ import (
 	"reflect"
 	"sort"
 	"strings"
-	"time"
 
 	mcfgv1 "github.com/openshift/api/machineconfiguration/v1"
 
@@ -26,7 +25,6 @@ import (
 	"github.com/openshift/machine-config-operator/pkg/apihelpers"
 	ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
 	kcc "github.com/openshift/machine-config-operator/pkg/controller/kubelet-config"
-	"github.com/openshift/machine-config-operator/pkg/helpers"
 )
 
 // syncVersion handles reporting the version to the clusteroperator
@@ -296,35 +294,6 @@ func (optr *Operator) syncUpgradeableStatus(co *configv1.ClusterOperator) error
 	return nil
 }
 
-func (optr *Operator) syncMetrics() error {
-	pools, err := optr.mcpLister.List(labels.Everything())
-	if err != nil {
-		return err
-	}
-	// set metrics per pool, we need to get the latest condition to log for the state
-	var latestTime metav1.Time
-	latestTime.Time = time.Time{}
-	var cond mcfgv1.MachineConfigPoolCondition
-	for _, pool := range pools {
-		for _, condition := range pool.Status.Conditions {
-			if condition.Status == corev1.ConditionTrue && condition.LastTransitionTime.After(latestTime.Time) {
-				cond = condition
-				latestTime = cond.LastTransitionTime
-			}
-		}
-
-		nodes, _ := helpers.GetNodesForPool(optr.mcpLister, optr.nodeLister, pool)
-		for _, node := range nodes {
-			mcoState.WithLabelValues(node.Name, pool.Name, string(cond.Type), cond.Reason).SetToCurrentTime()
-		}
-		mcoMachineCount.WithLabelValues(pool.Name).Set(float64(pool.Status.MachineCount))
-		mcoUpdatedMachineCount.WithLabelValues(pool.Name).Set(float64(pool.Status.UpdatedMachineCount))
-		mcoDegradedMachineCount.WithLabelValues(pool.Name).Set(float64(pool.Status.DegradedMachineCount))
-		mcoUnavailableMachineCount.WithLabelValues(pool.Name).Set(float64(pool.Status.UnavailableMachineCount))
-	}
-	return nil
-}
-
 func (optr *Operator) syncClusterFleetEvaluation(co *configv1.ClusterOperator) error {
 
 	unexpectedEvaluations, err := optr.generateClusterFleetEvaluations()

pkg/operator/operator_test.go

@@ -231,10 +231,6 @@ func (optr *Operator) syncAll(syncFuncs []syncFunc) error {
 		return fmt.Errorf("error updating cluster operator status: %w", syncClusterFleetEvaluationErr)
 	}
 
-	if err := optr.syncMetrics(); err != nil {
-		return fmt.Errorf("error syncing metrics: %w", err)
-	}
-
 	if optr.inClusterBringup && syncErr.err == nil {
 		klog.Infof("Initialization complete")
 		optr.inClusterBringup = false
@@ -1673,10 +1669,6 @@ func (optr *Operator) syncRequiredMachineConfigPools(config *renderConfig, co *c
 
 	// Let's start with a 10 minute timeout per "required" node.
 	if err := wait.PollUntilContextTimeout(ctx, time.Second, time.Duration(requiredMachineCount*10)*time.Minute, false, func(_ context.Context) (bool, error) {
-		if err := optr.syncMetrics(); err != nil {
-			return false, err
-		}
-
 		if lastErr != nil {
 			// In this case, only the status extension field is updated.
 			newCOStatus := co.Status.DeepCopy()