From 97fdd04f53bdf8721f8415068d9ef8c9d1b0c5ab Mon Sep 17 00:00:00 2001 From: Bazmahou Date: Thu, 6 Feb 2025 12:19:39 +0000 Subject: [PATCH 1/5] updating ami and adding test on pod creation --- README.md | 8 ++--- docs/.header.md | 6 ++-- tests/daemonset.yaml | 75 ++++++++++++++++++++++++++++++++++++++++++++ tests/k8s-test.sh | 14 +++++++++ tests/main.tf | 6 ++-- variables.tf | 2 +- 6 files changed, 100 insertions(+), 11 deletions(-) create mode 100644 tests/daemonset.yaml diff --git a/README.md b/README.md index cd39e18..b658356 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ module "captain" { node_pools = [ # { # "kubernetes_version" : "1.30", -# "ami_release_version" : "1.30.8-20250116", +# "ami_release_version" : "1.30.8-20250123", # "ami_type" : "AL2_x86_64", # "instance_type" : "t3a.large", # "name" : "glueops-platform-node-pool-1", @@ -49,7 +49,7 @@ module "captain" { # }, # { # "kubernetes_version" : "1.30", -# "ami_release_version" : "1.30.8-20250116", +# "ami_release_version" : "1.30.8-20250123", # "ami_type" : "AL2_x86_64", # "instance_type" : "t3a.small", # "name" : "glueops-platform-node-pool-argocd-app-controller-1", @@ -71,7 +71,7 @@ module "captain" { # }, # { # "kubernetes_version" : "1.30", -# "ami_release_version" : "1.30.8-20250116", +# "ami_release_version" : "1.30.8-20250123", # "ami_type" : "AL2_x86_64", # "instance_type" : "t3a.medium", # "name" : "clusterwide-node-pool-1", @@ -200,7 +200,7 @@ No requirements. | [eks\_version](#input\_eks\_version) | The version of EKS to deploy | `string` | `"1.30"` | no | | [iam\_role\_to\_assume](#input\_iam\_role\_to\_assume) | The full ARN of the IAM role to assume | `string` | n/a | yes | | [kube\_proxy\_version](#input\_kube\_proxy\_version) | You should grab the appropriate version number from: https://docs.aws.amazon.com/eks/latest/userguide/managing-kube-proxy.html | `string` | `"v1.30.7-eksbuild.2"` | no | -| [node\_pools](#input\_node\_pools) | node pool configurations:
- name (string): Name of the node pool. MUST BE UNIQUE! Recommended to use YYYYMMDD in the name
- node\_count (number): number of nodes to create in the node pool.
- instance\_type (string): Instance type to use for the nodes. ref: https://instances.vantage.sh/
- kubernetes\_version (string): Generally this is the same version as the EKS cluster. But if doing a node pool upgrade this may be a different version.
- ami\_release\_version (string): AMI Release version to use for EKS worker nodes. ref: https://github.com/awslabs/amazon-eks-ami/releases
- ami\_type (string): e.g. AMD64 or ARM
- spot (bool): Enable spot instances for the nodes. DO NOT ENABLE IN PROD!
- disk\_size\_gb (number): Disk size in GB for the nodes.
- max\_pods (number): max pods that can be scheduled per node.
- ssh\_key\_pair\_names (list(string)): List of SSH key pair names to associate with the nodes. ref: https://us-west-2.console.aws.amazon.com/ec2/home?region=us-west-2#KeyPairs:
- kubernetes\_labels (map(string)): Map of labels to apply to the nodes. ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
- kubernetes\_taints (list(object)): List of taints to apply to the nodes. ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ | 
list(object({
    name                = string
    node_count          = number
    instance_type       = string
    kubernetes_version  = string
    ami_release_version = string
    ami_type            = string
    spot                = bool
    disk_size_gb        = number
    max_pods            = number
    ssh_key_pair_names  = list(string)
    kubernetes_labels   = map(string)
    kubernetes_taints = list(object({
      key    = string
      value  = string
      effect = string
    }))

  }))
| 
[
  {
    "ami_release_version": "1.30.8-20250116",
    "ami_type": "AL2_x86_64",
    "disk_size_gb": 20,
    "instance_type": "t3a.large",
    "kubernetes_labels": {},
    "kubernetes_taints": [],
    "kubernetes_version": "1.30",
    "max_pods": 110,
    "name": "default-pool",
    "node_count": 1,
    "spot": false,
    "ssh_key_pair_names": []
  }
]
| no | +| [node\_pools](#input\_node\_pools) | node pool configurations:
- name (string): Name of the node pool. MUST BE UNIQUE! Recommended to use YYYYMMDD in the name
- node\_count (number): number of nodes to create in the node pool.
- instance\_type (string): Instance type to use for the nodes. ref: https://instances.vantage.sh/
- kubernetes\_version (string): Generally this is the same version as the EKS cluster. But if doing a node pool upgrade this may be a different version.
- ami\_release\_version (string): AMI Release version to use for EKS worker nodes. ref: https://github.com/awslabs/amazon-eks-ami/releases
- ami\_type (string): e.g. AMD64 or ARM
- spot (bool): Enable spot instances for the nodes. DO NOT ENABLE IN PROD!
- disk\_size\_gb (number): Disk size in GB for the nodes.
- max\_pods (number): max pods that can be scheduled per node.
- ssh\_key\_pair\_names (list(string)): List of SSH key pair names to associate with the nodes. ref: https://us-west-2.console.aws.amazon.com/ec2/home?region=us-west-2#KeyPairs:
- kubernetes\_labels (map(string)): Map of labels to apply to the nodes. ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
- kubernetes\_taints (list(object)): List of taints to apply to the nodes. ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ | 
list(object({
    name                = string
    node_count          = number
    instance_type       = string
    kubernetes_version  = string
    ami_release_version = string
    ami_type            = string
    spot                = bool
    disk_size_gb        = number
    max_pods            = number
    ssh_key_pair_names  = list(string)
    kubernetes_labels   = map(string)
    kubernetes_taints = list(object({
      key    = string
      value  = string
      effect = string
    }))

  }))
| 
[
  {
    "ami_release_version": "1.30.8-20250123",
    "ami_type": "AL2_x86_64",
    "disk_size_gb": 20,
    "instance_type": "t3a.large",
    "kubernetes_labels": {},
    "kubernetes_taints": [],
    "kubernetes_version": "1.30",
    "max_pods": 110,
    "name": "default-pool",
    "node_count": 1,
    "spot": false,
    "ssh_key_pair_names": []
  }
]
| no | | [peering\_configs](#input\_peering\_configs) | A list of maps containing VPC peering configuration details | 
list(object({
    vpc_peering_connection_id = string
    destination_cidr_block    = string
  }))
| `[]` | no | | [region](#input\_region) | The AWS region to deploy into | `string` | n/a | yes | | [vpc\_cidr\_block](#input\_vpc\_cidr\_block) | The CIDR block for the VPC | `string` | `"10.65.0.0/26"` | no | diff --git a/docs/.header.md b/docs/.header.md index 90b688b..9a5ad63 100644 --- a/docs/.header.md +++ b/docs/.header.md @@ -27,7 +27,7 @@ module "captain" { node_pools = [ # { # "kubernetes_version" : "1.30", -# "ami_release_version" : "1.30.8-20250116", +# "ami_release_version" : "1.30.8-20250123", # "ami_type" : "AL2_x86_64", # "instance_type" : "t3a.large", # "name" : "glueops-platform-node-pool-1", @@ -49,7 +49,7 @@ module "captain" { # }, # { # "kubernetes_version" : "1.30", -# "ami_release_version" : "1.30.8-20250116", +# "ami_release_version" : "1.30.8-20250123", # "ami_type" : "AL2_x86_64", # "instance_type" : "t3a.small", # "name" : "glueops-platform-node-pool-argocd-app-controller-1", @@ -71,7 +71,7 @@ module "captain" { # }, # { # "kubernetes_version" : "1.30", -# "ami_release_version" : "1.30.8-20250116", +# "ami_release_version" : "1.30.8-20250123", # "ami_type" : "AL2_x86_64", # "instance_type" : "t3a.medium", # "name" : "clusterwide-node-pool-1", diff --git a/tests/daemonset.yaml b/tests/daemonset.yaml new file mode 100644 index 0000000..d94d18d --- /dev/null +++ b/tests/daemonset.yaml @@ -0,0 +1,75 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: test-pods-creation + +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: alpine-daemonset-argocd + namespace: test-pods-creation +spec: + selector: + matchLabels: + app: alpine-test + template: + metadata: + labels: + app: alpine-test + spec: + tolerations: + - key: "glueops.dev/role" + operator: "Equal" + value: "glueops-platform-argocd-app-controller" + effect: "NoSchedule" + containers: + - name: alpine + image: alpine:latest + command: ["/bin/sh", "-c", "while true; do echo Running on $(hostname); sleep 3600; done"] +--- + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: alpine-daemonset-argocd + namespace: test-pods-creation +spec: + selector: + matchLabels: + app: alpine-test + template: + metadata: + labels: + app: alpine-test + spec: + tolerations: + - key: "glueops.dev/role" + operator: "Equal" + value: "glueops-platform" + effect: "NoSchedule" + containers: + - name: alpine + image: alpine:latest + command: ["/bin/sh", "-c", "while true; do echo Running on $(hostname); sleep 3600; done"] + +--- + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: alpine-daemonset-argocd + namespace: test-pods-creation +spec: + selector: + matchLabels: + app: alpine-test + template: + metadata: + labels: + app: alpine-test + spec: + containers: + - name: alpine + image: alpine:latest + command: ["/bin/sh", "-c", "while true; do echo Running on $(hostname); sleep 3600; done"] \ No newline at end of file diff --git a/tests/k8s-test.sh b/tests/k8s-test.sh index 085a16b..1ce4cb3 100755 --- a/tests/k8s-test.sh +++ b/tests/k8s-test.sh @@ -2,6 +2,20 @@ set -e +# Step 1: Verify pods can get created on the current ami_release_version +echo "::group::Creating daemonset on every node in the cluster" +kubectl apply -f daemonset.yaml +echo "::group::Checking the all pods are in running state" +POD_COUNT=$(kubectl get pods -n test-pods-creation --field-selector=status.phase=Running --no-headers | wc -l) + +echo "::group::Comparing number of running pods to the desired count" +if [ "$POD_COUNT" -ne 8 ]; then + echo "Expected 8 pods, but found $POD_COUNT." + exit 1 +else + echo "Pod count matches expected value: $POD_COUNT." +fi + # Step 1: Verify storage driver installation (Amazon EBS CSI Driver) echo "::group::Checking if the storage driver is installed..." kubectl get pods -n kube-system | grep "ebs-csi-" diff --git a/tests/main.tf b/tests/main.tf index 338c48f..a033ff4 100644 --- a/tests/main.tf +++ b/tests/main.tf @@ -11,7 +11,7 @@ module "captain" { node_pools = [ # { # "kubernetes_version" : "1.30", - # "ami_release_version" : "1.30.8-20250116", + # "ami_release_version" : "1.30.8-20250123", # "ami_type" : "AL2_x86_64", # "instance_type" : "t3a.large", # "name" : "glueops-platform-node-pool-1", @@ -33,7 +33,7 @@ module "captain" { # }, # { # "kubernetes_version" : "1.30", - # "ami_release_version" : "1.30.8-20250116", + # "ami_release_version" : "1.30.8-20250123", # "ami_type" : "AL2_x86_64", # "instance_type" : "t3a.small", # "name" : "glueops-platform-node-pool-argocd-app-controller-1", @@ -55,7 +55,7 @@ module "captain" { # }, # { # "kubernetes_version" : "1.30", - # "ami_release_version" : "1.30.8-20250116", + # "ami_release_version" : "1.30.8-20250123", # "ami_type" : "AL2_x86_64", # "instance_type" : "t3a.medium", # "name" : "clusterwide-node-pool-1", diff --git a/variables.tf b/variables.tf index 5793731..75a11d4 100644 --- a/variables.tf +++ b/variables.tf @@ -98,7 +98,7 @@ variable "node_pools" { name = "default-pool" node_count = 1 instance_type = "t3a.large" - ami_release_version = "1.30.8-20250116" + ami_release_version = "1.30.8-20250123" kubernetes_version = "1.30" ami_type = "AL2_x86_64" spot = false From 9a9f8bdce3952d5d4330286d6290aa78a57a05d2 Mon Sep 17 00:00:00 2001 From: Bazmahou Date: Thu, 6 Feb 2025 12:20:45 +0000 Subject: [PATCH 2/5] add clean up --- tests/k8s-test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/k8s-test.sh b/tests/k8s-test.sh index 1ce4cb3..78f57e1 100755 --- a/tests/k8s-test.sh +++ b/tests/k8s-test.sh @@ -15,6 +15,7 @@ if [ "$POD_COUNT" -ne 8 ]; then else echo "Pod count matches expected value: $POD_COUNT." fi +kubectl delete -f daemonset.yaml # Step 1: Verify storage driver installation (Amazon EBS CSI Driver) echo "::group::Checking if the storage driver is installed..." From 1fe29a9a473c247c48534f58f962e909178ed56e Mon Sep 17 00:00:00 2001 From: Bazmahou Date: Thu, 6 Feb 2025 12:23:22 +0000 Subject: [PATCH 3/5] fix daemonset duplication --- tests/daemonset.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/daemonset.yaml b/tests/daemonset.yaml index d94d18d..904c6bf 100644 --- a/tests/daemonset.yaml +++ b/tests/daemonset.yaml @@ -32,7 +32,7 @@ spec: apiVersion: apps/v1 kind: DaemonSet metadata: - name: alpine-daemonset-argocd + name: alpine-daemonset-glueops namespace: test-pods-creation spec: selector: @@ -58,7 +58,7 @@ spec: apiVersion: apps/v1 kind: DaemonSet metadata: - name: alpine-daemonset-argocd + name: alpine-daemonset namespace: test-pods-creation spec: selector: From 6510641c538aff4d956efd1071ed40ffbc5e729d Mon Sep 17 00:00:00 2001 From: Bazmahou Date: Thu, 6 Feb 2025 12:25:24 +0000 Subject: [PATCH 4/5] add a delay before checking pods --- tests/k8s-test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/k8s-test.sh b/tests/k8s-test.sh index 78f57e1..db6a5a5 100755 --- a/tests/k8s-test.sh +++ b/tests/k8s-test.sh @@ -8,6 +8,8 @@ kubectl apply -f daemonset.yaml echo "::group::Checking the all pods are in running state" POD_COUNT=$(kubectl get pods -n test-pods-creation --field-selector=status.phase=Running --no-headers | wc -l) +sleep 300 + echo "::group::Comparing number of running pods to the desired count" if [ "$POD_COUNT" -ne 8 ]; then echo "Expected 8 pods, but found $POD_COUNT." From 5004b4b9276e9993b73639ed7856232c8d2ba3cb Mon Sep 17 00:00:00 2001 From: Bazmahou Date: Thu, 6 Feb 2025 13:27:46 +0000 Subject: [PATCH 5/5] update: use pods instead of daemonset --- tests/daemonset.yaml | 75 -------------------------------------------- tests/k8s-test.sh | 26 +++++++++++---- 2 files changed, 20 insertions(+), 81 deletions(-) delete mode 100644 tests/daemonset.yaml diff --git a/tests/daemonset.yaml b/tests/daemonset.yaml deleted file mode 100644 index 904c6bf..0000000 --- a/tests/daemonset.yaml +++ /dev/null @@ -1,75 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: test-pods-creation - ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: alpine-daemonset-argocd - namespace: test-pods-creation -spec: - selector: - matchLabels: - app: alpine-test - template: - metadata: - labels: - app: alpine-test - spec: - tolerations: - - key: "glueops.dev/role" - operator: "Equal" - value: "glueops-platform-argocd-app-controller" - effect: "NoSchedule" - containers: - - name: alpine - image: alpine:latest - command: ["/bin/sh", "-c", "while true; do echo Running on $(hostname); sleep 3600; done"] ---- - -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: alpine-daemonset-glueops - namespace: test-pods-creation -spec: - selector: - matchLabels: - app: alpine-test - template: - metadata: - labels: - app: alpine-test - spec: - tolerations: - - key: "glueops.dev/role" - operator: "Equal" - value: "glueops-platform" - effect: "NoSchedule" - containers: - - name: alpine - image: alpine:latest - command: ["/bin/sh", "-c", "while true; do echo Running on $(hostname); sleep 3600; done"] - ---- - -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: alpine-daemonset - namespace: test-pods-creation -spec: - selector: - matchLabels: - app: alpine-test - template: - metadata: - labels: - app: alpine-test - spec: - containers: - - name: alpine - image: alpine:latest - command: ["/bin/sh", "-c", "while true; do echo Running on $(hostname); sleep 3600; done"] \ No newline at end of file diff --git a/tests/k8s-test.sh b/tests/k8s-test.sh index db6a5a5..111fdf7 100755 --- a/tests/k8s-test.sh +++ b/tests/k8s-test.sh @@ -3,13 +3,26 @@ set -e # Step 1: Verify pods can get created on the current ami_release_version -echo "::group::Creating daemonset on every node in the cluster" -kubectl apply -f daemonset.yaml +echo "::group::Creating pods on every node in the cluster" +for node in $(kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do + kubectl run -n test-pods-creation alpine-on-$node \ + --image=alpine:latest \ + --overrides='{ + "apiVersion": "v1", + "kind": "Pod", + "spec": { + "nodeName": "'"$node"'", + "containers": [{ + "name": "alpine", + "image": "alpine:latest", + "command": ["/bin/sh", "-c", "while true; do echo Running on $(hostname); sleep 3600; done"] + }] + } + }' +done +sleep 300 echo "::group::Checking the all pods are in running state" POD_COUNT=$(kubectl get pods -n test-pods-creation --field-selector=status.phase=Running --no-headers | wc -l) - -sleep 300 - echo "::group::Comparing number of running pods to the desired count" if [ "$POD_COUNT" -ne 8 ]; then echo "Expected 8 pods, but found $POD_COUNT." @@ -17,7 +30,8 @@ if [ "$POD_COUNT" -ne 8 ]; then else echo "Pod count matches expected value: $POD_COUNT." fi -kubectl delete -f daemonset.yaml +kubectl delete pods --all -n test-pods-creation +echo "::endgroup::" # Step 1: Verify storage driver installation (Amazon EBS CSI Driver) echo "::group::Checking if the storage driver is installed..."