From 4f97a4c14644c858ebbdaa9488436234d4efdce1 Mon Sep 17 00:00:00 2001 From: Dimitris Kargatzis Date: Mon, 28 Jul 2025 17:37:20 +0300 Subject: [PATCH 1/4] fix(ci): ensure correct Python setup and generate coverage report Signed-off-by: Dimitris Kargatzis --- .github/workflows/tests.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index bbb47e8..7a374d8 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -1,4 +1,4 @@ -name: "Unit and integration tests" +name: "Unit and Integration Tests" on: push: @@ -13,8 +13,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: - - "3.12" + python-version: ["3.12"] steps: # Checkout code @@ -28,25 +27,26 @@ jobs: version: "latest" # Set up Python - - name: "Set up Python" - run: uv python install "${{ matrix.python-version }}" + - name: "Set up Python ${{ matrix.python-version }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} # Install dependencies - name: "Install dependencies" run: uv sync --all-extras - # Run all tests + # Run all tests with coverage - name: "Run all tests" run: | - echo "Running unit tests..." - uv run pytest tests/unit/ -v --tb=short - echo "Running integration tests (mocked - no real API calls)..." - uv run pytest tests/integration/ -v --tb=short + echo "Running unit and integration tests with coverage..." + uv run pytest tests/unit/ tests/integration/ \ + --cov=. --cov-report=xml -v --tb=short # Upload coverage reports - name: "Upload coverage reports" uses: codecov/codecov-action@v4 - if: matrix.python-version == "3.12" + if: matrix.python-version == '3.12' with: file: ./coverage.xml fail_ci_if_error: false From 2f470a9068d60b8c3165534d8f5cbd8f4b2f0584 Mon Sep 17 00:00:00 2001 From: Dimitris Kargatzis Date: Tue, 29 Jul 2025 22:55:11 +0300 Subject: [PATCH 2/4] fix(ci): add auth token for Codecov integration - Add CODECOV_TOKEN to Codecov action - Create codecov.yml with 80% targets and minimal comment layout - Only comment on coverage changes to reduce noise Signed-off-by: Dimitris Kargatzis --- .github/workflows/tests.yaml | 2 + codecov.yml | 31 ++++++++++++++ docs/temp.md | 83 ++++++++++++++++++++++++++++++++++++ 3 files changed, 116 insertions(+) create mode 100644 codecov.yml create mode 100644 docs/temp.md diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 7a374d8..ad5761c 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -50,3 +50,5 @@ jobs: with: file: ./coverage.xml fail_ci_if_error: false + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000..d49cf9c --- /dev/null +++ b/codecov.yml @@ -0,0 +1,31 @@ +codecov: + require_ci_to_pass: yes + notify: + wait_for_ci: yes + +coverage: + precision: 1 + round: down + range: "80...100" + status: + project: + default: + target: 80% + threshold: 5% + patch: + default: + target: 80% + threshold: 5% + +parsers: + gcov: + branch_detection: + conditional: yes + loop: yes + method: no + macro: no + +comment: + layout: "diff,flags,footer" + behavior: default + require_changes: yes \ No newline at end of file diff --git a/docs/temp.md b/docs/temp.md new file mode 100644 index 0000000..c6f1d40 --- /dev/null +++ b/docs/temp.md @@ -0,0 +1,83 @@ +Here's a glossary-style documentation file based on the PDFs you provided, aligning terminology and concepts used across both the **Watchflow** brief and the **Agentic DevOps White Paper**. You can share this with your team to align while drafting documentation and README files. + +--- + +# Watchflow Glossary & Core Concepts + +### General Vision + +| Term | Definition | +|--------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Watchflow** | An open-source tool for real-time governance of DevOps operations (PRs, workflows, deployments). It serves as the core of Warestack’s agentic protection engine. | +| **Agentic DevOps** | A governance model where DevOps decisions are made dynamically and contextually by smart agents, not static rules. Inspired by the way Grammarly adapts to writing. | +| **Warestack** | The commercial SaaS tool built on top of Watchflow, providing enterprise-level dashboards, AI-driven rules, and integrations with GitHub, Slack, and Linear. | + +--- + +### Problem Framing + +| Term | Description | +|-----------------------------|-------------------------------------------------------------------------------------------------------------------------------------| +| **Static Protection Rules** | Traditional `config-as-code` logic (e.g., YAML-defined CI/CD checks) which are binary, hardcoded, and unaware of real-time context. | +| **Config-as-Code** | DevOps protection strategy using declarative YAML/JSON files—fast and predictable but lacks adaptability. | +| **Rule Drift** | The phenomenon where governance logic becomes outdated or inconsistent across multiple services/repos. | +| **Siloed Signals** | Lack of integration between development signals (e.g., PRs), communication tools (Slack), and tracking tools (Linear). | +| **Manual Enforcement** | Developers or DevOps engineers are responsible for checking and applying rules, which is time-consuming and error-prone. | + +--- + +### Watchflow Solution Components + +| Term | Description | +|-------------------------|---------------------------------------------------------------------------------------------------------------------------------| +| **Grammar for DevOps** | Human-readable rules written in plain English that define “how changes should happen.” Similar to writing grammar in Grammarly. | +| **Context-Aware Rules** | Rules that adapt based on dynamic attributes such as urgency, role (e.g., senior engineer), commit size, day/time, etc. | +| **Dynamic Rule Engine** | Engine that listens to GitHub events and applies context-based actions (block, comment, alert) in real time. | +| **Trace, Flag, Block** | Lifecycle actions applied to risky operations: continuously monitor, flag anomalies, and optionally block violations. | +| **Agentic Detection** | AI-assisted logic that observes behavioral patterns, PR content, and metadata to uncover subtle or complex rule breaches. | +| **Justified Actions** | Each enforcement decision is made with clear context-aware explanation for developer trust and auditability. | + +--- + +### Rules & Use Cases + +| Term | Example / Notes | +|-------------------------------|---------------------------------------------------------------------------------------------------------| +| **Protection Rule** | "Require 2 approvals for PRs to `main` unless it's a hotfix by a senior engineer on-call." | +| **Rule Adaptation** | Instead of `true/false`, Watchflow interprets developer roles and repo urgency to decide actions. | +| **Examples of Agentic Rules** | Stop self-approval, require issue links, ensure PR description clarity, enforce based on day/time/role. | +| **Hybrid Architecture** | Combines real-time event processing (webhooks) with AI rule reasoning and static fallback logic. | + +--- + +### Results from Evaluations + +| Metric | Result | +|-------------------------|------------------------------------------------------------------------------------------------------------------| +| **Violation Coverage** | 92% of violations caught (vs. \~13% with static rules). | +| **Precision** | 87% precision in detection—minimizing false positives. | +| **Coverage Gain** | Agentic system detected up to **7×** more issues than config-based rules in projects like Terraform, Kubernetes. | +| **Justified Decisions** | 89% of actions came with a clear rationale, aiding in audit trails and developer adoption. | + +--- + +### Integration Targets + +| Platform | Purpose | +|---------------------|--------------------------------------------------------------------------------| +| **GitHub** | Primary source of PR and deployment events. | +| **CI/CD Workflows** | Watchflow integrates at the protection rule level, not as a step in pipelines. | + +--- + +### Evaluation Benchmarks + +| Repositories Used | +|---------------------| +| Microsoft VSCode | +| Facebook React | +| HashiCorp Terraform | +| PyTorch | +| Kubernetes | + +100 PRs were analyzed using both static and agentic methods. Watchflow consistently surfaced more relevant violations. From 765e2fe0b9a8b0c750ad5158b7e71385e8a15c8f Mon Sep 17 00:00:00 2001 From: Dimitris Kargatzis Date: Tue, 29 Jul 2025 22:56:18 +0300 Subject: [PATCH 3/4] fix(ci): resolve lint issues in codecov config Signed-off-by: Dimitris Kargatzis --- codecov.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codecov.yml b/codecov.yml index d49cf9c..96d6083 100644 --- a/codecov.yml +++ b/codecov.yml @@ -28,4 +28,4 @@ parsers: comment: layout: "diff,flags,footer" behavior: default - require_changes: yes \ No newline at end of file + require_changes: yes From eadaf884cf2bc43fe951f51fa8e70cdf5657f2b5 Mon Sep 17 00:00:00 2001 From: Dimitris Kargatzis Date: Tue, 29 Jul 2025 23:57:00 +0300 Subject: [PATCH 4/4] chore: remove non-related documentation file Signed-off-by: Dimitris Kargatzis --- docs/temp.md | 83 ---------------------------------------------------- 1 file changed, 83 deletions(-) delete mode 100644 docs/temp.md diff --git a/docs/temp.md b/docs/temp.md deleted file mode 100644 index c6f1d40..0000000 --- a/docs/temp.md +++ /dev/null @@ -1,83 +0,0 @@ -Here's a glossary-style documentation file based on the PDFs you provided, aligning terminology and concepts used across both the **Watchflow** brief and the **Agentic DevOps White Paper**. You can share this with your team to align while drafting documentation and README files. - ---- - -# Watchflow Glossary & Core Concepts - -### General Vision - -| Term | Definition | -|--------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **Watchflow** | An open-source tool for real-time governance of DevOps operations (PRs, workflows, deployments). It serves as the core of Warestack’s agentic protection engine. | -| **Agentic DevOps** | A governance model where DevOps decisions are made dynamically and contextually by smart agents, not static rules. Inspired by the way Grammarly adapts to writing. | -| **Warestack** | The commercial SaaS tool built on top of Watchflow, providing enterprise-level dashboards, AI-driven rules, and integrations with GitHub, Slack, and Linear. | - ---- - -### Problem Framing - -| Term | Description | -|-----------------------------|-------------------------------------------------------------------------------------------------------------------------------------| -| **Static Protection Rules** | Traditional `config-as-code` logic (e.g., YAML-defined CI/CD checks) which are binary, hardcoded, and unaware of real-time context. | -| **Config-as-Code** | DevOps protection strategy using declarative YAML/JSON files—fast and predictable but lacks adaptability. | -| **Rule Drift** | The phenomenon where governance logic becomes outdated or inconsistent across multiple services/repos. | -| **Siloed Signals** | Lack of integration between development signals (e.g., PRs), communication tools (Slack), and tracking tools (Linear). | -| **Manual Enforcement** | Developers or DevOps engineers are responsible for checking and applying rules, which is time-consuming and error-prone. | - ---- - -### Watchflow Solution Components - -| Term | Description | -|-------------------------|---------------------------------------------------------------------------------------------------------------------------------| -| **Grammar for DevOps** | Human-readable rules written in plain English that define “how changes should happen.” Similar to writing grammar in Grammarly. | -| **Context-Aware Rules** | Rules that adapt based on dynamic attributes such as urgency, role (e.g., senior engineer), commit size, day/time, etc. | -| **Dynamic Rule Engine** | Engine that listens to GitHub events and applies context-based actions (block, comment, alert) in real time. | -| **Trace, Flag, Block** | Lifecycle actions applied to risky operations: continuously monitor, flag anomalies, and optionally block violations. | -| **Agentic Detection** | AI-assisted logic that observes behavioral patterns, PR content, and metadata to uncover subtle or complex rule breaches. | -| **Justified Actions** | Each enforcement decision is made with clear context-aware explanation for developer trust and auditability. | - ---- - -### Rules & Use Cases - -| Term | Example / Notes | -|-------------------------------|---------------------------------------------------------------------------------------------------------| -| **Protection Rule** | "Require 2 approvals for PRs to `main` unless it's a hotfix by a senior engineer on-call." | -| **Rule Adaptation** | Instead of `true/false`, Watchflow interprets developer roles and repo urgency to decide actions. | -| **Examples of Agentic Rules** | Stop self-approval, require issue links, ensure PR description clarity, enforce based on day/time/role. | -| **Hybrid Architecture** | Combines real-time event processing (webhooks) with AI rule reasoning and static fallback logic. | - ---- - -### Results from Evaluations - -| Metric | Result | -|-------------------------|------------------------------------------------------------------------------------------------------------------| -| **Violation Coverage** | 92% of violations caught (vs. \~13% with static rules). | -| **Precision** | 87% precision in detection—minimizing false positives. | -| **Coverage Gain** | Agentic system detected up to **7×** more issues than config-based rules in projects like Terraform, Kubernetes. | -| **Justified Decisions** | 89% of actions came with a clear rationale, aiding in audit trails and developer adoption. | - ---- - -### Integration Targets - -| Platform | Purpose | -|---------------------|--------------------------------------------------------------------------------| -| **GitHub** | Primary source of PR and deployment events. | -| **CI/CD Workflows** | Watchflow integrates at the protection rule level, not as a step in pipelines. | - ---- - -### Evaluation Benchmarks - -| Repositories Used | -|---------------------| -| Microsoft VSCode | -| Facebook React | -| HashiCorp Terraform | -| PyTorch | -| Kubernetes | - -100 PRs were analyzed using both static and agentic methods. Watchflow consistently surfaced more relevant violations.