From fa08153fea5f989790cb8fb192d5b0f409e68b02 Mon Sep 17 00:00:00 2001 From: dimeloper Date: Sun, 27 Jul 2025 00:01:07 +0000 Subject: [PATCH 01/13] feat: Implement comprehensive status check violation detection --- DEVELOPMENT.md | 10 ++++ README.md | 9 ++++ docs/getting-started/configuration.md | 17 ++++++ src/core/models.py | 1 + src/event_processors/factory.py | 2 + src/event_processors/pull_request.py | 12 +++++ src/integrations/github_api.py | 77 +++++++++++++++++++++++++-- src/main.py | 3 ++ src/rules/validators.py | 53 ++++++++++++++++-- 9 files changed, 178 insertions(+), 6 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index baa47e2..e14d82f 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -118,6 +118,7 @@ Create a GitHub App for development: - Issue comment - Pull request - Push + - Status 6. **Generate private key** and encode it: ```bash @@ -282,6 +283,15 @@ rules: event_types: [pull_request] parameters: test_param: "test_value" + + - id: status-check-required + name: Status Check Required + description: All PRs must pass required status checks + enabled: true + severity: high + event_types: [pull_request] + parameters: + required_checks: ["ci/test", "lint"] ``` ## Debugging diff --git a/README.md b/README.md index e4d657a..afdaa4a 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,15 @@ rules: parameters: min_approvals: 2 + - id: status-checks-required + name: Status Checks Required + description: All PRs must pass required CI/CD checks + enabled: true + severity: high + event_types: [pull_request] + parameters: + required_checks: ["ci/test", "lint"] + - id: no-deploy-weekends name: No Weekend Deployments description: Prevent deployments on weekends diff --git a/docs/getting-started/configuration.md b/docs/getting-started/configuration.md index ae54d4a..d9eaaaa 100644 --- a/docs/getting-started/configuration.md +++ b/docs/getting-started/configuration.md @@ -70,6 +70,23 @@ rules: message: "Deployments are not allowed on weekends" ``` +### Status Check Required Rule + +This rule ensures that pull requests pass required CI/CD checks: + +```yaml +rules: + - id: status-check-required + name: Status Check Required + description: All PRs must pass required status checks + enabled: true + severity: high + event_types: [pull_request] + parameters: + required_checks: ["ci/test", "lint", "build"] + message: "All required status checks must pass before merging" +``` + ### Large PR Rule This rule helps maintain code quality by flagging large changes: diff --git a/src/core/models.py b/src/core/models.py index 57a140b..bda595f 100644 --- a/src/core/models.py +++ b/src/core/models.py @@ -9,6 +9,7 @@ class EventType(Enum): ISSUE_COMMENT = "issue_comment" PULL_REQUEST = "pull_request" CHECK_RUN = "check_run" + STATUS = "status" DEPLOYMENT = "deployment" DEPLOYMENT_STATUS = "deployment_status" DEPLOYMENT_REVIEW = "deployment_review" diff --git a/src/event_processors/factory.py b/src/event_processors/factory.py index 463ac96..270adab 100644 --- a/src/event_processors/factory.py +++ b/src/event_processors/factory.py @@ -7,6 +7,7 @@ from .pull_request import PullRequestProcessor from .push import PushProcessor from .rule_creation import RuleCreationProcessor +from .status import StatusProcessor from .violation_acknowledgment import ViolationAcknowledgmentProcessor @@ -17,6 +18,7 @@ class EventProcessorFactory: "pull_request": PullRequestProcessor, "push": PushProcessor, "check_run": CheckRunProcessor, + "status": StatusProcessor, "deployment_review": DeploymentReviewProcessor, "deployment_status": DeploymentStatusProcessor, "deployment": DeploymentProcessor, # Process deployment events for rule checking diff --git a/src/event_processors/pull_request.py b/src/event_processors/pull_request.py index fcb2f9d..738929e 100644 --- a/src/event_processors/pull_request.py +++ b/src/event_processors/pull_request.py @@ -197,6 +197,12 @@ async def _prepare_event_data_for_agent(self, task: Task, github_token: str) -> ) event_data["files"] = files or [] + # Get checks and statuses for status check rules + checks = await self.github_client.get_pr_checks( + task.repo_full_name, pr_number, task.installation_id + ) + event_data["checks"] = checks or [] + except Exception as e: logger.warning(f"Error enriching event data: {e}") @@ -379,6 +385,12 @@ async def prepare_api_data(self, task: Task) -> dict[str, Any]: ) api_data["files"] = files or [] + # Fetch checks and statuses for status check rules + checks = await self.github_client.get_pr_checks( + task.repo_full_name, pr_number, task.installation_id + ) + api_data["checks"] = checks or [] + except Exception as e: logger.error(f"Error fetching API data: {e}") diff --git a/src/integrations/github_api.py b/src/integrations/github_api.py index c336e50..16fdfb9 100644 --- a/src/integrations/github_api.py +++ b/src/integrations/github_api.py @@ -116,9 +116,53 @@ async def get_pr_checks(self, repo_full_name: str, pr_number: int, installation_ """ Fetch the list of checks/statuses for a pull request. """ - logger.info(f"Fetching checks for PR #{pr_number} in {repo_full_name}") - # TODO: Implement actual GitHub API call - return [] + try: + # First get the PR to get the head SHA + pr_data = await self.get_pull_request(repo_full_name, pr_number, installation_id) + if not pr_data: + logger.warning(f"Could not get PR #{pr_number} in {repo_full_name}") + return [] + + head_sha = pr_data.get("head", {}).get("sha") + if not head_sha: + logger.warning(f"No head SHA found for PR #{pr_number} in {repo_full_name}") + return [] + + # Get both check runs and commit statuses + check_runs = await self.get_check_runs(repo_full_name, head_sha, installation_id) + commit_statuses = await self.get_commit_statuses(repo_full_name, head_sha, installation_id) + + # Combine and normalize the data + all_checks = [] + + # Add check runs + for check_run in check_runs: + all_checks.append({ + "type": "check_run", + "name": check_run.get("name"), + "status": check_run.get("status"), + "conclusion": check_run.get("conclusion"), + "state": check_run.get("conclusion"), # Map to legacy status API + "context": check_run.get("name"), # For compatibility + }) + + # Add commit statuses + for status in commit_statuses: + all_checks.append({ + "type": "status", + "name": status.get("context"), + "context": status.get("context"), + "state": status.get("state"), + "status": "completed" if status.get("state") in ["success", "failure", "error"] else "in_progress", + "conclusion": "success" if status.get("state") == "success" else "failure" if status.get("state") in ["failure", "error"] else None, + }) + + logger.info(f"Retrieved {len(all_checks)} checks/statuses for PR #{pr_number} in {repo_full_name}") + return all_checks + + except Exception as e: + logger.error(f"Error getting checks for PR #{pr_number} in {repo_full_name}: {e}") + return [] async def get_user_teams(self, repo: str, username: str, installation_id: int) -> list: """Fetch the teams a user belongs to in a repo's org.""" @@ -265,6 +309,33 @@ async def get_check_runs(self, repo: str, sha: str, installation_id: int) -> lis logger.error(f"Error getting check runs for {repo} commit {sha}: {e}") return [] + async def get_commit_statuses(self, repo: str, sha: str, installation_id: int) -> list[dict[str, Any]]: + """Get commit statuses for a commit.""" + try: + token = await self.get_installation_access_token(installation_id) + if not token: + logger.error(f"Failed to get installation token for {installation_id}") + return [] + + headers = {"Authorization": f"Bearer {token}", "Accept": "application/vnd.github.v3+json"} + + url = f"{config.github.api_base_url}/repos/{repo}/commits/{sha}/statuses" + + session = await self._get_session() + async with session.get(url, headers=headers) as response: + if response.status == 200: + data = await response.json() + return data if isinstance(data, list) else [] + else: + error_text = await response.text() + logger.error( + f"Failed to get commit statuses for {repo} commit {sha}. Status: {response.status}, Response: {error_text}" + ) + return [] + except Exception as e: + logger.error(f"Error getting commit statuses for {repo} commit {sha}: {e}") + return [] + async def get_pull_request_reviews(self, repo: str, pr_number: int, installation_id: int) -> list[dict[str, Any]]: """Get reviews for a pull request.""" try: diff --git a/src/main.py b/src/main.py index d3f96f3..198ff0f 100644 --- a/src/main.py +++ b/src/main.py @@ -20,6 +20,7 @@ from src.webhooks.handlers.issue_comment import IssueCommentEventHandler from src.webhooks.handlers.pull_request import PullRequestEventHandler from src.webhooks.handlers.push import PushEventHandler +from src.webhooks.handlers.status import StatusEventHandler from src.webhooks.router import router as webhook_router # --- Application Setup --- @@ -46,6 +47,7 @@ async def lifespan(_app: FastAPI): pull_request_handler = PullRequestEventHandler() push_handler = PushEventHandler() check_run_handler = CheckRunEventHandler() + status_handler = StatusEventHandler() issue_comment_handler = IssueCommentEventHandler() deployment_handler = DeploymentEventHandler() deployment_status_handler = DeploymentStatusEventHandler() @@ -55,6 +57,7 @@ async def lifespan(_app: FastAPI): dispatcher.register_handler(EventType.PULL_REQUEST, pull_request_handler) dispatcher.register_handler(EventType.PUSH, push_handler) dispatcher.register_handler(EventType.CHECK_RUN, check_run_handler) + dispatcher.register_handler(EventType.STATUS, status_handler) dispatcher.register_handler(EventType.ISSUE_COMMENT, issue_comment_handler) dispatcher.register_handler(EventType.DEPLOYMENT, deployment_handler) dispatcher.register_handler(EventType.DEPLOYMENT_STATUS, deployment_status_handler) diff --git a/src/rules/validators.py b/src/rules/validators.py index 467ec3c..070dd34 100644 --- a/src/rules/validators.py +++ b/src/rules/validators.py @@ -424,9 +424,56 @@ async def validate(self, parameters: dict[str, Any], event: dict[str, Any]) -> b if not required_checks: return True - # This would check if all required checks have passed - # For now, return True as placeholder - return True + # Check if this is a pull request event with check data + checks = event.get("checks", []) + if not checks: + # If no checks available, this might be a different event type + logger.debug("RequiredChecksValidator: No checks data available in event") + return True + + # Create a mapping of check names to their status + check_status = {} + for check in checks: + name = check.get("name") or check.get("context") + if name: + # Determine if check passed + # For check_runs: conclusion should be "success" + # For statuses: state should be "success" + conclusion = check.get("conclusion") + state = check.get("state") + + if conclusion == "success" or state == "success": + check_status[name] = "success" + elif conclusion in ["failure", "error", "cancelled", "timed_out"] or state in ["failure", "error"]: + check_status[name] = "failure" + else: + check_status[name] = "pending" + + # Check if all required checks have passed + failed_checks = [] + missing_checks = [] + + for required_check in required_checks: + if required_check not in check_status: + missing_checks.append(required_check) + elif check_status[required_check] != "success": + failed_checks.append(required_check) + + # Log detailed information for debugging + logger.debug(f"RequiredChecksValidator: Required checks: {required_checks}") + logger.debug(f"RequiredChecksValidator: Available checks: {list(check_status.keys())}") + logger.debug(f"RequiredChecksValidator: Failed checks: {failed_checks}") + logger.debug(f"RequiredChecksValidator: Missing checks: {missing_checks}") + + # Rule is violated if any required checks are missing or failed + violations_exist = len(failed_checks) > 0 or len(missing_checks) > 0 + + if violations_exist: + logger.info(f"RequiredChecksValidator: VIOLATION - Failed: {failed_checks}, Missing: {missing_checks}") + else: + logger.debug("RequiredChecksValidator: All required checks passed") + + return not violations_exist # Registry of all available validators From 23326322188fff5847cef2f5ae701df62d0d8719 Mon Sep 17 00:00:00 2001 From: dimeloper Date: Sun, 27 Jul 2025 00:05:15 +0000 Subject: [PATCH 02/13] feat: document precommit hooks setup --- DEVELOPMENT.md | 55 +++++++++++++++++++++------ src/event_processors/pull_request.py | 8 +--- src/integrations/github_api.py | 56 ++++++++++++++++------------ src/rules/validators.py | 6 +-- 4 files changed, 82 insertions(+), 43 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index e14d82f..1e2665e 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -22,6 +22,7 @@ cd watchflow ### 2. Create Virtual Environment Using uv (recommended): + ```bash # Create and activate virtual environment uv venv @@ -34,6 +35,7 @@ uv sync ``` Or using pip: + ```bash # Create virtual environment python -m venv .venv @@ -121,6 +123,7 @@ Create a GitHub App for development: - Status 6. **Generate private key** and encode it: + ```bash cat /path/to/private-key.pem | base64 | tr -d '\n' ``` @@ -146,6 +149,7 @@ For debugging AI agents: 1. Create a [LangSmith](https://langsmith.com/) account 2. Get your API key 3. Add to environment: + ```bash LANGCHAIN_TRACING_V2=true LANGCHAIN_API_KEY=your-langsmith-api-key @@ -194,6 +198,41 @@ uv run ruff format src/ uv run mypy src/ ``` +### Pre-commit Hooks + +This project uses pre-commit hooks to ensure code quality and consistency. The hooks automatically run on every commit and include: + +- **Trailing whitespace removal** - Cleans up extra whitespace +- **End of file fixer** - Ensures files end with newlines +- **YAML/JSON validation** - Checks syntax +- **Ruff formatting and linting** - Formats code and sorts imports +- **Conventional commit validation** - Ensures commit messages follow conventional format + +#### Setup Pre-commit Hooks + +```bash +# Install pre-commit hooks (run once after cloning) +uv run pre-commit install + +# Also install commit message validation +uv run pre-commit install --hook-type commit-msg +``` + +#### Using Pre-commit Hooks + +```bash +# Hooks run automatically on commit, but you can run them manually: +uv run pre-commit run --all-files + +# Run on specific files +uv run pre-commit run --files src/main.py + +# Skip hooks for a commit (not recommended) +git commit --no-verify -m "commit message" +``` + +The hooks will prevent commits if any issues are found. Most formatting issues are automatically fixed, so you just need to stage the changes and commit again. + ### Testing The project includes comprehensive tests that run **without making real API calls** by default: @@ -213,7 +252,7 @@ pytest tests/integration/ ### Test Structure -``` +```txt tests/ ├── unit/ # ⚡ Fast unit tests (mocked OpenAI) │ └── test_feasibility_agent.py @@ -236,16 +275,6 @@ pytest tests/integration/ -m integration _Note: Real API tests make actual OpenAI calls and will cost money. They're disabled by default in CI/CD._ -### Pre-commit Hooks - -```bash -# Install pre-commit hooks -uv run pre-commit install - -# Run manually -uv run pre-commit run --all-files -``` - ## Testing AI Agents ### Rule Evaluation Testing @@ -329,6 +358,7 @@ Test webhook delivery: ### GitHub App Permissions If webhooks aren't being received: + 1. Verify webhook URL is accessible 2. Check GitHub App permissions 3. Verify webhook secret matches @@ -336,6 +366,7 @@ If webhooks aren't being received: ### AI Agent Issues If agents aren't working: + 1. Verify OpenAI API key 2. Check LangSmith configuration 3. Review agent logs for errors @@ -343,6 +374,7 @@ If agents aren't working: ### Development Environment If dependencies aren't working: + 1. Ensure Python 3.12+ 2. Try recreating virtual environment 3. Check uv/pip installation @@ -362,6 +394,7 @@ locust -f load_test.py --host=http://localhost:8000 ### Agent Performance Monitor agent performance with LangSmith: + - Token usage per request - Response times - Error rates diff --git a/src/event_processors/pull_request.py b/src/event_processors/pull_request.py index 738929e..c388546 100644 --- a/src/event_processors/pull_request.py +++ b/src/event_processors/pull_request.py @@ -198,9 +198,7 @@ async def _prepare_event_data_for_agent(self, task: Task, github_token: str) -> event_data["files"] = files or [] # Get checks and statuses for status check rules - checks = await self.github_client.get_pr_checks( - task.repo_full_name, pr_number, task.installation_id - ) + checks = await self.github_client.get_pr_checks(task.repo_full_name, pr_number, task.installation_id) event_data["checks"] = checks or [] except Exception as e: @@ -386,9 +384,7 @@ async def prepare_api_data(self, task: Task) -> dict[str, Any]: api_data["files"] = files or [] # Fetch checks and statuses for status check rules - checks = await self.github_client.get_pr_checks( - task.repo_full_name, pr_number, task.installation_id - ) + checks = await self.github_client.get_pr_checks(task.repo_full_name, pr_number, task.installation_id) api_data["checks"] = checks or [] except Exception as e: diff --git a/src/integrations/github_api.py b/src/integrations/github_api.py index 16fdfb9..92fb1b0 100644 --- a/src/integrations/github_api.py +++ b/src/integrations/github_api.py @@ -122,44 +122,54 @@ async def get_pr_checks(self, repo_full_name: str, pr_number: int, installation_ if not pr_data: logger.warning(f"Could not get PR #{pr_number} in {repo_full_name}") return [] - + head_sha = pr_data.get("head", {}).get("sha") if not head_sha: logger.warning(f"No head SHA found for PR #{pr_number} in {repo_full_name}") return [] - + # Get both check runs and commit statuses check_runs = await self.get_check_runs(repo_full_name, head_sha, installation_id) commit_statuses = await self.get_commit_statuses(repo_full_name, head_sha, installation_id) - + # Combine and normalize the data all_checks = [] - + # Add check runs for check_run in check_runs: - all_checks.append({ - "type": "check_run", - "name": check_run.get("name"), - "status": check_run.get("status"), - "conclusion": check_run.get("conclusion"), - "state": check_run.get("conclusion"), # Map to legacy status API - "context": check_run.get("name"), # For compatibility - }) - + all_checks.append( + { + "type": "check_run", + "name": check_run.get("name"), + "status": check_run.get("status"), + "conclusion": check_run.get("conclusion"), + "state": check_run.get("conclusion"), # Map to legacy status API + "context": check_run.get("name"), # For compatibility + } + ) + # Add commit statuses for status in commit_statuses: - all_checks.append({ - "type": "status", - "name": status.get("context"), - "context": status.get("context"), - "state": status.get("state"), - "status": "completed" if status.get("state") in ["success", "failure", "error"] else "in_progress", - "conclusion": "success" if status.get("state") == "success" else "failure" if status.get("state") in ["failure", "error"] else None, - }) - + all_checks.append( + { + "type": "status", + "name": status.get("context"), + "context": status.get("context"), + "state": status.get("state"), + "status": "completed" + if status.get("state") in ["success", "failure", "error"] + else "in_progress", + "conclusion": "success" + if status.get("state") == "success" + else "failure" + if status.get("state") in ["failure", "error"] + else None, + } + ) + logger.info(f"Retrieved {len(all_checks)} checks/statuses for PR #{pr_number} in {repo_full_name}") return all_checks - + except Exception as e: logger.error(f"Error getting checks for PR #{pr_number} in {repo_full_name}: {e}") return [] diff --git a/src/rules/validators.py b/src/rules/validators.py index 070dd34..6c8fe75 100644 --- a/src/rules/validators.py +++ b/src/rules/validators.py @@ -441,7 +441,7 @@ async def validate(self, parameters: dict[str, Any], event: dict[str, Any]) -> b # For statuses: state should be "success" conclusion = check.get("conclusion") state = check.get("state") - + if conclusion == "success" or state == "success": check_status[name] = "success" elif conclusion in ["failure", "error", "cancelled", "timed_out"] or state in ["failure", "error"]: @@ -452,7 +452,7 @@ async def validate(self, parameters: dict[str, Any], event: dict[str, Any]) -> b # Check if all required checks have passed failed_checks = [] missing_checks = [] - + for required_check in required_checks: if required_check not in check_status: missing_checks.append(required_check) @@ -467,7 +467,7 @@ async def validate(self, parameters: dict[str, Any], event: dict[str, Any]) -> b # Rule is violated if any required checks are missing or failed violations_exist = len(failed_checks) > 0 or len(missing_checks) > 0 - + if violations_exist: logger.info(f"RequiredChecksValidator: VIOLATION - Failed: {failed_checks}, Missing: {missing_checks}") else: From 5ebcc17efaebc06d6eb706d9a83651c09e26caae Mon Sep 17 00:00:00 2001 From: dimeloper Date: Sun, 27 Jul 2025 00:05:45 +0000 Subject: [PATCH 03/13] feat: Implement comprehensive status check violation detection - part 2 --- docs/status-check-rules-guide.md | 222 ++++++++++++++++++++ src/event_processors/status.py | 145 +++++++++++++ src/webhooks/handlers/status.py | 29 +++ tests/unit/test_status_check_integration.py | 155 ++++++++++++++ 4 files changed, 551 insertions(+) create mode 100644 docs/status-check-rules-guide.md create mode 100644 src/event_processors/status.py create mode 100644 src/webhooks/handlers/status.py create mode 100644 tests/unit/test_status_check_integration.py diff --git a/docs/status-check-rules-guide.md b/docs/status-check-rules-guide.md new file mode 100644 index 0000000..2b528e9 --- /dev/null +++ b/docs/status-check-rules-guide.md @@ -0,0 +1,222 @@ +# Status Check Rules - Implementation Guide + +This document explains how to use the newly implemented status check functionality in Watchflow to enforce CI/CD pipeline requirements. + +## Overview + +Watchflow now supports monitoring and enforcing requirements for GitHub status checks (both modern check runs and legacy commit statuses). This allows you to create rules that ensure pull requests pass required CI/CD checks before they can be merged. + +## Supported Event Types + +- `check_run` - Modern GitHub check runs (recommended) +- `status` - Legacy GitHub commit status API +- `pull_request` - Includes status check validation during PR processing + +## Rule Configuration + +### Basic Status Check Rule + +```yaml +rules: + - id: ci-checks-required + name: CI Checks Required + description: All pull requests must pass CI tests and linting + enabled: true + severity: high + event_types: [pull_request] + parameters: + required_checks: ["ci/test", "lint"] +``` + +### Comprehensive Status Check Rule + +```yaml +rules: + - id: comprehensive-checks + name: Comprehensive CI/CD Checks + description: Enforce all required checks for production readiness + enabled: true + severity: critical + event_types: [pull_request] + parameters: + required_checks: + - "ci/test" + - "ci/lint" + - "ci/build" + - "security/scan" + - "codecov/patch" + - "continuous-integration/travis-ci/pr" + message: "All CI/CD checks must pass before merging to ensure code quality and security" +``` + +## How It Works + +### 1. Pull Request Processing + +When a pull request event occurs, Watchflow: + +1. Fetches the PR details from GitHub +2. Retrieves all check runs and commit statuses for the PR's head commit +3. Evaluates rules that include `required_checks` parameters +4. Creates violations for any failed or missing required checks + +### 2. Check Run/Status Processing + +When individual check runs or status updates occur, Watchflow: + +1. Processes the event if rules are configured for `check_run` or `status` events +2. Evaluates the check against any applicable rules +3. Can trigger additional actions based on rule configuration + +### 3. Status Check Validation + +The `RequiredChecksValidator` checks: + +- **Passing checks**: `conclusion: "success"` (check runs) or `state: "success"` (statuses) +- **Failing checks**: `conclusion: "failure"|"error"|"cancelled"|"timed_out"` or `state: "failure"|"error"` +- **Missing checks**: Required checks that don't appear in the current status list + +## Example Scenarios + +### Scenario 1: Basic CI Enforcement + +**Rule Configuration:** +```yaml +rules: + - id: basic-ci + name: Basic CI Required + description: Ensure tests pass before merge + enabled: true + severity: high + event_types: [pull_request] + parameters: + required_checks: ["ci/test"] +``` + +**Behavior:** +- ✅ **PR with passing tests**: No violations, PR can be merged +- ❌ **PR with failing tests**: Violation reported, PR blocked +- ❌ **PR without test results**: Violation reported, PR blocked + +### Scenario 2: Multi-Check Enforcement + +**Rule Configuration:** +```yaml +rules: + - id: production-ready + name: Production Ready Checks + description: All quality gates must pass + enabled: true + severity: critical + event_types: [pull_request] + parameters: + required_checks: ["ci/test", "lint", "security-scan", "build"] +``` + +**Behavior:** +- ✅ **All checks pass**: No violations +- ❌ **Any check fails**: Violation reported with details about which checks failed +- ❌ **Missing checks**: Violation reported with details about which checks are missing + +## Troubleshooting + +### Common Issues + +1. **Rules not triggering** + - Verify the rule `event_types` includes `pull_request` + - Check that `required_checks` parameter is properly configured + - Ensure the GitHub App has `Checks: Read` permissions + +2. **Check names not matching** + - Check the exact names of your CI/CD checks in GitHub + - Use the GitHub API or PR checks tab to verify check names + - Check names are case-sensitive + +3. **Legacy vs Modern Checks** + - Modern CI/CD tools use check runs (`ci/test`) + - Legacy tools use statuses (`continuous-integration/travis-ci/pr`) + - Watchflow supports both formats + +### Debugging + +Enable debug logging to see detailed check validation: + +```bash +LOG_LEVEL=DEBUG +``` + +This will show: +- Which checks were found for the PR +- Which required checks are missing or failing +- Detailed validation results + +## Migration Guide + +### From Static Branch Protection + +If you're currently using GitHub's branch protection rules for status checks: + +1. **Identify your current required checks** in GitHub branch protection settings +2. **Create equivalent Watchflow rules** using the check names +3. **Test the rules** on a test repository first +4. **Gradually migrate** by enabling Watchflow rules and disabling branch protection + +### Example Migration + +**Before (GitHub Branch Protection):** +- Required status checks: `ci/test`, `lint` + +**After (Watchflow Rule):** +```yaml +rules: + - id: migrate-from-branch-protection + name: Required CI Checks + description: Migrated from branch protection rules + enabled: true + severity: high + event_types: [pull_request] + parameters: + required_checks: ["ci/test", "lint"] +``` + +## Best Practices + +1. **Start Simple**: Begin with basic test requirements and add more checks gradually +2. **Use Descriptive Names**: Make rule IDs and names clear about what they enforce +3. **Set Appropriate Severity**: Use `critical` for security/build checks, `high` for tests, `medium` for linting +4. **Test Rules**: Always test new rules in a development environment first +5. **Monitor Performance**: Status check validation adds API calls, monitor for rate limits + +## API Integration + +### Fetching Check Data + +The system automatically fetches both check runs and commit statuses: + +```python +# This is handled automatically by the PullRequestProcessor +checks = await github_client.get_pr_checks(repo, pr_number, installation_id) +``` + +### Manual Check Validation + +You can also validate checks manually: + +```python +from src.rules.validators import RequiredChecksValidator + +validator = RequiredChecksValidator() +result = await validator.validate( + parameters={"required_checks": ["ci/test", "lint"]}, + event={"checks": checks_data} +) +``` + +## Support + +For issues or questions about status check rules: + +1. Check the debug logs for validation details +2. Verify check names match exactly what appears in GitHub +3. Ensure proper GitHub App permissions +4. Review the rule configuration syntax diff --git a/src/event_processors/status.py b/src/event_processors/status.py new file mode 100644 index 0000000..ba70c5f --- /dev/null +++ b/src/event_processors/status.py @@ -0,0 +1,145 @@ +import logging +import time +from typing import Any + +from src.agents.engine_agent.agent import RuleEngineAgent +from src.event_processors.base import BaseEventProcessor, ProcessingResult +from src.tasks.task_queue import Task + +logger = logging.getLogger(__name__) + + +class StatusProcessor(BaseEventProcessor): + """Processor for status events using hybrid agentic rule evaluation.""" + + def __init__(self): + # Call super class __init__ first + super().__init__() + + # Create instance of hybrid RuleEngineAgent + self.engine_agent = RuleEngineAgent() + + def get_event_type(self) -> str: + return "status" + + async def process(self, task: Task) -> ProcessingResult: + start_time = time.time() + payload = task.payload + status = payload.get("status", {}) + + # Ignore successful statuses for performance unless specifically configured + state = status.get("state", "") + if state == "success": + logger.info(f"Status '{status.get('context', 'unknown')}' succeeded - no rule evaluation needed") + return ProcessingResult( + success=True, violations=[], api_calls_made=0, processing_time_ms=int((time.time() - start_time) * 1000) + ) + + logger.info("=" * 80) + logger.info(f"🚀 Processing STATUS event for {task.repo_full_name}") + logger.info(f" Context: {status.get('context')}") + logger.info(f" State: {state}") + logger.info(f" Description: {status.get('description', '')}") + logger.info("=" * 80) + + # Prepare event_data for the agent + event_data = { + "status": status, + "repository": payload.get("repository", {}), + "organization": payload.get("organization", {}), + "commit": payload.get("commit", {}), + "branches": payload.get("branches", []), + "event_id": payload.get("event_id"), + "timestamp": payload.get("timestamp"), + } + + # Fetch rules + rules = await self.rule_provider.get_rules(task.repo_full_name, task.installation_id) + + # Convert rules to the new format expected by the agent + formatted_rules = self._convert_rules_to_new_format(rules) + + # Filter rules that apply to status events + status_rules = [rule for rule in formatted_rules if "status" in rule.get("event_types", [])] + + if not status_rules: + logger.info("No rules configured for status events") + return ProcessingResult( + success=True, violations=[], api_calls_made=1, processing_time_ms=int((time.time() - start_time) * 1000) + ) + + # Run agentic analysis using the instance + result = await self.engine_agent.execute( + event_type="status", + event_data=event_data, + rules=status_rules, + ) + + violations = result.data.get("violations", []) + + logger.info("=" * 80) + logger.info(f"🏁 STATUS processing completed in {int((time.time() - start_time) * 1000)}ms") + logger.info(f" Violations: {len(violations)}") + if violations: + logger.warning("🚨 VIOLATION SUMMARY:") + for i, violation in enumerate(violations, 1): + logger.warning( + f" {i}. {violation.get('rule_name', 'Unknown')} ({violation.get('severity', 'medium')})" + ) + logger.warning(f" {violation.get('message', '')}") + logger.info("=" * 80) + + return ProcessingResult( + success=(not violations), + violations=violations, + api_calls_made=1, + processing_time_ms=int((time.time() - start_time) * 1000), + ) + + def _convert_rules_to_new_format(self, rules: list[Any]) -> list[dict[str, Any]]: + """Convert Rule objects to the new flat schema format.""" + formatted_rules = [] + + for rule in rules: + # Convert Rule object to dict format + rule_dict = { + "id": rule.id, + "name": rule.name, + "description": rule.description, + "enabled": rule.enabled, + "severity": rule.severity.value if hasattr(rule.severity, "value") else rule.severity, + "event_types": [et.value if hasattr(et, "value") else et for et in rule.event_types], + "parameters": rule.parameters if hasattr(rule, "parameters") else {}, + } + + # If no parameters field, try to extract from conditions (backward compatibility) + if not rule_dict["parameters"] and hasattr(rule, "conditions"): + for condition in rule.conditions: + rule_dict["parameters"].update(condition.parameters) + + formatted_rules.append(rule_dict) + + return formatted_rules + + async def prepare_webhook_data(self, task: Task) -> dict[str, Any]: + """Prepare data from webhook payload.""" + status = task.payload.get("status", {}) + return { + "event_type": "status", + "repo_full_name": task.repo_full_name, + "status": { + "id": status.get("id"), + "context": status.get("context"), + "state": status.get("state"), + "description": status.get("description"), + "target_url": status.get("target_url"), + "created_at": status.get("created_at"), + "updated_at": status.get("updated_at"), + }, + "commit": task.payload.get("commit", {}), + "branches": task.payload.get("branches", []), + } + + async def prepare_api_data(self, task: Task) -> dict[str, Any]: + """Prepare data from GitHub API calls.""" + return {} diff --git a/src/webhooks/handlers/status.py b/src/webhooks/handlers/status.py new file mode 100644 index 0000000..2406bd3 --- /dev/null +++ b/src/webhooks/handlers/status.py @@ -0,0 +1,29 @@ +import logging + +from src.core.models import WebhookEvent +from src.tasks.task_queue import task_queue +from src.webhooks.handlers.base import EventHandler + +logger = logging.getLogger(__name__) + + +class StatusEventHandler(EventHandler): + """Handler for status webhook events using task queue.""" + + async def can_handle(self, event: WebhookEvent) -> bool: + return event.event_type.name == "STATUS" + + async def handle(self, event: WebhookEvent): + """Handle status events by enqueuing them for background processing.""" + logger.info(f"🔄 Enqueuing status event for {event.repo_full_name}") + + task_id = await task_queue.enqueue( + event_type="status", + repo_full_name=event.repo_full_name, + installation_id=event.installation_id, + payload=event.payload, + ) + + logger.info(f"✅ Status event enqueued with task ID: {task_id}") + + return {"status": "enqueued", "task_id": task_id, "message": "Status event has been queued for processing"} diff --git a/tests/unit/test_status_check_integration.py b/tests/unit/test_status_check_integration.py new file mode 100644 index 0000000..165af8e --- /dev/null +++ b/tests/unit/test_status_check_integration.py @@ -0,0 +1,155 @@ +from datetime import datetime +from unittest.mock import AsyncMock, patch + +import pytest + +from src.event_processors.pull_request import PullRequestProcessor +from src.rules.validators import RequiredChecksValidator +from src.tasks.task_queue import Task, TaskStatus + + +class TestStatusCheckIntegration: + """Test suite for status check requirement integration.""" + + @pytest.mark.asyncio + async def test_required_checks_validator_with_passing_checks(self): + """Test that RequiredChecksValidator correctly validates when all checks pass.""" + validator = RequiredChecksValidator() + + # Mock event data with passing checks + event_data = { + "checks": [ + {"name": "ci/test", "conclusion": "success", "status": "completed"}, + {"name": "lint", "conclusion": "success", "status": "completed"}, + {"context": "build", "state": "success"}, + ] + } + + parameters = {"required_checks": ["ci/test", "lint", "build"]} + + # Should return True (no violation) when all checks pass + result = await validator.validate(parameters, event_data) + assert result is True + + @pytest.mark.asyncio + async def test_required_checks_validator_with_failing_checks(self): + """Test that RequiredChecksValidator correctly identifies failing checks.""" + validator = RequiredChecksValidator() + + # Mock event data with failing checks + event_data = { + "checks": [ + {"name": "ci/test", "conclusion": "failure", "status": "completed"}, + {"name": "lint", "conclusion": "success", "status": "completed"}, + ] + } + + parameters = { + "required_checks": ["ci/test", "lint", "build"] # build is missing + } + + # Should return False (violation) when checks fail or are missing + result = await validator.validate(parameters, event_data) + assert result is False + + @pytest.mark.asyncio + async def test_required_checks_validator_with_missing_checks(self): + """Test that RequiredChecksValidator correctly identifies missing checks.""" + validator = RequiredChecksValidator() + + # Mock event data with missing required checks + event_data = {"checks": [{"name": "ci/test", "conclusion": "success", "status": "completed"}]} + + parameters = {"required_checks": ["ci/test", "lint", "security-scan"]} + + # Should return False (violation) when required checks are missing + result = await validator.validate(parameters, event_data) + assert result is False + + @pytest.mark.asyncio + async def test_required_checks_validator_with_legacy_status_api(self): + """Test that RequiredChecksValidator works with legacy status API.""" + validator = RequiredChecksValidator() + + # Mock event data with legacy status format + event_data = { + "checks": [ + {"context": "continuous-integration/travis-ci/pr", "state": "success"}, + {"context": "codecov/patch", "state": "failure"}, + ] + } + + parameters = {"required_checks": ["continuous-integration/travis-ci/pr", "codecov/patch"]} + + # Should return False (violation) when any check fails + result = await validator.validate(parameters, event_data) + assert result is False + + @pytest.mark.asyncio + async def test_required_checks_validator_no_required_checks(self): + """Test that RequiredChecksValidator passes when no checks are required.""" + validator = RequiredChecksValidator() + + event_data = {"checks": []} + + parameters = { + "required_checks": [] # No checks required + } + + # Should return True (no violation) when no checks are required + result = await validator.validate(parameters, event_data) + assert result is True + + @pytest.mark.asyncio + @patch("src.event_processors.pull_request.RuleEngineAgent") + async def test_pull_request_processor_fetches_checks(self, mock_agent_class): + """Test that PullRequestProcessor fetches check data for rules evaluation.""" + # Mock the RuleEngineAgent class to avoid OpenAI dependency + mock_agent = AsyncMock() + mock_agent_class.return_value = mock_agent + + # Create processor instance + processor = PullRequestProcessor() + + # Mock the GitHub client on the processor instance + processor.github_client = AsyncMock() + processor.github_client.get_pull_request_reviews.return_value = [] + processor.github_client.get_pull_request_files.return_value = [] + processor.github_client.get_pr_checks.return_value = [ + {"name": "ci/test", "conclusion": "failure", "status": "completed"} + ] + + # Create a mock task + task = Task( + id="test-task", + event_type="pull_request", + repo_full_name="owner/repo", + installation_id=123456, + payload={"pull_request": {"number": 42}}, + status=TaskStatus.PENDING, + created_at=datetime.now(), + ) + + # Test that checks are fetched via prepare_api_data + api_data = await processor.prepare_api_data(task) + + assert "checks" in api_data + assert len(api_data["checks"]) == 1 + assert api_data["checks"][0]["name"] == "ci/test" + assert api_data["checks"][0]["conclusion"] == "failure" + + # Test that checks are also fetched via _prepare_event_data_for_agent + event_data = await processor._prepare_event_data_for_agent(task, "mock-token") + + assert "checks" in event_data + assert len(event_data["checks"]) == 1 + assert event_data["checks"][0]["name"] == "ci/test" + assert event_data["checks"][0]["conclusion"] == "failure" + + # Verify the GitHub API was called twice (once for each method) + assert processor.github_client.get_pr_checks.call_count == 2 + processor.github_client.get_pr_checks.assert_called_with("owner/repo", 42, 123456) + + +if __name__ == "__main__": + pytest.main([__file__]) From 7c40d1052e97f214e38ee2f4836a754f189a211f Mon Sep 17 00:00:00 2001 From: dimeloper Date: Sun, 27 Jul 2025 00:11:50 +0000 Subject: [PATCH 04/13] feat: include watchflow rules --- .watchflow/rules.yaml | 62 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 .watchflow/rules.yaml diff --git a/.watchflow/rules.yaml b/.watchflow/rules.yaml new file mode 100644 index 0000000..3b70002 --- /dev/null +++ b/.watchflow/rules.yaml @@ -0,0 +1,62 @@ +rules: + - id: status-check-ci-tests + name: "Require CI Tests" + description: "All CI tests must pass before merging" + enabled: true + severity: high + event_types: ["pull_request"] + conditions: + required_checks: + - "ci/test" + - "build" + - "lint" + + - id: status-check-security + name: "Security Checks Required" + description: "Security scans must pass before merging" + enabled: true + severity: high + event_types: ["pull_request"] + conditions: + required_checks: + - "security/scan" + - "vulnerability-check" + + - id: status-check-code-quality + name: "Code Quality Gates" + description: "Code quality checks must pass before merging" + enabled: true + severity: medium + event_types: ["pull_request"] + conditions: + required_checks: + - "codecov/patch" + - "codeclimate" + - "sonarqube" + + - id: conventional-commits + name: "Conventional Commit Format" + description: "Commit messages must follow conventional commit format" + enabled: true + severity: medium + event_types: ["push"] + conditions: + commit_message_pattern: "^(feat|fix|docs|style|refactor|test|chore)(\\(.+\\))?: .{1,50}" + + - id: pr-description-required + name: "Pull Request Description Required" + description: "Pull requests must have a meaningful description" + enabled: true + severity: low + event_types: ["pull_request"] + conditions: + min_description_length: 20 + + - id: no-direct-main-push + name: "No Direct Push to Main" + description: "Prevent direct pushes to main branch - use pull requests" + enabled: true + severity: high + event_types: ["push"] + conditions: + blocked_branches: ["main", "master"] From 2c4d99e7c7540a3141022d8e61f5f1104935aaf8 Mon Sep 17 00:00:00 2001 From: dimeloper Date: Sun, 27 Jul 2025 00:14:45 +0000 Subject: [PATCH 05/13] test: problematic code --- src/main.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main.py b/src/main.py index 198ff0f..f55abe2 100644 --- a/src/main.py +++ b/src/main.py @@ -25,6 +25,14 @@ # --- Application Setup --- +# TODO: This is intentionally problematic code to test our status check rules! +# This will cause CI failures that should be caught by our new status check validation +unused_variable = "This will cause linting issues" +def broken_function(): + # Missing docstring, poor formatting, unused function + x=1+2 # Poor spacing + return x + logging.basicConfig( level=logging.INFO, format="%(asctime)s %(levelname)8s %(message)s", From dc2a5996a19a617af505f0b9fd6fc6863fe789d5 Mon Sep 17 00:00:00 2001 From: dimeloper Date: Sun, 27 Jul 2025 00:16:58 +0000 Subject: [PATCH 06/13] test: check rules on push --- .watchflow/rules.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.watchflow/rules.yaml b/.watchflow/rules.yaml index 3b70002..464fcd1 100644 --- a/.watchflow/rules.yaml +++ b/.watchflow/rules.yaml @@ -4,7 +4,7 @@ rules: description: "All CI tests must pass before merging" enabled: true severity: high - event_types: ["pull_request"] + event_types: ["pull_request", "push", "status"] conditions: required_checks: - "ci/test" @@ -16,7 +16,7 @@ rules: description: "Security scans must pass before merging" enabled: true severity: high - event_types: ["pull_request"] + event_types: ["pull_request", "push", "status"] conditions: required_checks: - "security/scan" @@ -27,7 +27,7 @@ rules: description: "Code quality checks must pass before merging" enabled: true severity: medium - event_types: ["pull_request"] + event_types: ["pull_request", "push", "status"] conditions: required_checks: - "codecov/patch" From c4a23e26b1e3d2e07cdff9e5ded8f2f0a770066f Mon Sep 17 00:00:00 2001 From: dimeloper Date: Sun, 27 Jul 2025 00:19:32 +0000 Subject: [PATCH 07/13] test: check rules on status --- .watchflow/rules.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.watchflow/rules.yaml b/.watchflow/rules.yaml index 464fcd1..6bf472f 100644 --- a/.watchflow/rules.yaml +++ b/.watchflow/rules.yaml @@ -4,19 +4,18 @@ rules: description: "All CI tests must pass before merging" enabled: true severity: high - event_types: ["pull_request", "push", "status"] + event_types: ["pull_request", "status"] conditions: required_checks: - - "ci/test" - - "build" - - "lint" + - "Lint code and README files" + - "Run pre-commit hooks / Lint code and README files" - id: status-check-security name: "Security Checks Required" description: "Security scans must pass before merging" enabled: true severity: high - event_types: ["pull_request", "push", "status"] + event_types: ["status"] conditions: required_checks: - "security/scan" @@ -27,7 +26,7 @@ rules: description: "Code quality checks must pass before merging" enabled: true severity: medium - event_types: ["pull_request", "push", "status"] + event_types: ["status"] conditions: required_checks: - "codecov/patch" From 0a6bae802ec967693d8cce4a8d0e78074a35abe2 Mon Sep 17 00:00:00 2001 From: dimeloper Date: Sun, 27 Jul 2025 00:21:36 +0000 Subject: [PATCH 08/13] test: fix rule structure --- .watchflow/rules.yaml | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/.watchflow/rules.yaml b/.watchflow/rules.yaml index 6bf472f..715d558 100644 --- a/.watchflow/rules.yaml +++ b/.watchflow/rules.yaml @@ -6,9 +6,11 @@ rules: severity: high event_types: ["pull_request", "status"] conditions: - required_checks: - - "Lint code and README files" - - "Run pre-commit hooks / Lint code and README files" + - type: "required_checks" + parameters: + required_checks: + - "Lint code and README files" + - "Run pre-commit hooks / Lint code and README files" - id: status-check-security name: "Security Checks Required" @@ -17,9 +19,11 @@ rules: severity: high event_types: ["status"] conditions: - required_checks: - - "security/scan" - - "vulnerability-check" + - type: "required_checks" + parameters: + required_checks: + - "security/scan" + - "vulnerability-check" - id: status-check-code-quality name: "Code Quality Gates" @@ -28,8 +32,10 @@ rules: severity: medium event_types: ["status"] conditions: - required_checks: - - "codecov/patch" + - type: "required_checks" + parameters: + required_checks: + - "codecov/patch" - "codeclimate" - "sonarqube" From 685f22a85c62e60fb79cb83b950c230070460368 Mon Sep 17 00:00:00 2001 From: dimeloper Date: Sun, 27 Jul 2025 00:24:45 +0000 Subject: [PATCH 09/13] test: fix rule structure --- .watchflow/rules.yaml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.watchflow/rules.yaml b/.watchflow/rules.yaml index 715d558..13523f0 100644 --- a/.watchflow/rules.yaml +++ b/.watchflow/rules.yaml @@ -36,17 +36,19 @@ rules: parameters: required_checks: - "codecov/patch" - - "codeclimate" - - "sonarqube" + - "codeclimate" + - "sonarqube" - id: conventional-commits name: "Conventional Commit Format" description: "Commit messages must follow conventional commit format" - enabled: true + enabled: false # Disabled - no validator implemented yet severity: medium event_types: ["push"] conditions: - commit_message_pattern: "^(feat|fix|docs|style|refactor|test|chore)(\\(.+\\))?: .{1,50}" + - type: "commit_message_pattern" + parameters: + commit_message_pattern: "^(feat|fix|docs|style|refactor|test|chore)(\\(.+\\))?: .{1,50}" - id: pr-description-required name: "Pull Request Description Required" @@ -55,7 +57,9 @@ rules: severity: low event_types: ["pull_request"] conditions: - min_description_length: 20 + - type: "min_description_length" + parameters: + min_description_length: 20 - id: no-direct-main-push name: "No Direct Push to Main" @@ -64,4 +68,6 @@ rules: severity: high event_types: ["push"] conditions: - blocked_branches: ["main", "master"] + - type: "protected_branches" + parameters: + protected_branches: ["main", "master"] From 042a6ec65f9f7675f15a73a722d15bfccdff8718 Mon Sep 17 00:00:00 2001 From: dimeloper Date: Sun, 27 Jul 2025 00:26:45 +0000 Subject: [PATCH 10/13] test: fail if no checks found --- src/rules/validators.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/rules/validators.py b/src/rules/validators.py index 6c8fe75..15f9c45 100644 --- a/src/rules/validators.py +++ b/src/rules/validators.py @@ -427,9 +427,10 @@ async def validate(self, parameters: dict[str, Any], event: dict[str, Any]) -> b # Check if this is a pull request event with check data checks = event.get("checks", []) if not checks: - # If no checks available, this might be a different event type - logger.debug("RequiredChecksValidator: No checks data available in event") - return True + # If no checks available but required checks are specified, + # this is a violation - required checks are missing + logger.debug("RequiredChecksValidator: No checks data available in event - VIOLATION") + return False # Create a mapping of check names to their status check_status = {} From 2209e73d0d782e8ea1ed577420c47e2a09f2197f Mon Sep 17 00:00:00 2001 From: dimeloper Date: Sun, 27 Jul 2025 00:28:03 +0000 Subject: [PATCH 11/13] test: test rule triggers --- .watchflow/rules.yaml | 11 +++++++++++ src/rules/validators.py | 11 ++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/.watchflow/rules.yaml b/.watchflow/rules.yaml index 13523f0..4b38fdc 100644 --- a/.watchflow/rules.yaml +++ b/.watchflow/rules.yaml @@ -1,4 +1,15 @@ rules: + - id: test-always-fail + name: "Test Rule - Always Fail" + description: "This rule should always fail to test if rule evaluation is working" + enabled: true + severity: high + event_types: ["pull_request"] + conditions: + - type: "min_description_length" + parameters: + min_description_length: 999999 # Impossibly high requirement + - id: status-check-ci-tests name: "Require CI Tests" description: "All CI tests must pass before merging" diff --git a/src/rules/validators.py b/src/rules/validators.py index 15f9c45..33a0a4f 100644 --- a/src/rules/validators.py +++ b/src/rules/validators.py @@ -420,16 +420,25 @@ async def validate(self, parameters: dict[str, Any], event: dict[str, Any]) -> b class RequiredChecksValidator(ConditionValidator): async def validate(self, parameters: dict[str, Any], event: dict[str, Any]) -> bool: + # Add detailed logging to understand what's happening + logger.info(f"RequiredChecksValidator: CALLED with parameters: {parameters}") + logger.info(f"RequiredChecksValidator: Event keys available: {list(event.keys())}") + required_checks = parameters.get("required_checks", []) + logger.info(f"RequiredChecksValidator: Required checks: {required_checks}") + if not required_checks: + logger.info("RequiredChecksValidator: No required checks specified - PASSING") return True # Check if this is a pull request event with check data checks = event.get("checks", []) + logger.info(f"RequiredChecksValidator: Found {len(checks)} checks in event data") + if not checks: # If no checks available but required checks are specified, # this is a violation - required checks are missing - logger.debug("RequiredChecksValidator: No checks data available in event - VIOLATION") + logger.error("RequiredChecksValidator: No checks data available in event - VIOLATION") return False # Create a mapping of check names to their status From ca2825b1ad30b5ee5ceea2cdc7390aae4c88fbe1 Mon Sep 17 00:00:00 2001 From: dimeloper Date: Sun, 27 Jul 2025 16:57:22 +0000 Subject: [PATCH 12/13] feat: include local setup detailed guide. --- DEVELOPMENT.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 1e2665e..f547f74 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -2,6 +2,12 @@ This guide covers setting up the Watchflow development environment for local development and testing. +## Quick Start + +🚀 **New to Watchflow?** Start with our [Local Development Setup Guide](./LOCAL_SETUP.md) for a complete end-to-end setup including GitHub App configuration and webhook testing. + +This document covers advanced development topics and workflow. + ## Prerequisites - Python 3.12 or higher From c91fc267d6ea0b28da5033773f1c0b27cc5fc0c9 Mon Sep 17 00:00:00 2001 From: dimeloper Date: Sun, 27 Jul 2025 16:57:34 +0000 Subject: [PATCH 13/13] feat: include local setup detailed guide. --- LOCAL_SETUP.md | 388 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 388 insertions(+) create mode 100644 LOCAL_SETUP.md diff --git a/LOCAL_SETUP.md b/LOCAL_SETUP.md new file mode 100644 index 0000000..627e879 --- /dev/null +++ b/LOCAL_SETUP.md @@ -0,0 +1,388 @@ +# Local Development Setup Guide + +This guide covers setting up Watchflow for complete end-to-end local development, including GitHub App configuration, webhook setup, and API integration. + +## Why This Setup? + +**Personal GitHub App vs. Marketplace App**: For local development, creating your own GitHub App instead of using the production Watchflow app from the marketplace provides several critical advantages: + +- **Isolated development environment**: Your local testing won't interfere with production Watchflow instances +- **Full control over webhooks**: You can point webhooks to your local ngrok tunnel instead of production servers +- **Custom configuration**: You can modify permissions and settings without affecting the production app +- **Safe experimentation**: Test new features, rule changes, and integrations without risk to live systems +- **Independent debugging**: Monitor webhook deliveries and debug issues in isolation +- **No rate limiting conflicts**: Avoid hitting GitHub API rate limits that might affect production usage + +**ngrok for Local Development**: Since GitHub webhooks need to reach your local development server, and your localhost isn't accessible from the internet, ngrok creates a secure tunnel that: + +- **Exposes your local server** to GitHub's webhook delivery system +- **Provides HTTPS endpoints** required by GitHub for webhook URLs +- **Offers request inspection** tools to debug webhook payloads +- **Eliminates complex firewall configuration** or port forwarding + +## Prerequisites + +- Python 3.12 or higher +- [uv](https://docs.astral.sh/uv/) package manager (recommended) or pip +- [ngrok](https://ngrok.com/) installed for webhook tunneling +- GitHub organization or user account with admin access +- [OpenAI API key](https://platform.openai.com/api-keys) for AI agent functionality +- [LangSmith account](https://langsmith.com/) for AI agent debugging (optional) + +## Step 1: Create a GitHub App + +1. Navigate to your GitHub organization settings or [personal settings](https://github.com/settings/apps) +2. Go to **"Developer settings"** → **"GitHub Apps"** → **"New GitHub App"** +3. Fill in the basic app information: + - **App name**: `watchflow-dev` (or your preferred name) + - **Homepage URL**: `http://localhost:8000` + - **Webhook URL**: `https://placeholder.ngrok.io/webhooks/github` (we'll update this in Step 3) + - **Webhook secret**: Generate a secure random string and save it + - **Description**: "Local development instance of Watchflow" + +## Step 2: Configure GitHub App Permissions + +### Repository Permissions + +Set the following permissions for your GitHub App: + +- **Actions**: Read-only +- **Checks**: Read and write +- **Contents**: Read-only +- **Deployments**: Read and write +- **Environments**: Read-only +- **Issues**: Read and write +- **Metadata**: Read-only (mandatory) +- **Pull requests**: Read and write +- **Commit statuses**: Read and write + +### Organization Permissions + +- **Members**: Read-only + +### Subscribe to Events + +Check the following webhook events: + +- ✅ **Check run** +- ✅ **Commit comment** +- ✅ **Deployment** +- ✅ **Deployment protection rule** +- ✅ **Deployment review** +- ✅ **Deployment status** +- ✅ **Issue comment** +- ✅ **Issues** +- ✅ **Pull request** +- ✅ **Pull request review** +- ✅ **Pull request review comment** +- ✅ **Pull request review thread** +- ✅ **Push** +- ✅ **Status** +- ✅ **Workflow dispatch** +- ✅ **Workflow job** +- ✅ **Workflow run** + +### Generate and Download Private Key + +1. After creating the app, scroll down to **"Private keys"** +2. Click **"Generate a private key"** +3. Download the `.pem` file and save it securely +4. Note down your **App ID** from the app settings page + +## Step 3: Clone and Setup Watchflow + +### 3.1 Clone the Repository + +```bash +git clone https://github.com/watchflow/watchflow.git +cd watchflow +``` + +### 3.2 Create Virtual Environment + +Using uv (recommended): + +```bash +# Create and activate virtual environment +uv venv +source .venv/bin/activate # On macOS/Linux +# or +.venv\Scripts\activate # On Windows + +# Install dependencies +uv sync +``` + +Or using pip: + +```bash +# Create virtual environment +python -m venv .venv + +# Activate virtual environment +source .venv/bin/activate # On macOS/Linux +# or +.venv\Scripts\activate # On Windows + +# Install dependencies +pip install -e ".[dev]" +``` + +### 3.3 Install and Setup ngrok + +```bash +# Install ngrok +# On macOS with Homebrew: +brew install ngrok + +# On other systems, download from https://ngrok.com/download + +# Start ngrok tunnel to your local API +ngrok http 8000 +``` + +Copy the ngrok HTTPS URL from the terminal output (e.g., `https://abc123.ngrok.io`) + +### 3.4 Update GitHub App Webhook URL + +1. Go back to your GitHub App settings +2. Update the **Webhook URL** to: `https://your-ngrok-url.ngrok.io/webhooks/github` +3. Save the changes + +## Step 4: Configure Environment Variables + +### 4.1 Create Environment File + +```bash +cp .env.example .env +``` + +### 4.2 Configure Required Variables + +Edit your `.env` file with the following configuration: + +```bash +# GitHub App Configuration +APP_NAME_GITHUB=watchflow-dev +CLIENT_ID_GITHUB=your_app_id_from_github_app_settings +CLIENT_SECRET_GITHUB=your_client_secret_from_github_app_settings +PRIVATE_KEY_BASE64_GITHUB=your_base64_encoded_private_key +REDIRECT_URI_GITHUB=http://localhost:3000 + +# GitHub Webhook Configuration +WEBHOOK_SECRET_GITHUB=your_webhook_secret_from_step_1 + +# OpenAI API Configuration +OPENAI_API_KEY=your-openai-api-key + +# LangChain Configuration (Optional - for AI debugging) +LANGCHAIN_TRACING_V2=true +LANGCHAIN_ENDPOINT=https://api.smith.langchain.com +LANGCHAIN_API_KEY=your-langsmith-api-key +LANGCHAIN_PROJECT=watchflow-dev + +# Application Configuration +ENVIRONMENT=development + +# CORS Configuration +CORS_HEADERS=["*"] +CORS_ORIGINS='["http://localhost:3000", "http://127.0.0.1:3000"]' + +# AWS Configuration (if using AWS services) +AWS_ACCESS_KEY_ID=your_aws_access_key_id +AWS_SECRET_ACCESS_KEY=your_aws_secret_access_key +``` + +### 4.3 Encode Your Private Key + +Convert your GitHub App private key to base64: + +**Option 1: Using command line (recommended):** + +```bash +# Encode the private key file +cat /path/to/your-private-key.pem | base64 | tr -d '\n' +``` + +**Option 2: Using online tools:** + +If you prefer using a web interface, you can use online base64 encoding tools like [base64encode.org](https://www.base64encode.org): + +1. Open the `.pem` file in a text editor +2. Copy the entire content (including the BEGIN/END lines) +3. Paste it into the online encoder +4. Copy the base64 output + +Copy the base64 output and use it as the value for `PRIVATE_KEY_BASE64_GITHUB` in your `.env` file. + +**Security Note**: When using online tools, ensure you're using a reputable service and understand that your private key content will be processed by their servers. For maximum security, prefer the command-line method. + +## Step 5: Install GitHub App on Repositories + +### 5.1 Install the App + +1. In your GitHub App settings, click on **"Install App"** in the left sidebar +2. Choose to install on your **organization** or **personal account** +3. Select **"Selected repositories"** and choose the repositories you want to monitor +4. Alternatively, select **"All repositories"** for organization-wide installation +5. Click **"Install"** to complete the installation + +### 5.2 Verify Installation + +1. Go to the repository settings of an installed repository +2. Navigate to **"Integrations"** → **"GitHub Apps"** +3. Verify that your `watchflow-dev` app is listed and active + +## Step 6: Start Local Development Environment + +### 6.1 Start the Watchflow API + +```bash +# Using uv (recommended) +uv run uvicorn src.main:app --reload --host 0.0.0.0 --port 8000 + +# Using pip +uvicorn src.main:app --reload --host 0.0.0.0 --port 8000 +``` + +The API will be available at `http://localhost:8000` + +### 6.2 Verify API is Running + +Open your browser and navigate to: + +- `http://localhost:8000/docs` - Interactive API documentation +- `http://localhost:8000/health` - Health check endpoint + +## Step 7: Test the Setup + +### 7.1 Verify Webhook Connection + +1. Ensure your local API is running on port 8000 +2. Verify ngrok tunnel is active and forwarding to localhost:8000 +3. Check the ngrok dashboard at `http://localhost:4040` for incoming requests + +### 7.2 Test with Real Events + +1. Go to one of your monitored repositories +2. Create a new pull request or push a commit +3. Check your local API logs to confirm webhook events are being received +4. Monitor the ngrok dashboard for incoming webhook requests + +### 7.3 Test Rule Evaluation + +Create a test rule in a monitored repository by adding `.watchflow/rules.yaml`: + +```yaml +rules: + - id: test-rule + name: Test Rule for Local Development + description: Simple rule to test local setup + enabled: true + severity: medium + event_types: [pull_request] + parameters: + test_param: "local_test" + + - id: pr-approval-required + name: PR Approval Required + description: All pull requests must have at least 1 approval + enabled: true + severity: high + event_types: [pull_request] + parameters: + min_approvals: 1 +``` + +## Troubleshooting + +### Common Issues + +#### Webhook not receiving events + +- Verify ngrok is running and the URL is correct in GitHub App settings +- Check that the webhook secret matches your environment configuration +- Ensure your local API endpoint `/webhooks/github` is properly configured +- Check GitHub App webhook delivery logs in the app settings + +#### Permission errors + +- Double-check that all required permissions are granted to the GitHub App +- Verify the app is installed on the correct repositories/organization +- Ensure the private key is correctly encoded and configured + +#### ngrok tunnel expires + +- Free ngrok tunnels expire after 8 hours +- Restart ngrok and update the webhook URL in your GitHub App settings +- Consider upgrading to ngrok Pro for persistent URLs + +#### AI Agent not working + +- Verify your OpenAI API key is valid and has sufficient credits +- Check the AI model name in your configuration +- Review API logs for OpenAI-related errors + +### Logs and Debugging + +Monitor the following for debugging: + +1. **Local API server logs** - Check your terminal running uvicorn +2. **ngrok request logs** - Run `ngrok http 8000 --log=stdout` +3. **GitHub App webhook delivery logs** - Available in GitHub App settings +4. **LangSmith traces** - If configured, view at [LangSmith Dashboard](https://smith.langchain.com/) + +### Additional Debugging Commands + +```bash +# Check if API is responding +curl http://localhost:8000/health + +# Test rule evaluation endpoint +curl -X POST "http://localhost:8000/api/v1/rules/evaluate" \ + -H "Content-Type: application/json" \ + -d '{ + "rule_text": "All pull requests must have at least 2 approvals" + }' + +# View detailed logs +tail -f logs/watchflow.log +``` + +## Security Notes + +- **Never commit your private keys or webhook secrets to version control** +- Use environment variables or secure secret management for all credentials +- Rotate webhook secrets periodically +- Limit GitHub App installation scope to only necessary repositories during development +- Keep your ngrok tunnel URL private and don't share it publicly +- Use separate GitHub Apps for development and production environments + +## Next Steps + +Once your local setup is working: + +1. **Explore the API documentation** at `http://localhost:8000/docs` +2. **Create custom rules** in your test repositories +3. **Set up LangSmith** for AI agent debugging and monitoring +4. **Run the test suite** to verify everything is working: `pytest` +5. **Read the main development guide** in `DEVELOPMENT.md` for advanced topics + +## Development Workflow Integration + +After completing this setup, you can integrate with the standard development workflow: + +```bash +# Format and lint code +uv run ruff format src/ +uv run ruff check src/ + +# Run tests +pytest + +# Install pre-commit hooks for code quality +uv run pre-commit install +uv run pre-commit install --hook-type commit-msg +``` + +For more advanced development topics, testing strategies, and deployment options, refer to the main [DEVELOPMENT.md](./DEVELOPMENT.md) guide.