diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index ba634ecc8..1258ae687 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -17,7 +17,7 @@ resources to help you get started. Do Your Homework ---------------- -Before adding a contribution or create a new issue, take a look at the project’s +Before adding a contribution or create a new issue, take a look at the project's `README `_, read through our `documentation `_, and browse existing `issues `_, @@ -73,7 +73,7 @@ overlooked. We value any suggestions to improve .. tip:: Our documentation is treated like code. Make sure to check our - `writing guidelines `_ + `writing guidelines `_ to help guide new users. Other Ways @@ -87,7 +87,7 @@ questions, and interact with us and other community members on Helpful Resources ----------------- -- Review our `comprehensive guide `_ +- Review our `comprehensive guide `_ for more details on how to add quality contributions to our codebase and documentation - Check this free resource on `How to contribute to an open source project on github `_ - Follow `this wiki page `_ diff --git a/docs/source/tutorial_add_importer_pipeline.rst b/docs/source/tutorial_add_importer_pipeline.rst index 3f5b6b785..ff22746b5 100644 --- a/docs/source/tutorial_add_importer_pipeline.rst +++ b/docs/source/tutorial_add_importer_pipeline.rst @@ -152,7 +152,7 @@ At this point, an example importer will look like this: .. code-block:: python :caption: vulnerabilities/pipelines/example_importer.py :linenos: - :emphasize-lines: 16-17, 20-21, 23-24 + :emphasize-lines: 17-18, 21-22, 24-25 from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline @@ -165,6 +165,7 @@ At this point, an example importer will look like this: license_url = "https://exmaple.org/license/" spdx_license_expression = "CC-BY-4.0" importer_name = "Example Importer" + run_once = False @classmethod def steps(cls): @@ -196,7 +197,7 @@ version management from `univers `_. .. code-block:: python :caption: vulnerabilities/pipelines/example_importer.py :linenos: - :emphasize-lines: 34-35, 37-40 + :emphasize-lines: 35-36, 38-41 from datetime import datetime from datetime import timezone @@ -223,6 +224,7 @@ version management from `univers `_. license_url = "https://example.org/license/" spdx_license_expression = "CC-BY-4.0" importer_name = "Example Importer" + run_once = False @classmethod def steps(cls): @@ -303,6 +305,17 @@ version management from `univers `_. Implement ``on_failure`` to handle cleanup in case of pipeline failure. Cleanup of downloaded archives or cloned repos is necessary to avoid potential resource leakage. +.. tip:: + + Set ``run_once`` to ``True`` if pipeline is meant to be run once. + + - To rerun onetime pipeline, reset ``is_active`` to ``True`` via a migration, pipeline will + run one more time and then deactivate. + + - To convert a onetime pipeline to a regular pipeline, set the ``run_once`` class variable + to ``False`` and reset ``is_active` field to ``True`` via a migration. + + .. note:: | Use ``make valid`` to format your new code using black and isort automatically. diff --git a/vulnerabilities/management/commands/run_scheduler.py b/vulnerabilities/management/commands/run_scheduler.py index f7583ed41..476eb4816 100644 --- a/vulnerabilities/management/commands/run_scheduler.py +++ b/vulnerabilities/management/commands/run_scheduler.py @@ -17,16 +17,21 @@ def init_pipeline_scheduled(): - """Initialize schedule jobs for active PipelineSchedule.""" - active_pipeline_qs = models.PipelineSchedule.objects.filter(is_active=True).order_by( - "created_date" - ) - for pipeline_schedule in active_pipeline_qs: - if scheduled_job_exists(pipeline_schedule.schedule_work_id): - continue - new_id = pipeline_schedule.create_new_job() - pipeline_schedule.schedule_work_id = new_id - pipeline_schedule.save(update_fields=["schedule_work_id"]) + """ + Initialize schedule jobs for active PipelineSchedule. + - Create new schedule if there is no schedule for active pipeline + - Create new schedule if schedule is corrupted for an active pipeline + - Delete schedule for inactive pipeline + """ + pipeline_qs = models.PipelineSchedule.objects.order_by("created_date") + for pipeline in pipeline_qs: + reset_schedule = pipeline.is_active != bool(pipeline.schedule_work_id) + if not scheduled_job_exists(pipeline.schedule_work_id): + reset_schedule = True + + if reset_schedule: + pipeline.schedule_work_id = pipeline.create_new_job() + pipeline.save(update_fields=["schedule_work_id"]) class Command(rqscheduler.Command): diff --git a/vulnerabilities/migrations/0110_pipelineschedule_is_run_once.py b/vulnerabilities/migrations/0110_pipelineschedule_is_run_once.py new file mode 100644 index 000000000..e9265e2ae --- /dev/null +++ b/vulnerabilities/migrations/0110_pipelineschedule_is_run_once.py @@ -0,0 +1,22 @@ +# Generated by Django 4.2.22 on 2026-01-08 13:41 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0109_alter_advisoryseverity_scoring_elements_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="pipelineschedule", + name="is_run_once", + field=models.BooleanField( + db_index=True, + default=False, + help_text="When set to True, this Pipeline will run only once.", + ), + ), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 38f4ebdc3..02a3f4098 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -2273,6 +2273,13 @@ class PipelineSchedule(models.Model): ), ) + is_run_once = models.BooleanField( + null=False, + db_index=True, + default=False, + help_text=("When set to True, this Pipeline will run only once."), + ) + live_logging = models.BooleanField( null=False, db_index=True, diff --git a/vulnerabilities/pipelines/__init__.py b/vulnerabilities/pipelines/__init__.py index c642748b0..9efd58c05 100644 --- a/vulnerabilities/pipelines/__init__.py +++ b/vulnerabilities/pipelines/__init__.py @@ -169,6 +169,10 @@ class VulnerableCodeBaseImporterPipeline(VulnerableCodePipeline): importer_name = None advisory_confidence = MAX_CONFIDENCE + # When set to true pipeline is run only once. + # To rerun onetime pipeline reset is_active field to True via migration. + run_once = False + @classmethod def steps(cls): return ( @@ -262,6 +266,10 @@ class VulnerableCodeBaseImporterPipelineV2(VulnerableCodePipeline): repo_url = None ignorable_versions = [] + # When set to true pipeline is run only once. + # To rerun onetime pipeline reset is_active field to True via migration. + run_once = False + @classmethod def steps(cls): return ( diff --git a/vulnerabilities/schedules.py b/vulnerabilities/schedules.py index 2c2e5366b..8ae3bbb93 100644 --- a/vulnerabilities/schedules.py +++ b/vulnerabilities/schedules.py @@ -89,7 +89,15 @@ def update_pipeline_schedule(): from vulnerabilities.improvers import IMPROVERS_REGISTRY from vulnerabilities.models import PipelineSchedule - pipeline_ids = [*IMPORTERS_REGISTRY.keys(), *IMPROVERS_REGISTRY.keys()] - - PipelineSchedule.objects.exclude(pipeline_id__in=pipeline_ids).delete() - [PipelineSchedule.objects.get_or_create(pipeline_id=id) for id in pipeline_ids] + pipelines = IMPORTERS_REGISTRY | IMPROVERS_REGISTRY + + PipelineSchedule.objects.exclude(pipeline_id__in=pipelines.keys()).delete() + for id, pipeline_class in pipelines.items(): + run_once = getattr(pipeline_class, "run_once", False) + + PipelineSchedule.objects.get_or_create( + pipeline_id=id, + defaults={ + "is_run_once": run_once, + }, + ) diff --git a/vulnerabilities/tasks.py b/vulnerabilities/tasks.py index e035985a2..2e7ac2b10 100644 --- a/vulnerabilities/tasks.py +++ b/vulnerabilities/tasks.py @@ -17,7 +17,6 @@ from vulnerabilities import models from vulnerabilities.importer import Importer from vulnerabilities.improver import Improver -from vulnerablecode.settings import VULNERABLECODE_PIPELINE_TIMEOUT logger = logging.getLogger(__name__) @@ -48,6 +47,13 @@ def execute_pipeline(pipeline_id, run_id): exitcode = 1 run.set_run_ended(exitcode=exitcode, output=output) + + # Onetime pipeline are inactive after first execution. + pipeline = run.pipeline + if pipeline.is_run_once: + pipeline.is_active = False + pipeline.save() + logger.info("Update Run instance with exitcode, output, and end_date") diff --git a/vulnerabilities/templates/pipeline_dashboard.html b/vulnerabilities/templates/pipeline_dashboard.html index d8b03a91a..a7f4139a4 100644 --- a/vulnerabilities/templates/pipeline_dashboard.html +++ b/vulnerabilities/templates/pipeline_dashboard.html @@ -80,7 +80,11 @@

Pipeline Dashboard

{{ schedule.pipeline_id }}
{{ schedule.is_active|yesno:"Yes,No" }}
- {{ schedule.run_interval }} hour{{ schedule.run_interval|pluralize }} + {% if schedule.is_run_once %} + Once + {% else %} + {{ schedule.run_interval }} hour{{ schedule.run_interval|pluralize }} + {% endif %}
@@ -95,7 +99,11 @@

Pipeline Dashboard

{% endif %}
- {{ schedule.next_run_date|date:"Y-m-d" }} + {% if schedule.next_run_date %} + {{ schedule.next_run_date|date:"Y-m-d" }} + {% else %} + N/A + {% endif %}