From dc9f88e299c05f16848356443e263518cba009b8 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Tue, 17 Jun 2025 20:52:16 +0500 Subject: [PATCH 1/2] fix to run result sbmission(with copy to predictions dir) --- compute_worker/compute_worker.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py index 85a2cbe06..e2edafc10 100644 --- a/compute_worker/compute_worker.py +++ b/compute_worker/compute_worker.py @@ -588,7 +588,7 @@ async def _run_program_directory(self, program_dir, kind): Function responsible for running program directory Args: - - program_dir : can be either ingestion program or program/submission + - program_dir : can be either ingestion program or program(submission or scoring) - kind : either `program` or `ingestion` """ # If the directory doesn't even exist, move on @@ -609,6 +609,9 @@ async def _run_program_directory(self, program_dir, kind): logger.info( "Program directory missing metadata, assuming it's going to be handled by ingestion" ) + # Copy program dir content to output directory because in case of only result submission, + # we need to copy the result submission files because the scoring program will use these as predictions + shutil.copytree(program_dir, self.output_dir) return else: raise SubmissionException("Program directory missing 'metadata.yaml/metadata'") From d5e355741f90ad5cdb04037259b8211c59cdcc6d Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Thu, 19 Jun 2025 12:47:48 +0500 Subject: [PATCH 2/2] a new attribute added to phase to distinguish between result only phases and other phases. Submissions submitted to result only phases are considered result submissiona and their content is copied to predictions output for scoring, for other the submisison content is not copied to save space. Some cleanup and comments are added. Functionality added to set result only submissions to true/false from the interface --- compute_worker/compute_worker.py | 25 ++++++++++++++++--- src/apps/api/serializers/competitions.py | 2 ++ src/apps/api/views/competitions.py | 1 + ...9_phase_accepts_only_result_submissions.py | 18 +++++++++++++ src/apps/competitions/models.py | 2 ++ src/apps/competitions/tasks.py | 2 ++ .../competitions/tests/unpacker_test_data.py | 2 ++ src/apps/competitions/unpackers/v1.py | 1 + src/apps/competitions/unpackers/v2.py | 1 + .../riot/competitions/editor/_phases.tag | 12 +++++++++ 10 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 src/apps/competitions/migrations/0059_phase_accepts_only_result_submissions.py diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py index e2edafc10..f416f1c18 100644 --- a/compute_worker/compute_worker.py +++ b/compute_worker/compute_worker.py @@ -219,6 +219,7 @@ def __init__(self, run_args): # Details for submission self.is_scoring = run_args["is_scoring"] + self.submission_is_result_only = run_args["submission_is_result_only"] # true when the submission phase only accepts results submissions self.user_pk = run_args["user_pk"] self.submission_id = run_args["id"] self.submissions_api_url = run_args["submissions_api_url"] @@ -585,7 +586,16 @@ def _get_host_path(self, *paths): async def _run_program_directory(self, program_dir, kind): """ - Function responsible for running program directory + This function is executed 2 times during the ingestion and scoring. + Ingestion: + - During ingestion this function is called for program(submission) and ingestion + - For ingestion, a metadata file is expected and if not provided, an error is raised + - For program(submission), a metadata file is optional + + Scoring: + - During scoring this function is called for program(scoring) and ingestion + - Because there is no ingestion during scoring, the function returns without executing ingestion + - For program(scoring), a metadata file is optional Args: - program_dir : can be either ingestion program or program(submission or scoring) @@ -599,6 +609,9 @@ async def _run_program_directory(self, program_dir, kind): self.completed_program_counter += 1 return + # metadata or metadata.yaml file is optional for program(submission/scoring) + # result submission is not supposed to have metadata file because it has no program to run just some predictions + # if a code submission has no metadata file, it means the ingestion knows how to run the code submission without a metadata file if os.path.exists(os.path.join(program_dir, "metadata.yaml")): metadata_path = 'metadata.yaml' elif os.path.exists(os.path.join(program_dir, "metadata")): @@ -609,13 +622,19 @@ async def _run_program_directory(self, program_dir, kind): logger.info( "Program directory missing metadata, assuming it's going to be handled by ingestion" ) - # Copy program dir content to output directory because in case of only result submission, + # Copy only if is result only phase i.e. self.submission_is_result_only is True + # Copy program dir content to output directory because in case of only result submission, # we need to copy the result submission files because the scoring program will use these as predictions - shutil.copytree(program_dir, self.output_dir) + print(f"---IHSAN---{self.submission_is_result_only}") + if self.submission_is_result_only: + shutil.copytree(program_dir, self.output_dir) return else: raise SubmissionException("Program directory missing 'metadata.yaml/metadata'") + # If metadata file is found, then we check for command in the metadata + # If command is not there for ingestion, an error is raised + # If command is not there for submission/scoring, a warning is displayed logger.info(f"Metadata path is {os.path.join(program_dir, metadata_path)}") with open(os.path.join(program_dir, metadata_path), 'r') as metadata_file: try: # try to find a command in the metadata, in other cases set metadata to None diff --git a/src/apps/api/serializers/competitions.py b/src/apps/api/serializers/competitions.py index d35fed024..cc39b4f52 100644 --- a/src/apps/api/serializers/competitions.py +++ b/src/apps/api/serializers/competitions.py @@ -45,6 +45,7 @@ class Meta: 'hide_output', 'hide_prediction_output', 'hide_score_output', + 'accepts_only_result_submissions', 'leaderboard', 'public_data', 'starting_kit', @@ -128,6 +129,7 @@ class Meta: 'hide_output', 'hide_prediction_output', 'hide_score_output', + 'accepts_only_result_submissions', # no leaderboard 'public_data', 'starting_kit', diff --git a/src/apps/api/views/competitions.py b/src/apps/api/views/competitions.py index f5d464761..b281d593d 100644 --- a/src/apps/api/views/competitions.py +++ b/src/apps/api/views/competitions.py @@ -277,6 +277,7 @@ def update(self, request, *args, **kwargs): hide_output=phase["hide_output"], hide_prediction_output=phase["hide_prediction_output"], hide_score_output=phase["hide_score_output"], + accepts_only_result_submissions=phase["accepts_only_result_submissions"], competition=Competition.objects.get(id=data['id']) ) # Get phase id diff --git a/src/apps/competitions/migrations/0059_phase_accepts_only_result_submissions.py b/src/apps/competitions/migrations/0059_phase_accepts_only_result_submissions.py new file mode 100644 index 000000000..0aa82f144 --- /dev/null +++ b/src/apps/competitions/migrations/0059_phase_accepts_only_result_submissions.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.28 on 2025-06-19 07:15 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('competitions', '0058_phase_hide_prediction_output'), + ] + + operations = [ + migrations.AddField( + model_name='phase', + name='accepts_only_result_submissions', + field=models.BooleanField(default=False), + ), + ] diff --git a/src/apps/competitions/models.py b/src/apps/competitions/models.py index d8ce83dad..7291a402e 100644 --- a/src/apps/competitions/models.py +++ b/src/apps/competitions/models.py @@ -357,6 +357,8 @@ class Phase(ChaHubSaveMixin, models.Model): public_data = models.ForeignKey('datasets.Data', on_delete=models.SET_NULL, null=True, blank=True, related_name="phase_public_data") starting_kit = models.ForeignKey('datasets.Data', on_delete=models.SET_NULL, null=True, blank=True, related_name="phase_starting_kit") + accepts_only_result_submissions = models.BooleanField(default=False) + class Meta: ordering = ('index',) diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py index 9b685d1d7..edd8258cf 100644 --- a/src/apps/competitions/tasks.py +++ b/src/apps/competitions/tasks.py @@ -84,6 +84,7 @@ 'hide_output', 'hide_prediction_output', 'hide_score_output', + 'accepts_only_result_submissions', ] PHASE_FILES = [ "input_data", @@ -130,6 +131,7 @@ def _send_to_compute_worker(submission, is_scoring): "execution_time_limit": min(MAX_EXECUTION_TIME_LIMIT, submission.phase.execution_time_limit), "id": submission.pk, "is_scoring": is_scoring, + "submission_is_result_only": submission.phase.accepts_only_result_submissions } if not submission.detailed_result.name and submission.phase.competition.enable_detailed_results: diff --git a/src/apps/competitions/tests/unpacker_test_data.py b/src/apps/competitions/tests/unpacker_test_data.py index b2ee7e075..dcf99c4c4 100644 --- a/src/apps/competitions/tests/unpacker_test_data.py +++ b/src/apps/competitions/tests/unpacker_test_data.py @@ -216,6 +216,7 @@ 'hide_output': False, 'hide_prediction_output': False, 'hide_score_output': False, + 'accepts_only_result_submissions': False }, { 'index': 1, @@ -236,6 +237,7 @@ 'hide_output': False, 'hide_prediction_output': False, 'hide_score_output': False, + 'accepts_only_result_submissions': False } ] diff --git a/src/apps/competitions/unpackers/v1.py b/src/apps/competitions/unpackers/v1.py index 535e9e2d5..6185b0bd5 100644 --- a/src/apps/competitions/unpackers/v1.py +++ b/src/apps/competitions/unpackers/v1.py @@ -91,6 +91,7 @@ def _unpack_phases(self): 'hide_output': phase.get('hide_output', False), 'hide_prediction_output': phase.get('hide_prediction_output', False), 'hide_score_output': phase.get('hide_score_output', False), + 'accepts_only_result_submissions': phase.get('accepts_only_result_submissions', False) } execution_time_limit = phase.get('execution_time_limit') if execution_time_limit: diff --git a/src/apps/competitions/unpackers/v2.py b/src/apps/competitions/unpackers/v2.py index 508479eba..5aa3d56f4 100644 --- a/src/apps/competitions/unpackers/v2.py +++ b/src/apps/competitions/unpackers/v2.py @@ -200,6 +200,7 @@ def _unpack_phases(self): 'hide_output': phase_data.get('hide_output', False), 'hide_prediction_output': phase_data.get('hide_prediction_output', False), 'hide_score_output': phase_data.get('hide_score_output', False), + 'accepts_only_result_submissions': phase_data.get('accepts_only_result_submissions', False) } try: new_phase['tasks'] = phase_data['tasks'] diff --git a/src/static/riot/competitions/editor/_phases.tag b/src/static/riot/competitions/editor/_phases.tag index a9c318c28..72d591adf 100644 --- a/src/static/riot/competitions/editor/_phases.tag +++ b/src/static/riot/competitions/editor/_phases.tag @@ -216,6 +216,16 @@ +
+
+ + +
+
+
@@ -663,6 +673,7 @@ self.refs.hide_output.checked = phase.hide_output self.refs.hide_prediction_output.checked = phase.hide_prediction_output self.refs.hide_score_output.checked = phase.hide_score_output + self.refs.accepts_only_result_submissions.checked = phase.accepts_only_result_submissions // Setting description in markdown editor self.simple_markdown_editor.value(self.phases[index].description || '') @@ -829,6 +840,7 @@ data.hide_output = self.refs.hide_output.checked data.hide_prediction_output = self.refs.hide_prediction_output.checked data.hide_score_output = self.refs.hide_score_output.checked + data.accepts_only_result_submissions = self.refs.accepts_only_result_submissions.checked _.forEach(number_fields, field => { let str = _.get(data, field) if (str) {