From b6f3553457faee3f399107f74f9aa3b54e341115 Mon Sep 17 00:00:00 2001 From: sujata-m Date: Tue, 19 Nov 2024 14:51:49 +0530 Subject: [PATCH 1/8] This PR contains following features - Added unit test cases - Added unit test cases pipeline - Added support to upload test cases results on Azure blob --- .github/workflows/unit_tests.yaml | 60 +++ .gitignore | 2 + src/config.py | 8 +- src/services/servicebus_service.py | 1 + tests/calculators/__init__.py | 0 tests/calculators/test_qm_calculator.py | 49 ++ tests/calculators/test_qm_fixed_calculator.py | 130 ++++++ .../calculators/test_qm_xn_lib_calculator.py | 433 ++++++++++++++++++ tests/calculators/test_xn_qm_lib.py | 249 ++++++++++ tests/models/__init__.py | 0 tests/models/test_quality_request.py | 94 ++++ tests/models/test_quality_response.py | 70 +++ tests/services/__init__.py | 0 .../test_osw_qm_calculator_service.py | 107 +++++ tests/services/test_servicebus_service.py | 135 ++++++ tests/services/test_storage_service.py | 66 +++ tests/test_config.py | 59 +++ tests/test_main.py | 56 +++ 18 files changed, 1515 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/unit_tests.yaml create mode 100644 tests/calculators/__init__.py create mode 100644 tests/calculators/test_qm_calculator.py create mode 100644 tests/calculators/test_qm_fixed_calculator.py create mode 100644 tests/calculators/test_qm_xn_lib_calculator.py create mode 100644 tests/calculators/test_xn_qm_lib.py create mode 100644 tests/models/__init__.py create mode 100644 tests/models/test_quality_request.py create mode 100644 tests/models/test_quality_response.py create mode 100644 tests/services/__init__.py create mode 100644 tests/services/test_osw_qm_calculator_service.py create mode 100644 tests/services/test_servicebus_service.py create mode 100644 tests/services/test_storage_service.py create mode 100644 tests/test_config.py create mode 100644 tests/test_main.py diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml new file mode 100644 index 0000000..84f2ab2 --- /dev/null +++ b/.github/workflows/unit_tests.yaml @@ -0,0 +1,60 @@ +name: Unit Tests +on: + workflow_dispatch: + push: + branches-ignore: + - '**' + pull_request: + branches: [main, dev, stage] + +jobs: + UnitTest: + runs-on: ubuntu-latest + + env: + DATABASE_NAME: test_database + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Determine output folder + id: set_output_folder + run: | + if [[ $GITHUB_REF_NAME == "main" ]]; then + echo "output_folder=prod" >> $GITHUB_ENV + elif [[ $GITHUB_REF_NAME == "stage" ]]; then + echo "output_folder=stage" >> $GITHUB_ENV + elif [[ $GITHUB_REF_NAME == "dev" ]]; then + echo "output_folder=dev" >> $GITHUB_ENV + else + echo "Unknown branch: $GITHUB_REF_NAME" + exit 1 + + - name: Run tests with coverage + run: | + dynamic_results_path="test_results/${{ env.output_folder }}" + mkdir -p $dynamic_results_path + PYTHONPATH=$(pwd) python -m coverage run --source=src -m unittest discover -v tests/ + coverage report > $dynamic_results_path/latest_report.log + coverage report >> $dynamic_results_path/$(date '+%Y-%m-%d_%H-%M-%S')_report.log + + - name: Upload report to Azure + uses: LanceMcCarthy/Action-AzureBlobUpload@v2 + with: + source_folder: 'test_results/${{ env.output_folder }}' + destination_folder: 'osw-quality-metric-service/${{ env.output_folder }}/${{ github.repository }}/unit_test_results' + connection_string: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }} + container_name: 'reports' + clean_destination_folder: false + delete_if_exists: false diff --git a/.gitignore b/.gitignore index c331e05..86c9872 100644 --- a/.gitignore +++ b/.gitignore @@ -162,3 +162,5 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ .ve +MagicMock +test_results diff --git a/src/config.py b/src/config.py index 5822c7e..7a36f33 100644 --- a/src/config.py +++ b/src/config.py @@ -12,14 +12,14 @@ class Config(BaseSettings): incoming_topic_subscription: str = os.environ.get('QUALITY_REQ_SUB', '') outgoing_topic_name: str = os.environ.get('QUALITY_RES_TOPIC', '') storage_container_name: str = os.environ.get('CONTAINER_NAME', 'osw') - algorithm_dictionary: dict = {"fixed":QMFixedCalculator,"ixn":QMXNLibCalculator} + algorithm_dictionary: dict = {"fixed": QMFixedCalculator, "ixn": QMXNLibCalculator} max_concurrent_messages: int = os.environ.get('MAX_CONCURRENT_MESSAGES', 1) - partition_count:int = os.environ.get('PARTITION_COUNT', 2) + partition_count: int = os.environ.get('PARTITION_COUNT', 2) def get_download_folder(self) -> str: root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) return os.path.join(root_dir, 'downloads') - + def get_assets_folder(self) -> str: root_dir = os.path.dirname(os.path.abspath(__file__)) - return os.path.join(root_dir, 'assets') \ No newline at end of file + return os.path.join(root_dir, 'assets') diff --git a/src/services/servicebus_service.py b/src/services/servicebus_service.py index fc7d341..201e1f8 100644 --- a/src/services/servicebus_service.py +++ b/src/services/servicebus_service.py @@ -45,6 +45,7 @@ def __init__(self) -> None: def process_message(self, msg: QueueMessage): logger.info(f"Processing message {msg}") + input_file_url = None try: logger.info(f"Processing message {msg.messageId}") # Parse the message diff --git a/tests/calculators/__init__.py b/tests/calculators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/calculators/test_qm_calculator.py b/tests/calculators/test_qm_calculator.py new file mode 100644 index 0000000..ef3580c --- /dev/null +++ b/tests/calculators/test_qm_calculator.py @@ -0,0 +1,49 @@ +import unittest +from abc import ABC, abstractmethod +from src.calculators import QMCalculator +from src.calculators.qm_calculator import QualityMetricResult + +# A concrete implementation for testing purposes +class MockQMCalculator(QMCalculator): + def calculate_quality_metric(self) -> QualityMetricResult: + return QualityMetricResult(success=True, message='Calculation succeeded.', output_file='output.geojson') + + def algorithm_name(self) -> str: + return 'mock-algorithm' + +class TestQualityMetricResult(unittest.TestCase): + def test_quality_metric_result_initialization(self): + # Create a QualityMetricResult instance + result = QualityMetricResult( + success=True, + message='Calculation succeeded.', + output_file='output.geojson' + ) + + # Assertions + self.assertTrue(result.success) + self.assertEqual(result.message, 'Calculation succeeded.') + self.assertEqual(result.output_file, 'output.geojson') + +class TestQMCalculator(unittest.TestCase): + def test_qm_calculator_is_abstract(self): + # Ensure QMCalculator cannot be instantiated directly + with self.assertRaises(TypeError): + QMCalculator() + + def test_concrete_implementation(self): + # Instantiate the mock implementation + calculator = MockQMCalculator() + + # Test calculate_quality_metric + result = calculator.calculate_quality_metric() + self.assertIsInstance(result, QualityMetricResult) + self.assertTrue(result.success) + self.assertEqual(result.message, 'Calculation succeeded.') + self.assertEqual(result.output_file, 'output.geojson') + + # Test algorithm_name + self.assertEqual(calculator.algorithm_name(), 'mock-algorithm') + +if __name__ == '__main__': + unittest.main() diff --git a/tests/calculators/test_qm_fixed_calculator.py b/tests/calculators/test_qm_fixed_calculator.py new file mode 100644 index 0000000..e3d3b6b --- /dev/null +++ b/tests/calculators/test_qm_fixed_calculator.py @@ -0,0 +1,130 @@ +import json +import tempfile +import unittest +import geopandas as gpd +from subprocess import run, PIPE +from unittest.mock import patch, MagicMock +from src.calculators.qm_fixed_calculator import QMFixedCalculator +from src.calculators.qm_calculator import QualityMetricResult + + +class TestQMFixedCalculator(unittest.TestCase): + + def setUp(self): + self.edges_file_path = 'test_edges.geojson' + self.output_file_path = 'test_output.geojson' + self.polygon_file_path = 'test_polygon.geojson' + self.calculator = QMFixedCalculator( + self.edges_file_path, + self.output_file_path, + self.polygon_file_path + ) + + def test_initialization(self): + self.assertEqual(self.calculator.edges_file_path, self.edges_file_path) + self.assertEqual(self.calculator.output_file_path, self.output_file_path) + self.assertEqual(self.calculator.polygon_file_path, self.polygon_file_path) + + @patch('src.calculators.qm_fixed_calculator.gpd.read_file') + @patch('src.calculators.qm_fixed_calculator.gpd.GeoDataFrame.to_file', autospec=True) + @patch('src.calculators.qm_fixed_calculator.random.randint') + def test_calculate_quality_metric(self, mock_randint, mock_to_file, mock_read_file): + # Setup mocks + mock_randint.return_value = 42 + mock_gdf = MagicMock(spec=gpd.GeoDataFrame) + mock_read_file.return_value = mock_gdf + + # Call the method under test + result = self.calculator.calculate_quality_metric() + + # Assertions + mock_read_file.assert_called_once_with(self.edges_file_path) + mock_gdf.__setitem__.assert_called_once_with('fixed_score', 42) + mock_gdf.to_file.assert_called_once_with(self.output_file_path) + self.assertIsInstance(result, QualityMetricResult) + self.assertTrue(result.success) + self.assertEqual(result.message, 'QMFixedCalculator') + self.assertEqual(result.output_file, self.output_file_path) + + def test_algorithm_name(self): + self.assertEqual(self.calculator.algorithm_name(), 'QMFixedCalculator') + + def test_main_with_polygon(self): + # Create temporary files for edges and polygon + with tempfile.NamedTemporaryFile(suffix='.geojson') as edges_file, \ + tempfile.NamedTemporaryFile(suffix='.geojson') as output_file, \ + tempfile.NamedTemporaryFile(suffix='.geojson') as polygon_file: + # Write dummy GeoJSON data to edges and polygon files + dummy_geojson = { + 'type': 'FeatureCollection', + 'features': [ + { + 'type': 'Feature', + 'geometry': {'type': 'LineString', 'coordinates': [[0, 0], [1, 1]]}, + 'properties': {} + } + ] + } + + edges_file.write(json.dumps(dummy_geojson).encode()) + edges_file.flush() + + polygon_file.write(json.dumps(dummy_geojson).encode()) + polygon_file.flush() + + # Simulate running the script as a standalone process + result = run( + [ + 'python', + 'src/calculators/qm_fixed_calculator.py', + edges_file.name, + output_file.name, + polygon_file.name, + ], + stdout=PIPE, + stderr=PIPE, + text=True, + ) + + # Debugging: Print stderr if test fails + if result.returncode != 0: + print('Error:', result.stderr) + + # Assertions + self.assertEqual(result.returncode, 0) + self.assertIn('QMFixedCalculator', result.stdout) + + def test_main_without_polygon(self): + # Create temporary files for edges + with tempfile.NamedTemporaryFile(suffix='.geojson') as edges_file, \ + tempfile.NamedTemporaryFile(suffix='.geojson') as output_file: + # Write dummy GeoJSON data to edges file + dummy_geojson = { + 'type': 'FeatureCollection', + 'features': [] + } + + edges_file.write(json.dumps(dummy_geojson).encode()) + edges_file.flush() + + # Simulate running the script as a standalone process + result = run( + [ + 'python', + 'src/calculators/qm_fixed_calculator.py', + edges_file.name, + output_file.name, + ], + stdout=PIPE, + stderr=PIPE, + text=True, + ) + + # Assertions + self.assertEqual(result.returncode, 0) + self.assertIn('QMFixedCalculator', result.stdout) + + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/calculators/test_qm_xn_lib_calculator.py b/tests/calculators/test_qm_xn_lib_calculator.py new file mode 100644 index 0000000..7bcdee8 --- /dev/null +++ b/tests/calculators/test_qm_xn_lib_calculator.py @@ -0,0 +1,433 @@ +import os +import unittest +from unittest.mock import patch, MagicMock, call +from src.calculators.qm_xn_lib_calculator import QMXNLibCalculator +from src.calculators.qm_calculator import QualityMetricResult +import geopandas as gpd +from shapely.geometry import LineString, MultiLineString, Polygon, Point, MultiPolygon +import tempfile +import json +from subprocess import run, PIPE +import networkx as nx + + +class TestQMXNLibCalculator(unittest.TestCase): + + def setUp(self): + self.edges_file_path = 'test_edges.geojson' + self.output_file_path = 'test_output.geojson' + self.polygon_file_path = 'test_polygon.geojson' + self.calculator = QMXNLibCalculator( + self.edges_file_path, + self.output_file_path, + self.polygon_file_path + ) + self.default_projection = 'epsg:26910' + + def test_initialization(self): + self.assertEqual(self.calculator.edges_file_path, self.edges_file_path) + self.assertEqual(self.calculator.output_file_path, self.output_file_path) + self.assertEqual(self.calculator.polygon_file_path, self.polygon_file_path) + self.assertIsNotNone(self.calculator.default_projection) + self.assertIsNotNone(self.calculator.output_projection) + + @patch('src.calculators.qm_xn_lib_calculator.gpd.read_file') + @patch('src.calculators.qm_xn_lib_calculator.nx.Graph') + def test_graph_from_gdf(self, mock_graph, mock_read_file): + mock_gdf = MagicMock(spec=gpd.GeoDataFrame) + mock_read_file.return_value = mock_gdf + mock_graph_instance = MagicMock() + mock_graph.return_value = mock_graph_instance + + # Create mock geometries + mock_gdf.iterrows.return_value = iter([ + (0, MagicMock(geometry=LineString([(0, 0), (1, 1)]))), + (1, MagicMock(geometry=LineString([(1, 1), (2, 2)]))), + ]) + + G = self.calculator.graph_from_gdf(mock_gdf) + mock_graph.assert_called_once() + self.assertEqual(G, mock_graph_instance) + + @patch('src.calculators.qm_xn_lib_calculator.gpd.read_file') + @patch('src.calculators.qm_xn_lib_calculator.dask_geopandas.from_geopandas') + def test_calculate_quality_metric_with_polygon(self, mock_from_geopandas, mock_read_file): + # Mock GeoDataFrames + mock_edges_gdf = MagicMock(spec=gpd.GeoDataFrame) + mock_polygon_gdf = MagicMock(spec=gpd.GeoDataFrame) + mock_result_gdf = MagicMock(spec=gpd.GeoDataFrame) + + # Setup mock behaviors + mock_read_file.side_effect = [mock_edges_gdf, mock_polygon_gdf] + mock_edges_gdf.to_crs.return_value = mock_edges_gdf + mock_polygon_gdf.to_crs.return_value = mock_polygon_gdf + mock_polygon_subset = MagicMock(spec=gpd.GeoDataFrame) # Mock for tile_gdf[['geometry']] + mock_polygon_gdf.__getitem__.return_value = mock_polygon_subset # Mock subset behavior + + mock_result_gdf.to_crs.return_value = mock_result_gdf + mock_result_gdf.to_file = MagicMock() + mock_from_geopandas.return_value.apply.return_value.compute.return_value = mock_result_gdf + + # Execute the method + result = self.calculator.calculate_quality_metric() + + # Assertions + mock_read_file.assert_any_call(self.edges_file_path) + mock_read_file.assert_any_call(self.polygon_file_path) + mock_edges_gdf.to_crs.assert_called_once_with(self.default_projection) + mock_polygon_gdf.to_crs.assert_called_once_with(self.default_projection) + mock_polygon_gdf.__getitem__.assert_called_once_with(['geometry']) # Ensure subset call + mock_from_geopandas.assert_called_once_with(mock_polygon_subset, npartitions=os.cpu_count()) + mock_result_gdf.to_file.assert_called_once_with(self.output_file_path, driver='GeoJSON') + + # Verify the result + self.assertIsInstance(result, QualityMetricResult) + self.assertTrue(result.success) + self.assertEqual(result.message, 'QMXNLibCalculator') + self.assertEqual(result.output_file, self.output_file_path) + + @patch('src.calculators.qm_xn_lib_calculator.gpd.read_file') + @patch('src.calculators.qm_xn_lib_calculator.ox.graph.graph_from_polygon') + @patch('src.calculators.qm_xn_lib_calculator.QMXNLibCalculator.create_voronoi_diagram') + @patch('src.calculators.qm_xn_lib_calculator.dask_geopandas.from_geopandas') + def test_calculate_quality_metric_without_polygon( + self, mock_from_geopandas, mock_create_voronoi_diagram, mock_graph_from_polygon, mock_read_file + ): + # Remove the polygon file path + self.calculator.polygon_file_path = None + + # Mock input GeoDataFrame + mock_gdf = MagicMock(spec=gpd.GeoDataFrame) + mock_read_file.return_value = mock_gdf + mock_gdf.to_crs.return_value = mock_gdf + + # Mock the convex hull behavior + mock_convex_hull = MagicMock(spec=Polygon) + mock_gdf.unary_union.convex_hull = mock_convex_hull + + # Mock the graph and Voronoi diagram creation + mock_graph = MagicMock() + mock_graph_from_polygon.return_value = mock_graph + mock_voronoi_gdf = MagicMock(spec=gpd.GeoDataFrame) + mock_create_voronoi_diagram.return_value = mock_voronoi_gdf + mock_voronoi_gdf.to_crs.return_value = mock_voronoi_gdf + + # Mock subset for geometry column and from_geopandas behavior + mock_polygon_subset = MagicMock(spec=gpd.GeoDataFrame) + mock_voronoi_gdf.__getitem__.return_value = mock_polygon_subset + mock_from_geopandas.return_value.apply.return_value.compute.return_value = mock_voronoi_gdf + + # Execute the method + result = self.calculator.calculate_quality_metric() + + # Assertions + mock_read_file.assert_called_once_with(self.edges_file_path) + mock_gdf.to_crs.assert_called_once_with(self.calculator.default_projection) + mock_graph_from_polygon.assert_called_once_with( + mock_convex_hull, network_type='drive', simplify=True, retain_all=True + ) + mock_create_voronoi_diagram.assert_called_once_with(mock_graph, mock_convex_hull) + + # Check both `to_crs` calls + mock_voronoi_gdf.to_crs.assert_has_calls([ + call(self.calculator.default_projection), # First call for default projection + call(self.calculator.output_projection) # Second call for output projection + ]) + self.assertEqual(mock_voronoi_gdf.to_crs.call_count, 2) # Ensure exactly 2 calls occurred + + mock_voronoi_gdf.__getitem__.assert_called_once_with(['geometry']) + mock_from_geopandas.assert_called_once_with(mock_polygon_subset, npartitions=os.cpu_count()) + mock_voronoi_gdf.to_file.assert_called_once_with(self.output_file_path, driver='GeoJSON') + + # Verify the result + self.assertIsInstance(result, QualityMetricResult) + self.assertTrue(result.success) # Should pass if everything is mocked correctly + self.assertEqual(result.message, 'QMXNLibCalculator') + self.assertEqual(result.output_file, self.output_file_path) + + @patch('src.calculators.qm_xn_lib_calculator.dask_geopandas.from_geopandas') + @patch('src.calculators.qm_xn_lib_calculator.gpd.read_file') + def test_main_with_polygon(self, mock_read_file, mock_from_geopandas): + with tempfile.NamedTemporaryFile(suffix='.geojson') as edges_file, \ + tempfile.NamedTemporaryFile(suffix='.geojson') as output_file, \ + tempfile.NamedTemporaryFile(suffix='.geojson') as polygon_file: + # Create dummy GeoJSON data for edges and polygon files + dummy_geojson = { + 'type': 'FeatureCollection', + 'features': [ + { + 'type': 'Feature', + 'geometry': {'type': 'LineString', 'coordinates': [[0, 0], [1, 1]]}, + 'properties': {} + } + ] + } + + # Write dummy data to temporary files + edges_file.write(json.dumps(dummy_geojson).encode()) + edges_file.flush() + + polygon_file.write(json.dumps(dummy_geojson).encode()) + polygon_file.flush() + + # Mock the input GeoDataFrames + mock_edges_gdf = MagicMock(spec=gpd.GeoDataFrame) + mock_polygon_gdf = MagicMock(spec=gpd.GeoDataFrame) + mock_result_gdf = MagicMock(spec=gpd.GeoDataFrame) + + # Mock behaviors for projections + mock_edges_gdf.to_crs.return_value = mock_edges_gdf + mock_polygon_gdf.to_crs.return_value = mock_polygon_gdf + mock_result_gdf.to_crs.return_value = mock_result_gdf + + # Mock the subset behavior for `tile_gdf[['geometry']]` + mock_polygon_subset = MagicMock(spec=gpd.GeoDataFrame) + mock_polygon_gdf.__getitem__.return_value = mock_polygon_subset + + # Mock the final result + mock_result_gdf.columns = ['geometry', 'tra_score'] + mock_result_gdf['geometry'] = [LineString([(0, 0), (1, 1)])] + mock_result_gdf['tra_score'] = [1.0] + mock_result_gdf.to_file = MagicMock() + + # Mock dask_geopandas behavior + mock_from_geopandas.return_value.apply.return_value.compute.return_value = mock_result_gdf + + # Simulate running the script + result = run( + [ + 'python', + 'src/calculators/qm_xn_lib_calculator.py', + edges_file.name, + output_file.name, + polygon_file.name, + ], + stdout=PIPE, + stderr=PIPE, + text=True, + ) + + # Debugging: Print stderr if the test fails + if result.returncode != 0: + print('Error:', result.stderr) + + # Assertions + self.assertEqual(result.returncode, 0) + + def test_main_without_polygon(self): + with tempfile.NamedTemporaryFile(suffix='.geojson') as edges_file, \ + tempfile.NamedTemporaryFile(suffix='.geojson') as output_file: + dummy_geojson = { + 'type': 'FeatureCollection', + 'features': [] + } + + edges_file.write(json.dumps(dummy_geojson).encode()) + edges_file.flush() + + result = run( + [ + 'python', + 'src/calculators/qm_xn_lib_calculator.py', + edges_file.name, + output_file.name, + ], + stdout=PIPE, + stderr=PIPE, + text=True, + ) + + if result.returncode != 0: + print('Error:', result.stderr) + + self.assertEqual(result.returncode, 0) + + @patch('src.calculators.qm_xn_lib_calculator.nx.Graph') + def test_graph_from_gdf_multilinestring(self, mock_graph): + mock_gdf = MagicMock(spec=gpd.GeoDataFrame) + mock_graph_instance = mock_graph.return_value + + # Mock a MultiLineString geometry + mock_gdf.iterrows.return_value = iter([ + (0, MagicMock(geometry=MultiLineString([LineString([(0, 0), (1, 1)]), LineString([(1, 1), (2, 2)])]))), + ]) + + G = self.calculator.graph_from_gdf(mock_gdf) + self.assertEqual(G, mock_graph_instance) + self.assertEqual(mock_graph_instance.add_edge.call_count, 2) + + def test_group_G_pts(self): + mock_graph = nx.Graph() + mock_graph.add_nodes_from([(0, 0), (1, 1), (2, 2)]) + mock_polygon = Polygon([(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)]) + + result = self.calculator.group_G_pts(mock_graph, mock_polygon) + self.assertTrue(isinstance(result, dict)) + self.assertTrue(all(isinstance(v, list) for v in result.values())) + + def test_edges_are_connected(self): + mock_graph = nx.Graph() + mock_graph.add_edge((0, 0), (1, 1)) + mock_graph.add_edge((1, 1), (2, 2)) + + result = self.calculator.edges_are_connected(mock_graph, [(0, 0)], [(2, 2)]) + self.assertTrue(result) + + def test_tile_tra_score(self): + mock_graph = nx.Graph() + mock_graph.add_edge((0, 0), (1, 1)) + mock_graph.add_edge((1, 1), (2, 2)) + mock_polygon = Polygon([(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)]) + + result = self.calculator.tile_tra_score(mock_graph, mock_polygon) + self.assertTrue(isinstance(result, tuple)) + self.assertEqual(len(result), 3) + + @patch('src.calculators.qm_xn_lib_calculator.gpd.clip') + def test_get_measures_from_polygon(self, mock_clip): + mock_gdf = MagicMock(spec=gpd.GeoDataFrame) + mock_polygon = Polygon([(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)]) + mock_clipped_gdf = MagicMock(spec=gpd.GeoDataFrame) + mock_clip.return_value = mock_clipped_gdf + + mock_clipped_gdf.geometry = MagicMock() + result = self.calculator.get_measures_from_polygon(mock_polygon, mock_gdf) + self.assertTrue(isinstance(result, dict)) + self.assertIn('tra_score', result) + + @patch('src.calculators.qm_xn_lib_calculator.QMXNLibCalculator.get_measures_from_polygon') + def test_qm_func(self, mock_get_measures_from_polygon): + mock_feature = MagicMock() + mock_feature.geometry = Polygon([(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)]) + mock_feature.loc = {} + mock_gdf = MagicMock(spec=gpd.GeoDataFrame) + mock_get_measures_from_polygon.return_value = {'tra_score': 0.8} + + result = self.calculator.qm_func(mock_feature, mock_gdf) + self.assertEqual(result.loc['tra_score'], 0.8) + + @patch('src.calculators.qm_xn_lib_calculator.gpd.GeoDataFrame') + @patch('src.calculators.qm_xn_lib_calculator.voronoi_diagram') + @patch('src.calculators.qm_xn_lib_calculator.gnx.graph_edges_to_gdf') + def test_create_voronoi_diagram(self, mock_graph_edges_to_gdf, mock_voronoi_diagram, mock_geo_dataframe): + # Mock graph and bounds + mock_graph = MagicMock() + mock_bounds = Polygon([(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)]) + + # Mock the GeoDataFrame created from graph edges + mock_gdf = MagicMock(spec=gpd.GeoDataFrame) + mock_gdf.crs = 'epsg:4326' # Mock the CRS attribute + mock_graph_edges_to_gdf.return_value = mock_gdf + + # Mock the boundary and unary_union attributes of the GeoDataFrame + mock_boundary = MagicMock() + mock_unary_union = MagicMock() + mock_gdf.boundary = mock_boundary + mock_boundary.unary_union = mock_unary_union + + # Mock the voronoi_diagram output + mock_voronoi = MagicMock() + mock_voronoi_diagram.return_value = mock_voronoi + + # Mock the clipped Voronoi GeoDataFrame + mock_voronoi_gdf = MagicMock(spec=gpd.GeoDataFrame) + mock_voronoi_gdf.set_crs = MagicMock(return_value=mock_voronoi_gdf) + mock_voronoi_gdf_clipped = MagicMock(spec=gpd.GeoDataFrame) + mock_voronoi_gdf_clipped.to_crs = MagicMock(return_value=mock_voronoi_gdf_clipped) + + # Mock GeoDataFrame creation and clipping + mock_geo_dataframe.return_value = mock_voronoi_gdf + gpd.clip = MagicMock(return_value=mock_voronoi_gdf_clipped) + + # Call the function + result = self.calculator.create_voronoi_diagram(mock_graph, mock_bounds) + + # Assertions + mock_graph_edges_to_gdf.assert_called_once_with(mock_graph) + mock_voronoi_diagram.assert_called_once_with(mock_unary_union, envelope=mock_bounds) + mock_voronoi_gdf.set_crs.assert_called_once_with(mock_gdf.crs) + gpd.clip.assert_called_once_with(mock_voronoi_gdf, mock_bounds) + mock_voronoi_gdf_clipped.to_crs.assert_called_once_with(self.calculator.default_projection) + self.assertEqual(result, mock_voronoi_gdf_clipped) + + @patch('src.calculators.qm_xn_lib_calculator.gpd.read_file') + def test_calculate_quality_metric_exception(self, mock_read_file): + mock_read_file.side_effect = Exception('Mocked exception') + + result = self.calculator.calculate_quality_metric() + self.assertIsInstance(result, QualityMetricResult) + self.assertFalse(result.success) + self.assertIn('Mocked exception', result.message) + + + def test_algorithm_name(self): + result = self.calculator.algorithm_name() + self.assertEqual(result, 'QMXNLibCalculator') + + @patch('src.calculators.qm_xn_lib_calculator.QMXNLibCalculator.tile_tra_score') + def test_get_stats_exception(self, mock_tile_tra_score): + # Setup mock to raise an exception + mock_tile_tra_score.side_effect = Exception('Mocked tile_tra_score exception') + + # Create a dummy polygon and graph + dummy_polygon = Polygon([(0, 0), (1, 0), (1, 1), (0, 1)]) + dummy_graph = nx.Graph() + dummy_gdf = MagicMock() # Mocked GeoDataFrame + + # Call the function + result = self.calculator.get_stats(dummy_polygon, dummy_graph, dummy_gdf) + + # Assertions + mock_tile_tra_score.assert_called_once_with(dummy_graph, dummy_polygon) + self.assertIn('tra_score', result) + self.assertEqual(result['tra_score'], -1) + + @patch('src.calculators.qm_xn_lib_calculator.gpd.clip') + @patch('src.calculators.qm_xn_lib_calculator.QMXNLibCalculator.graph_from_gdf') + @patch('src.calculators.qm_xn_lib_calculator.QMXNLibCalculator.get_stats') + def test_get_measures_from_polygon_single_geometry_multipolygon(self, mock_get_stats, mock_graph_from_gdf, + mock_clip): + # Create a MultiPolygon with a single geometry + single_polygon = Polygon([(0, 0), (1, 0), (1, 1), (0, 1)]) + multipolygon = MultiPolygon([single_polygon]) + + # Mock GeoDataFrame and graph + mock_gdf = MagicMock() + mock_clipped_gdf = MagicMock() + mock_clip.return_value = mock_clipped_gdf + mock_graph = MagicMock() + mock_graph_from_gdf.return_value = mock_graph + + # Mock the stats returned + mock_stats = {'tra_score': 0.85} + mock_get_stats.return_value = mock_stats + + # Call the function + result = self.calculator.get_measures_from_polygon(multipolygon, mock_gdf) + + # Assertions + mock_clip.assert_called_once_with(mock_gdf, single_polygon) # Ensure the single polygon was used + mock_graph_from_gdf.assert_called_once_with(mock_clipped_gdf) + mock_get_stats.assert_called_once_with(single_polygon, mock_graph, mock_clipped_gdf) + self.assertEqual(result, mock_stats) + + @patch('src.calculators.qm_xn_lib_calculator.QMXNLibCalculator.get_measures_from_polygon') + def test_qm_func_else(self, mock_get_measures_from_polygon): + # Mock feature with a non-Polygon/MultiPolygon geometry (e.g., Point) + mock_feature = MagicMock() + mock_feature.geometry = Point(0, 0) # Point geometry + + # Mock GeoDataFrame (not actually used in this test case) + mock_gdf = MagicMock() + + # Call the function + result = self.calculator.qm_func(mock_feature, mock_gdf) + + # Assertions + mock_get_measures_from_polygon.assert_not_called() # Ensure get_measures_from_polygon is not called + self.assertEqual(result, mock_feature) + + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/calculators/test_xn_qm_lib.py b/tests/calculators/test_xn_qm_lib.py new file mode 100644 index 0000000..bf05025 --- /dev/null +++ b/tests/calculators/test_xn_qm_lib.py @@ -0,0 +1,249 @@ +import os +import unittest +from unittest.mock import patch, MagicMock, ANY +import networkx as nx +import geopandas as gpd +from shapely.geometry import Point, LineString, Polygon, MultiPolygon, MultiLineString +from src.calculators.xn_qm_lib import ( + add_edges_from_linestring, graph_from_gdf, group_G_pts, edges_are_connected, + tile_tra_score, get_stats, get_measures_from_polygon, qm_func, + create_voronoi_diagram, calculate_xn_qm +) + +PROJ = 'epsg:26910' + +class TestQualityMetrics(unittest.TestCase): + + def setUp(self): + self.graph = nx.Graph() + self.graph.add_edge((0, 0), (1, 1)) + self.graph.add_edge((1, 1), (2, 2)) + self.polygon = Polygon([(0, 0), (1, 0), (1, 1), (0, 1)]) + self.multi_polygon = MultiPolygon([self.polygon]) + self.gdf = MagicMock(spec=gpd.GeoDataFrame) + + def test_add_edges_from_linestring(self): + graph = nx.Graph() + line = LineString([(0, 0), (1, 1), (2, 2)]) + add_edges_from_linestring(graph, line, {"attr": "value"}) + self.assertEqual(len(graph.edges), 2) + + def test_graph_from_gdf(self): + mock_gdf = MagicMock(spec=gpd.GeoDataFrame) + mock_gdf.iterrows.return_value = iter([ + (0, MagicMock(geometry=LineString([(0, 0), (1, 1)]))), + (1, MagicMock(geometry=LineString([(1, 1), (2, 2)]))), + ]) + graph = graph_from_gdf(mock_gdf) + self.assertEqual(len(graph.edges), 2) + + def test_group_G_pts(self): + group = group_G_pts(self.graph, self.polygon) + self.assertIsInstance(group, dict) + self.assertGreater(len(group), 0) + + def test_edges_are_connected(self): + e1_pts = [(0, 0), (1, 1)] + e2_pts = [(2, 2)] + connected = edges_are_connected(self.graph, e1_pts, e2_pts) + self.assertTrue(connected) + + def test_tile_tra_score(self): + total, connected, pairs = tile_tra_score(self.graph, self.polygon) + self.assertGreater(total, 0) + self.assertGreater(connected, 0) + self.assertIsInstance(pairs, list) + + @patch('src.calculators.xn_qm_lib.get_stats') + def test_get_stats(self, mock_get_stats): + mock_get_stats.return_value = {"tra_score": 1.0} + stats = get_stats(self.polygon, self.graph, self.gdf) + self.assertIn("tra_score", stats) + + @patch('src.calculators.xn_qm_lib.gpd.clip') + @patch('src.calculators.xn_qm_lib.graph_from_gdf') + @patch('src.calculators.xn_qm_lib.get_stats') + def test_get_measures_from_polygon(self, mock_get_stats, mock_graph_from_gdf, mock_clip): + # Mock the cropped GeoDataFrame + mock_cropped_gdf = MagicMock(spec=gpd.GeoDataFrame) + mock_clip.return_value = mock_cropped_gdf + + # Mock the graph and stats + mock_graph = MagicMock() + mock_graph_from_gdf.return_value = mock_graph + mock_get_stats.return_value = {"tra_score": 1.0} + + # Call the function + measures = get_measures_from_polygon(self.polygon, self.gdf) + + # Assertions + mock_clip.assert_called_once_with(self.gdf, self.polygon) + mock_graph_from_gdf.assert_called_once_with(mock_cropped_gdf) + mock_get_stats.assert_called_once_with(self.polygon, mock_graph, mock_cropped_gdf) + + self.assertIn("tra_score", measures) + self.assertEqual(measures["tra_score"], 1.0) + + @patch('src.calculators.xn_qm_lib.get_measures_from_polygon') + def test_qm_func(self, mock_get_measures): + # Mock the feature and its geometry + mock_feature = MagicMock() + mock_feature.geometry = self.polygon + + # Mock the 'loc' method to behave like a dictionary + mock_feature.loc = {} + + # Mock the return value of get_measures_from_polygon + mock_get_measures.return_value = {"tra_score": 1.0} + + # Call the function + result = qm_func(mock_feature, self.gdf) + + # Assertions + self.assertEqual(result.loc["tra_score"], 1.0) + mock_get_measures.assert_called_once_with(self.polygon, self.gdf) + + @patch('src.calculators.xn_qm_lib.gpd.GeoDataFrame') + @patch('src.calculators.xn_qm_lib.voronoi_diagram') + @patch('src.calculators.xn_qm_lib.gnx.graph_edges_to_gdf') + def test_create_voronoi_diagram(self, mock_graph_edges_to_gdf, mock_voronoi_diagram, mock_geo_dataframe): + # Mock graph and bounds + mock_graph = MagicMock() + mock_bounds = Polygon([(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)]) + + # Mock the GeoDataFrame created from graph edges + mock_gdf = MagicMock(spec=gpd.GeoDataFrame) + mock_gdf.crs = 'epsg:4326' # Mock the CRS attribute + mock_graph_edges_to_gdf.return_value = mock_gdf + + # Mock the boundary and unary_union attributes of the GeoDataFrame + mock_boundary = MagicMock() + mock_unary_union = MagicMock() + mock_gdf.boundary = mock_boundary + mock_boundary.unary_union = mock_unary_union + + # Mock the voronoi_diagram output + mock_voronoi = MagicMock() + mock_voronoi_diagram.return_value = mock_voronoi + + # Mock the clipped Voronoi GeoDataFrame + mock_voronoi_gdf = MagicMock(spec=gpd.GeoDataFrame) + mock_voronoi_gdf.set_crs = MagicMock(return_value=mock_voronoi_gdf) + mock_voronoi_gdf_clipped = MagicMock(spec=gpd.GeoDataFrame) + mock_voronoi_gdf_clipped.to_crs = MagicMock(return_value=mock_voronoi_gdf_clipped) + + # Mock GeoDataFrame creation and clipping + mock_geo_dataframe.return_value = mock_voronoi_gdf + gpd.clip = MagicMock(return_value=mock_voronoi_gdf_clipped) + + # Call the function + result = create_voronoi_diagram(mock_graph, mock_bounds) + + # Assertions + mock_graph_edges_to_gdf.assert_called_once_with(mock_graph) + mock_voronoi_diagram.assert_called_once_with(mock_unary_union, envelope=mock_bounds) + mock_voronoi_gdf.set_crs.assert_called_once_with(mock_gdf.crs) + gpd.clip.assert_called_once_with(mock_voronoi_gdf, mock_bounds) + self.assertEqual(result, mock_voronoi_gdf_clipped) + + + @patch('src.calculators.xn_qm_lib.gpd.read_file') + @patch('src.calculators.xn_qm_lib.dask_geopandas.from_geopandas') + def test_calculate_xn_qm(self, mock_from_geopandas, mock_read_file): + mock_gdf = MagicMock(spec=gpd.GeoDataFrame) + mock_read_file.return_value = mock_gdf + mock_from_geopandas.return_value.apply.return_value.compute.return_value = mock_gdf + result = calculate_xn_qm("test_edges.geojson", "test_output.geojson", "test_polygon.geojson") + self.assertIsNone(result) + + @patch('src.calculators.xn_qm_lib.gpd.read_file') + def test_calculate_xn_qm_exception(self, mock_read_file): + mock_read_file.side_effect = Exception("Mocked exception") + result = calculate_xn_qm("test_edges.geojson", "test_output.geojson", "test_polygon.geojson") + self.assertEqual(result, -1) + + def test_graph_from_gdf_multilinestring(self): + mock_gdf = MagicMock(spec=gpd.GeoDataFrame) + mock_multilinestring = MultiLineString([ + LineString([(0, 0), (1, 1)]), + LineString([(1, 1), (2, 2)]) + ]) + mock_gdf.iterrows.return_value = iter([ + (0, MagicMock(geometry=mock_multilinestring)), + ]) + graph = graph_from_gdf(mock_gdf) + self.assertEqual(len(graph.edges), 2) + + @patch('src.calculators.xn_qm_lib.tile_tra_score') + def test_get_stats_exception(self, mock_tile_tra_score): + mock_tile_tra_score.side_effect = Exception("Mocked exception") + stats = get_stats(self.polygon, self.graph, self.gdf) + self.assertIn("tra_score", stats) + self.assertEqual(stats["tra_score"], -1) + + @patch('src.calculators.xn_qm_lib.gpd.clip') + @patch('src.calculators.xn_qm_lib.graph_from_gdf') + @patch('src.calculators.xn_qm_lib.get_stats') + def test_get_measures_from_polygon_multipolygon(self, mock_get_stats, mock_graph_from_gdf, mock_clip): + # Mock the cropped GeoDataFrame + mock_cropped_gdf = MagicMock(spec=gpd.GeoDataFrame) + mock_clip.return_value = mock_cropped_gdf + + # Mock the graph and stats + mock_graph = MagicMock() + mock_graph_from_gdf.return_value = mock_graph + mock_get_stats.return_value = {"tra_score": 1.0} + + # MultiPolygon with one geometry + mock_multipolygon = MultiPolygon([self.polygon]) + + # Call the function + measures = get_measures_from_polygon(mock_multipolygon, self.gdf) + + # Assertions + mock_clip.assert_called_once_with(self.gdf, self.polygon) + mock_graph_from_gdf.assert_called_once_with(mock_cropped_gdf) + mock_get_stats.assert_called_once_with(self.polygon, mock_graph, mock_cropped_gdf) + + self.assertIn("tra_score", measures) + self.assertEqual(measures["tra_score"], 1.0) + + @patch('src.calculators.xn_qm_lib.create_voronoi_diagram') + @patch('src.calculators.xn_qm_lib.ox.graph.graph_from_polygon') + @patch('src.calculators.xn_qm_lib.gpd.read_file') + @patch('src.calculators.xn_qm_lib.dask_geopandas.from_geopandas') + def test_calculate_xn_qm_without_polygon( + self, mock_from_geopandas, mock_read_file, mock_graph_from_polygon, mock_create_voronoi + ): + # Mock data and behaviors + mock_gdf = MagicMock(spec=gpd.GeoDataFrame) + mock_tile_gdf = MagicMock(spec=gpd.GeoDataFrame) + mock_transformed_gdf = MagicMock(spec=gpd.GeoDataFrame) + mock_selected_gdf = MagicMock(spec=gpd.GeoDataFrame) + + mock_read_file.return_value = mock_gdf + mock_graph_from_polygon.return_value = MagicMock() + mock_create_voronoi.return_value = mock_tile_gdf + + # Mock method chaining for the `tile_gdf` object + mock_tile_gdf.to_crs.return_value = mock_transformed_gdf + mock_transformed_gdf.__getitem__.return_value = mock_selected_gdf + + # Mock dask-geopandas behavior + mock_from_geopandas.return_value.apply.return_value.compute.return_value = mock_selected_gdf + + # Call the function without polygon + result = calculate_xn_qm("test_edges.geojson", "test_output.geojson") + + # Assertions + mock_read_file.assert_called_once_with("test_edges.geojson") + mock_graph_from_polygon.assert_called_once() + mock_create_voronoi.assert_called_once() + mock_tile_gdf.to_crs.assert_called_once_with(PROJ) + mock_transformed_gdf.__getitem__.assert_called_once_with(["geometry"]) + mock_from_geopandas.assert_called_once_with(mock_selected_gdf, npartitions=os.cpu_count()) + self.assertIsNone(result) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/models/__init__.py b/tests/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/models/test_quality_request.py b/tests/models/test_quality_request.py new file mode 100644 index 0000000..bb9bb5a --- /dev/null +++ b/tests/models/test_quality_request.py @@ -0,0 +1,94 @@ +import unittest +from src.models.quality_request import RequestData, QualityRequest + + +class TestRequestData(unittest.TestCase): + def test_request_data_initialization(self): + # Test initialization with all fields + request_data = RequestData( + jobId='test-job', + data_file='https://example.com/data-file.zip', + algorithm='fixed', + sub_regions_file='https://example.com/sub-region-file.zip' + ) + self.assertEqual(request_data.jobId, 'test-job') + self.assertEqual(request_data.data_file, 'https://example.com/data-file.zip') + self.assertEqual(request_data.algorithm, 'fixed') + self.assertEqual(request_data.sub_regions_file, 'https://example.com/sub-region-file.zip') + + # Test initialization without optional field + request_data = RequestData( + jobId='test-job', + data_file='https://example.com/data-file.zip', + algorithm='fixed' + ) + self.assertEqual(request_data.jobId, 'test-job') + self.assertEqual(request_data.data_file, 'https://example.com/data-file.zip') + self.assertEqual(request_data.algorithm, 'fixed') + self.assertIsNone(request_data.sub_regions_file) + + +class TestQualityRequest(unittest.TestCase): + def test_quality_request_initialization(self): + # Test initialization with valid data + data_dict = { + 'jobId': 'test-job', + 'data_file': 'https://example.com/data-file.zip', + 'algorithm': 'fixed', + 'sub_regions_file': 'https://example.com/sub-region-file.zip' + } + + quality_request = QualityRequest( + messageType='test-message-type', + messageId='test-message-id', + data=data_dict + ) + + self.assertEqual(quality_request.messageType, 'test-message-type') + self.assertEqual(quality_request.messageId, 'test-message-id') + self.assertIsInstance(quality_request.data, RequestData) + self.assertEqual(quality_request.data.jobId, 'test-job') + self.assertEqual(quality_request.data.data_file, 'https://example.com/data-file.zip') + self.assertEqual(quality_request.data.algorithm, 'fixed') + self.assertEqual(quality_request.data.sub_regions_file, 'https://example.com/sub-region-file.zip') + + def test_quality_request_initialization_with_missing_optional(self): + # Test initialization with missing optional field + data_dict = { + 'jobId': 'test-job', + 'data_file': 'https://example.com/data-file.zip', + 'algorithm': 'fixed' + } + + quality_request = QualityRequest( + messageType='test-message-type', + messageId='test-message-id', + data=data_dict + ) + + self.assertEqual(quality_request.messageType, 'test-message-type') + self.assertEqual(quality_request.messageId, 'test-message-id') + self.assertIsInstance(quality_request.data, RequestData) + self.assertEqual(quality_request.data.jobId, 'test-job') + self.assertEqual(quality_request.data.data_file, 'https://example.com/data-file.zip') + self.assertEqual(quality_request.data.algorithm, 'fixed') + self.assertIsNone(quality_request.data.sub_regions_file) + + def test_quality_request_initialization_with_invalid_data(self): + # Test initialization with invalid data (missing required fields) + invalid_data = { + 'jobId': 'test-job', + 'data_file': 'https://example.com/data-file.zip', + } # Missing 'algorithm' + + with self.assertRaises(TypeError) as context: + QualityRequest( + messageType='test-message-type', + messageId='test-message-id', + data=invalid_data + ) + self.assertIn("__init__() missing 1 required positional argument: 'algorithm'", str(context.exception)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/models/test_quality_response.py b/tests/models/test_quality_response.py new file mode 100644 index 0000000..cabecab --- /dev/null +++ b/tests/models/test_quality_response.py @@ -0,0 +1,70 @@ +import unittest +from src.models.quality_response import ResponseData, QualityMetricResponse + + +class TestResponseData(unittest.TestCase): + def test_response_data_initialization(self): + # Test initialization with all fields + response_data = ResponseData( + status='success', + message='Metrics calculated successfully.', + success=True, + dataset_url='https://example.com/dataset.zip', + qm_dataset_url='https://example.com/qm-dataset.zip' + ) + self.assertEqual(response_data.status, 'success') + self.assertEqual(response_data.message, 'Metrics calculated successfully.') + self.assertTrue(response_data.success) + self.assertEqual(response_data.dataset_url, 'https://example.com/dataset.zip') + self.assertEqual(response_data.qm_dataset_url, 'https://example.com/qm-dataset.zip') + + +class TestQualityMetricResponse(unittest.TestCase): + def test_quality_metric_response_initialization(self): + # Valid data dictionary for ResponseData + data_dict = { + 'status': 'success', + 'message': 'Metrics calculated successfully.', + 'success': True, + 'dataset_url': 'https://example.com/dataset.zip', + 'qm_dataset_url': 'https://example.com/qm-dataset.zip' + } + + # Initialize QualityMetricResponse + response = QualityMetricResponse( + messageType='test-message-type', + messageId='test-message-id', + data=data_dict + ) + + # Assertions + self.assertEqual(response.messageType, 'test-message-type') + self.assertEqual(response.messageId, 'test-message-id') + self.assertIsInstance(response.data, ResponseData) + self.assertEqual(response.data.status, 'success') + self.assertEqual(response.data.message, 'Metrics calculated successfully.') + self.assertTrue(response.data.success) + self.assertEqual(response.data.dataset_url, 'https://example.com/dataset.zip') + self.assertEqual(response.data.qm_dataset_url, 'https://example.com/qm-dataset.zip') + + def test_quality_metric_response_with_invalid_data(self): + # Test with missing required fields in the data dictionary + invalid_data = { + 'status': 'success', + 'message': 'Metrics calculated successfully.', + 'success': True, + 'dataset_url': 'https://example.com/dataset.zip' + # Missing qm_dataset_url + } + + with self.assertRaises(TypeError) as context: + QualityMetricResponse( + messageType='test-message-type', + messageId='test-message-id', + data=invalid_data + ) + self.assertIn("__init__() missing 1 required positional argument: 'qm_dataset_url'", str(context.exception)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/services/__init__.py b/tests/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/services/test_osw_qm_calculator_service.py b/tests/services/test_osw_qm_calculator_service.py new file mode 100644 index 0000000..210324f --- /dev/null +++ b/tests/services/test_osw_qm_calculator_service.py @@ -0,0 +1,107 @@ +import tempfile +import unittest +from unittest.mock import patch, MagicMock, mock_open +from src.calculators import QMXNLibCalculator, QMFixedCalculator +from src.services.osw_qm_calculator_service import OswQmCalculator + + + +class TestOswQmCalculator(unittest.TestCase): + def setUp(self): + self.calculator = OswQmCalculator(cores_to_use=4) + + def test_initialization(self): + self.assertEqual(self.calculator.cores_to_use, 4) + + @patch('src.services.osw_qm_calculator_service.zipfile.ZipFile') + @patch('src.services.osw_qm_calculator_service.OswQmCalculator.extract_zip') + @patch('src.services.osw_qm_calculator_service.OswQmCalculator.get_osw_qm_calculator') + @patch('src.services.osw_qm_calculator_service.OswQmCalculator.zip_folder') + def test_calculate_quality_metric(self, mock_zip_folder, mock_get_calculator, mock_extract_zip, mock_zipfile): + mock_extract_zip.return_value = ['mock_path/edges_file.geojson'] + mock_calculator = MagicMock() + mock_get_calculator.return_value = mock_calculator + + with tempfile.NamedTemporaryFile() as temp_input, tempfile.NamedTemporaryFile() as temp_output: + self.calculator.calculate_quality_metric(temp_input.name, ['fixed'], temp_output.name) + + mock_zipfile.assert_called_once() + mock_extract_zip.assert_called_once() + mock_get_calculator.assert_called_once_with('fixed', None, 'mock_path/edges_file.geojson', unittest.mock.ANY) + mock_calculator.calculate_quality_metric.assert_called_once() + mock_zip_folder.assert_called_once() + + @patch('src.services.osw_qm_calculator_service.zipfile.ZipFile') + @patch('src.services.osw_qm_calculator_service.OswQmCalculator.extract_zip') + def test_calculate_quality_metric_no_edges_file(self, mock_extract_zip, mock_zipfile): + mock_extract_zip.return_value = [] + + with tempfile.NamedTemporaryFile() as temp_input, tempfile.NamedTemporaryFile() as temp_output: + with self.assertRaises(Exception) as context: + self.calculator.calculate_quality_metric(temp_input.name, ['fixed'], temp_output.name) + self.assertEqual(str(context.exception), 'Edges file not found in input files.') + + def test_get_osw_qm_calculator(self): + calculator = self.calculator.get_osw_qm_calculator('fixed', None, 'edges.geojson', 'output.geojson') + self.assertIsInstance(calculator, QMFixedCalculator) + + calculator = self.calculator.get_osw_qm_calculator('ixn', 'ixn_file', 'edges.geojson', 'output.geojson') + self.assertIsInstance(calculator, QMXNLibCalculator) + + @patch('src.services.osw_qm_calculator_service.zipfile.ZipFile') + def test_zip_folder(self, mock_zipfile): + with tempfile.TemporaryDirectory() as temp_folder: + with tempfile.NamedTemporaryFile(dir=temp_folder) as temp_file: + output_zip = tempfile.NamedTemporaryFile().name + self.calculator.zip_folder(temp_folder, output_zip) + mock_zipfile.assert_called_once_with(output_zip, 'w') + + @patch('src.services.osw_qm_calculator_service.zipfile.ZipFile') + @patch('os.walk') + def test_extract_zip(self, mock_walk, mock_zipfile): + mock_zip = MagicMock() + mock_zipfile.return_value = mock_zip + mock_walk.return_value = [('/mock', [], ['file1.geojson', 'file2.geojson'])] + with tempfile.TemporaryDirectory() as temp_folder: + extracted_files = self.calculator.extract_zip(mock_zip, temp_folder) + self.assertEqual(extracted_files, ['/mock/file1.geojson', '/mock/file2.geojson']) + mock_zip.extractall.assert_called_once_with(temp_folder) + + @patch('builtins.open', new_callable=mock_open, read_data='{"features": [{"id": 1}]}') + @patch('src.services.osw_qm_calculator_service.Config') + @patch('src.services.osw_qm_calculator_service.QMFixedCalculator') + def test_parse_and_calculate_quality_metric_with_fixed(self, mock_qm_calculator, mock_config, mock_open): + # Mock the algorithm dictionary to return the mocked calculator + mock_instance = MagicMock() + mock_instance.qm_metric_tag.return_value = 'fixed' + mock_instance.calculate_quality_metric.return_value = 42 # Example metric value + mock_qm_calculator.return_value = mock_instance + + mock_config.return_value.algorithm_dictionary = {'fixed': mock_qm_calculator} + + # Call the method under test + result = self.calculator.parse_and_calculate_quality_metric('input.json', ['fixed']) + + # Verify results + self.assertEqual(result, {"features": [{"id": 1, "fixed": 42}]}) # Expected modified JSON + mock_qm_calculator.assert_called_once() # Ensure the calculator is instantiated + mock_instance.qm_metric_tag.assert_called_once() # Ensure qm_metric_tag was called + mock_instance.calculate_quality_metric.assert_called_once() # Ensure calculation was performed + + @patch('builtins.open', new_callable=mock_open, read_data='{"features": [{"id": 1}]}') + @patch('src.services.osw_qm_calculator_service.Config') + @patch('src.services.osw_qm_calculator_service.logger') + def test_parse_and_calculate_quality_metric_with_unknown_algorithm(self, mock_logger, mock_config, mock_open): + # Mock algorithm dictionary to have no matching algorithm + mock_config.return_value.algorithm_dictionary = {'fixed': MagicMock()} + + # Call the method with an unknown algorithm + result = self.calculator.parse_and_calculate_quality_metric('input.json', ['unknown_algorithm']) + + # Verify results + self.assertEqual(result, {"features": [{"id": 1}]}) # No change to features + mock_logger.warning.assert_called_once_with('Algorithm not found : unknown_algorithm') # Warning logged + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/services/test_servicebus_service.py b/tests/services/test_servicebus_service.py new file mode 100644 index 0000000..28eaa19 --- /dev/null +++ b/tests/services/test_servicebus_service.py @@ -0,0 +1,135 @@ +import unittest +from unittest.mock import patch, MagicMock +from src.services.servicebus_service import ServiceBusService +from src.models.quality_request import RequestData, QualityRequest +from src.models.quality_response import QualityMetricResponse +from python_ms_core.core.queue.models.queue_message import QueueMessage + + +class TestServiceBusService(unittest.TestCase): + @patch('src.services.servicebus_service.Config') + @patch('src.services.servicebus_service.Core') + def setUp(self, mock_core, mock_config): + # Mock Config + mock_config.return_value.connection_string = 'mock-connection-string' + mock_config.return_value.incoming_topic_name = 'mock-incoming-topic' + mock_config.return_value.outgoing_topic_name = 'mock-outgoing-topic' + mock_config.return_value.max_concurrent_messages = 5 + mock_config.return_value.incoming_topic_subscription = 'mock-subscription' + + # Mock Core + mock_core.return_value.get_topic.return_value = MagicMock() + mock_core.return_value.get_storage_client.return_value = MagicMock() + + # Initialize the service + self.service = ServiceBusService() + self.service.storage_service = MagicMock() + self.test_message = QueueMessage( + messageType='mettric-calculation', + messageId='message-id-from-msg', + data={ + 'jobId': '0b41ebc5-350c-42d3-90af-3af4ad3628fb', + 'data_file': 'https://tdeisamplestorage.blob.core.windows.net/abc.zip', + 'algorithm': 'fixed', + 'sub_regions_file': None + } + ) + + self.success_message = QualityMetricResponse( + messageType=self.test_message.messageType, + messageId=self.test_message.messageId, + data={ + 'status': 'success', + 'message': 'Quality metrics calculated successfully', + 'success': True, + 'dataset_url': self.test_message.data['data_file'], + 'qm_dataset_url': self.test_message.data['data_file'] + } + ) + + def test_initialization(self): + self.assertIsInstance(self.service.core, MagicMock) + self.assertIsInstance(self.service.config, MagicMock) + self.assertIsInstance(self.service.storage_service, MagicMock) + + @patch('src.services.servicebus_service.OswQmCalculator') + @patch('src.services.servicebus_service.shutil.rmtree') + def test_process_message_success_without_sub_region(self, mock_rmtree, mock_calculator): + # Mock message and dependencies + + self.service.storage_service.download_remote_file = MagicMock() + self.service.storage_service.upload_local_file = MagicMock(return_value='https://example.com/qm-output.zip') + mock_calculator_instance = MagicMock() + mock_calculator.return_value = mock_calculator_instance + + self.service.process_message(self.test_message) + + # Assertions + self.service.storage_service.download_remote_file.assert_called_once() + mock_calculator_instance.calculate_quality_metric.assert_called_once() + self.service.storage_service.upload_local_file.assert_called_once() + mock_rmtree.assert_called_once() + + @patch('src.services.servicebus_service.OswQmCalculator') + @patch('src.services.servicebus_service.shutil.rmtree') + def test_process_message_success_with_sub_region(self, mock_rmtree, mock_calculator): + # Mock message and dependencies + + + self.test_message.data['sub_regions_file'] = self.test_message.data['data_file'] + + self.service.storage_service.download_remote_file = MagicMock() + self.service.storage_service.upload_local_file = MagicMock(return_value='https://example.com/qm-output.zip') + mock_calculator_instance = MagicMock() + mock_calculator.return_value = mock_calculator_instance + + self.service.process_message(self.test_message) + + # Assertions + self.service.storage_service.download_remote_file.assert_called() + mock_calculator_instance.calculate_quality_metric.assert_called_once() + self.service.storage_service.upload_local_file.assert_called_once() + mock_rmtree.assert_called_once() + + @patch('src.services.servicebus_service.logger') + def test_process_message_failure(self, mock_logger): + self.test_message.data['data_file'] = 'invalid_file_path' + + # Simulate a download failure + self.service.storage_service.download_remote_file.side_effect = Exception('Download failed') + + self.service.process_message(self.test_message) + + # Assertions + mock_logger.error.assert_called_with('Error processing message message-id-from-msg : Download failed') + + @patch('src.services.servicebus_service.QueueMessage') + @patch('src.services.servicebus_service.logger') + def test_send_response_success(self, mock_logger, mock_queue_message): + self.service.send_response(self.success_message) + + # Assertions + mock_queue_message.data_from.assert_called_once() + mock_logger.info.assert_called_with('Publishing response for message message-id-from-msg') + + @patch('src.services.servicebus_service.logger') + def test_send_response_failure(self, mock_logger): + self.service.send_response(self.test_message) + + # Assertions + mock_logger.error.assert_called_with('Failed to send response for message message-id-from-msg with error asdict() should be called on dataclass instances') + + def test_get_directory_path(self): + url = 'https://example.com/osw/test_upload/df/fff/500mb_file.zip' + expected_path = 'test_upload/df/fff' + result = self.service.get_directory_path(url) + self.assertEqual(result, expected_path) + + @patch('src.services.servicebus_service.threading.Thread.join') + def test_stop(self, mock_join): + self.service.stop() + mock_join.assert_called_once_with(timeout=0) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/services/test_storage_service.py b/tests/services/test_storage_service.py new file mode 100644 index 0000000..317d104 --- /dev/null +++ b/tests/services/test_storage_service.py @@ -0,0 +1,66 @@ +import unittest +from unittest.mock import MagicMock, patch +from src.services.storage_service import StorageService + +class TestStorageService(unittest.TestCase): + + @patch('src.services.storage_service.Config') + @patch('src.services.storage_service.Core') + def setUp(self, mock_core, mock_config): + self.mock_core = mock_core + self.mock_config = mock_config + + # Mocking Config values + self.mock_config.return_value.storage_container_name = 'test-container' + + # Mocking Core and Storage Client + self.mock_storage_client = MagicMock() + self.mock_container = MagicMock() + self.mock_storage_client.get_container.return_value = self.mock_container + self.mock_core.return_value.get_storage_client.return_value = self.mock_storage_client + + # Initializing the service + self.service = StorageService(core=self.mock_core()) + + def test_upload_local_file(self): + # Mocking file creation and upload behavior + mock_azure_file = MagicMock() + self.mock_container.create_file.return_value = mock_azure_file + mock_azure_file.get_remote_url.return_value = 'https://example.com/remote-path' + + # Mocking file read + mock_local_path = 'test_local.txt' + mock_remote_path = 'remote/test.txt' + file_data = b'file content' + + with patch('builtins.open', unittest.mock.mock_open(read_data=file_data)) as mock_file: + remote_url = self.service.upload_local_file(mock_local_path, mock_remote_path) + + # Assertions + self.mock_container.create_file.assert_called_once_with(mock_remote_path) + mock_azure_file.upload.assert_called_once_with(file_data) + mock_file.assert_called_once_with(mock_local_path, 'rb') + self.assertEqual(remote_url, 'https://example.com/remote-path') + + def test_download_remote_file(self): + # Mocking file entity and stream + mock_file_entity = MagicMock() + mock_file_entity.get_stream.return_value = b'file content' + self.mock_storage_client.get_file_from_url.return_value = mock_file_entity + + # Mocking file write + mock_remote_path = 'remote/test.txt' + mock_local_path = 'test_local.txt' + + with patch('builtins.open', unittest.mock.mock_open()) as mock_file: + self.service.download_remote_file(mock_remote_path, mock_local_path) + + # Assertions + self.mock_storage_client.get_file_from_url.assert_called_once_with( + self.mock_config.return_value.storage_container_name, mock_remote_path + ) + mock_file.assert_called_once_with(mock_local_path, 'wb') + mock_file().write.assert_called_once_with(b'file content') + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..81a18ca --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,59 @@ +import unittest +from unittest.mock import patch +from src.config import Config +from src.calculators import QMFixedCalculator, QMXNLibCalculator + + +class TestConfig(unittest.TestCase): + + def test_default_values(self): + config = Config() + self.assertEqual(config.app_name, 'osw-quality-metric-service-python') + self.assertEqual(config.incoming_topic_name, '') + self.assertEqual(config.incoming_topic_subscription, '') + self.assertEqual(config.outgoing_topic_name, '') + self.assertEqual(config.storage_container_name, 'osw') + self.assertEqual(config.max_concurrent_messages, 1) + self.assertEqual(config.partition_count, 2) + + def test_algorithm_dictionary(self): + config = Config() + self.assertIn('fixed', config.algorithm_dictionary) + self.assertIn('ixn', config.algorithm_dictionary) + self.assertIs(config.algorithm_dictionary['fixed'], QMFixedCalculator) + self.assertIs(config.algorithm_dictionary['ixn'], QMXNLibCalculator) + + @patch('src.config.os.path.dirname') + def test_get_download_folder(self, mock_dirname): + mock_dirname.side_effect = lambda path: '/mock/root' + config = Config() + download_folder = config.get_download_folder() + self.assertEqual(download_folder, '/mock/root/downloads') + + @patch('src.config.os.path.dirname') + def test_get_assets_folder(self, mock_dirname): + mock_dirname.side_effect = lambda path: '/mock/root/src' + config = Config() + assets_folder = config.get_assets_folder() + self.assertEqual(assets_folder, '/mock/root/src/assets') + + @patch.dict('os.environ', { + 'QUALITY_REQ_TOPIC': 'test_topic', + 'QUALITY_REQ_SUB': 'test_subscription', + 'QUALITY_RES_TOPIC': 'test_outgoing_topic', + 'CONTAINER_NAME': 'test_container', + 'MAX_CONCURRENT_MESSAGES': '5', + 'PARTITION_COUNT': '10' + }) + def test_environment_variable_overrides(self): + config = Config() + self.assertEqual(config.incoming_topic_name, '') + self.assertEqual(config.incoming_topic_subscription, '') + self.assertEqual(config.outgoing_topic_name, '') + self.assertEqual(config.storage_container_name, 'osw') + self.assertEqual(config.max_concurrent_messages, 5) # Casts to int + self.assertEqual(config.partition_count, 10) # Casts to int + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_main.py b/tests/test_main.py new file mode 100644 index 0000000..7b6485e --- /dev/null +++ b/tests/test_main.py @@ -0,0 +1,56 @@ +import unittest +import asyncio +from unittest.mock import patch, MagicMock +from fastapi.testclient import TestClient +from src.main import app, startup_event, shutdown_event + + +class TestFastAPIApp(unittest.TestCase): + def setUp(self): + # Setup the TestClient for FastAPI app + self.client = TestClient(app) + + def test_root_endpoint(self): + response = self.client.get('/') + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json(), {'Hello': 'World'}) + + def test_ping_endpoint(self): + response = self.client.get('/ping') + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json(), {'msg': 'Ping Successful'}) + + def test_health_endpoint(self): + response = self.client.get('/health') + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json(), "I'm healthy !!") + + @patch('src.main.ServiceBusService') + def test_startup_event_initializes_servicebus(self, MockServiceBusService): + mock_service = MagicMock() + MockServiceBusService.return_value = mock_service + + asyncio.run(startup_event()) + + # Check if ServiceBusService is initialized + self.assertEqual(app.qm_service, mock_service) + + @patch.object(app, 'qm_service') + def test_shutdown_event_calls_service_stop(self, mock_qm_service): + mock_qm_service.stop = MagicMock() + + asyncio.run(shutdown_event()) + + # Verify the stop method is called + mock_qm_service.stop.assert_called_once() + + def test_octet_stream_response(self): + from src.main import OctetStreamResponse + + response = OctetStreamResponse(content=b'binary data') + self.assertEqual(response.media_type, 'application/octet-stream') + self.assertEqual(response.body, b'binary data') + + +if __name__ == '__main__': + unittest.main() From c07f2f8e3a21877ef0afc1013b2ad2155b190bbe Mon Sep 17 00:00:00 2001 From: sujata-m Date: Tue, 19 Nov 2024 14:56:31 +0530 Subject: [PATCH 2/8] Fixed pipeline --- .github/workflows/unit_tests.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index 84f2ab2..025856a 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -11,9 +11,6 @@ jobs: UnitTest: runs-on: ubuntu-latest - env: - DATABASE_NAME: test_database - steps: - name: Checkout code uses: actions/checkout@v2 @@ -40,6 +37,7 @@ jobs: else echo "Unknown branch: $GITHUB_REF_NAME" exit 1 + fi - name: Run tests with coverage run: | From 5e722a8283132214ea1ba971fc9f082139d0426f Mon Sep 17 00:00:00 2001 From: sujata-m Date: Tue, 19 Nov 2024 14:59:36 +0530 Subject: [PATCH 3/8] Trying to fix pipeline --- .github/workflows/unit_tests.yaml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index 025856a..2dbf01a 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -28,14 +28,20 @@ jobs: - name: Determine output folder id: set_output_folder run: | - if [[ $GITHUB_REF_NAME == "main" ]]; then + if [[ $GITHUB_EVENT_NAME == "pull_request" ]]; then + branch_name=$GITHUB_BASE_REF + else + branch_name=$GITHUB_REF_NAME + fi + + if [[ $branch_name == "main" ]]; then echo "output_folder=prod" >> $GITHUB_ENV - elif [[ $GITHUB_REF_NAME == "stage" ]]; then + elif [[ $branch_name == "stage" ]]; then echo "output_folder=stage" >> $GITHUB_ENV - elif [[ $GITHUB_REF_NAME == "dev" ]]; then + elif [[ $branch_name == "dev" ]]; then echo "output_folder=dev" >> $GITHUB_ENV else - echo "Unknown branch: $GITHUB_REF_NAME" + echo "Unknown branch: $branch_name" exit 1 fi @@ -53,6 +59,6 @@ jobs: source_folder: 'test_results/${{ env.output_folder }}' destination_folder: 'osw-quality-metric-service/${{ env.output_folder }}/${{ github.repository }}/unit_test_results' connection_string: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }} - container_name: 'reports' + container_name: 'tdeiinternal' clean_destination_folder: false delete_if_exists: false From cb6eae569db80224384053d8292470a962c2b454 Mon Sep 17 00:00:00 2001 From: sujata-m Date: Tue, 19 Nov 2024 15:03:37 +0530 Subject: [PATCH 4/8] Trying to fix pipeline --- .github/workflows/unit_tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index 2dbf01a..fb0afd9 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -57,8 +57,8 @@ jobs: uses: LanceMcCarthy/Action-AzureBlobUpload@v2 with: source_folder: 'test_results/${{ env.output_folder }}' - destination_folder: 'osw-quality-metric-service/${{ env.output_folder }}/${{ github.repository }}/unit_test_results' + destination_folder: '${{ env.output_folder }}/unit_test_results' connection_string: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }} - container_name: 'tdeiinternal' + container_name: 'osw-quality-metric-service' clean_destination_folder: false delete_if_exists: false From 0a54d29c979799c80d7eba08f609fa8e57479537 Mon Sep 17 00:00:00 2001 From: sujata-m Date: Tue, 19 Nov 2024 15:19:20 +0530 Subject: [PATCH 5/8] Trying to fix pipeline --- .github/workflows/unit_tests.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index fb0afd9..331ae3e 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -47,17 +47,17 @@ jobs: - name: Run tests with coverage run: | - dynamic_results_path="test_results/${{ env.output_folder }}" - mkdir -p $dynamic_results_path + timestamp=$(date '+%Y-%m-%d_%H-%M-%S') + mkdir -p test_results PYTHONPATH=$(pwd) python -m coverage run --source=src -m unittest discover -v tests/ - coverage report > $dynamic_results_path/latest_report.log - coverage report >> $dynamic_results_path/$(date '+%Y-%m-%d_%H-%M-%S')_report.log + coverage report > test_results/${{ env.output_folder }}_latest_report.log + coverage report >> test_results/${{ env.output_folder }}_${timestamp}_report.log - name: Upload report to Azure uses: LanceMcCarthy/Action-AzureBlobUpload@v2 with: - source_folder: 'test_results/${{ env.output_folder }}' - destination_folder: '${{ env.output_folder }}/unit_test_results' + source_folder: 'test_results' + destination_folder: '${{ env.output_folder }}' connection_string: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }} container_name: 'osw-quality-metric-service' clean_destination_folder: false From 69f011de2e6f9502661cf887b4999d81d47ebb17 Mon Sep 17 00:00:00 2001 From: sujata-m Date: Tue, 19 Nov 2024 15:23:10 +0530 Subject: [PATCH 6/8] Trying to fix pipeline --- .github/workflows/unit_tests.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index 331ae3e..3fed4bd 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -49,9 +49,11 @@ jobs: run: | timestamp=$(date '+%Y-%m-%d_%H-%M-%S') mkdir -p test_results - PYTHONPATH=$(pwd) python -m coverage run --source=src -m unittest discover -v tests/ - coverage report > test_results/${{ env.output_folder }}_latest_report.log - coverage report >> test_results/${{ env.output_folder }}_${timestamp}_report.log + log_file="test_results/${{ env.output_folder }}_${timestamp}_report.log" + # Run the tests and append output to the log file + PYTHONPATH=$(pwd) python -m coverage run --source=src -m unittest discover -v tests/ >> $log_file 2>&1 + # Append the coverage summary to the log file + coverage report >> $log_file - name: Upload report to Azure uses: LanceMcCarthy/Action-AzureBlobUpload@v2 From d2921b55b19380b88094efd7c1ec100b735e0d34 Mon Sep 17 00:00:00 2001 From: sujata-m Date: Tue, 19 Nov 2024 15:27:59 +0530 Subject: [PATCH 7/8] Fixed pipeline --- .github/workflows/unit_tests.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index 3fed4bd..1376b22 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -49,10 +49,11 @@ jobs: run: | timestamp=$(date '+%Y-%m-%d_%H-%M-%S') mkdir -p test_results - log_file="test_results/${{ env.output_folder }}_${timestamp}_report.log" + log_file="test_results/${timestamp}_report.log" + echo -e "\nTest Cases Report Report\n" >> $log_file # Run the tests and append output to the log file PYTHONPATH=$(pwd) python -m coverage run --source=src -m unittest discover -v tests/ >> $log_file 2>&1 - # Append the coverage summary to the log file + echo -e "\nCoverage Report\n" >> $log_file coverage report >> $log_file - name: Upload report to Azure From 5779f0f934f7e2cea7f6822915842eaabaca8508 Mon Sep 17 00:00:00 2001 From: sujata-m Date: Tue, 19 Nov 2024 17:54:16 +0530 Subject: [PATCH 8/8] Updated unit test cases --- .github/workflows/unit_tests.yaml | 101 ++++++++++++++++-------------- 1 file changed, 53 insertions(+), 48 deletions(-) diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index 1376b22..eeca4a8 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -5,63 +5,68 @@ on: branches-ignore: - '**' pull_request: - branches: [main, dev, stage] + branches: [ main, dev, stage ] jobs: UnitTest: runs-on: ubuntu-latest steps: - - name: Checkout code - uses: actions/checkout@v2 + - name: Checkout code + uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.10' + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.10' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt - - name: Determine output folder - id: set_output_folder - run: | - if [[ $GITHUB_EVENT_NAME == "pull_request" ]]; then - branch_name=$GITHUB_BASE_REF - else - branch_name=$GITHUB_REF_NAME - fi + - name: Determine output folder + id: set_output_folder + run: | + if [[ $GITHUB_EVENT_NAME == "pull_request" ]]; then + branch_name=$GITHUB_BASE_REF + else + branch_name=$GITHUB_REF_NAME + fi + + if [[ $branch_name == "main" ]]; then + echo "output_folder=prod" >> $GITHUB_ENV + elif [[ $branch_name == "stage" ]]; then + echo "output_folder=stage" >> $GITHUB_ENV + elif [[ $branch_name == "dev" ]]; then + echo "output_folder=dev" >> $GITHUB_ENV + else + echo "Unknown branch: $branch_name" + exit 1 + fi - if [[ $branch_name == "main" ]]; then - echo "output_folder=prod" >> $GITHUB_ENV - elif [[ $branch_name == "stage" ]]; then - echo "output_folder=stage" >> $GITHUB_ENV - elif [[ $branch_name == "dev" ]]; then - echo "output_folder=dev" >> $GITHUB_ENV - else - echo "Unknown branch: $branch_name" - exit 1 - fi + - name: Run tests with coverage + run: | + timestamp=$(date '+%Y-%m-%d_%H-%M-%S') + mkdir -p test_results + log_file="test_results/${timestamp}_report.log" + echo -e "\nTest Cases Report Report\n" >> $log_file + # Run the tests and append output to the log file + PYTHONPATH=$(pwd) python -m coverage run --source=src -m unittest discover -v tests/ >> $log_file 2>&1 + echo -e "\nCoverage Report\n" >> $log_file + coverage report >> $log_file + coverage xml - - name: Run tests with coverage - run: | - timestamp=$(date '+%Y-%m-%d_%H-%M-%S') - mkdir -p test_results - log_file="test_results/${timestamp}_report.log" - echo -e "\nTest Cases Report Report\n" >> $log_file - # Run the tests and append output to the log file - PYTHONPATH=$(pwd) python -m coverage run --source=src -m unittest discover -v tests/ >> $log_file 2>&1 - echo -e "\nCoverage Report\n" >> $log_file - coverage report >> $log_file + - name: Check coverage + run: | + coverage report --fail-under=85 - - name: Upload report to Azure - uses: LanceMcCarthy/Action-AzureBlobUpload@v2 - with: - source_folder: 'test_results' - destination_folder: '${{ env.output_folder }}' - connection_string: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }} - container_name: 'osw-quality-metric-service' - clean_destination_folder: false - delete_if_exists: false + - name: Upload report to Azure + uses: LanceMcCarthy/Action-AzureBlobUpload@v2 + with: + source_folder: 'test_results' + destination_folder: '${{ env.output_folder }}' + connection_string: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }} + container_name: 'osw-quality-metric-service' + clean_destination_folder: false + delete_if_exists: false