Merge pull request #26 from TaskarCenterAtUW/feature-memory-pref

Fixed memory issue

Author: sujata-m

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Change log
 
+### 0.2.7
+- Switch to `jsonschema_rs` for performance enhancement, instead of `jsonschema` package
+- Refactor code for improve memory utilization
+- Added garbage collector
+
+
 ### 0.2.6
 - Add garbage collection to free up memory after validation
 

requirements.txt

@@ -1,4 +1,4 @@
-jsonschema
+jsonschema_rs
 zipfile36
 coverage
 geopandas

setup.py

@@ -19,7 +19,7 @@
     long_description_content_type='text/markdown',
     url='https://github.com/TaskarCenterAtUW/TDEI-python-lib-osw-validation',
     install_requires=[
-        'jsonschema',
+        'jsonschema_rs',
         'zipfile36',
         'geopandas'
     ],

src/python_osw_validation/__init__.py

@@ -1,10 +1,10 @@
 import os
 import gc
 import json
-import jsonschema
-from typing import Dict, Any, Optional, List
+import jsonschema_rs
 import geopandas as gpd
 from .zipfile_handler import ZipFileHandler
+from typing import Dict, Any, Optional, List
 from .extracted_data_validator import ExtractedDataValidator, OSW_DATASET_FILES
 from .version import __version__
 
@@ -37,7 +37,7 @@ def load_osw_schema(self, schema_path: str) -> Dict[str, Any]:
         except Exception as e:
             self.errors.append(f'Invalid or missing schema file: {e}')
             raise Exception(f'Invalid or missing schema file: {e}')
-        
+
     def are_ids_unique(self, gdf):
         """Check for duplicate values in the _id field"""
         duplicates = gdf[gdf.duplicated('_id', keep=False)]['_id'].unique()
@@ -47,6 +47,7 @@ def are_ids_unique(self, gdf):
     def validate(self, max_errors=20) -> ValidationResult:
         zip_handler = None
         OSW_DATASET = {}
+        validator = None
         try:
             # Extract the zipfile
             zip_handler = ZipFileHandler(self.zipfile_path)
@@ -73,7 +74,8 @@ def validate(self, max_errors=20) -> ValidationResult:
             # Validate data integrity
             for file in validator.files:
                 file_path = os.path.join(file)
-                osw_file = next((osw_file_any for osw_file_any in OSW_DATASET_FILES.keys() if osw_file_any in file_path), '')
+                osw_file = next(
+                    (osw_file_any for osw_file_any in OSW_DATASET_FILES.keys() if osw_file_any in file_path), '')
                 OSW_DATASET[osw_file] = gpd.read_file(file_path)
 
             # Are all id's unique in each file? No need to check uniqueness across files yet since we do not have a global OSW ID format yet
@@ -104,26 +106,32 @@ def validate(self, max_errors=20) -> ValidationResult:
             unmatched = node_ids_edges_u - node_ids
             is_valid = len(unmatched) == 0
             if not is_valid:
-                self.errors.append(f"All _u_id's in edges should be part of _id's mentioned in nodes, _u_id's not in nodes are: {unmatched}")
+                self.errors.append(
+                    f"All _u_id's in edges should be part of _id's mentioned in nodes, _u_id's not in nodes are: {unmatched}")
 
             # Do all node references in _v_id exist in nodes?
             unmatched = node_ids_edges_v - node_ids
             is_valid = len(unmatched) == 0
             if not is_valid:
-                self.errors.append(f"All _v_id's in edges should be part of _id's mentioned in nodes, _v_id's not in nodes are: {unmatched}")
+                self.errors.append(
+                    f"All _v_id's in edges should be part of _id's mentioned in nodes, _v_id's not in nodes are: {unmatched}")
 
             # Do all node references in _w_id exist in nodes?
             unmatched = node_ids_zones_w - node_ids
             is_valid = len(unmatched) == 0
             if not is_valid:
-                self.errors.append(f"All _w_id's in zones should be part of _id's mentioned in nodes, _w_id's not in nodes are: {unmatched}")
+                self.errors.append(
+                    f"All _w_id's in zones should be part of _id's mentioned in nodes, _w_id's not in nodes are: {unmatched}")
 
             # Geometry validation: check geometry type in each file and test if coordinates make a shape that is reasonable geometric shape according to the Simple Feature Access standard
             for osw_file in OSW_DATASET:
-                invalid_geojson = OSW_DATASET[osw_file][(OSW_DATASET[osw_file].geometry.type != OSW_DATASET_FILES[osw_file]['geometry']) | (OSW_DATASET[osw_file].is_valid == False)]
+                invalid_geojson = OSW_DATASET[osw_file][
+                    (OSW_DATASET[osw_file].geometry.type != OSW_DATASET_FILES[osw_file]['geometry']) | (
+                                OSW_DATASET[osw_file].is_valid == False)]
                 is_valid = len(invalid_geojson) == 0
                 if not is_valid:
-                    self.errors.append(f"Invalid {osw_file} geometries found, id's of invalid geometries: {set(invalid_geojson['_id'])}")
+                    self.errors.append(
+                        f"Invalid {osw_file} geometries found, id's of invalid geometries: {set(invalid_geojson['_id'])}")
 
             # Validate OSW external extensions
             for file in validator.externalExtensions:
@@ -132,7 +140,8 @@ def validate(self, max_errors=20) -> ValidationResult:
                 invalid_geojson = extensionFile[extensionFile.is_valid == False]
                 is_valid = len(invalid_geojson) == 0
                 if not is_valid:
-                    self.errors.append(f"Invalid geometries found in extension file {file}, list of invalid geometries: {invalid_geojson.to_json()}")
+                    self.errors.append(
+                        f"Invalid geometries found in extension file {file}, list of invalid geometries: {invalid_geojson.to_json()}")
 
             if self.errors:
                 return ValidationResult(False, self.errors)
@@ -145,6 +154,16 @@ def validate(self, max_errors=20) -> ValidationResult:
             del OSW_DATASET
             if zip_handler:
                 zip_handler.remove_extracted_files()
+
+            # Force garbage collection to free memory
+            gc.collect()
+
+            # Additional memory cleanup for geopandas dataframes
+            if validator:
+                for osw_file in validator.files:
+                    if osw_file in locals():
+                        del osw_file
+                del validator
             gc.collect()
 
     def load_osw_file(self, graph_geojson_path: str) -> Dict[str, Any]:
@@ -155,7 +174,7 @@ def load_osw_file(self, graph_geojson_path: str) -> Dict[str, Any]:
     def validate_osw_errors(self, file_path: str, max_errors: int) -> bool:
         """Validate OSW Data against the schema and process all errors"""
         geojson_data = self.load_osw_file(file_path)
-        validator = jsonschema.Draft7Validator(self.load_osw_schema(self.schema_file_path))
+        validator = jsonschema_rs.Draft7Validator(self.load_osw_schema(self.schema_file_path))
 
         for error in validator.iter_errors(geojson_data):
             self.errors.append(f'Validation error: {error.message}')

src/python_osw_validation/extracted_data_validator.py

@@ -1,4 +1,5 @@
 import os
+import gc
 import glob
 
 OSW_DATASET_FILES = {
@@ -57,44 +58,55 @@ def is_valid(self) -> bool:
         optional_files = [key for key, value in OSW_DATASET_FILES.items() if not value['required']]
         missing_files = []
         duplicate_files = []
-        for required_file in required_files:
-            file_count = 0
-            for filename in geojson_files:
-                base_name = os.path.basename(filename)
-                if required_file in base_name and base_name.endswith('.geojson'):
-                    file_count += 1
-                    save_filename = filename
-            if file_count == 0:
-                # Missing required file
-                missing_files.append(required_file)
-            elif file_count == 1:
-                self.files.append(save_filename)
-            else:
-                # Duplicate file
-                duplicate_files.append(required_file)
+        save_filename = None  # Initialize this variable
 
-        for optional_file in optional_files:
-            file_count = 0
-            for filename in geojson_files:
-                base_name = os.path.basename(filename)
-                if optional_file in base_name and base_name.endswith('.geojson'):
-                    file_count += 1
-                    save_filename = filename
-            if file_count == 1:
-                self.files.append(save_filename)
-            elif file_count > 1:
-                # Duplicate file
-                duplicate_files.append(optional_file)
+        try:
+            # Process required files
+            for required_file in required_files:
+                file_count = 0
+                for filename in geojson_files:
+                    base_name = os.path.basename(filename)
+                    if required_file in base_name and base_name.endswith('.geojson'):
+                        file_count += 1
+                        save_filename = filename
+                if file_count == 0:
+                    # Missing required file
+                    missing_files.append(required_file)
+                elif file_count == 1:
+                    self.files.append(save_filename)
+                else:
+                    # Duplicate file
+                    duplicate_files.append(required_file)
 
-        if missing_files:
-            self.error = f'Missing required .geojson files: {", ".join(missing_files)}.'
-            return False
+            # Process optional files
+            for optional_file in optional_files:
+                file_count = 0
+                for filename in geojson_files:
+                    base_name = os.path.basename(filename)
+                    if optional_file in base_name and base_name.endswith('.geojson'):
+                        file_count += 1
+                        save_filename = filename
+                if file_count == 1:
+                    self.files.append(save_filename)
+                elif file_count > 1:
+                    # Duplicate file
+                    duplicate_files.append(optional_file)
 
-        if duplicate_files:
-            self.error = f'Multiple .geojson files of the same type found: {", ".join(duplicate_files)}.'
-            return False
+            # Check for missing or duplicate files
+            if missing_files:
+                self.error = f'Missing required .geojson files: {", ".join(missing_files)}.'
+                return False
+
+            if duplicate_files:
+                self.error = f'Multiple .geojson files of the same type found: {", ".join(duplicate_files)}.'
+                return False
+
+            # Add OSW external extensions, GeoJSON files we know nothing about
+            self.externalExtensions.extend([item for item in geojson_files if item not in self.files])
 
-        # Add OSW external extensions, GeoJSON files we know nothing about
-        self.externalExtensions.extend([item for item in geojson_files if item not in self.files])
+        finally:
+            # Cleanup large lists and call garbage collector
+            del geojson_files, required_files, optional_files, missing_files, duplicate_files
+            gc.collect()
 
         return True

src/python_osw_validation/version.py

@@ -1 +1 @@
-__version__ = '0.2.6'
+__version__ = '0.2.7'