Merge pull request #8 from TaskarCenterAtUW/develop

version 0.0.5

Author: sujata-m

CHANGELOG.md

@@ -8,4 +8,13 @@
 - Updated README file
 
 ### 0.0.3
-- Added schema file to package
+- Added schema file to package
+
+### 0.0.4
+- Points are not required for a valid OSW dataset
+
+### 0.0.5
+- Support for multi-level geojson file
+- Now handles the following two folder structures when unzipped abc.zip
+  1. abc\{nodes, edges, points}.geojson
+  2. {nodes, edges, points}.geojson

README.md

@@ -11,8 +11,8 @@ This package validates the OSW geojson file. Package requires a OSW zip file pat
 ## What this package does?
 
 - It unzip the provided zip files
-- Check for nodes, edges and points geojson files inside the unzipped folder
-- Validate each file against schema, schema can be found here
+- Check for the required nodes and edges geojson files inside the unzipped folder
+- Validate each file (nodes, edges and points) against schema, schema can be found here
 - Return true or false according to validation
 - you can check the error if it returned false.
 
@@ -73,6 +73,9 @@ test_invalid_geometry_zipfile (test_osw_validation.TestOSWValidation) ... ok
 test_invalid_zipfile (test_osw_validation.TestOSWValidation) ... ok
 test_invalid_zipfile_with_invalid_schema (test_osw_validation.TestOSWValidation) ... ok
 test_invalid_zipfile_with_schema (test_osw_validation.TestOSWValidation) ... ok
+test_minimal_zipfile (test_osw_validation.TestOSWValidation) ... ok
+test_minimal_zipfile_with_invalid_schema (test_osw_validation.TestOSWValidation) ... ok
+test_minimal_zipfile_with_schema (test_osw_validation.TestOSWValidation) ... ok
 test_missing_files_inside_zipfile (test_osw_validation.TestOSWValidation) ... ok
 test_missing_files_inside_zipfile_with_invalid_schema (test_osw_validation.TestOSWValidation) ... ok
 test_missing_files_inside_zipfile_with_schema (test_osw_validation.TestOSWValidation) ... ok
@@ -93,7 +96,7 @@ test_extract_valid_zip (test_zipfile_handler.TestZipFileHandler) ... ok
 test_remove_extracted_files (test_zipfile_handler.TestZipFileHandler) ... ok
 
 ----------------------------------------------------------------------
-Ran 31 tests in 193.358s
+Ran 34 tests in 121.220s
 
 OK
 ```

freeze_version.py

@@ -11,7 +11,7 @@
 
 build_date = date.today().strftime('%Y-%m-%d')
 
-version = '0.0.3'
+version = '0.0.5'
 
 with open(version_file_path, 'w+') as version_file:
     version_file.write("version = '{}'\n".format(version))

src/python_osw_validation/extracted_data_validator.py

@@ -9,26 +9,39 @@ def __init__(self, extracted_dir: str):
         self.error = None
 
     def is_valid(self) -> bool:
-        if not os.path.exists(self.extracted_dir) or not os.listdir(self.extracted_dir):
-            self.error = 'Folder is empty or does not exist.'
+        # Check if the directory exists
+        if not os.path.exists(self.extracted_dir):
+            self.error = 'Directory does not exist.'
             return False
 
+        # Look for required files at the root level
         geojson_files = glob.glob(os.path.join(self.extracted_dir, '*.geojson'))
-        if len(geojson_files) < 3:
-            self.error = 'There are not enough .geojson files in the folder.'
+
+        # If not found at the root, check inside folders
+        if not geojson_files:
+            geojson_files = glob.glob(os.path.join(self.extracted_dir, '*', '*.geojson'))
+
+        if not geojson_files:
+            self.error = 'No .geojson files found in the specified directory or its subdirectories.'
             return False
 
+        required_files = {'nodes', 'edges'}
+        optional_files = {'points'}
         for filename in geojson_files:
             base_name = os.path.basename(filename)
-            if 'edges' in base_name and base_name.endswith('.geojson'):
-                self.files.append(filename)
-            elif 'nodes' in base_name and base_name.endswith('.geojson'):
-                self.files.append(filename)
-            elif 'points' in base_name and base_name.endswith('.geojson'):
-                self.files.append(filename)
-
-        if len(self.files) != 3:
-            self.error = 'Missing one or more required .geojson files.'
+            for required_file in required_files:
+                if required_file in base_name and base_name.endswith('.geojson'):
+                    self.files.append(filename)
+                    required_files.remove(required_file)
+                    break
+            for optional_file in optional_files:
+                if optional_file in base_name and base_name.endswith('.geojson'):
+                    self.files.append(filename)
+                    optional_files.remove(optional_file)
+                    break
+
+        if required_files:
+            self.error = f'Missing required .geojson files: {", ".join(required_files)}.'
             return False
 
         return True

tests/assets/minimal.zip

@@ -6,62 +6,63 @@
 
 
 class TestExtractedDataValidator(unittest.TestCase):
-
     def setUp(self):
-        self.valid_dir = tempfile.mkdtemp()
-        self.invalid_empty_dir = tempfile.mkdtemp()
-        self.invalid_missing_files_dir = tempfile.mkdtemp()
-        self.invalid_missing_required_files_dir = tempfile.mkdtemp()
+        # Create a temporary directory for testing
+        self.test_dir = tempfile.mkdtemp()
 
     def tearDown(self):
-        shutil.rmtree(self.valid_dir)
-        shutil.rmtree(self.invalid_empty_dir)
-        shutil.rmtree(self.invalid_missing_files_dir)
-        shutil.rmtree(self.invalid_missing_required_files_dir)
+        # Remove the temporary directory and its contents
+        shutil.rmtree(self.test_dir)
 
-    def create_valid_directory_structure(self):
-        # Create a valid directory structure with 3 geojson files
-        valid_files = ['edges.geojson', 'nodes.geojson', 'points.geojson']
-        for file in valid_files:
-            open(os.path.join(self.valid_dir, file), 'w').close()
+    def create_files(self, files):
+        for file in files:
+            with open(os.path.join(self.test_dir, file), 'w') as f:
+                f.write('Test content')
 
-    def create_invalid_empty_directory(self):
-        # Create an empty directory
-        pass
+    def test_valid_data_at_root(self):
+        # Test when required files are at the root level
+        validator = ExtractedDataValidator(self.test_dir)
+        self.create_files(['a.nodes.geojson', 'a.edges.geojson', 'a.points.geojson'])
+        self.assertTrue(validator.is_valid())
 
-    def create_invalid_missing_files_directory(self):
-        # Create a directory with only one geojson file
-        open(os.path.join(self.invalid_missing_files_dir, 'edges.geojson'), 'w').close()
+    def test_valid_data_inside_folder(self):
+        # Test when required files are inside a folder
+        validator = ExtractedDataValidator(self.test_dir)
+        os.makedirs(os.path.join(self.test_dir, 'abc'))
+        self.create_files(['abc/a.nodes.geojson', 'abc/a.edges.geojson', 'abc/a.points.geojson'])
+        self.assertTrue(validator.is_valid())
 
-    def create_invalid_missing_required_files_directory(self):
-        # Create a directory with three geojson files, but not all required ones
-        invalid_files = ['edges.geojson', 'invalid.geojson', 'nodes.geojson']
-        for file in invalid_files:
-            open(os.path.join(self.invalid_missing_required_files_dir, file), 'w').close()
+    def test_missing_required_files(self):
+        # Test when one of the required files is missing
+        validator = ExtractedDataValidator(self.test_dir)
+        self.create_files(['a.nodes.geojson', 'a.points.geojson'])
+        self.assertFalse(validator.is_valid())
+        self.assertEqual(validator.error, 'Missing required .geojson files: edges.')
 
-    def test_valid_directory_structure(self):
-        self.create_valid_directory_structure()
-        validator = ExtractedDataValidator(self.valid_dir)
+    def test_missing_optional_file(self):
+        # Test when optional file is missing
+        validator = ExtractedDataValidator(self.test_dir)
+        self.create_files(['a.nodes.geojson', 'a.edges.geojson'])
         self.assertTrue(validator.is_valid())
-        self.assertEqual(len(validator.files), 3)
 
-    def test_invalid_empty_directory(self):
-        self.create_invalid_empty_directory()
-        validator = ExtractedDataValidator(self.invalid_empty_dir)
+    def test_no_geojson_files(self):
+        # Test when no .geojson files are present
+        validator = ExtractedDataValidator(self.test_dir)
+        self.create_files(['some.txt', 'another.txt'])
         self.assertFalse(validator.is_valid())
-        self.assertEqual(validator.error, 'Folder is empty or does not exist.')
+        self.assertEqual(validator.error, 'No .geojson files found in the specified directory or its subdirectories.')
 
-    def test_invalid_missing_files_directory(self):
-        self.create_invalid_missing_files_directory()
-        validator = ExtractedDataValidator(self.invalid_missing_files_dir)
+    def test_invalid_directory(self):
+        # Test when the specified directory does not exist
+        validator = ExtractedDataValidator('nonexistent_directory')
         self.assertFalse(validator.is_valid())
-        self.assertEqual(validator.error, 'There are not enough .geojson files in the folder.')
+        self.assertEqual(validator.error, 'Directory does not exist.')
 
-    def test_invalid_missing_required_files_directory(self):
-        self.create_invalid_missing_required_files_directory()
-        validator = ExtractedDataValidator(self.invalid_missing_required_files_dir)
+    def test_empty_directory(self):
+        # Test when the specified directory is empty
+        validator = ExtractedDataValidator(self.test_dir)
         self.assertFalse(validator.is_valid())
-        self.assertEqual(validator.error, 'Missing one or more required .geojson files.')
+        self.assertEqual(validator.error, 'No .geojson files found in the specified directory or its subdirectories.')
 
 
 if __name__ == '__main__':

tests/unit_tests/test_extracted_data_validator.py

@@ -14,6 +14,7 @@ class TestOSWValidation(unittest.TestCase):
 
     def setUp(self):
         self.valid_zipfile = os.path.join(ASSETS_PATH, 'valid.zip')
+        self.minimal_zipfile = os.path.join(ASSETS_PATH, 'minimal.zip')
         self.invalid_zipfile = os.path.join(ASSETS_PATH, 'invalid.zip')
         self.nodes_invalid_zipfile = os.path.join(ASSETS_PATH, 'nodes_invalid.zip')
         self.edges_invalid_zipfile = os.path.join(ASSETS_PATH, 'edges_invalid.zip')
@@ -46,6 +47,23 @@ def test_valid_zipfile_with_invalid_schema(self):
         result = validation.validate()
         self.assertTrue(len(result.errors) > 0)
 
+    def test_minimal_zipfile(self):
+        validation = OSWValidation(zipfile_path=self.minimal_zipfile)
+        result = validation.validate()
+        self.assertTrue(result.is_valid)
+        self.assertIsNone(result.errors)
+
+    def test_minimal_zipfile_with_schema(self):
+        validation = OSWValidation(zipfile_path=self.minimal_zipfile, schema_file_path=self.schema_file_path)
+        result = validation.validate()
+        self.assertTrue(result.is_valid)
+        self.assertIsNone(result.errors)
+
+    def test_minimal_zipfile_with_invalid_schema(self):
+        validation = OSWValidation(zipfile_path=self.minimal_zipfile, schema_file_path=self.invalid_schema_file_path)
+        result = validation.validate()
+        self.assertTrue(len(result.errors) > 0)
+
     def test_invalid_zipfile(self):
         validation = OSWValidation(zipfile_path=self.invalid_zipfile)
         result = validation.validate()