- Added garbage collector

- Updated incline in batch of 1000 and after each batch freeing some memory

Author: sujata-m

src/osw_incline/__init__.py

@@ -7,6 +7,7 @@
 from .version import __version__
 from .dem_processor import DEMProcessor
 
+
 class OSWIncline:
     def __init__(self, dem_files: List[str], nodes_file: str, edges_file: str, debug=False):
         self.dem_files = dem_files
@@ -16,7 +17,6 @@ def __init__(self, dem_files: List[str], nodes_file: str, edges_file: str, debug
         if self.debug:
             Logger.debug('Debug mode is enabled')
 
-
     def calculate(self):
         try:
             if self.debug:
@@ -35,10 +35,13 @@ def calculate(self):
                 nodes_path=graph_nodes_path,
                 edges_path=graph_edges_path
             )
+
+            # Delete osm_graph and dem_processor to force garbage collection
+            del osm_graph, dem_processor
+            gc.collect()
+
             end_time = time.time()
             time_taken = end_time - start_time
-            del osm_graph
-            del dem_processor
             if self.debug:
                 Logger.info(f'Entire processing took: {time_taken} seconds')
             return True
@@ -50,5 +53,4 @@ def calculate(self):
             gc.collect()
 
 
-
-OSWIncline.__version__ = __version__
+OSWIncline.__version__ = __version__

src/osw_incline/dem_processor.py

@@ -23,24 +23,17 @@ def __init__(self, osm_graph: OSMGraph, dem_files: List[str], debug=False):
         self.debug = debug
 
     def process(self, nodes_path, edges_path):
+        gc.disable()
         for dem_file in self.dem_files:
             dem_file_path = Path(dem_file)
             if self.debug:
                 Logger.debug(f'Processing DEM tile: {dem_file_path}')
 
             try:
                 with rasterio.open(dem_file_path) as dem:
-                    for u, v, d in self.OG.G.edges(data=True):
-                        if 'geometry' in d:
-                            incline = self.infer_incline(linestring=d['geometry'], dem=dem, precision=3)
-                            if incline is not None:
-                                # Add incline to the edge properties
-                                d['incline'] = incline
-                        else:
-                            if self.debug:
-                                Logger.info(f'No geometry found for edge {u}-{v}')
-
-                dem.close()
+                    edges = list(self.OG.G.edges(data=True))  # Get all edges, even if fewer than batch_size
+                    self._process_in_batches(edges, dem, batch_size=1000)
+
                 self.OG.to_geojson(nodes_path, edges_path)
             except rasterio.errors.RasterioIOError:
                 if self.debug:
@@ -53,8 +46,19 @@ def process(self, nodes_path, edges_path):
             finally:
                 gc.collect()
 
-        del self.OG
-        gc.collect()
+        gc.disable()
+
+    def _process_in_batches(self, edges, dem, batch_size=1000):
+        # Process edges in batches
+        for i in range(0, len(edges), batch_size):
+            batch = edges[i:i + batch_size]
+            for u, v, d in batch:
+                if 'geometry' in d:
+                    incline = self.infer_incline(linestring=d['geometry'], dem=dem, precision=3)
+                    if incline is not None:
+                        d['incline'] = incline
+            # Trigger garbage collection after each batch
+            gc.collect()
 
     def infer_incline(self, linestring, dem, precision=3):
         first_point = linestring.coords[0]
@@ -161,6 +165,7 @@ def interpolated_value(self, x, y, dem, method='idw', scaling_factor=1.0):
 
         interpolated = interpolator(dx, dy, dem_arr)
 
+        del dem_arr
         if interpolated is None:
             return interpolated
         else:
@@ -194,6 +199,7 @@ def idw(self, dx, dy, masked_array):
 
         value = weighted_values.sum()
 
+        del xs, ys, values_masked, weighted_values
 
         if np.isnan(value):
             return None
@@ -217,4 +223,4 @@ def bivariate_spline(self, dx, dy, arr):
         spline = RectBivariateSpline(
             np.array(range(ncol)), np.array(range(nrow)), arr, kx=kx, ky=ky
         )
-        return spline(dx, dy)[0][0]
+        return spline(dx, dy)[0][0]

src/osw_incline/osm_graph.py

@@ -4,6 +4,7 @@
 import networkx as nx
 from shapely.geometry import shape, mapping
 
+SCHEMA = 'https://sidewalks.washington.edu/opensidewalks/0.2/schema.json'
 
 class OSMGraph:
     def __init__(self, G=None):
@@ -30,14 +31,16 @@ def from_geojson(cls, nodes_path, edges_path):
             props['geometry'] = shape(node_feature['geometry'])
             G.add_node(n, **props)
 
+        del nodes_fc
+        gc.collect()
+
         for edge_feature in edges_fc['features']:
             props = edge_feature['properties']
             u = props.pop('_u_id')
             v = props.pop('_v_id')
             props['geometry'] = shape(edge_feature['geometry'])
             G.add_edge(u, v, **props)
 
-        del nodes_fc
         del edges_fc
         gc.collect()
 
@@ -46,15 +49,6 @@ def from_geojson(cls, nodes_path, edges_path):
     def to_geojson(self, *args):
         nodes_path = args[0]
         edges_path = args[1]
-
-        # Load the original files to retain the original top-level keys
-        with open(nodes_path) as f:
-            original_nodes_fc = json.load(f)
-
-        with open(edges_path) as f:
-            original_edges_fc = json.load(f)
-
-        # Process the edges
         edge_features = []
         for u, v, d in self.G.edges(data=True):
             d_copy = {**d}
@@ -72,11 +66,19 @@ def to_geojson(self, *args):
                 'geometry': geometry,
                 'properties': d_copy
             })
+        edges_fc = {
+            'type': 'FeatureCollection',
+            'features': edge_features,
+            '$schema': SCHEMA
+        }
 
-        # Update the original edges feature collection
-        original_edges_fc['features'] = edge_features
+        with open(edges_path, 'w') as f:
+            json.dump(edges_fc, f)
+
+        # Delete edge_features and force garbage collection
+        del edge_features, edges_fc
+        gc.collect()
 
-        # Process the nodes
         node_features = []
         for n, d in self.G.nodes(data=True):
             d_copy = {**d}
@@ -99,18 +101,19 @@ def to_geojson(self, *args):
                     'geometry': geometry,
                     'properties': d_copy
                 })
-
-        # Update the original nodes feature collection
-        original_nodes_fc['features'] = node_features
-
-        # Write the updated nodes and edges to the files
-        with open(edges_path, 'w') as f:
-            json.dump(original_edges_fc, f)
+        nodes_fc = {
+            'type': 'FeatureCollection',
+            'features': node_features,
+            '$schema': SCHEMA
+        }
 
         with open(nodes_path, 'w') as f:
-            json.dump(original_nodes_fc, f)
+            json.dump(nodes_fc, f)
+
+        # Delete node_features and force garbage collection
+        del node_features, nodes_fc
+        gc.collect()
 
-        # Handle points if the third argument (points_path) is provided
         if len(args) == 3:
             points_path = args[2]
             point_features = []
@@ -137,10 +140,15 @@ def to_geojson(self, *args):
                         'geometry': geometry,
                         'properties': d_copy
                     })
+            points_fc = {
+                'type': 'FeatureCollection',
+                'features': point_features,
+                '$schema': SCHEMA
+            }
 
             with open(points_path, 'w') as f:
-                json.dump({'type': 'FeatureCollection', 'features': point_features}, f)
+                json.dump(points_fc, f)
 
-        del original_nodes_fc
-        del original_edges_fc
-        gc.collect()
+            # Delete point_features and force garbage collection
+            del point_features, points_fc
+            gc.collect()