Expanded tags and packets to accept any Python data type and use Arrow-based datastore

.gitignore

@@ -5,6 +5,10 @@ notebooks/**/*.parquet
 notebooks/**/*.pkl
 notebooks/**/*.db
 
+
+# Ignore any notebook that starts with an underscore
+notebooks/**/_*.ipynb
+
 # Ignore vscode settings
 .vscode/
 

pyproject.toml

@@ -15,6 +15,7 @@ dependencies = [
     "pyyaml>=6.0.2",
     "pyarrow>=20.0.0",
     "polars>=1.30.0",
+    "beartype>=0.21.0",
 ]
 readme = "README.md"
 requires-python = ">=3.10"

src/orcabridge/__init__.py

@@ -1,9 +1,11 @@
-from . import hashing, mappers, pod, sources, store, streams
-from .mappers import Join, MapPackets, MapTags, packet, tag
+from .core import operators, sources, streams
+from .core.streams import SyncStreamFromLists, SyncStreamFromGenerator
+from . import hashing, pod, store
+from .core.operators import Join, MapPackets, MapTags, packet, tag
 from .pod import FunctionPod, function_pod
-from .sources import GlobSource
+from .core.sources import GlobSource
 from .store import DirDataStore, SafeDirDataStore
-from .pipeline import GraphTracker
+from .core.tracker import GraphTracker
 
 DEFAULT_TRACKER = GraphTracker()
 DEFAULT_TRACKER.activate()
@@ -13,8 +15,7 @@
     "hashing",
     "store",
     "pod",
-    "dir_data_store",
-    "mappers",
+    "operators",
     "streams",
     "sources",
     "MapTags",
@@ -29,4 +30,5 @@
     "SafeDirDataStore",
     "DEFAULT_TRACKER",
     "SyncStreamFromLists",
+    "SyncStreamFromGenerator",
 ]