[FIX] solve import issue, isolate dependency and requirements (#135)

* update import issue; isolate env and evaluator; isolate agent and model

* fix habitat import; seperate requirements

* fix req

* fix req; add simple agent

* add depth image

* fix requires

* model READY, test agent runable

* update requires

* key fix to resolve parse issue, remove run parse when import server

* update habitat scripts

* requires ready

Author: kew6688

.gitignore

@@ -150,3 +150,5 @@ logs/
 /results/
 checkpoints
 internnav/model/basemodel/LongCLIP/
+.gradio/
+result/

.pre-commit-config.yaml

@@ -37,7 +37,7 @@ repos:
       - id: trailing-whitespace
       - id: check-yaml
       - id: end-of-file-fixer
-      - id: requirements-txt-fixer
+      # - id: requirements-txt-fixer
       - id: check-merge-conflict
       - id: fix-encoding-pragma
         args: ["--remove"]

internnav/agent/base.py

@@ -10,10 +10,10 @@ def __init__(self, config: AgentCfg):
         self.config = config
 
     def step(self, obs: Dict[str, Any]):
-        pass
+        raise NotImplementedError("This function is not implemented yet.")
 
     def reset(self):
-        pass
+        raise NotImplementedError("This function is not implemented yet.")
 
     @classmethod
     def register(cls, agent_type: str):

internnav/agent/rdp_agent.py

@@ -9,7 +9,6 @@
 from internnav.configs.agent import AgentCfg
 from internnav.configs.model.base_encoders import ModelCfg
 from internnav.model import get_config, get_policy
-from internnav.model.basemodel.LongCLIP.model import longclip
 from internnav.model.basemodel.rdp.utils import (
     FixedLengthStack,
     compute_actions,
@@ -19,7 +18,6 @@
     quat_to_euler_angles,
     to_local_coords_batch,
 )
-from internnav.model.utils.bert_token import BertTokenizer
 from internnav.model.utils.feature_extract import (
     extract_image_features,
     extract_instruction_tokens,
@@ -67,13 +65,17 @@ def __init__(self, config: AgentCfg):
 
         if self.use_clip_encoders:
             if self._model_settings.text_encoder.type == 'roberta':
+                from internnav.model.utils.bert_token import BertTokenizer
+
                 self.bert_tokenizer = BertTokenizer(
                     max_length=self._model_settings.instruction_encoder.max_length,
                     load_model=self._model_settings.instruction_encoder.load_model,
                     device=self.device,
                 )
                 self.use_bert = True
             elif self._model_settings.text_encoder.type == 'clip-long':
+                from internnav.model.basemodel.LongCLIP.model import longclip
+
                 self.bert_tokenizer = longclip.tokenize
                 self.use_bert = True
                 self.is_clip_long = True

internnav/agent/simple_agent.py

@@ -0,0 +1,54 @@
+import time
+from typing import Any, Dict
+
+import torch
+
+from internnav.agent import Agent
+from internnav.configs.agent import AgentCfg
+from internnav.model import get_config, get_policy
+
+
+class SimpleAgent(Agent):
+    """
+    agent template, override the functions for custom policy
+    """
+
+    def __init__(self, agent_config: AgentCfg):
+        self.agent_config = agent_config
+        self.device = torch.device('cuda', 0)
+
+        # get policy by name
+        policy = get_policy(agent_config.model_settings.policy_name)
+
+        # load policy checkpoints
+        self.policy = policy.from_pretrained(
+            agent_config.ckpt_path,
+            config=get_config(agent_config.model_settings.policy_name)(
+                model_cfg={'model': agent_config.model_settings.model_dump()}
+            ),
+        ).to(self.device)
+
+    def convert_input(self, obs):
+        return obs
+
+    def convert_output(self, action):
+        return action
+
+    def inference(self, input):
+        return self.policy(input)
+
+    def step(self, obs: Dict[str, Any]):
+        print(f'{self.config.model_name} Agent step')
+        start = time.time()
+
+        # convert obs to model input
+        obs = self.convert_input(obs)
+        action = self.inference(obs)
+        action = self.convert_output(action)
+
+        end = time.time()
+        print(f'time: {round(end-start, 4)}s')
+        return action
+
+    def reset(self):
+        pass

…av/evaluator/utils/vln_default_config.py …/configs/evaluator/vln_default_config.pyinternnav/evaluator/utils/vln_default_config.py renamed to internnav/configs/evaluator/vln_default_config.py internnav/evaluator/utils/vln_default_config.py renamed to internnav/configs/evaluator/vln_default_config.py

@@ -0,0 +1 @@
+from .vln_camera import VLNCameraCfg

internnav/configs/trainer/__init__.py

@@ -0,0 +1 @@
+from .vln_eval_task import VLNEvalTaskCfg

internnav/env/utils/internutopia_extension/configs/sensors/__init__.py

@@ -6,7 +6,6 @@
 from collections import defaultdict
 
 import numpy as np
-from internutopia.core.util import is_in_container
 from PIL import Image, ImageDraw
 from scipy.ndimage import binary_dilation
 
@@ -243,6 +242,8 @@ def load_data(dataset_root_dir, split, filter_same_trajectory=True, filter_stair
 
 def load_scene_usd(mp3d_data_dir, scan):
     """Load scene USD based on the scan"""
+    from internutopia.core.util import is_in_container
+
     find_flag = False
     for root, dirs, files in os.walk(os.path.join(mp3d_data_dir, scan)):
         target_file_name = 'fixed_docker.usd' if is_in_container() else 'fixed.usd'