Drop config.yml->repos and LocalRepo support (#3352)

Author: un-def

docs/docs/reference/api/python/index.md

@@ -14,7 +14,7 @@ from dstack.api import Task, GPU, Client, Resources
 client = Client.from_config()
 
 task = Task(
-    name="my-awesome-run",  # If not specified, a random name is assigned 
+    name="my-awesome-run",  # If not specified, a random name is assigned
     image="ghcr.io/huggingface/text-generation-inference:latest",
     env={"MODEL_ID": "TheBloke/Llama-2-13B-chat-GPTQ"},
     commands=[
@@ -42,7 +42,7 @@ finally:
 ```
 
 !!! info "NOTE:"
-    1. The `configuration` argument in the `apply_configuration` method can be either `dstack.api.Task`, `dstack.api.Service`, or `dstack.api.DevEnvironment`. 
+    1. The `configuration` argument in the `apply_configuration` method can be either `dstack.api.Task`, `dstack.api.Service`, or `dstack.api.DevEnvironment`.
     2. When you create `dstack.api.Task`, `dstack.api.Service`, or `dstack.api.DevEnvironment`, you can specify the `image` argument. If `image` isn't specified, the default image will be used. For a private Docker registry, ensure you also pass the `registry_auth` argument.
     3. The `repo` argument in the `apply_configuration` method allows the mounting of a local folder, a remote repo, or a
        programmatically created repo. In this case, the `commands` argument can refer to the files within this repo.
@@ -173,15 +173,6 @@ finally:
         memory: dstack.api.Memory
         Range: dstack.api.Range
 
-### `dstack.api.LocalRepo` { #dstack.api.LocalRepo data-toc-label="LocalRepo" }
-
-::: dstack.api.LocalRepo
-    options:
-      show_bases: false
-      show_root_heading: false
-      show_root_toc_entry: false
-      heading_level: 4
-
 ### `dstack.api.RemoteRepo` { #dstack.api.RemoteRepo data-toc-label="RemoteRepo" }
 
 ::: dstack.api.RemoteRepo

src/dstack/_internal/cli/commands/init.py

@@ -6,13 +6,11 @@
 from dstack._internal.cli.commands import BaseCommand
 from dstack._internal.cli.services.repos import (
     get_repo_from_dir,
+    get_repo_from_url,
     is_git_repo_url,
     register_init_repo_args,
 )
-from dstack._internal.cli.utils.common import confirm_ask, console, warn
-from dstack._internal.core.errors import ConfigurationError
-from dstack._internal.core.models.repos.remote import RemoteRepo
-from dstack._internal.core.services.configs import ConfigManager
+from dstack._internal.cli.utils.common import console
 from dstack.api import Client
 
 
@@ -36,20 +34,6 @@ def _register(self):
             dest="repo",
         )
         register_init_repo_args(self._parser)
-        # Deprecated since 0.19.25, ignored
-        self._parser.add_argument(
-            "--ssh-identity",
-            metavar="SSH_PRIVATE_KEY",
-            help=argparse.SUPPRESS,
-            type=Path,
-            dest="ssh_identity_file",
-        )
-        # A hidden mode for transitional period only, remove it with repos in `config.yml`
-        self._parser.add_argument(
-            "--remove",
-            help=argparse.SUPPRESS,
-            action="store_true",
-        )
 
     def _command(self, args: argparse.Namespace):
         super()._command(args)
@@ -65,45 +49,10 @@ def _command(self, args: argparse.Namespace):
         else:
             repo_path = Path.cwd()
 
-        if args.remove:
-            if repo_url is not None:
-                raise ConfigurationError(f"Local path expected, got URL: {repo_url}")
-            assert repo_path is not None
-            config_manager = ConfigManager()
-            repo_config = config_manager.get_repo_config(repo_path)
-            if repo_config is None:
-                raise ConfigurationError("Repo record not found, nothing to remove")
-            console.print(
-                f"You are about to remove the repo {repo_path}\n"
-                "Only the record about the repo will be removed,"
-                " the repo files will remain intact\n"
-            )
-            if not confirm_ask("Remove the repo?"):
-                return
-            config_manager.delete_repo_config(repo_config.repo_id)
-            config_manager.save()
-            console.print("Repo has been removed")
-            return
-
-        local: bool = args.local
-        if local:
-            warn(
-                "Local repos are deprecated since 0.19.25 and will be removed soon. Consider"
-                " using [code]files[/code] instead: https://dstack.ai/docs/concepts/tasks/#files"
-            )
-        if args.ssh_identity_file:
-            warn(
-                "[code]--ssh-identity[/code] in [code]dstack init[/code] is deprecated and ignored"
-                " since 0.19.25. Use this option with [code]dstack apply[/code]"
-                " and [code]dstack attach[/code] instead"
-            )
-
         if repo_url is not None:
-            # Dummy repo branch to avoid autodetection that fails on private repos.
-            # We don't need branch/hash for repo_id anyway.
-            repo = RemoteRepo.from_url(repo_url, repo_branch="master")
+            repo = get_repo_from_url(repo_url)
         elif repo_path is not None:
-            repo = get_repo_from_dir(repo_path, local=local)
+            repo = get_repo_from_dir(repo_path)
         else:
             assert False, "should not reach here"
         api = Client.from_config(project_name=args.project)

src/dstack/_internal/cli/services/configurators/run.py

@@ -18,11 +18,12 @@
 from dstack._internal.cli.services.profile import apply_profile_args, register_profile_args
 from dstack._internal.cli.services.repos import (
     get_repo_from_dir,
+    get_repo_from_url,
     init_default_virtual_repo,
     is_git_repo_url,
     register_init_repo_args,
 )
-from dstack._internal.cli.utils.common import confirm_ask, console, warn
+from dstack._internal.cli.utils.common import confirm_ask, console
 from dstack._internal.cli.utils.rich import MultiItemStatus
 from dstack._internal.cli.utils.run import get_runs_table, print_run_plan
 from dstack._internal.core.errors import (
@@ -44,17 +45,13 @@
     TaskConfiguration,
 )
 from dstack._internal.core.models.repos import RepoHeadWithCreds
-from dstack._internal.core.models.repos.base import Repo
-from dstack._internal.core.models.repos.local import LocalRepo
 from dstack._internal.core.models.repos.remote import RemoteRepo, RemoteRepoCreds
 from dstack._internal.core.models.resources import CPUSpec
 from dstack._internal.core.models.runs import JobStatus, JobSubmission, RunSpec, RunStatus
-from dstack._internal.core.services.configs import ConfigManager
 from dstack._internal.core.services.diff import diff_models
 from dstack._internal.core.services.repos import (
     InvalidRepoCredentialsError,
     get_repo_creds_and_default_branch,
-    load_repo,
 )
 from dstack._internal.utils.common import local_time
 from dstack._internal.utils.interpolator import InterpolatorError, VariablesInterpolator
@@ -96,8 +93,9 @@ def apply_configuration(
         if conf.working_dir is not None and not is_absolute_posix_path(conf.working_dir):
             raise ConfigurationError("working_dir must be absolute")
 
-        config_manager = ConfigManager()
-        repo = self.get_repo(conf, configuration_path, configurator_args, config_manager)
+        repo = self.get_repo(conf, configuration_path, configurator_args)
+        if repo is None:
+            repo = init_default_virtual_repo(api=self.api)
         profile = load_profile(Path.cwd(), configurator_args.profile)
         with console.status("Getting apply plan..."):
             run_plan = self.api.runs.get_run_plan(
@@ -475,12 +473,11 @@ def get_repo(
         conf: RunConfigurationT,
         configuration_path: str,
         configurator_args: argparse.Namespace,
-        config_manager: ConfigManager,
-    ) -> Repo:
+    ) -> Optional[RemoteRepo]:
         if configurator_args.no_repo:
-            return init_default_virtual_repo(api=self.api)
+            return None
 
-        repo: Optional[Repo] = None
+        repo: Optional[RemoteRepo] = None
         repo_head: Optional[RepoHeadWithCreds] = None
         repo_branch: Optional[str] = configurator_args.repo_branch
         repo_hash: Optional[str] = configurator_args.repo_hash
@@ -497,8 +494,6 @@ def get_repo(
         local_path: Optional[Path] = None
         # dummy value, safe to join with any path
         root_dir = Path(".")
-        # True if no repo specified, but we found one in `config.yml`
-        legacy_local_path = False
         if repo_arg := configurator_args.repo:
             if is_git_repo_url(repo_arg):
                 url = repo_arg
@@ -521,84 +516,49 @@ def get_repo(
             if repo_hash is None:
                 repo_hash = repo_spec.hash
         else:
-            local_path = Path.cwd()
-            legacy_local_path = True
+            return None
+
         if url:
-            repo = RemoteRepo.from_url(repo_url=url)
+            repo = get_repo_from_url(url)
             repo_head = self.api.repos.get(repo_id=repo.repo_id, with_creds=True)
         elif local_path:
-            if legacy_local_path:
-                if repo_config := config_manager.get_repo_config(local_path):
-                    repo = load_repo(repo_config)
-                    repo_head = self.api.repos.get(repo_id=repo.repo_id, with_creds=True)
-                    if repo_head is not None:
-                        warn(
-                            "The repo is not specified but found and will be used in the run\n"
-                            "Future versions will not load repos automatically\n"
-                            "To prepare for future versions and get rid of this warning:\n"
-                            "- If you need the repo in the run, either specify [code]repos[/code]"
-                            " in the configuration or use [code]--repo .[/code]\n"
-                            "- If you don't need the repo in the run, either run"
-                            " [code]dstack init --remove[/code] once (it removes only the record"
-                            " about the repo, the repo files will remain intact)"
-                            " or use [code]--no-repo[/code]"
-                        )
-                    else:
-                        # ignore stale entries in `config.yml`
-                        repo = None
-                        init = False
-            else:
-                original_local_path = local_path
-                local_path = local_path.expanduser()
-                if not local_path.is_absolute():
-                    local_path = (root_dir / local_path).resolve()
-                if not local_path.exists():
-                    raise ConfigurationError(
-                        f"Invalid repo path: {original_local_path} -> {local_path}"
-                    )
-                local: bool = configurator_args.local
-                repo = get_repo_from_dir(local_path, local=local)
-                repo_head = self.api.repos.get(repo_id=repo.repo_id, with_creds=True)
-            if isinstance(repo, RemoteRepo):
-                repo_branch = repo.run_repo_data.repo_branch
-                repo_hash = repo.run_repo_data.repo_hash
+            original_local_path = local_path
+            local_path = local_path.expanduser()
+            if not local_path.is_absolute():
+                local_path = (root_dir / local_path).resolve()
+            if not local_path.exists():
+                raise ConfigurationError(
+                    f"Invalid repo path: {original_local_path} -> {local_path}"
+                )
+            repo = get_repo_from_dir(local_path)
+            repo_head = self.api.repos.get(repo_id=repo.repo_id, with_creds=True)
+            repo_branch = repo.run_repo_data.repo_branch
+            repo_hash = repo.run_repo_data.repo_hash
         else:
             assert False, "should not reach here"
 
-        if repo is None:
-            return init_default_virtual_repo(api=self.api)
+        assert repo.repo_url is not None
 
-        if isinstance(repo, RemoteRepo):
-            assert repo.repo_url is not None
+        if repo_head is not None and repo_head.repo_creds is not None:
+            if git_identity_file is None and oauth_token is None:
+                git_private_key = repo_head.repo_creds.private_key
+                oauth_token = repo_head.repo_creds.oauth_token
+        else:
+            init = True
 
-            if repo_head is not None and repo_head.repo_creds is not None:
-                if git_identity_file is None and oauth_token is None:
-                    git_private_key = repo_head.repo_creds.private_key
-                    oauth_token = repo_head.repo_creds.oauth_token
-            else:
-                init = True
+        try:
+            repo_creds, _ = get_repo_creds_and_default_branch(
+                repo_url=repo.repo_url,
+                identity_file=git_identity_file,
+                private_key=git_private_key,
+                oauth_token=oauth_token,
+            )
+        except InvalidRepoCredentialsError as e:
+            raise CLIError(*e.args) from e
 
-            try:
-                repo_creds, default_repo_branch = get_repo_creds_and_default_branch(
-                    repo_url=repo.repo_url,
-                    identity_file=git_identity_file,
-                    private_key=git_private_key,
-                    oauth_token=oauth_token,
-                )
-            except InvalidRepoCredentialsError as e:
-                raise CLIError(*e.args) from e
-
-            if repo_branch is None and repo_hash is None:
-                if default_repo_branch is None:
-                    raise CLIError(
-                        "Failed to automatically detect remote repo branch."
-                        " Specify branch or hash."
-                    )
-                # TODO: remove in 0.20. Currently `default_repo_branch` is sent only for backward compatibility of `dstack-runner`.
-                repo_branch = default_repo_branch
-            repo.run_repo_data.repo_branch = repo_branch
-            if repo_hash is not None:
-                repo.run_repo_data.repo_hash = repo_hash
+        repo.run_repo_data.repo_branch = repo_branch
+        if repo_hash is not None:
+            repo.run_repo_data.repo_hash = repo_hash
 
         if init:
             self.api.repos.init(
@@ -608,15 +568,6 @@ def get_repo(
                 creds=repo_creds,
             )
 
-        if isinstance(repo, LocalRepo):
-            warn(
-                f"{repo.repo_dir} is a local repo\n"
-                "Local repos are deprecated since 0.19.25 and will be removed soon\n"
-                "There are two options:\n"
-                "- Migrate to [code]files[/code]: https://dstack.ai/docs/concepts/tasks/#files\n"
-                "- Specify [code]--no-repo[/code] if you don't need the repo at all"
-            )
-
         return repo
 
 

src/dstack/_internal/cli/services/repos.py

@@ -1,11 +1,9 @@
-import argparse
-from typing import Literal, Union, overload
+from pathlib import Path
 
 import git
 
 from dstack._internal.cli.services.configurators.base import ArgsParser
 from dstack._internal.core.errors import CLIError
-from dstack._internal.core.models.repos.local import LocalRepo
 from dstack._internal.core.models.repos.remote import GitRepoURL, RemoteRepo, RepoError
 from dstack._internal.core.models.repos.virtual import VirtualRepo
 from dstack._internal.utils.path import PathLike
@@ -28,12 +26,6 @@ def register_init_repo_args(parser: ArgsParser):
         type=str,
         dest="git_identity_file",
     )
-    # Deprecated since 0.19.25
-    parser.add_argument(
-        "--local",
-        action="store_true",
-        help=argparse.SUPPRESS,
-    )
 
 
 def init_default_virtual_repo(api: Client) -> VirtualRepo:
@@ -42,17 +34,12 @@ def init_default_virtual_repo(api: Client) -> VirtualRepo:
     return repo
 
 
-@overload
-def get_repo_from_dir(repo_dir: PathLike, local: Literal[False] = False) -> RemoteRepo: ...
-
-
-@overload
-def get_repo_from_dir(repo_dir: PathLike, local: Literal[True]) -> LocalRepo: ...
-
-
-def get_repo_from_dir(repo_dir: PathLike, local: bool = False) -> Union[RemoteRepo, LocalRepo]:
-    if local:
-        return LocalRepo.from_dir(repo_dir)
+def get_repo_from_dir(repo_dir: PathLike) -> RemoteRepo:
+    repo_dir = Path(repo_dir)
+    if not repo_dir.exists():
+        raise CLIError(f"Path does not exist: {repo_dir}")
+    if not repo_dir.is_dir():
+        raise CLIError(f"Path is not a directory: {repo_dir}")
     try:
         return RemoteRepo.from_dir(repo_dir)
     except git.InvalidGitRepositoryError:
@@ -61,6 +48,17 @@ def get_repo_from_dir(repo_dir: PathLike, local: bool = False) -> Union[RemoteRe
             "Use `files` to mount an arbitrary directory:"
             " https://dstack.ai/docs/concepts/tasks/#files"
         )
+    except git.GitError as e:
+        raise CLIError(f"{e.__class__.__name__}: {e}") from e
+    except RepoError as e:
+        raise CLIError(str(e)) from e
+
+
+def get_repo_from_url(repo_url: str) -> RemoteRepo:
+    try:
+        return RemoteRepo.from_url(repo_url)
+    except git.GitError as e:
+        raise CLIError(f"{e.__class__.__name__}: {e}") from e
     except RepoError as e:
         raise CLIError(str(e)) from e