Skip to content

Drop config.yml->repos and LocalRepo support #3352

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
un-def merged 1 commit into from
Dec 8, 2025
Merged

Drop config.yml->repos and LocalRepo support #3352

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 2 additions & 11 deletions docs/docs/reference/api/python/index.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ from dstack.api import Task, GPU, Client, Resources
client = Client.from_config()

task = Task(
name="my-awesome-run", # If not specified, a random name is assigned
name="my-awesome-run", # If not specified, a random name is assigned
image="ghcr.io/huggingface/text-generation-inference:latest",
env={"MODEL_ID": "TheBloke/Llama-2-13B-chat-GPTQ"},
commands=[
Expand Down Expand Up @@ -42,7 +42,7 @@ finally:
```

!!! info "NOTE:"
1. The `configuration` argument in the `apply_configuration` method can be either `dstack.api.Task`, `dstack.api.Service`, or `dstack.api.DevEnvironment`.
1. The `configuration` argument in the `apply_configuration` method can be either `dstack.api.Task`, `dstack.api.Service`, or `dstack.api.DevEnvironment`.
2. When you create `dstack.api.Task`, `dstack.api.Service`, or `dstack.api.DevEnvironment`, you can specify the `image` argument. If `image` isn't specified, the default image will be used. For a private Docker registry, ensure you also pass the `registry_auth` argument.
3. The `repo` argument in the `apply_configuration` method allows the mounting of a local folder, a remote repo, or a
programmatically created repo. In this case, the `commands` argument can refer to the files within this repo.
Expand Down Expand Up @@ -173,15 +173,6 @@ finally:
memory: dstack.api.Memory
Range: dstack.api.Range

### `dstack.api.LocalRepo` { #dstack.api.LocalRepo data-toc-label="LocalRepo" }

::: dstack.api.LocalRepo
options:
show_bases: false
show_root_heading: false
show_root_toc_entry: false
heading_level: 4

### `dstack.api.RemoteRepo` { #dstack.api.RemoteRepo data-toc-label="RemoteRepo" }

::: dstack.api.RemoteRepo
Expand Down
59 changes: 4 additions & 55 deletions src/dstack/_internal/cli/commands/init.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,11 @@
from dstack._internal.cli.commands import BaseCommand
from dstack._internal.cli.services.repos import (
get_repo_from_dir,
get_repo_from_url,
is_git_repo_url,
register_init_repo_args,
)
from dstack._internal.cli.utils.common import confirm_ask, console, warn
from dstack._internal.core.errors import ConfigurationError
from dstack._internal.core.models.repos.remote import RemoteRepo
from dstack._internal.core.services.configs import ConfigManager
from dstack._internal.cli.utils.common import console
from dstack.api import Client


Expand All @@ -36,20 +34,6 @@ def _register(self):
dest="repo",
)
register_init_repo_args(self._parser)
# Deprecated since 0.19.25, ignored
self._parser.add_argument(
"--ssh-identity",
metavar="SSH_PRIVATE_KEY",
help=argparse.SUPPRESS,
type=Path,
dest="ssh_identity_file",
)
# A hidden mode for transitional period only, remove it with repos in `config.yml`
self._parser.add_argument(
"--remove",
help=argparse.SUPPRESS,
action="store_true",
)

def _command(self, args: argparse.Namespace):
super()._command(args)
Expand All @@ -65,45 +49,10 @@ def _command(self, args: argparse.Namespace):
else:
repo_path = Path.cwd()

if args.remove:
if repo_url is not None:
raise ConfigurationError(f"Local path expected, got URL: {repo_url}")
assert repo_path is not None
config_manager = ConfigManager()
repo_config = config_manager.get_repo_config(repo_path)
if repo_config is None:
raise ConfigurationError("Repo record not found, nothing to remove")
console.print(
f"You are about to remove the repo {repo_path}\n"
"Only the record about the repo will be removed,"
" the repo files will remain intact\n"
)
if not confirm_ask("Remove the repo?"):
return
config_manager.delete_repo_config(repo_config.repo_id)
config_manager.save()
console.print("Repo has been removed")
return

local: bool = args.local
if local:
warn(
"Local repos are deprecated since 0.19.25 and will be removed soon. Consider"
" using [code]files[/code] instead: https://dstack.ai/docs/concepts/tasks/#files"
)
if args.ssh_identity_file:
warn(
"[code]--ssh-identity[/code] in [code]dstack init[/code] is deprecated and ignored"
" since 0.19.25. Use this option with [code]dstack apply[/code]"
" and [code]dstack attach[/code] instead"
)

if repo_url is not None:
# Dummy repo branch to avoid autodetection that fails on private repos.
# We don't need branch/hash for repo_id anyway.
repo = RemoteRepo.from_url(repo_url, repo_branch="master")
repo = get_repo_from_url(repo_url)
elif repo_path is not None:
repo = get_repo_from_dir(repo_path, local=local)
repo = get_repo_from_dir(repo_path)
else:
assert False, "should not reach here"
api = Client.from_config(project_name=args.project)
Expand Down
133 changes: 42 additions & 91 deletions src/dstack/_internal/cli/services/configurators/run.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,12 @@
from dstack._internal.cli.services.profile import apply_profile_args, register_profile_args
from dstack._internal.cli.services.repos import (
get_repo_from_dir,
get_repo_from_url,
init_default_virtual_repo,
is_git_repo_url,
register_init_repo_args,
)
from dstack._internal.cli.utils.common import confirm_ask, console, warn
from dstack._internal.cli.utils.common import confirm_ask, console
from dstack._internal.cli.utils.rich import MultiItemStatus
from dstack._internal.cli.utils.run import get_runs_table, print_run_plan
from dstack._internal.core.errors import (
Expand All @@ -44,17 +45,13 @@
TaskConfiguration,
)
from dstack._internal.core.models.repos import RepoHeadWithCreds
from dstack._internal.core.models.repos.base import Repo
from dstack._internal.core.models.repos.local import LocalRepo
from dstack._internal.core.models.repos.remote import RemoteRepo, RemoteRepoCreds
from dstack._internal.core.models.resources import CPUSpec
from dstack._internal.core.models.runs import JobStatus, JobSubmission, RunSpec, RunStatus
from dstack._internal.core.services.configs import ConfigManager
from dstack._internal.core.services.diff import diff_models
from dstack._internal.core.services.repos import (
InvalidRepoCredentialsError,
get_repo_creds_and_default_branch,
load_repo,
)
from dstack._internal.utils.common import local_time
from dstack._internal.utils.interpolator import InterpolatorError, VariablesInterpolator
Expand Down Expand Up @@ -96,8 +93,9 @@ def apply_configuration(
if conf.working_dir is not None and not is_absolute_posix_path(conf.working_dir):
raise ConfigurationError("working_dir must be absolute")

config_manager = ConfigManager()
repo = self.get_repo(conf, configuration_path, configurator_args, config_manager)
repo = self.get_repo(conf, configuration_path, configurator_args)
if repo is None:
repo = init_default_virtual_repo(api=self.api)
profile = load_profile(Path.cwd(), configurator_args.profile)
with console.status("Getting apply plan..."):
run_plan = self.api.runs.get_run_plan(
Expand Down Expand Up @@ -475,12 +473,11 @@ def get_repo(
conf: RunConfigurationT,
configuration_path: str,
configurator_args: argparse.Namespace,
config_manager: ConfigManager,
) -> Repo:
) -> Optional[RemoteRepo]:
if configurator_args.no_repo:
return init_default_virtual_repo(api=self.api)
return None

repo: Optional[Repo] = None
repo: Optional[RemoteRepo] = None
repo_head: Optional[RepoHeadWithCreds] = None
repo_branch: Optional[str] = configurator_args.repo_branch
repo_hash: Optional[str] = configurator_args.repo_hash
Expand All @@ -497,8 +494,6 @@ def get_repo(
local_path: Optional[Path] = None
# dummy value, safe to join with any path
root_dir = Path(".")
# True if no repo specified, but we found one in `config.yml`
legacy_local_path = False
if repo_arg := configurator_args.repo:
if is_git_repo_url(repo_arg):
url = repo_arg
Expand All @@ -521,84 +516,49 @@ def get_repo(
if repo_hash is None:
repo_hash = repo_spec.hash
else:
local_path = Path.cwd()
legacy_local_path = True
return None

if url:
repo = RemoteRepo.from_url(repo_url=url)
repo = get_repo_from_url(url)
repo_head = self.api.repos.get(repo_id=repo.repo_id, with_creds=True)
elif local_path:
if legacy_local_path:
if repo_config := config_manager.get_repo_config(local_path):
repo = load_repo(repo_config)
repo_head = self.api.repos.get(repo_id=repo.repo_id, with_creds=True)
if repo_head is not None:
warn(
"The repo is not specified but found and will be used in the run\n"
"Future versions will not load repos automatically\n"
"To prepare for future versions and get rid of this warning:\n"
"- If you need the repo in the run, either specify [code]repos[/code]"
" in the configuration or use [code]--repo .[/code]\n"
"- If you don't need the repo in the run, either run"
" [code]dstack init --remove[/code] once (it removes only the record"
" about the repo, the repo files will remain intact)"
" or use [code]--no-repo[/code]"
)
else:
# ignore stale entries in `config.yml`
repo = None
init = False
else:
original_local_path = local_path
local_path = local_path.expanduser()
if not local_path.is_absolute():
local_path = (root_dir / local_path).resolve()
if not local_path.exists():
raise ConfigurationError(
f"Invalid repo path: {original_local_path} -> {local_path}"
)
local: bool = configurator_args.local
repo = get_repo_from_dir(local_path, local=local)
repo_head = self.api.repos.get(repo_id=repo.repo_id, with_creds=True)
if isinstance(repo, RemoteRepo):
repo_branch = repo.run_repo_data.repo_branch
repo_hash = repo.run_repo_data.repo_hash
original_local_path = local_path
local_path = local_path.expanduser()
if not local_path.is_absolute():
local_path = (root_dir / local_path).resolve()
if not local_path.exists():
raise ConfigurationError(
f"Invalid repo path: {original_local_path} -> {local_path}"
)
repo = get_repo_from_dir(local_path)
repo_head = self.api.repos.get(repo_id=repo.repo_id, with_creds=True)
repo_branch = repo.run_repo_data.repo_branch
repo_hash = repo.run_repo_data.repo_hash
else:
assert False, "should not reach here"

if repo is None:
return init_default_virtual_repo(api=self.api)
assert repo.repo_url is not None

if isinstance(repo, RemoteRepo):
assert repo.repo_url is not None
if repo_head is not None and repo_head.repo_creds is not None:
if git_identity_file is None and oauth_token is None:
git_private_key = repo_head.repo_creds.private_key
oauth_token = repo_head.repo_creds.oauth_token
else:
init = True

if repo_head is not None and repo_head.repo_creds is not None:
if git_identity_file is None and oauth_token is None:
git_private_key = repo_head.repo_creds.private_key
oauth_token = repo_head.repo_creds.oauth_token
else:
init = True
try:
repo_creds, _ = get_repo_creds_and_default_branch(
repo_url=repo.repo_url,
identity_file=git_identity_file,
private_key=git_private_key,
oauth_token=oauth_token,
)
except InvalidRepoCredentialsError as e:
raise CLIError(*e.args) from e

try:
repo_creds, default_repo_branch = get_repo_creds_and_default_branch(
repo_url=repo.repo_url,
identity_file=git_identity_file,
private_key=git_private_key,
oauth_token=oauth_token,
)
except InvalidRepoCredentialsError as e:
raise CLIError(*e.args) from e

if repo_branch is None and repo_hash is None:
if default_repo_branch is None:
raise CLIError(
"Failed to automatically detect remote repo branch."
" Specify branch or hash."
)
# TODO: remove in 0.20. Currently `default_repo_branch` is sent only for backward compatibility of `dstack-runner`.
repo_branch = default_repo_branch
repo.run_repo_data.repo_branch = repo_branch
if repo_hash is not None:
repo.run_repo_data.repo_hash = repo_hash
repo.run_repo_data.repo_branch = repo_branch
if repo_hash is not None:
repo.run_repo_data.repo_hash = repo_hash

if init:
self.api.repos.init(
Expand All @@ -608,15 +568,6 @@ def get_repo(
creds=repo_creds,
)

if isinstance(repo, LocalRepo):
warn(
f"{repo.repo_dir} is a local repo\n"
"Local repos are deprecated since 0.19.25 and will be removed soon\n"
"There are two options:\n"
"- Migrate to [code]files[/code]: https://dstack.ai/docs/concepts/tasks/#files\n"
"- Specify [code]--no-repo[/code] if you don't need the repo at all"
)

return repo


Expand Down
38 changes: 18 additions & 20 deletions src/dstack/_internal/cli/services/repos.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
import argparse
from typing import Literal, Union, overload
from pathlib import Path

import git

from dstack._internal.cli.services.configurators.base import ArgsParser
from dstack._internal.core.errors import CLIError
from dstack._internal.core.models.repos.local import LocalRepo
from dstack._internal.core.models.repos.remote import GitRepoURL, RemoteRepo, RepoError
from dstack._internal.core.models.repos.virtual import VirtualRepo
from dstack._internal.utils.path import PathLike
Expand All @@ -28,12 +26,6 @@ def register_init_repo_args(parser: ArgsParser):
type=str,
dest="git_identity_file",
)
# Deprecated since 0.19.25
parser.add_argument(
"--local",
action="store_true",
help=argparse.SUPPRESS,
)


def init_default_virtual_repo(api: Client) -> VirtualRepo:
Expand All @@ -42,17 +34,12 @@ def init_default_virtual_repo(api: Client) -> VirtualRepo:
return repo


@overload
def get_repo_from_dir(repo_dir: PathLike, local: Literal[False] = False) -> RemoteRepo: ...


@overload
def get_repo_from_dir(repo_dir: PathLike, local: Literal[True]) -> LocalRepo: ...


def get_repo_from_dir(repo_dir: PathLike, local: bool = False) -> Union[RemoteRepo, LocalRepo]:
if local:
return LocalRepo.from_dir(repo_dir)
def get_repo_from_dir(repo_dir: PathLike) -> RemoteRepo:
repo_dir = Path(repo_dir)
if not repo_dir.exists():
raise CLIError(f"Path does not exist: {repo_dir}")
if not repo_dir.is_dir():
raise CLIError(f"Path is not a directory: {repo_dir}")
try:
return RemoteRepo.from_dir(repo_dir)
except git.InvalidGitRepositoryError:
Expand All @@ -61,6 +48,17 @@ def get_repo_from_dir(repo_dir: PathLike, local: bool = False) -> Union[RemoteRe
"Use `files` to mount an arbitrary directory:"
" https://dstack.ai/docs/concepts/tasks/#files"
)
except git.GitError as e:
raise CLIError(f"{e.__class__.__name__}: {e}") from e
except RepoError as e:
raise CLIError(str(e)) from e


def get_repo_from_url(repo_url: str) -> RemoteRepo:
try:
return RemoteRepo.from_url(repo_url)
except git.GitError as e:
raise CLIError(f"{e.__class__.__name__}: {e}") from e
except RepoError as e:
raise CLIError(str(e)) from e

Expand Down
Loading