Compare commits

..

4 Commits

Author SHA1 Message Date
alex 998665a4bf feat: small refactorings
Test and coverage / build (push) Failing after 56s
2026-06-28 01:18:08 +03:00
alex 17aaa41101 feat[logging]: flexible logging support 2026-06-28 00:48:50 +03:00
alex 206a8730a8 feat!: token support for cloning 2026-06-27 22:57:58 +03:00
alex 28f1fda048 feat[config]: Use toml for configs 2026-06-27 22:44:25 +03:00
20 changed files with 1313 additions and 137 deletions
+11 -16
View File
@@ -20,26 +20,21 @@ Other methods are not supporting:
Which is acceptable for full account mirroring.
**Security notice.**
This application uses SSH as git transport layer.
It is safe enough with right use,
and for right use you need to save
git server ssh digest (~/.ssh/known_hosts file).
To do this you just need to clone any repository over ssh first
This application can use SSH or HTTP(S) as git transport layer.
With SSH, you need to save git server ssh digest
(~/.ssh/known_hosts file). To do this you just need to clone any
repository over ssh first.
With HTTP(S), credentials are passed to git through environment-based
configuration and are not written into repository remote URLs.
**Config**. We use single config for this application.
It is slightly ancient solution for modern Docker/Kubernetes backends,
but provides configuration in one place and _secure enough_ place to save token.
Example config:
```ini
[main]
endpoint=https://example.com
token=XXXXX
format={owner}/{name}
out_dir=/home/user/repositories
ssh_key=/home/user/id_rsa
```
Example config is available in [config.example.toml](config.example.toml).
Use `main.api_token` for Gitea API access. Use `fetch.ssh_key` for SSH
cloning, or `fetch.user` and `fetch.token` for HTTP(S) cloning.
### Native
@@ -50,9 +45,9 @@ removing the ability to specify a user
2. Install dependencies (`pip3 install -r requirements.txt`).
Venv-level is recommended.
3. Install git (`sudo apt install git`)
4. And run it with path to ini config.
4. And run it with path to TOML config.
```bash
python gitea-mirror.py config.ini
python gitea-mirror.py config.toml
```
+35
View File
@@ -0,0 +1,35 @@
[main]
endpoint = "https://example.com"
api_token = "XXXXX"
format = "{owner}/{name}"
out_dir = "/home/user/repositories"
[fetch]
ssh_key = "/home/user/id_rsa"
# Use this fetch config instead of ssh_key to clone over HTTP(S):
# user = "gitea-user"
# token = "XXXXX"
[logging]
level = "INFO"
# Human-friendly logs in stdout.
[[logging.outputs]]
type = "console"
format = "plain"
stream = "stdout"
# Docker-friendly structured logs in stderr.
[[logging.outputs]]
type = "console"
format = "json"
stream = "stderr"
# Rotating file logs with selectable format.
[[logging.outputs]]
type = "file"
path = "/var/log/gitea-mirror/gitea-mirror.log"
format = "json"
max_size = 10485760
backup_count = 5
@@ -0,0 +1,131 @@
# Hardening CLI And Sync Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Make Gitea mirror fail reliably on API/git/config/path errors and expose a proper package CLI entrypoint.
**Architecture:** Keep the existing module layout, but move executable orchestration into an importable `src.cli` module. Preserve `gitea-mirror.py` as a compatibility wrapper while making `process_repo` return success/failure for aggregate exit handling.
**Tech Stack:** Python 3.12, requests, pydantic v2, pytest, ruff, uv.
---
### Task 1: API Errors Are Fatal
**Files:**
- Modify: `src/gitea_api.py`
- Test: `tests/test_gitea_api.py`
- [ ] **Step 1: Write failing tests**
```python
def test_get_repositories_raises_on_http_error(monkeypatch):
api = GiteaApi(endpoint="https://example.com", api_token="token")
session = FakeSession([FakeResponse(status_code=500, data=[])])
monkeypatch.setattr(requests, "session", lambda: session)
with pytest.raises(GiteaApiError):
api.get_repositories()
```
- [ ] **Step 2: Run test to verify it fails**
Run: `uv run pytest tests/test_gitea_api.py -v`
Expected: FAIL because `GiteaApiError` does not exist or is not raised.
- [ ] **Step 3: Implement minimal code**
Add `GiteaApiError`, request timeout, network exception handling, JSON/schema exception handling, and raise instead of returning `[]`.
- [ ] **Step 4: Run test to verify it passes**
Run: `uv run pytest tests/test_gitea_api.py -v`
Expected: PASS.
### Task 2: Git Failures Affect Exit Code
**Files:**
- Create: `src/cli.py`
- Modify: `gitea-mirror.py`
- Test: `tests/test_mirror_process.py`, `tests/test_cli.py`
- [ ] **Step 1: Write failing tests**
```python
def test_process_repo_returns_false_when_clone_fails(monkeypatch, tmp_path):
monkeypatch.setattr(cli, "git_clone", lambda **kwargs: False)
assert not cli.process_repo(config=make_config(tmp_path, FetchConfig(ssh_key="/tmp/key")), repo=make_repo())
```
- [ ] **Step 2: Run test to verify it fails**
Run: `uv run pytest tests/test_mirror_process.py tests/test_cli.py -v`
Expected: FAIL because `process_repo` currently returns `None` and no importable CLI module exists.
- [ ] **Step 3: Implement minimal code**
Move orchestration to `src.cli`, return booleans from `process_repo`, aggregate failures in `run`, and make `main` return non-zero for failures.
- [ ] **Step 4: Run test to verify it passes**
Run: `uv run pytest tests/test_mirror_process.py tests/test_cli.py -v`
Expected: PASS.
### Task 3: Validate Repository Paths And Config
**Files:**
- Modify: `src/models.py`
- Modify: `src/repository_name.py`
- Test: `tests/test_config.py`, `tests/test_repository_name.py`
- [ ] **Step 1: Write failing tests**
```python
def test_output_path_rejects_parent_traversal(tmp_path):
repo = make_repo()
with pytest.raises(ValueError):
get_repository_path(str(tmp_path), "../{name}", repo)
```
- [ ] **Step 2: Run test to verify it fails**
Run: `uv run pytest tests/test_config.py tests/test_repository_name.py -v`
Expected: FAIL because validation is missing.
- [ ] **Step 3: Implement minimal code**
Add repository format validation in `Config`, log level validation in `LoggingConfig`, and a safe output path helper that rejects absolute paths and paths escaping `out_dir`.
- [ ] **Step 4: Run test to verify it passes**
Run: `uv run pytest tests/test_config.py tests/test_repository_name.py -v`
Expected: PASS.
### Task 4: Package CLI Entrypoint
**Files:**
- Modify: `pyproject.toml`
- Modify: `gitea-mirror.py`
- Test: `tests/test_cli.py`
- [ ] **Step 1: Write failing test**
```python
def test_project_defines_console_script():
data = tomllib.loads(Path("pyproject.toml").read_text())
assert data["project"]["scripts"]["gitea-mirror"] == "src.cli:main"
```
- [ ] **Step 2: Run test to verify it fails**
Run: `uv run pytest tests/test_cli.py -v`
Expected: FAIL because `[project.scripts]` is absent.
- [ ] **Step 3: Implement minimal code**
Add `[project.scripts] gitea-mirror = "src.cli:main"` and make `gitea-mirror.py` delegate to `src.cli.main`.
- [ ] **Step 4: Run full verification**
Run: `make check`
Expected: PASS.
+2 -51
View File
@@ -1,55 +1,6 @@
import os.path
import sys
from os import makedirs
from src.config import Config, read_ini_config
from src.git import git_clone, git_pull
from src.gitea_api import GiteaApi
from src.models import GiteaRepository
from src.repository_name import get_repository_name, is_valid_repository_names
BASE_PATH = "out"
FORMAT = "{owner}/{name}"
def process_repo(config: Config, repo: GiteaRepository):
path = get_repository_name(name_format=config.repository_format, r=repo)
out_path = os.path.join(config.out_dir, path)
makedirs(out_path, exist_ok=True)
if os.path.exists(os.path.join(out_path, ".git")):
git_pull(out_path, ssh_key="fake")
return
print(f"New repository: {path}")
git_clone(ssh_url=repo.ssh_url, repository=out_path, ssh_key="fake")
def main():
if len(sys.argv) < 2:
print("Usage: python gitea-mirror.py CONFIG_PATH")
sys.exit(1)
try:
config = read_ini_config(sys.argv[1])
except RuntimeError as err:
print(f"Invalid config: {err}")
sys.exit(1)
api = GiteaApi(
endpoint=config.endpoint,
token=config.token,
)
repos = api.get_repositories()
print(f"total {len(repos)} repositories")
if not is_valid_repository_names(
name_format=config.repository_format,
repos=repos,
):
print("Format string is not valid, duplicates are not allowed")
sys.exit(1)
for repo in repos:
process_repo(config=config, repo=repo)
from src.cli import main
if __name__ == "__main__":
main()
sys.exit(main())
+3
View File
@@ -9,6 +9,9 @@ dependencies = [
"requests>=2.32",
]
[project.scripts]
gitea-mirror = "src.cli:main"
[dependency-groups]
dev = [
"pre-commit>=4.0",
+119
View File
@@ -0,0 +1,119 @@
import logging
import os.path
import sys
from os import makedirs
from .config import Config, read_toml_config
from .git import git_clone, git_pull, http_git_env, ssh_git_env
from .gitea_api import GiteaApi, GiteaApiError
from .log_fields import (
OPERATION,
PATH,
REPOSITORY,
REPOSITORY_COUNT,
REPOSITORY_URL,
)
from .logging_config import setup_logging
from .models import GiteaRepository
from .repository_name import get_repository_path, is_valid_repository_names
logger = logging.getLogger(__name__)
def get_fetch_params(config: Config, repo: GiteaRepository):
if config.fetch.ssh_key_path:
return repo.ssh_url, ssh_git_env(config.fetch.ssh_key_path)
return repo.clone_url, http_git_env(
user=config.fetch.user,
token=config.fetch.token,
)
def process_repo(config: Config, repo: GiteaRepository) -> bool:
out_path = get_repository_path(
out_dir=config.out_dir,
name_format=config.repository_format,
repo=repo,
)
path = os.path.relpath(out_path, config.out_dir)
repository_url, git_env = get_fetch_params(config=config, repo=repo)
log_context = {
REPOSITORY: path,
PATH: out_path,
REPOSITORY_URL: repository_url,
}
makedirs(out_path, exist_ok=True)
if os.path.exists(os.path.join(out_path, ".git")):
logger.info(
"repository pull started",
extra=log_context | {OPERATION: "pull"},
)
return git_pull(out_path, env=git_env)
logger.info(
"repository clone started",
extra=log_context | {OPERATION: "clone"},
)
return git_clone(
repository_url=repository_url,
repository=out_path,
env=git_env,
)
def run(config: Config) -> int:
api = GiteaApi(
endpoint=config.endpoint,
api_token=config.api_token,
)
try:
repos = api.get_repositories()
except GiteaApiError as err:
logger.error(
"failed to fetch repositories",
extra={OPERATION: "mirror"},
)
logger.debug("gitea api error", exc_info=err)
return 1
logger.info(
"repositories ready",
extra={OPERATION: "mirror", REPOSITORY_COUNT: len(repos)},
)
if not is_valid_repository_names(
name_format=config.repository_format,
repos=repos,
):
logger.error(
"repository name format creates duplicates",
extra={OPERATION: "validate_repository_names"},
)
return 1
has_failure = False
for repo in repos:
if not process_repo(config=config, repo=repo):
has_failure = True
return 1 if has_failure else 0
def main(argv: list[str] | None = None) -> int:
args = sys.argv[1:] if argv is None else argv
if len(args) < 1:
logging.basicConfig(
level=logging.ERROR,
format="%(levelname)s %(message)s",
)
logger.error("Usage: gitea-mirror CONFIG_PATH")
return 1
try:
config = read_toml_config(args[0])
except RuntimeError as err:
logging.basicConfig(
level=logging.ERROR,
format="%(levelname)s %(message)s",
)
logger.error("invalid config: %s", err)
return 1
setup_logging(config.logging)
return run(config)
+33 -22
View File
@@ -2,43 +2,54 @@
Token should be treated as password,
files are more secure in general than command-line arguments
.ini config example
TOML config example
[main]
endpoint=https://example.com/gitea
token=something
format={owner}/{name}
out_dir=/home/user/repositories
ssh_key=/home/user/.ssh/id_rsa.pub
endpoint = "https://example.com/gitea"
api_token = "something"
format = "{owner}/{name}"
out_dir = "/home/user/repositories"
[fetch]
ssh_key = "/home/user/.ssh/id_rsa"
"""
import configparser
import os
import tomllib
from .models import Config
from pydantic import ValidationError
from .models import Config, FetchConfig, LoggingConfig
MAIN_SECTION = "main"
def read_ini_config(path: str) -> Config:
def read_toml_config(path: str) -> Config:
if not os.path.exists(path):
raise RuntimeError("INI config path is not exists")
raise RuntimeError("TOML config path does not exist")
parser = configparser.ConfigParser()
parser.read(path)
try:
endpoint = parser[MAIN_SECTION]["endpoint"]
token = parser[MAIN_SECTION]["token"]
repository_format = parser[MAIN_SECTION]["format"]
out_dir = parser[MAIN_SECTION]["out_dir"]
ssh_key_path = parser[MAIN_SECTION]["ssh_key"]
except KeyError as err:
raise RuntimeError(f"No value for section: {err}")
with open(path, "rb") as config_file:
data = tomllib.load(config_file)
main = data[MAIN_SECTION]
endpoint = main["endpoint"]
api_token = main["api_token"]
repository_format = main["format"]
out_dir = main["out_dir"]
fetch = FetchConfig(**data["fetch"])
logging_config = LoggingConfig(**data.get("logging", {}))
return Config(
repository_format=repository_format,
endpoint=endpoint,
token=token,
api_token=api_token,
out_dir=out_dir,
ssh_key_path=ssh_key_path,
fetch=fetch,
logging=logging_config,
)
except (
KeyError,
ValueError,
ValidationError,
tomllib.TOMLDecodeError,
) as err:
raise RuntimeError(f"Invalid TOML config: {err}") from err
+59 -10
View File
@@ -1,24 +1,73 @@
import base64
import logging
import os
import shlex
import subprocess
from os import makedirs
from .log_fields import EXIT_CODE, OPERATION, PATH, REPOSITORY_URL
def git_clone(ssh_url: str, repository: str, ssh_key: str) -> bool:
logger = logging.getLogger(__name__)
def ssh_git_env(ssh_key_path: str) -> dict[str, str]:
return {
"GIT_SSH_COMMAND": (
f"ssh -i {shlex.quote(ssh_key_path)} -o IdentitiesOnly=yes"
),
}
def http_git_env(user: str, token: str) -> dict[str, str]:
credentials = base64.b64encode(f"{user}:{token}".encode("utf-8")).decode(
"ascii"
)
return {
"GIT_CONFIG_COUNT": "1",
"GIT_CONFIG_KEY_0": "http.extraHeader",
"GIT_CONFIG_VALUE_0": f"Authorization: Basic {credentials}",
}
def git_clone(
repository_url: str,
repository: str,
env: dict[str, str] | None = None,
) -> bool:
makedirs(repository, exist_ok=True)
git_env = os.environ | (env or {})
try:
subprocess.check_call(["git", "clone", ssh_url, "."], cwd=repository)
except subprocess.CalledProcessError:
print(
"Unable to clone repository "
f"{repository} with key {ssh_key} from {ssh_url}"
subprocess.check_call(
["git", "clone", repository_url, "."],
cwd=repository,
env=git_env,
)
except subprocess.CalledProcessError as err:
logger.error(
"git clone failed",
extra={
OPERATION: "clone",
PATH: repository,
REPOSITORY_URL: repository_url,
EXIT_CODE: err.returncode,
},
)
return False
return True
def git_pull(repository: str, ssh_key: str) -> bool:
def git_pull(repository: str, env: dict[str, str] | None = None) -> bool:
git_env = os.environ | (env or {})
try:
subprocess.check_call(["git", "pull"], cwd=repository)
except subprocess.CalledProcessError:
print(f"Unable to pull repository {repository} with key {ssh_key}")
subprocess.check_call(["git", "pull"], cwd=repository, env=git_env)
except subprocess.CalledProcessError as err:
logger.error(
"git pull failed",
extra={
OPERATION: "pull",
PATH: repository,
EXIT_CODE: err.returncode,
},
)
return False
return True
+48 -7
View File
@@ -1,43 +1,84 @@
import logging
from json import JSONDecodeError
from urllib.parse import urljoin
import requests
from pydantic import TypeAdapter
from pydantic import TypeAdapter, ValidationError
from .log_fields import OPERATION, PAGE, REPOSITORY_COUNT, STATUS_CODE
from .models import GiteaRepository
REPOSITORIES_ADAPTER = TypeAdapter(list[GiteaRepository])
logger = logging.getLogger(__name__)
REQUEST_TIMEOUT = (5, 30)
class GiteaApiError(RuntimeError):
pass
class GiteaApi:
def __init__(self, endpoint: str, token: str):
self._endpoint = endpoint
self._token = token
def __init__(self, endpoint: str, api_token: str):
self._endpoint = str(endpoint)
self._api_token = api_token
def get_repositories(self, page_size=10) -> list[GiteaRepository]:
"""
For mirroring input user is not important.
"""
session = requests.session()
session.headers.update({"Authorization": "token " + self._token})
session.headers.update({"Authorization": "token " + self._api_token})
all_repos = {} # hack for unique repositories in result
page_id = 1
while True:
try:
r = session.get(
urljoin(
self._endpoint,
"/api/v1/user/repos",
),
params={"limit": page_size, "page": page_id},
timeout=REQUEST_TIMEOUT,
)
except requests.RequestException as err:
raise GiteaApiError(
f"request failed while fetching page {page_id}: {err}"
) from err
if r.status_code != 200:
print(f"Failed request, code {r.status_code}")
return []
logger.error(
"failed to fetch repositories",
extra={
OPERATION: "fetch_repositories",
STATUS_CODE: r.status_code,
PAGE: page_id,
},
)
raise GiteaApiError(
f"failed to fetch repositories: status {r.status_code}"
)
try:
repos_data = r.json()
except (JSONDecodeError, ValueError) as err:
raise GiteaApiError(
f"invalid JSON while fetching page {page_id}: {err}"
) from err
if not repos_data:
break
else:
page_id += 1
try:
cur_repos = REPOSITORIES_ADAPTER.validate_python(repos_data)
except ValidationError as err:
raise GiteaApiError(
f"invalid repository payload on page {page_id - 1}: {err}"
) from err
for repo in cur_repos:
all_repos[repo.repo_id] = repo
logger.info(
"repositories fetched",
extra={
OPERATION: "fetch_repositories",
REPOSITORY_COUNT: len(all_repos),
},
)
return list(all_repos.values())
+8
View File
@@ -0,0 +1,8 @@
OPERATION = "operation"
REPOSITORY = "repository"
PATH = "path"
REPOSITORY_URL = "repository_url"
EXIT_CODE = "exit_code"
STATUS_CODE = "status_code"
PAGE = "page"
REPOSITORY_COUNT = "repository_count"
+82
View File
@@ -0,0 +1,82 @@
import json
import logging
import logging.handlers
import sys
from datetime import UTC, datetime
from typing import Any
from .models import LoggingConfig, LoggingOutputConfig
RESERVED_LOG_RECORD_KEYS = set(logging.makeLogRecord({}).__dict__)
def _context(record: logging.LogRecord) -> dict[str, Any]:
return {
key: value
for key, value in record.__dict__.items()
if key not in RESERVED_LOG_RECORD_KEYS and not key.startswith("_")
}
class JsonFormatter(logging.Formatter):
def format(self, record: logging.LogRecord) -> str:
data = {
"timestamp": datetime.fromtimestamp(
record.created,
tz=UTC,
).isoformat(),
"level": record.levelname,
"logger": record.name,
"message": record.getMessage(),
}
data.update(_context(record))
if record.exc_info:
data["exception"] = self.formatException(record.exc_info)
return json.dumps(data, ensure_ascii=False, default=str)
class PlainFormatter(logging.Formatter):
def format(self, record: logging.LogRecord) -> str:
message = f"{record.levelname} {record.getMessage()}"
context = _context(record)
if context:
fields = " ".join(
f"{key}={value}" for key, value in context.items()
)
message = f"{message} {fields}"
if record.exc_info:
message = f"{message}\n{self.formatException(record.exc_info)}"
return message
def _formatter(output: LoggingOutputConfig) -> logging.Formatter:
if output.format == "json":
return JsonFormatter()
return PlainFormatter()
def _handler(output: LoggingOutputConfig) -> logging.Handler:
if output.type == "file":
handler = logging.handlers.RotatingFileHandler(
filename=output.path,
maxBytes=output.max_size,
backupCount=output.backup_count,
)
else:
stream = sys.stderr if output.stream == "stderr" else sys.stdout
handler = logging.StreamHandler(stream)
handler.setFormatter(_formatter(output))
return handler
def setup_logging(
config: LoggingConfig,
logger: logging.Logger | None = None,
) -> logging.Logger:
configured_logger = logger or logging.getLogger()
configured_logger.handlers.clear()
configured_logger.setLevel(config.level)
configured_logger.propagate = False
for output in config.outputs:
configured_logger.addHandler(_handler(output))
return configured_logger
+67 -3
View File
@@ -1,14 +1,77 @@
import datetime
import logging
from typing import Literal
from pydantic import BaseModel, Field, HttpUrl
from pydantic import BaseModel, Field, HttpUrl, field_validator, model_validator
class FetchConfig(BaseModel):
ssh_key_path: str | None = Field(default=None, alias="ssh_key")
user: str | None = None
token: str | None = None
@model_validator(mode="after")
def validate_fetch_method(self):
has_ssh = self.ssh_key_path is not None
has_http = self.user is not None or self.token is not None
if has_ssh and has_http:
raise ValueError("fetch must use either ssh_key or user/token")
if not has_ssh and not (self.user and self.token):
raise ValueError("fetch must define ssh_key or user/token")
return self
class Config(BaseModel):
repository_format: str
ssh_key_path: str
endpoint: HttpUrl
token: str
api_token: str
out_dir: str
fetch: FetchConfig
logging: "LoggingConfig" = Field(default_factory=lambda: LoggingConfig())
@field_validator("repository_format")
@classmethod
def validate_repository_format(cls, value: str) -> str:
try:
value.format(
name="repo",
repository_id=1,
owner="owner",
owner_id=1,
)
except KeyError:
raise ValueError("repository format contains unknown fields")
return value
class LoggingOutputConfig(BaseModel):
type: Literal["console", "file"] = "console"
format: Literal["plain", "json"] = "plain"
stream: Literal["stdout", "stderr"] = "stdout"
path: str | None = None
max_size: int = 10 * 1024 * 1024
backup_count: int = 5
@model_validator(mode="after")
def validate_output(self):
if self.type == "file" and not self.path:
raise ValueError("file logging output requires path")
return self
class LoggingConfig(BaseModel):
level: str = "INFO"
outputs: list[LoggingOutputConfig] = Field(
default_factory=lambda: [LoggingOutputConfig()]
)
@field_validator("level")
@classmethod
def validate_level(cls, value: str) -> str:
if not isinstance(logging.getLevelName(value), int):
raise ValueError(f"unknown logging level: {value}")
return value
class GiteaUser(BaseModel):
@@ -19,6 +82,7 @@ class GiteaUser(BaseModel):
class GiteaRepository(BaseModel):
ssh_url: str
clone_url: str
name: str
repo_id: int = Field(alias="id")
updated_at: datetime.datetime
+18
View File
@@ -1,4 +1,5 @@
import datetime
import os
from typing import List
from .models import GiteaRepository, GiteaUser
@@ -7,6 +8,7 @@ from .models import GiteaRepository, GiteaUser
def _get_test_repository() -> GiteaRepository:
return GiteaRepository(
ssh_url="ssh://git@example.com/project/name",
clone_url="https://example.com/project/name.git",
name="test name",
id=42,
updated_at=datetime.datetime.now(),
@@ -35,6 +37,22 @@ def get_repository_name(name_format: str, r: GiteaRepository) -> str:
)
def get_repository_path(
out_dir: str,
name_format: str,
repo: GiteaRepository,
) -> str:
name = get_repository_name(name_format, repo)
if os.path.isabs(name):
raise ValueError("repository path must be relative")
out_dir_path = os.path.abspath(out_dir)
repo_path = os.path.abspath(os.path.join(out_dir_path, name))
if os.path.commonpath([out_dir_path, repo_path]) != out_dir_path:
raise ValueError("repository path must stay inside out_dir")
return repo_path
def is_valid_repository_names(name_format: str, repos: List[GiteaRepository]):
names = set(get_repository_name(name_format, r) for r in repos)
return len(names) == len(repos) # all names must be unique
+111
View File
@@ -0,0 +1,111 @@
import datetime
import tomllib
from pathlib import Path
from src import cli
from src.config import Config, FetchConfig
from src.gitea_api import GiteaApiError
from src.models import GiteaRepository, GiteaUser
def make_repo(repo_id=42, name="repo") -> GiteaRepository:
return GiteaRepository(
ssh_url="ssh://git@example.com/alex/repo.git",
clone_url="https://example.com/alex/repo.git",
name=name,
id=repo_id,
updated_at=datetime.datetime.now(),
owner=GiteaUser(
id=23,
login="alex",
email="alex@example.com",
),
)
def make_config(tmp_path, fetch: FetchConfig | None = None) -> Config:
return Config(
api_token="api-token",
endpoint="https://example.com",
repository_format="{owner}/{name}",
out_dir=str(tmp_path),
fetch=fetch or FetchConfig(ssh_key="/tmp/test_key"),
)
def test_project_defines_console_script():
data = tomllib.loads(Path("pyproject.toml").read_text())
assert data["project"]["scripts"]["gitea-mirror"] == "src.cli:main"
def test_run_returns_failure_when_api_fetch_fails(monkeypatch, tmp_path):
class FakeApi:
def __init__(self, endpoint, api_token):
pass
def get_repositories(self):
raise GiteaApiError("request failed")
monkeypatch.setattr(cli, "GiteaApi", FakeApi)
assert cli.run(make_config(tmp_path)) == 1
def test_run_returns_failure_when_any_repo_fails(monkeypatch, tmp_path):
class FakeApi:
def __init__(self, endpoint, api_token):
pass
def get_repositories(self):
return [
make_repo(repo_id=1, name="repo-1"),
make_repo(repo_id=2, name="repo-2"),
]
calls = []
def fake_process_repo(config, repo):
calls.append(repo.repo_id)
return repo.repo_id == 1
monkeypatch.setattr(cli, "GiteaApi", FakeApi)
monkeypatch.setattr(cli, "process_repo", fake_process_repo)
assert cli.run(make_config(tmp_path)) == 1
assert calls == [1, 2]
def test_run_returns_success_when_all_repos_succeed(monkeypatch, tmp_path):
class FakeApi:
def __init__(self, endpoint, api_token):
pass
def get_repositories(self):
return [make_repo()]
monkeypatch.setattr(cli, "GiteaApi", FakeApi)
monkeypatch.setattr(cli, "process_repo", lambda config, repo: True)
assert cli.run(make_config(tmp_path)) == 0
def test_main_returns_failure_for_invalid_config(monkeypatch):
def fake_read_toml_config(path):
raise RuntimeError("bad")
monkeypatch.setattr(cli, "read_toml_config", fake_read_toml_config)
assert cli.main(["bad.toml"]) == 1
def test_main_uses_config_path(monkeypatch, tmp_path):
config = make_config(tmp_path)
calls = []
monkeypatch.setattr(cli, "read_toml_config", lambda path: config)
monkeypatch.setattr(cli, "setup_logging", lambda logging_config: None)
monkeypatch.setattr(cli, "run", lambda config: calls.append(config) or 0)
assert cli.main(["config.toml"]) == 0
assert calls == [config]
+134 -13
View File
@@ -2,41 +2,162 @@ from tempfile import NamedTemporaryFile
import pytest
from src.config import Config, read_ini_config
from src.config import (
Config,
FetchConfig,
LoggingConfig,
read_toml_config,
)
from src.models import LoggingOutputConfig
@pytest.mark.parametrize(
"config_data, expected",
[
(
"[main]\ntoken=something\n"
"format={owner}/{name}\n"
"ssh_key=/tmp/no_key\n"
"endpoint=https://example.com\n"
"out_dir=/home/user/repositories",
'[main]\napi_token = "something"\n'
'format = "{owner}/{name}"\n'
'endpoint = "https://example.com"\n'
'out_dir = "/home/user/repositories"\n'
"\n"
"[fetch]\n"
'ssh_key = "/tmp/no_key"',
Config(
token="something",
api_token="something",
repository_format="{owner}/{name}",
out_dir="/home/user/repositories",
endpoint="https://example.com",
ssh_key_path="/tmp/no_key",
fetch=FetchConfig(ssh_key="/tmp/no_key"),
logging=LoggingConfig(),
),
),
(
'[main]\napi_token = "something"\n'
'format = "{owner}/{name}"\n'
'endpoint = "https://example.com"\n'
'out_dir = "/home/user/repositories"\n'
"\n"
"[fetch]\n"
'user = "alex"\n'
'token = "repo-token"',
Config(
api_token="something",
repository_format="{owner}/{name}",
out_dir="/home/user/repositories",
endpoint="https://example.com",
fetch=FetchConfig(user="alex", token="repo-token"),
logging=LoggingConfig(),
),
),
(
'[main]\napi_token = "something"\n'
'format = "{owner}/{name}"\n'
'endpoint = "https://example.com"\n'
'out_dir = "/home/user/repositories"\n'
"\n"
"[fetch]\n"
'ssh_key = "/tmp/no_key"\n'
"\n"
"[logging]\n"
'level = "DEBUG"\n'
"\n"
"[[logging.outputs]]\n"
'type = "console"\n'
'format = "plain"\n'
'stream = "stdout"\n'
"\n"
"[[logging.outputs]]\n"
'type = "console"\n'
'format = "json"\n'
'stream = "stderr"\n'
"\n"
"[[logging.outputs]]\n"
'type = "file"\n'
'path = "/tmp/gitea-mirror.log"\n'
'format = "json"\n'
"max_size = 1048576\n"
"backup_count = 3",
Config(
api_token="something",
repository_format="{owner}/{name}",
out_dir="/home/user/repositories",
endpoint="https://example.com",
fetch=FetchConfig(ssh_key="/tmp/no_key"),
logging=LoggingConfig(
level="DEBUG",
outputs=[
LoggingOutputConfig(
type="console",
format="plain",
stream="stdout",
),
LoggingOutputConfig(
type="console",
format="json",
stream="stderr",
),
LoggingOutputConfig(
type="file",
path="/tmp/gitea-mirror.log",
format="json",
max_size=1048576,
backup_count=3,
),
],
),
),
),
("[main]", None),
(
'[main]\napi_token = "something"\n'
'format = "{owner}/{name}"\n'
'endpoint = "https://example.com"\n'
'out_dir = "/home/user/repositories"\n'
"\n"
"[fetch]\n"
'ssh_key = "/tmp/no_key"\n'
'user = "alex"\n'
'token = "repo-token"',
None,
),
(
'[main]\napi_token = "something"\n'
'format = "{unknown}"\n'
'endpoint = "https://example.com"\n'
'out_dir = "/home/user/repositories"\n'
"\n"
"[fetch]\n"
'ssh_key = "/tmp/no_key"',
None,
),
(
'[main]\napi_token = "something"\n'
'format = "{owner}/{name}"\n'
'endpoint = "https://example.com"\n'
'out_dir = "/home/user/repositories"\n'
"\n"
"[fetch]\n"
'ssh_key = "/tmp/no_key"\n'
"\n"
"[logging]\n"
'level = "INF"',
None,
),
("not valid toml", None),
],
)
def test_ini_config(config_data, expected):
def test_toml_config(config_data, expected):
with NamedTemporaryFile() as tf:
if config_data:
tf.write(config_data.encode("utf-8"))
tf.flush()
if expected:
assert read_ini_config(tf.name) == expected
assert read_toml_config(tf.name) == expected
else:
with pytest.raises(RuntimeError):
read_ini_config(tf.name)
read_toml_config(tf.name)
def test_ini_config_not_exists():
def test_toml_config_not_exists():
with pytest.raises(RuntimeError):
read_ini_config("not_existing_file")
read_toml_config("not_existing_file")
+72
View File
@@ -0,0 +1,72 @@
import base64
import os
import subprocess
from src.git import git_clone, git_pull, http_git_env, ssh_git_env
def test_ssh_git_env_uses_requested_identity_file():
env = ssh_git_env("/tmp/test_key")
assert env == {
"GIT_SSH_COMMAND": "ssh -i /tmp/test_key -o IdentitiesOnly=yes"
}
def test_ssh_git_env_quotes_identity_file_path():
env = ssh_git_env("/tmp/test key")
assert env == {
"GIT_SSH_COMMAND": "ssh -i '/tmp/test key' -o IdentitiesOnly=yes"
}
def test_http_git_env_uses_basic_auth_header_without_url_credentials():
env = http_git_env(user="alex", token="secret")
expected_credentials = base64.b64encode(b"alex:secret").decode("ascii")
assert env == {
"GIT_CONFIG_COUNT": "1",
"GIT_CONFIG_KEY_0": "http.extraHeader",
"GIT_CONFIG_VALUE_0": f"Authorization: Basic {expected_credentials}",
}
def test_git_clone_passes_url_and_env_to_git(monkeypatch, tmp_path):
calls = []
env = {"GIT_SSH_COMMAND": "ssh -i /tmp/test_key -o IdentitiesOnly=yes"}
def fake_check_call(args, cwd, env):
calls.append((args, cwd, env))
monkeypatch.setattr(subprocess, "check_call", fake_check_call)
assert git_clone(
repository_url="ssh://git@example.com/owner/repo.git",
repository=str(tmp_path),
env=env,
)
args, cwd, actual_env = calls[0]
assert args == ["git", "clone", "ssh://git@example.com/owner/repo.git", "."]
assert cwd == str(tmp_path)
assert os.environ.items() <= actual_env.items()
assert env.items() <= actual_env.items()
def test_git_pull_passes_env_to_git(monkeypatch, tmp_path):
calls = []
env = {"GIT_CONFIG_COUNT": "1"}
def fake_check_call(args, cwd, env):
calls.append((args, cwd, env))
monkeypatch.setattr(subprocess, "check_call", fake_check_call)
assert git_pull(repository=str(tmp_path), env=env)
args, cwd, actual_env = calls[0]
assert args == ["git", "pull"]
assert cwd == str(tmp_path)
assert os.environ.items() <= actual_env.items()
assert env.items() <= actual_env.items()
+101
View File
@@ -0,0 +1,101 @@
import datetime
import pytest
import requests
from pydantic import ValidationError
from src.gitea_api import GiteaApi, GiteaApiError
def repo_data(repo_id=42):
return {
"id": repo_id,
"ssh_url": "ssh://git@example.com/alex/repo.git",
"clone_url": "https://example.com/alex/repo.git",
"name": "repo",
"updated_at": datetime.datetime.now(datetime.UTC).isoformat(),
"owner": {
"id": 23,
"login": "alex",
"email": "alex@example.com",
},
}
class FakeResponse:
def __init__(self, status_code=200, data=None, json_error=None):
self.status_code = status_code
self._data = data
self._json_error = json_error
def json(self):
if self._json_error:
raise self._json_error
return self._data
class FakeSession:
def __init__(self, responses=None, request_error=None):
self.headers = {}
self.responses = list(responses or [])
self.request_error = request_error
self.calls = []
def get(self, url, params, timeout):
self.calls.append((url, params, timeout))
if self.request_error:
raise self.request_error
return self.responses.pop(0)
def test_get_repositories_fetches_all_pages_with_timeout(monkeypatch):
session = FakeSession(
[
FakeResponse(data=[repo_data(repo_id=1)]),
FakeResponse(data=[repo_data(repo_id=2)]),
FakeResponse(data=[]),
]
)
monkeypatch.setattr(requests, "session", lambda: session)
repos = GiteaApi(
endpoint="https://example.com",
api_token="api-token",
).get_repositories(page_size=1)
assert [repo.repo_id for repo in repos] == [1, 2]
assert session.headers == {"Authorization": "token api-token"}
assert session.calls[0][2] == (5, 30)
def test_get_repositories_raises_on_http_error(monkeypatch):
session = FakeSession([FakeResponse(status_code=500, data=[])])
monkeypatch.setattr(requests, "session", lambda: session)
with pytest.raises(GiteaApiError, match="status 500"):
GiteaApi(
endpoint="https://example.com",
api_token="api-token",
).get_repositories()
def test_get_repositories_raises_on_network_error(monkeypatch):
session = FakeSession(request_error=requests.Timeout("timed out"))
monkeypatch.setattr(requests, "session", lambda: session)
with pytest.raises(GiteaApiError, match="request failed"):
GiteaApi(
endpoint="https://example.com",
api_token="api-token",
).get_repositories()
def test_get_repositories_raises_on_invalid_payload(monkeypatch):
session = FakeSession([FakeResponse(data=[{"id": 1}])])
monkeypatch.setattr(requests, "session", lambda: session)
with pytest.raises((GiteaApiError, ValidationError)):
GiteaApi(
endpoint="https://example.com",
api_token="api-token",
).get_repositories()
+112
View File
@@ -0,0 +1,112 @@
import io
import json
import logging
import logging.handlers
import sys
from src.log_fields import EXIT_CODE, OPERATION, PATH, REPOSITORY
from src.logging_config import JsonFormatter, PlainFormatter, setup_logging
from src.models import LoggingConfig, LoggingOutputConfig
def test_json_formatter_keeps_context_fields():
record = logging.LogRecord(
name="gitea_mirror.git",
level=logging.ERROR,
pathname=__file__,
lineno=10,
msg="git command failed",
args=(),
exc_info=None,
)
setattr(record, OPERATION, "pull")
setattr(record, REPOSITORY, "alex/repo")
setattr(record, PATH, "/backup/alex/repo")
setattr(record, EXIT_CODE, 128)
data = json.loads(JsonFormatter().format(record))
assert data["level"] == "ERROR"
assert data["logger"] == "gitea_mirror.git"
assert data["message"] == "git command failed"
assert data[OPERATION] == "pull"
assert data[REPOSITORY] == "alex/repo"
assert data[PATH] == "/backup/alex/repo"
assert data[EXIT_CODE] == 128
assert "timestamp" in data
def test_plain_formatter_appends_context_fields():
record = logging.LogRecord(
name="gitea_mirror.git",
level=logging.INFO,
pathname=__file__,
lineno=10,
msg="repository cloned",
args=(),
exc_info=None,
)
setattr(record, OPERATION, "clone")
setattr(record, REPOSITORY, "alex/repo")
message = PlainFormatter().format(record)
assert message == (
f"INFO repository cloned {OPERATION}=clone {REPOSITORY}=alex/repo"
)
def test_setup_logging_configures_console_and_rotating_file(tmp_path):
log_path = tmp_path / "gitea-mirror.log"
logger = logging.getLogger("test_setup_logging")
setup_logging(
LoggingConfig(
level="DEBUG",
outputs=[
LoggingOutputConfig(
type="console",
format="plain",
stream="stdout",
),
LoggingOutputConfig(
type="console",
format="json",
stream="stderr",
),
LoggingOutputConfig(
type="file",
path=str(log_path),
format="json",
max_size=1024,
backup_count=2,
),
],
),
logger=logger,
)
assert logger.level == logging.DEBUG
assert len(logger.handlers) == 3
assert logger.propagate is False
stdout_handler, stderr_handler, file_handler = logger.handlers
assert stdout_handler.stream is sys.stdout
assert isinstance(stdout_handler.formatter, PlainFormatter)
assert stderr_handler.stream is sys.stderr
assert isinstance(stderr_handler.formatter, JsonFormatter)
assert isinstance(file_handler, logging.handlers.RotatingFileHandler)
assert isinstance(file_handler.formatter, JsonFormatter)
assert file_handler.maxBytes == 1024
assert file_handler.backupCount == 2
def test_setup_logging_replaces_existing_handlers():
logger = logging.getLogger("test_setup_logging_replaces")
old_stream = io.StringIO()
logger.addHandler(logging.StreamHandler(old_stream))
setup_logging(LoggingConfig(), logger=logger)
assert len(logger.handlers) == 1
assert isinstance(logger.handlers[0].formatter, PlainFormatter)
+127
View File
@@ -0,0 +1,127 @@
import datetime
from src import cli
from src.config import Config, FetchConfig
from src.models import GiteaRepository, GiteaUser
def make_repo() -> GiteaRepository:
return GiteaRepository(
ssh_url="ssh://git@example.com/alex/repo.git",
clone_url="https://example.com/alex/repo.git",
name="repo",
id=42,
updated_at=datetime.datetime.now(),
owner=GiteaUser(
id=23,
login="alex",
email="alex@example.com",
),
)
def make_config(tmp_path, fetch: FetchConfig) -> Config:
return Config(
api_token="api-token",
endpoint="https://example.com",
repository_format="{owner}/{name}",
out_dir=str(tmp_path),
fetch=fetch,
)
def test_process_repo_clones_with_ssh_url_and_key_env(monkeypatch, tmp_path):
calls = []
def fake_git_clone(repository_url, repository, env):
calls.append((repository_url, repository, env))
return True
monkeypatch.setattr(cli, "git_clone", fake_git_clone)
assert cli.process_repo(
config=make_config(tmp_path, FetchConfig(ssh_key="/tmp/test_key")),
repo=make_repo(),
)
assert calls == [
(
"ssh://git@example.com/alex/repo.git",
str(tmp_path / "alex" / "repo"),
{
"GIT_SSH_COMMAND": (
"ssh -i /tmp/test_key -o IdentitiesOnly=yes"
),
},
)
]
def test_process_repo_clones_with_https_url_and_auth_env(monkeypatch, tmp_path):
calls = []
def fake_git_clone(repository_url, repository, env):
calls.append((repository_url, repository, env))
return True
monkeypatch.setattr(cli, "git_clone", fake_git_clone)
assert cli.process_repo(
config=make_config(tmp_path, FetchConfig(user="alex", token="secret")),
repo=make_repo(),
)
repository_url, repository, env = calls[0]
assert repository_url == "https://example.com/alex/repo.git"
assert repository == str(tmp_path / "alex" / "repo")
assert env["GIT_CONFIG_COUNT"] == "1"
assert env["GIT_CONFIG_KEY_0"] == "http.extraHeader"
assert env["GIT_CONFIG_VALUE_0"].startswith("Authorization: Basic ")
def test_process_repo_pulls_existing_repo_with_fetch_env(monkeypatch, tmp_path):
calls = []
repo_path = tmp_path / "alex" / "repo" / ".git"
repo_path.mkdir(parents=True)
def fake_git_pull(repository, env):
calls.append((repository, env))
return True
monkeypatch.setattr(cli, "git_pull", fake_git_pull)
assert cli.process_repo(
config=make_config(tmp_path, FetchConfig(ssh_key="/tmp/test_key")),
repo=make_repo(),
)
assert calls == [
(
str(tmp_path / "alex" / "repo"),
{
"GIT_SSH_COMMAND": (
"ssh -i /tmp/test_key -o IdentitiesOnly=yes"
),
},
)
]
def test_process_repo_returns_false_when_clone_fails(monkeypatch, tmp_path):
monkeypatch.setattr(cli, "git_clone", lambda **kwargs: False)
assert not cli.process_repo(
config=make_config(tmp_path, FetchConfig(ssh_key="/tmp/test_key")),
repo=make_repo(),
)
def test_process_repo_returns_false_when_pull_fails(monkeypatch, tmp_path):
repo_path = tmp_path / "alex" / "repo" / ".git"
repo_path.mkdir(parents=True)
monkeypatch.setattr(cli, "git_pull", lambda repository, env: False)
assert not cli.process_repo(
config=make_config(tmp_path, FetchConfig(ssh_key="/tmp/test_key")),
repo=make_repo(),
)
+26 -1
View File
@@ -1,6 +1,7 @@
import pytest
from src.repository_name import is_valid_format
from src.repository_name import get_repository_path, is_valid_format
from tests.test_mirror_process import make_repo
@pytest.mark.parametrize(
@@ -13,3 +14,27 @@ from src.repository_name import is_valid_format
)
def test_name_formatting(name_format, expected):
assert is_valid_format(name_format) == expected
@pytest.mark.parametrize(
"name_format",
[
"../{name}",
"/tmp/{name}",
],
)
def test_repository_path_rejects_paths_outside_out_dir(tmp_path, name_format):
with pytest.raises(ValueError):
get_repository_path(
out_dir=str(tmp_path),
name_format=name_format,
repo=make_repo(),
)
def test_repository_path_returns_path_inside_out_dir(tmp_path):
assert get_repository_path(
out_dir=str(tmp_path),
name_format="{owner}/{name}",
repo=make_repo(),
) == str(tmp_path / "alex" / "repo")