Files
publish-to-gitea/gitea_push.py
T

590 lines
24 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""推送到 Gitea 仓库 —— 通用 / 鲁棒版。
用法:
python3 gitea_push.py <repo_name> <description> [paths...] [options]
典型场景:
1. 新目录 → 新仓库
python3 gitea_push.py remind-me "飞书加急提醒" ./skills/remind-me/
2. 增量更新已有仓库(同样命令,自动拉远程历史后提交)
python3 gitea_push.py remind-me "飞书加急提醒" ./skills/remind-me/
3. 推送一个已有 git 仓库(保留完整提交历史)
python3 gitea_push.py my-proj "我的项目" --from-git /root/my-proj
4. 只检查不推送
python3 gitea_push.py my-proj "我的项目" ./src --dry-run
环境变量:
GITEA_URL 实例地址(默认 https://git.yoresee.cc)
GITEA_USER 用户名(默认 NightStar)
GITEA_EMAIL 提交邮箱(默认 nightstar@yoresee.cc)
GITEA_SSH_HOST SSH 主机(默认 git.yoresee.cc)
退出码: 0 成功 / 1 一般错误 / 2 隐私检查未通过 / 3 推送被拒(重试后仍失败)
"""
from __future__ import annotations
import argparse
import fnmatch
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path
# ─────────────────────────── 配置 ───────────────────────────
GITEA_URL = os.environ.get("GITEA_URL", "https://git.yoresee.cc").rstrip("/")
GITEA_USER = os.environ.get("GITEA_USER", "NightStar")
GITEA_EMAIL = os.environ.get("GITEA_EMAIL", "nightstar@yoresee.cc")
GITEA_SSH_HOST = os.environ.get("GITEA_SSH_HOST", "git.yoresee.cc")
DEFAULT_BRANCH = "master"
PUSH_RETRIES = 3
# 隐私模式:命中即拦(--allow-leaks 可放行)
SECRET_PATTERNS = [
(r"ghp_[A-Za-z0-9]{20,}", "GitHub PAT (ghp_)"),
(r"github_pat_[A-Za-z0-9_]{20,}", "GitHub fine-grained PAT"),
(r"sk-[A-Za-z0-9]{20,}", "API key (sk-)"),
(r"sk-sp-[A-Za-z0-9\-_]{20,}", "Token Plan key (sk-sp-)"),
(r"AKIA[0-9A-Z]{16}", "AWS Access Key ID"),
(r"xox[baprs]-[A-Za-z0-9\-]{10,}", "Slack token"),
(r"cli_[a-zA-Z0-9]{10,}", "飞书 app_id (cli_)"),
(r"ou_[a-f0-9]{20,}", "飞书 open_id (ou_)"),
(r"t-[a-zA-Z0-9]{20,}", "token (t-)"),
(r"Bearer\s+[A-Za-z0-9\-_.]{20,}", "Bearer 凭据"),
(r"-----BEGIN [A-Z ]*PRIVATE KEY-----", "私钥文件"),
(r"(?i)\b(password|passwd|secret|api[_-]?key|access[_-]?token)\s*[:=]\s*[\"']?[^\s\"',{}]{12,}", "明文口令/密钥赋值"),
]
# IP:仅拦公网地址(保留 127./10./192.168./172.16-31./0.0.0.0/255. 等常见非敏感写法)
IP_RE = re.compile(r"\b(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})\b")
# 扫描的文本类扩展名(旧版漏了 .html/.css/.vue 等,这里补齐)
SCAN_EXTS = {
".py", ".pyi", ".md", ".markdown", ".json", ".json5", ".yaml", ".yml",
".sh", ".bash", ".zsh", ".fish", ".txt", ".toml", ".ini", ".cfg", ".conf", ".properties",
".js", ".mjs", ".cjs", ".ts", ".tsx", ".jsx", ".vue", ".svelte",
".html", ".htm", ".css", ".scss", ".less", ".xml", ".svg",
".go", ".rs", ".java", ".kt", ".rb", ".php", ".sql", ".pl", ".lua",
".env", ".envrc", ".tf", ".tfvars", ".gradle", ".cmake",
}
SCAN_NAMES = {
"Dockerfile", "Makefile", "Rakefile", "Gemfile", "Procfile",
".env", ".env.local", ".env.production", ".npmrc", ".pypirc", ".netrc",
}
ALWAYS_EXCLUDE_DIRS = {".git", "node_modules", "__pycache__", ".venv", "venv", ".mypy_cache",
".pytest_cache", ".ruff_cache", "dist", "build", ".next", ".nuxt",
"target", ".idea", ".vscode", "vendor"}
ALWAYS_EXCLUDE_FILES = {".DS_Store", "Thumbs.db"}
MAX_WARN_BYTES = 5 * 1024 * 1024 # 单文件超 5MB 提醒
class PushError(Exception):
def __init__(self, msg: str, code: int = 1):
super().__init__(msg)
self.code = code
# ─────────────────────────── 日志 ───────────────────────────
QUIET = False
def log(msg: str = "", *, err: bool = True) -> None:
if not QUIET:
print(msg, file=sys.stderr if err else sys.stdout)
def step(n: int, total: int, msg: str) -> None:
log(f"[{n}/{total}] {msg}")
# ─────────────────────────── 子进程 ───────────────────────────
def _ssh_env() -> dict:
env = dict(os.environ)
env["GIT_SSH_COMMAND"] = env.get(
"GIT_SSH_COMMAND", "ssh -o StrictHostKeyChecking=accept-new"
)
return env
def run(cmd: list[str], cwd: Path | None = None, *, check: bool = True,
env: dict | None = None) -> subprocess.CompletedProcess:
res = subprocess.run(cmd, cwd=str(cwd) if cwd else None, capture_output=True,
text=True, env=env)
if check and res.returncode != 0:
raise PushError(f"命令失败: {' '.join(cmd)}\n{res.stderr.strip()}")
return res
def git(args: list[str], cwd: Path, *, check: bool = True) -> subprocess.CompletedProcess:
return run(["git", *args], cwd=cwd, check=check, env=_ssh_env())
def tea(args: list[str], *, check: bool = True) -> subprocess.CompletedProcess:
return run(["tea", *args], check=check)
def has_commits(cwd: Path) -> bool:
return git(["rev-parse", "--verify", "HEAD"], cwd, check=False).returncode == 0
def remote_branch_exists(cwd: Path, branch: str) -> bool:
return git(["rev-parse", "--verify", f"origin/{branch}"], cwd,
check=False).returncode == 0
# ─────────────────────────── 隐私检查 ───────────────────────────
def _is_scannable(p: Path) -> bool:
if p.name in SCAN_NAMES:
return True
return p.suffix.lower() in SCAN_EXTS
# 跳过不算敏感的四类:本机/私网/保留段、以及 RFC 5737 文档专用网段
_DOC_RANGES = ((192, 0, 2), (198, 51, 100), (203, 0, 113))
def _public_ip(match: re.Match) -> bool:
"""判定是否公网 IP(跳过本机/私网/环回/广播/文档专用段等无害写法)。"""
a, b, c = int(match.group(1)), int(match.group(2)), int(match.group(3))
if a == 0 or a == 127 or a == 255:
return False
if a == 10 or (a == 192 and b == 168) or (a == 172 and 16 <= b <= 31):
return False
if a == 169 and b == 254:
return False
if (a, b, c) in _DOC_RANGES:
return False
# 其余一律报出,交人工判断(版本号式写法也宁可多报)
return True
def scan_paths(paths: list[Path]) -> list[dict]:
"""扫描文件中的疑似隐私。返回命中列表。"""
findings: list[dict] = []
for root in paths:
files = [root] if root.is_file() else [f for f in root.rglob("*") if f.is_file()]
for f in files:
if any(part in ALWAYS_EXCLUDE_DIRS for part in f.parts):
continue
if not _is_scannable(f):
continue
try:
text = f.read_text(encoding="utf-8", errors="ignore")
except OSError:
continue
hits: list[str] = []
for pattern, label in SECRET_PATTERNS:
n = len(re.findall(pattern, text))
if n:
hits.append(f"{label} × {n}")
pub_ips = [m.group(0) for m in IP_RE.finditer(text) if _public_ip(m)]
if pub_ips:
uniq = sorted(set(pub_ips))
hits.append(f"公网 IP × {len(uniq)}: {', '.join(uniq[:3])}"
+ (" …" if len(uniq) > 3 else ""))
if hits:
findings.append({"file": str(f), "hits": hits})
return findings
# ─────────────────────────── 文件收集 ───────────────────────────
def _fnmatch_fallback(patterns: list[str], rel: str) -> bool:
"""pathspec 不可用时的兜底匹配(够用即可)。"""
for pat in patterns:
pat = pat.rstrip("/")
if fnmatch.fnmatch(rel, pat) or fnmatch.fnmatch(rel, f"{pat}/*"):
return True
if pat.startswith("**/") and fnmatch.fnmatch(rel, pat[3:]):
return True
return False
def load_ignore_spec(sources: list[Path]):
"""收集各源目录向上查找的 .gitignore 规则。"""
patterns: list[str] = []
seen: set[str] = set()
for src in sources:
p = src.resolve()
if p.is_file():
p = p.parent
while True:
gi = p / ".gitignore"
if str(gi) not in seen and gi.is_file():
seen.add(str(gi))
try:
for line in gi.read_text(encoding="utf-8", errors="ignore").splitlines():
line = line.strip()
if line and not line.startswith("#") and line != ".gitignore":
patterns.append(line)
except OSError:
pass
if p.parent == p:
break
p = p.parent
try:
import pathspec # type: ignore
return pathspec.PathSpec.from_lines("gitwildmatch", patterns)
except ImportError:
log(" (pathspec 未安装,使用 fnmatch 兜底匹配)")
return patterns
def is_ignored(spec, rel: str) -> bool:
if isinstance(spec, list):
return _fnmatch_fallback(spec, rel)
try:
return spec.match_file(rel)
except Exception:
return False
def collect_files(sources: list[Path], work_dir: Path, spec, extra_excludes: list[str]) -> int:
"""把源文件复制进工作区(保持相对结构)。返回文件数。"""
count = 0
for src in sources:
src = src.resolve()
if src.is_file():
(work_dir / src.name).write_bytes(src.read_bytes())
count += 1
continue
if not src.is_dir():
log(f" ⚠️ 跳过不存在的路径: {src}")
continue
for f in sorted(src.rglob("*")):
if not f.is_file():
continue
if any(part in ALWAYS_EXCLUDE_DIRS for part in f.parts):
continue
if f.name in ALWAYS_EXCLUDE_FILES or f.name.endswith((".pyc", ".pyo")):
continue
rel = str(f.relative_to(src))
if is_ignored(spec, rel):
continue
if any(fnmatch.fnmatch(rel, pat) for pat in extra_excludes):
continue
dest = work_dir / rel
dest.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(f, dest)
size = f.stat().st_size
if size > MAX_WARN_BYTES:
log(f" ⚠️ 大文件 {rel} ({size // 1024 // 1024} MB)")
count += 1
return count
# ─────────────────────────── 仓库准备 ───────────────────────────
def login_user() -> str:
"""取 tea 当前登录用户名(失败回退到 GITEA_USER)。"""
res = tea(["login", "list", "--output", "json"], check=False)
if res.returncode == 0:
try:
entries = json.loads(res.stdout or "[]")
for e in entries:
if e.get("user"):
return str(e["user"])
except (json.JSONDecodeError, TypeError):
pass
return GITEA_USER
def ensure_repo(repo: str, description: str, owner: str, branch: str,
private: bool, dry_run: bool) -> bool:
"""确保仓库存在。返回是否新建。
⚠️ tea 0.15.0 的坑:给自己命名空间建仓时传 `--owner <自己>` 会报
`Error: not found`;必须省略 --owner。只有 owner ≠ 登录用户(建到
组织/他人命名空间)时才传。
"""
if dry_run:
log(" (dry-run:跳过建仓)")
return False
existed = False
self_owner = login_user().lower()
cmd = ["repos", "create", "--name", repo, "--description", description,
"--branch", branch, "--init"]
if owner.lower() != self_owner:
cmd += ["--owner", owner]
if private:
cmd.append("--private")
res = tea(cmd, check=False)
if res.returncode == 0:
out = (res.stdout or "") + (res.stderr or "")
if "not found" in out.lower():
raise PushError(
f"创建仓库返回 not found(owner={owner} 可能不存在或无权访问):\n{out.strip()[:300]}")
return True
err = (res.stderr + res.stdout).lower()
if "already exists" in err or "409" in err:
existed = True
else:
# 二次确认:列一下 owner 的仓库
listing = tea(["repos", "list", "--owner", owner, "--output", "json",
"--limit", "200"], check=False)
if listing.returncode == 0:
try:
names = {r.get("name") for r in json.loads(listing.stdout or "[]")}
if repo in names:
existed = True
except json.JSONDecodeError:
pass
if not existed:
raise PushError(f"创建仓库失败: {res.stderr.strip() or res.stdout.strip()}")
log(f" 仓库 {owner}/{repo} 已存在,走增量提交(不 force)")
return False
def prepare_worktree(sources: list[Path], repo: str, owner: str, branch: str,
from_git: Path | None, extra_excludes: list[str]) -> tuple[Path, bool]:
"""准备工作区。返回 (路径, 是否为临时目录)。"""
if from_git:
work = from_git.resolve()
if not (work / ".git").exists():
raise PushError(f"{work} 不是 git 仓库(--from-git 需要已有 .git)")
remotes = git(["remote"], work, check=False).stdout.split()
url = f"git@{GITEA_SSH_HOST}:{owner}/{repo}.git"
if "origin" in remotes:
git(["remote", "set-url", "origin", url], work)
else:
git(["remote", "add", "origin", url], work)
return work, False
work = Path(tempfile.mkdtemp(prefix=f"gitea_{repo}_"))
run(["git", "init", "-b", branch], work)
git(["config", "user.name", GITEA_USER], work)
git(["config", "user.email", GITEA_EMAIL], work)
git(["remote", "add", "origin", f"git@{GITEA_SSH_HOST}:{owner}/{repo}.git"], work)
spec = load_ignore_spec(sources)
n = collect_files(sources, work, spec, extra_excludes)
log(f" 共收集 {n} 个文件")
return work, True
def align_with_remote(work: Path, branch: str, *, from_git: bool) -> str:
"""把本地分支对齐到远程历史,避免 'fetch first' 拒绝。
非 --from-git 场景用 `reset --mixed`(而非 --hard):只移动 HEAD、
不碰工作区,避免覆盖同名文件的用户内容。
"""
git(["fetch", "origin", branch], work, check=False)
if not remote_branch_exists(work, branch):
return "no-remote-branch"
if not has_commits(work):
# 未出生分支:继承远程历史,但保留工作区文件(--mixed 不碰文件)
git(["reset", "--mixed", f"origin/{branch}"], work)
return "adopted-remote"
if from_git:
res = git(["rebase", f"origin/{branch}"], work, check=False)
if res.returncode != 0:
git(["rebase", "--abort"], work, check=False)
raise PushError(
"rebase 冲突,请手动解决后重试:\n" + res.stderr.strip()[:500], code=3)
return "rebased"
return "kept-local"
# ─────────────────────────── 提交 & 推送 ───────────────────────────
def commit_if_changed(work: Path, message: str, *, add_all: bool = True) -> bool:
if add_all:
git(["add", "-A"], work)
if git(["diff", "--cached", "--quiet"], work, check=False).returncode == 0:
log(" 无内容变更,跳过 commit")
return False
res = git(["commit", "-m", message], work, check=False)
if res.returncode != 0:
raise PushError(f"commit 失败:\n{res.stderr.strip() or res.stdout.strip()}")
log(f" ✓ {git(['log', '--oneline', '-1'], work).stdout.strip()}")
return True
def push_with_retry(work: Path, branch: str) -> None:
last = ""
for attempt in range(1, PUSH_RETRIES + 1):
res = git(["push", "-u", "origin", branch], work, check=False)
if res.returncode == 0:
log(" ✓ 推送成功")
return
err = res.stderr.strip()
last = err
if any(k in err.lower() for k in ("rejected", "fetch first", "non-fast-forward", "behind")):
log(f" 推送被拒(第 {attempt} 次),拉取远程后重试…")
git(["fetch", "origin", branch], work, check=False)
if remote_branch_exists(work, branch):
rb = git(["rebase", f"origin/{branch}"], work, check=False)
if rb.returncode != 0:
git(["rebase", "--abort"], work, check=False)
raise PushError("rebase 冲突,无法自动重试:\n" + rb.stderr.strip()[:500],
code=3)
continue
raise PushError(f"推送失败:\n{err[:500]}", code=3)
raise PushError(f"推送失败(已重试 {PUSH_RETRIES} 次):\n{last[:500]}", code=3)
# ─────────────────────────── 主流程 ───────────────────────────
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(
prog="gitea_push.py",
description="推送到 Gitea 仓库(通用鲁棒版)",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""示例:
gitea_push.py remind-me "飞书加急提醒" ./skills/remind-me/
gitea_push.py my-proj "我的项目" --from-git /root/my-proj
gitea_push.py my-proj "我的项目" ./src --dry-run
""")
p.add_argument("repo_name", help="仓库名(kebab-case)")
p.add_argument("description", help="仓库描述 / 提交信息")
p.add_argument("paths", nargs="*", help="要推送的文件或目录(--from-git 时可省)")
p.add_argument("--from-git", metavar="DIR", default=None,
help="推送一个已有 git 仓库(保留提交历史)")
p.add_argument("--owner", default=GITEA_USER, help=f"仓库所有者(默认 {GITEA_USER})")
p.add_argument("--branch", default=DEFAULT_BRANCH, help=f"分支(默认 {DEFAULT_BRANCH})")
p.add_argument("--message", default=None, help="自定义提交信息(默认用 description)")
p.add_argument("--private", action="store_true", help="建私有仓库(默认公开)")
p.add_argument("--exclude", action="append", default=[], metavar="PATTERN",
help="额外排除(glob,可重复)")
p.add_argument("--allow-leaks", action="store_true", help="隐私检查命中时仍继续")
p.add_argument("--no-scan", action="store_true", help="跳过隐私检查")
p.add_argument("--dry-run", action="store_true", help="只检查与暂存,不建仓不推送")
p.add_argument("--quiet", action="store_true", help="静默(只输出结果 JSON)")
return p
def main(argv: list[str] | None = None) -> int:
global QUIET
args = build_parser().parse_args(argv)
QUIET = args.quiet
branch = args.branch
message = args.message or f"feat: {args.description}"
dry = args.dry_run
total_steps = 4
try:
if shutil.which("git") is None:
raise PushError("找不到 git")
if shutil.which("tea") is None and not dry:
raise PushError("找不到 tea CLI(Gitea 命令行)")
# 源路径
if args.from_git:
sources = [Path(args.from_git)]
else:
if not args.paths:
raise PushError("请至少指定一个路径,或用 --from-git 指定仓库目录")
sources = [Path(p) for p in args.paths]
missing = [str(s) for s in sources if not s.exists()]
if missing:
raise PushError("路径不存在: " + ", ".join(missing))
# 1. 隐私检查
step(1, total_steps, "隐私检查…")
if args.no_scan:
log(" (已跳过)")
else:
if args.from_git:
scan_targets = [p for p in Path(args.from_git).iterdir()
if p.name != ".git"]
else:
scan_targets = sources
findings = scan_paths(scan_targets)
if findings:
log("")
for item in findings:
log(f" ⚠️ {item['file']}")
for h in item["hits"]:
log(f" - {h}")
log("")
if not args.allow_leaks:
raise PushError(
f"发现 {len(findings)} 个文件含疑似隐私,已中止。"
"确认无误可加 --allow-leaks 继续。", code=2)
log(" --allow-leaks 已指定,继续推送")
else:
log(" ✓ 未发现敏感内容")
# 2. 仓库准备
step(2, total_steps, f"确保仓库 {args.owner}/{args.repo_name}…")
is_new = ensure_repo(args.repo_name, args.description, args.owner,
branch, args.private, dry)
log(" ✓ 新建仓库" if is_new else " ✓ 复用现有仓库")
# 3. 工作区 + 对齐远程
step(3, total_steps, "准备工作区并对齐远程历史…")
work, is_temp = prepare_worktree(sources, args.repo_name, args.owner,
branch, Path(args.from_git) if args.from_git else None,
args.exclude)
if not args.from_git:
git(["config", "user.name", GITEA_USER], work)
git(["config", "user.email", GITEA_EMAIL], work)
mode = align_with_remote(work, branch, from_git=bool(args.from_git))
log(f" 远程对齐方式: {mode}")
if args.from_git:
# --from-git:推送仓库已有的提交,不自动提交未完成的工作区改动
dirty = git(["status", "--porcelain"], work, check=False).stdout.strip()
if dirty:
log(" ⚠️ 工作区有未提交改动,不会被推送(请先自行 commit)")
changed = False
else:
# 普通目录推送:全部纳入暂存区(含新增/修改/删除)
changed = commit_if_changed(work, message, add_all=True)
# 4. 推送
if dry:
step(4, total_steps, "dry-run:跳过推送")
log(f" 工作区: {work}")
else:
step(4, total_steps, f"推送到 {args.owner}/{args.repo_name} ({branch})…")
if args.from_git:
ahead = git(["rev-list", "--count", f"origin/{branch}..HEAD"],
work, check=False).stdout.strip()
if ahead in ("", "0"):
log(" 本地无新提交,远程已是最新")
else:
log(f" 本地领先远程 {ahead} 个提交")
push_with_retry(work, branch)
else:
push_with_retry(work, branch)
browse = f"{GITEA_URL}/{args.owner}/{args.repo_name}"
log("")
log(f"✅ 完成!浏览: {browse}")
print(json.dumps({
"ok": True,
"repo": args.repo_name,
"owner": args.owner,
"url": browse,
"clone": f"git@{GITEA_SSH_HOST}:{args.owner}/{args.repo_name}.git",
"branch": branch,
"new_repo": is_new,
"dry_run": dry,
"work_dir": str(work) if is_temp else None,
"from_git": bool(args.from_git),
}, ensure_ascii=False))
return 0
except PushError as exc:
log(f"\n❌ {exc}")
print(json.dumps({"ok": False, "error": str(exc)}, ensure_ascii=False))
return exc.code
except KeyboardInterrupt:
log("\n已中断")
return 1
if __name__ == "__main__":
sys.exit(main())