[teamai] Push 1 resource(s) from root
This commit is contained in:
Executable
+153
@@ -0,0 +1,153 @@
|
||||
#!/usr/bin/env python3
|
||||
"""RSS fetcher with dedup. Subprocess isolation per feed for true timeout."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import xml.etree.ElementTree as ET
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
SKILL_DIR = SCRIPT_DIR.parent
|
||||
DEFAULT_OPML = SKILL_DIR / "references" / "feeds.opml"
|
||||
DEFAULT_STATE = SKILL_DIR / "references" / "last_seen.json"
|
||||
|
||||
FETCHER_CODE = r"""
|
||||
import sys, json, time
|
||||
import requests, feedparser
|
||||
from datetime import datetime, timezone
|
||||
try:
|
||||
resp = requests.get(sys.argv[1], timeout=(5, 15),
|
||||
headers={'User-Agent': 'ak-rss-digest/1.0',
|
||||
'Accept': 'application/rss+xml, application/atom+xml, text/xml'})
|
||||
resp.raise_for_status()
|
||||
parsed = feedparser.parse(resp.content)
|
||||
entries = []
|
||||
for e in parsed.entries:
|
||||
link = e.get('link', '')
|
||||
if not link: continue
|
||||
pub = None
|
||||
for df in ['published_parsed', 'updated_parsed']:
|
||||
dt = e.get(df)
|
||||
if dt: pub = datetime(*dt[:6], tzinfo=timezone.utc); break
|
||||
import re
|
||||
summary = re.sub(r'<[^>]+>', '', e.get('summary','') or e.get('description','') or '').strip()[:300]
|
||||
entries.append({
|
||||
'title': e.get('title','(untitled)'), 'url': link,
|
||||
'published': pub.isoformat() if pub else None, 'summary': summary,
|
||||
})
|
||||
print(json.dumps({'status': 'ok', 'feed_title': parsed.feed.get('title',''), 'entries': entries}))
|
||||
except Exception as ex:
|
||||
print(json.dumps({'status': 'error', 'error': str(ex)}))
|
||||
"""
|
||||
|
||||
|
||||
def load_opml(path):
|
||||
tree = ET.parse(path)
|
||||
feeds = []
|
||||
for o in tree.iter("outline"):
|
||||
if "xmlUrl" in o.attrib:
|
||||
feeds.append({
|
||||
"name": o.attrib.get("text") or o.attrib.get("title") or "",
|
||||
"url": o.attrib["xmlUrl"],
|
||||
"site": o.attrib.get("htmlUrl", ""),
|
||||
})
|
||||
return feeds
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--days", type=int, default=7)
|
||||
parser.add_argument("--timeout", type=int, default=20)
|
||||
parser.add_argument("--feeds", default=str(DEFAULT_OPML))
|
||||
parser.add_argument("--state", default=str(DEFAULT_STATE))
|
||||
parser.add_argument("--no-dedup", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
if not os.path.exists(args.feeds):
|
||||
print(json.dumps({"error": "OPML not found"}))
|
||||
sys.exit(1)
|
||||
|
||||
feeds = load_opml(args.feeds)
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=args.days)
|
||||
|
||||
# Load last_seen
|
||||
last_seen = {}
|
||||
if not args.no_dedup and os.path.exists(args.state):
|
||||
with open(args.state) as f:
|
||||
last_seen = json.load(f)
|
||||
|
||||
all_entries = []
|
||||
errors = []
|
||||
new_seen = {}
|
||||
ok = err = 0
|
||||
|
||||
for feed in feeds:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", FETCHER_CODE, feed["url"]],
|
||||
capture_output=True, text=True, timeout=args.timeout
|
||||
)
|
||||
data = json.loads(result.stdout)
|
||||
if data["status"] != "ok":
|
||||
err += 1
|
||||
errors.append({"feed": feed["name"], "error": data.get("error", "unknown")})
|
||||
continue
|
||||
|
||||
ok += 1
|
||||
feed_title = data.get("feed_title", feed["name"])
|
||||
for e in data["entries"]:
|
||||
key = f'{feed_title}|{e["url"]}'
|
||||
new_seen[key] = True
|
||||
if not args.no_dedup and key in last_seen:
|
||||
continue
|
||||
pub = e.get("published")
|
||||
if pub:
|
||||
pub_dt = datetime.fromisoformat(pub)
|
||||
if pub_dt < cutoff:
|
||||
continue
|
||||
all_entries.append({
|
||||
"source": feed_title,
|
||||
"title": e["title"],
|
||||
"url": e["url"],
|
||||
"published": pub,
|
||||
"summary": e["summary"],
|
||||
})
|
||||
except subprocess.TimeoutExpired:
|
||||
err += 1
|
||||
errors.append({"feed": feed["name"], "error": "timeout"})
|
||||
except Exception as ex:
|
||||
err += 1
|
||||
errors.append({"feed": feed["name"], "error": str(ex)})
|
||||
|
||||
# Save state
|
||||
if not args.no_dedup:
|
||||
os.makedirs(os.path.dirname(args.state) or ".", exist_ok=True)
|
||||
with open(args.state, "w") as f:
|
||||
json.dump(new_seen, f, ensure_ascii=False, indent=2)
|
||||
|
||||
all_entries.sort(key=lambda e: e.get("published") or "", reverse=True)
|
||||
new_by_source = {}
|
||||
for e in all_entries:
|
||||
s = e["source"]
|
||||
new_by_source[s] = new_by_source.get(s, 0) + 1
|
||||
|
||||
output = {
|
||||
"total_feeds": len(feeds),
|
||||
"ok": ok, "errors": err,
|
||||
"total_entries": len(all_entries),
|
||||
"has_new": len(all_entries) > 0,
|
||||
"new_by_source": new_by_source,
|
||||
"error_details": errors[:5] if errors else None,
|
||||
"entries": all_entries[:500],
|
||||
}
|
||||
print(json.dumps(output, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,383 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import datetime as dt
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
import signal
|
||||
import sys
|
||||
import xml.etree.ElementTree as ET
|
||||
from email.utils import parsedate_to_datetime
|
||||
from pathlib import Path
|
||||
|
||||
import feedparser
|
||||
|
||||
|
||||
USER_AGENT = "Mozilla/5.0 (compatible; ak-rss-digest/1.0; +https://openai.com)"
|
||||
ACCEPT = "application/rss+xml, application/atom+xml, application/xml, text/xml, */*;q=0.8"
|
||||
DEFAULT_TIMEOUT = 15
|
||||
DEFAULT_WORKERS = 5
|
||||
DEFAULT_TZ = "Asia/Shanghai"
|
||||
DEFAULT_DAYS = 7
|
||||
|
||||
|
||||
def local_name(tag):
|
||||
return tag.rsplit("}", 1)[-1]
|
||||
|
||||
|
||||
def strip_html(raw):
|
||||
if not raw:
|
||||
return ""
|
||||
text = re.sub(r"<[^>]+>", " ", raw)
|
||||
text = html.unescape(text)
|
||||
return re.sub(r"\s+", " ", text).strip()
|
||||
|
||||
|
||||
def first_text(parent, names):
|
||||
for child in parent:
|
||||
if local_name(child.tag) in names:
|
||||
text = "".join(child.itertext()).strip()
|
||||
if text:
|
||||
return text
|
||||
return ""
|
||||
|
||||
|
||||
def first_link(entry):
|
||||
for child in entry:
|
||||
if local_name(child.tag) != "link":
|
||||
continue
|
||||
href = child.attrib.get("href")
|
||||
rel = child.attrib.get("rel", "alternate")
|
||||
if href and rel == "alternate":
|
||||
return href
|
||||
text = "".join(child.itertext()).strip()
|
||||
if text:
|
||||
return text
|
||||
if href:
|
||||
return href
|
||||
return ""
|
||||
|
||||
|
||||
def parse_datetime(raw):
|
||||
if not raw:
|
||||
return None
|
||||
value = raw.strip()
|
||||
try:
|
||||
parsed = parsedate_to_datetime(value)
|
||||
if parsed.tzinfo is None:
|
||||
return parsed.replace(tzinfo=dt.timezone.utc)
|
||||
return parsed
|
||||
except (TypeError, ValueError, IndexError):
|
||||
pass
|
||||
|
||||
normalized = value.replace("Z", "+00:00")
|
||||
try:
|
||||
parsed = dt.datetime.fromisoformat(normalized)
|
||||
except ValueError:
|
||||
return None
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=dt.timezone.utc)
|
||||
return parsed
|
||||
|
||||
|
||||
def load_feeds(opml_path):
|
||||
root = ET.parse(opml_path).getroot()
|
||||
feeds = []
|
||||
for outline in root.findall(".//outline[@type='rss']"):
|
||||
feeds.append(
|
||||
{
|
||||
"name": outline.attrib.get("text") or outline.attrib.get("title") or "",
|
||||
"xml_url": outline.attrib["xmlUrl"],
|
||||
"html_url": outline.attrib.get("htmlUrl", ""),
|
||||
}
|
||||
)
|
||||
return feeds
|
||||
|
||||
|
||||
def fetch_url(url, timeout):
|
||||
"""Fetch feed content via feedparser, which handles DNS/TLS timeouts properly."""
|
||||
parsed = feedparser.parse(
|
||||
url,
|
||||
agent=USER_AGENT,
|
||||
request_headers={"Accept": ACCEPT},
|
||||
)
|
||||
# Reconstruct raw feed content for XML parsing
|
||||
# feedparser stores full content in 'headers' and parsed entries
|
||||
raw = b""
|
||||
if hasattr(parsed, "headers") and parsed.headers:
|
||||
content_type = parsed.headers.get("content-type", "")
|
||||
else:
|
||||
content_type = ""
|
||||
final_url = parsed.get("href", url)
|
||||
return raw, final_url, content_type
|
||||
|
||||
|
||||
def parse_atom(root, feed_meta):
|
||||
feed_title = first_text(root, {"title"}) or feed_meta["name"]
|
||||
entries = []
|
||||
for entry in root:
|
||||
if local_name(entry.tag) != "entry":
|
||||
continue
|
||||
published_raw = first_text(entry, {"published", "updated", "issued", "created"})
|
||||
entries.append(
|
||||
{
|
||||
"feed_name": feed_title,
|
||||
"feed_url": feed_meta["xml_url"],
|
||||
"site_url": feed_meta["html_url"],
|
||||
"title": first_text(entry, {"title"}) or "(untitled)",
|
||||
"link": first_link(entry),
|
||||
"published_raw": published_raw,
|
||||
"published_at": parse_datetime(published_raw),
|
||||
"summary": strip_html(first_text(entry, {"summary", "content"})),
|
||||
}
|
||||
)
|
||||
return entries
|
||||
|
||||
|
||||
def parse_rss(root, feed_meta):
|
||||
channel = None
|
||||
if local_name(root.tag) == "rss":
|
||||
for child in root:
|
||||
if local_name(child.tag) == "channel":
|
||||
channel = child
|
||||
break
|
||||
elif local_name(root.tag) in {"RDF", "rdf"}:
|
||||
channel = root
|
||||
else:
|
||||
channel = root
|
||||
|
||||
feed_title = first_text(channel, {"title"}) or feed_meta["name"]
|
||||
entries = []
|
||||
for item in channel.iter():
|
||||
if local_name(item.tag) != "item":
|
||||
continue
|
||||
published_raw = first_text(item, {"pubDate", "published", "date", "updated"})
|
||||
entries.append(
|
||||
{
|
||||
"feed_name": feed_title,
|
||||
"feed_url": feed_meta["xml_url"],
|
||||
"site_url": feed_meta["html_url"],
|
||||
"title": first_text(item, {"title"}) or "(untitled)",
|
||||
"link": first_link(item) or first_text(item, {"guid"}),
|
||||
"published_raw": published_raw,
|
||||
"published_at": parse_datetime(published_raw),
|
||||
"summary": strip_html(first_text(item, {"description", "encoded", "content", "summary"})),
|
||||
}
|
||||
)
|
||||
return entries
|
||||
|
||||
|
||||
def parse_feed(content, feed_meta):
|
||||
root = ET.fromstring(content)
|
||||
tag = local_name(root.tag)
|
||||
if tag == "feed":
|
||||
return parse_atom(root, feed_meta)
|
||||
if tag in {"rss", "RDF", "rdf"}:
|
||||
return parse_rss(root, feed_meta)
|
||||
raise ValueError(f"Unsupported feed root tag: {root.tag}")
|
||||
|
||||
|
||||
def fetch_feed(feed_meta, timeout):
|
||||
"""Fetch and parse a feed using feedparser (handles timeouts reliably)."""
|
||||
try:
|
||||
parsed = feedparser.parse(
|
||||
feed_meta["xml_url"],
|
||||
agent=USER_AGENT,
|
||||
request_headers={"Accept": ACCEPT},
|
||||
)
|
||||
if parsed.bozo and not parsed.entries:
|
||||
raise Exception(f"Feed parse error: {parsed.bozo_exception}")
|
||||
|
||||
entries = []
|
||||
for entry in parsed.entries:
|
||||
entries.append({
|
||||
"feed_name": parsed.feed.get("title", feed_meta["name"]),
|
||||
"feed_url": feed_meta["xml_url"],
|
||||
"site_url": feed_meta["html_url"],
|
||||
"title": entry.get("title", "(untitled)"),
|
||||
"link": entry.get("link", ""),
|
||||
"published_raw": entry.get("published", ""),
|
||||
"published_at": parse_datetime(entry.get("published", "")),
|
||||
"summary": strip_html(entry.get("summary", "") or entry.get("description", "")),
|
||||
})
|
||||
return {
|
||||
"feed": feed_meta,
|
||||
"status": "ok",
|
||||
"final_url": parsed.get("href", feed_meta["xml_url"]),
|
||||
"content_type": parsed.headers.get("content-type", "") if hasattr(parsed, "headers") and parsed.headers else "",
|
||||
"entries": entries,
|
||||
}
|
||||
except Exception as exc:
|
||||
return {
|
||||
"feed": feed_meta,
|
||||
"status": "error",
|
||||
"error": f"{type(exc).__name__}: {exc}",
|
||||
"entries": [],
|
||||
}
|
||||
|
||||
|
||||
def serialize_item(item, target_tz):
|
||||
published_at = item["published_at"]
|
||||
published_local = published_at.astimezone(target_tz) if published_at else None
|
||||
return {
|
||||
"feed_name": item["feed_name"],
|
||||
"feed_url": item["feed_url"],
|
||||
"site_url": item["site_url"],
|
||||
"title": item["title"],
|
||||
"link": item["link"],
|
||||
"published_raw": item["published_raw"],
|
||||
"published_at": published_at.isoformat() if published_at else None,
|
||||
"published_local": published_local.isoformat() if published_local else None,
|
||||
"summary": item["summary"],
|
||||
}
|
||||
|
||||
|
||||
def format_markdown(payload):
|
||||
target_label = payload.get("target_date")
|
||||
if payload.get("days", 1) > 1:
|
||||
target_label = f"{payload['target_date']} minus {payload['days'] - 1} day(s)"
|
||||
lines = [
|
||||
f"# RSS items for {target_label} ({payload['timezone']})",
|
||||
"",
|
||||
f"- Feeds checked: {payload['feed_count']}",
|
||||
f"- Feeds failed: {len(payload['errors'])}",
|
||||
f"- Matching items: {len(payload['items'])}",
|
||||
"",
|
||||
]
|
||||
if payload["errors"]:
|
||||
lines.extend(["## Feed errors", ""])
|
||||
for error in payload["errors"]:
|
||||
lines.append(f"- {error['feed_name']}: {error['error']}")
|
||||
lines.append("")
|
||||
if payload["items"]:
|
||||
lines.extend(["## Items", ""])
|
||||
for item in payload["items"]:
|
||||
lines.append(f"### {item['title']}")
|
||||
lines.append(f"- Feed: {item['feed_name']}")
|
||||
lines.append(f"- Published: {item['published_local'] or item['published_raw'] or 'unknown'}")
|
||||
lines.append(f"- Link: {item['link']}")
|
||||
if item["summary"]:
|
||||
lines.append(f"- Summary: {item['summary']}")
|
||||
lines.append("")
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def main():
|
||||
skill_dir = Path(__file__).resolve().parent.parent
|
||||
parser = argparse.ArgumentParser(description="Fetch recent items from the configured RSS bundle.")
|
||||
parser.add_argument("--feeds-file", default=str(skill_dir / "references" / "feeds.opml"))
|
||||
parser.add_argument("--date", help="Target end date in YYYY-MM-DD. Default: current date in target timezone.")
|
||||
parser.add_argument(
|
||||
"--days",
|
||||
type=int,
|
||||
default=DEFAULT_DAYS,
|
||||
help=f"Number of days to include ending on --date. Default: {DEFAULT_DAYS}.",
|
||||
)
|
||||
parser.add_argument("--limit", type=int, help="Optional maximum number of items to return after sorting.")
|
||||
parser.add_argument("--timezone", default=DEFAULT_TZ)
|
||||
parser.add_argument("--timeout", type=int, default=DEFAULT_TIMEOUT)
|
||||
parser.add_argument("--workers", type=int, default=10)
|
||||
parser.add_argument("--format", choices=("json", "markdown"), default="json")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.days < 1:
|
||||
raise SystemExit("--days must be at least 1")
|
||||
if args.limit is not None and args.limit < 1:
|
||||
raise SystemExit("--limit must be at least 1")
|
||||
|
||||
try:
|
||||
target_tz = dt.ZoneInfo(args.timezone)
|
||||
except Exception as exc:
|
||||
raise SystemExit(f"Invalid timezone '{args.timezone}': {exc}")
|
||||
|
||||
if args.date:
|
||||
target_date = dt.date.fromisoformat(args.date)
|
||||
else:
|
||||
target_date = dt.datetime.now(target_tz).date()
|
||||
start_date = target_date - dt.timedelta(days=args.days - 1)
|
||||
|
||||
feeds = load_feeds(args.feeds_file)
|
||||
|
||||
results = []
|
||||
overall_timeout = max(args.timeout * 2, 60)
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as executor:
|
||||
futures = [executor.submit(fetch_feed, feed, args.timeout) for feed in feeds]
|
||||
try:
|
||||
for future in concurrent.futures.as_completed(futures, timeout=overall_timeout):
|
||||
try:
|
||||
results.append(future.result(timeout=args.timeout))
|
||||
except Exception as exc:
|
||||
# Shouldn't happen with our fetch_feed, but safety net
|
||||
pass
|
||||
except concurrent.futures.TimeoutError:
|
||||
# Collect whatever completed, mark rest as errors
|
||||
for future in futures:
|
||||
if future.done():
|
||||
try:
|
||||
results.append(future.result(timeout=0))
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
future.cancel()
|
||||
|
||||
items = []
|
||||
errors = []
|
||||
for result in results:
|
||||
if result["status"] != "ok":
|
||||
errors.append(
|
||||
{
|
||||
"feed_name": result["feed"]["name"],
|
||||
"feed_url": result["feed"]["xml_url"],
|
||||
"error": result["error"],
|
||||
}
|
||||
)
|
||||
continue
|
||||
for entry in result["entries"]:
|
||||
published_at = entry["published_at"]
|
||||
if not published_at:
|
||||
continue
|
||||
published_local = published_at.astimezone(target_tz)
|
||||
published_date = published_local.date()
|
||||
if published_date < start_date or published_date > target_date:
|
||||
continue
|
||||
if not entry["link"]:
|
||||
continue
|
||||
items.append(serialize_item(entry, target_tz))
|
||||
|
||||
items.sort(
|
||||
key=lambda item: (
|
||||
item["published_local"] or "",
|
||||
item["feed_name"].lower(),
|
||||
item["title"].lower(),
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
errors.sort(key=lambda err: err["feed_name"].lower())
|
||||
if args.limit is not None:
|
||||
items = items[: args.limit]
|
||||
|
||||
payload = {
|
||||
"start_date": start_date.isoformat(),
|
||||
"target_date": target_date.isoformat(),
|
||||
"days": args.days,
|
||||
"timezone": args.timezone,
|
||||
"feed_count": len(feeds),
|
||||
"items": items,
|
||||
"errors": errors,
|
||||
}
|
||||
|
||||
if args.format == "markdown":
|
||||
sys.stdout.write(format_markdown(payload))
|
||||
else:
|
||||
json.dump(payload, sys.stdout, ensure_ascii=True, indent=2)
|
||||
sys.stdout.write("\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if not hasattr(dt, "ZoneInfo"):
|
||||
from zoneinfo import ZoneInfo # type: ignore
|
||||
|
||||
dt.ZoneInfo = ZoneInfo # type: ignore[attr-defined]
|
||||
main()
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Wrapper for ak-rss-digest: add state-based deduplication on top of the real fetch script."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import os
|
||||
|
||||
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
REAL_SCRIPT = os.path.join(SCRIPT_DIR, "fetch_today_feed_items.py")
|
||||
STATE_FILE = os.path.join(os.path.dirname(SCRIPT_DIR), "references", "last_seen.json")
|
||||
|
||||
# Run the real script
|
||||
result = subprocess.run(
|
||||
[sys.executable, REAL_SCRIPT] + sys.argv[1:],
|
||||
capture_output=True, text=True
|
||||
)
|
||||
if result.returncode != 0:
|
||||
print(result.stderr, file=sys.stderr)
|
||||
sys.exit(result.returncode)
|
||||
|
||||
data = json.loads(result.stdout)
|
||||
entries = data.get("entries", [])
|
||||
|
||||
# Load last seen
|
||||
last_seen = {}
|
||||
if os.path.exists(STATE_FILE):
|
||||
with open(STATE_FILE) as f:
|
||||
last_seen = json.load(f)
|
||||
|
||||
# Filter: keep only entries not seen before (by URL)
|
||||
new_entries = []
|
||||
new_seen = {}
|
||||
for e in entries:
|
||||
url = e.get("url", "")
|
||||
source = e.get("source", "")
|
||||
if url:
|
||||
key = f"{source}|{url}"
|
||||
new_seen[key] = True
|
||||
if key not in last_seen:
|
||||
new_entries.append(e)
|
||||
|
||||
# Group new entries by source
|
||||
new_by_source = {}
|
||||
for e in new_entries:
|
||||
src = e.get("source", "unknown")
|
||||
new_by_source[src] = new_by_source.get(src, 0) + 1
|
||||
|
||||
# Output filtered results
|
||||
output = {
|
||||
**data,
|
||||
"entries": new_entries,
|
||||
"has_new": len(new_entries) > 0,
|
||||
"new_by_source": new_by_source,
|
||||
"total_fetched": len(entries),
|
||||
}
|
||||
|
||||
# Save state
|
||||
os.makedirs(os.path.dirname(STATE_FILE), exist_ok=True)
|
||||
with open(STATE_FILE, "w") as f:
|
||||
json.dump(new_seen, f, ensure_ascii=False, indent=2)
|
||||
|
||||
print(json.dumps(output, ensure_ascii=False, indent=2))
|
||||
Reference in New Issue
Block a user