Sync main branch to develop

.env.example

@@ -76,19 +76,6 @@ DATABASE_URL=postgres://user:password@localhost:5432/boost_dashboard
 # Slack webhook URL (get from Slack: https://api.slack.com/messaging/webhooks)
 # SLACK_WEBHOOK_URL=https://hooks.slack.com/services/YOUR/WEBHOOK/URL
 
-# =============================================================================
-# Clang GitHub Tracker
-# =============================================================================
-# GitHub repo to sync (default: llvm/llvm-project).
-# CLANG_GITHUB_OWNER=llvm
-# CLANG_GITHUB_REPO=llvm-project
-#
-# Private repo for Markdown export (optional).
-# Issues/PRs are exported to: issues/YYYY/YYYY-MM/#N - title.md
-# If unset, upload is skipped and an error is logged.
-# CLANG_GITHUB_TRACKER_PRIVATE_REPO_OWNER=your-org
-# CLANG_GITHUB_TRACKER_PRIVATE_REPO_NAME=your-private-repo
-# CLANG_GITHUB_TRACKER_PRIVATE_REPO_BRANCH=main
 
 # =============================================================================
 # GitHub tokens (multiple use cases)
@@ -108,6 +95,13 @@ DATABASE_URL=postgres://user:password@localhost:5432/boost_dashboard
 # GitHub repo to sync (default: llvm/llvm-project).
 # CLANG_GITHUB_OWNER=llvm
 # CLANG_GITHUB_REPO=llvm-project
+# Markdown publish target (optional; see also Clang section above).
+# CLANG_GITHUB_CONTEXT_REPO_OWNER=your-org
+# CLANG_GITHUB_CONTEXT_REPO_NAME=your-repo
+# CLANG_GITHUB_CONTEXT_REPO_BRANCH=main
+# If that repo is private: set GITHUB_TOKEN_WRITE to a PAT that can read+push it
+# (classic: repo scope; fine-grained: grant this repository). Publish uses the
+# write token, not GITHUB_TOKENS_SCRAPING.
 # Pinecone sync (run_cppa_pinecone_sync) — app_type and namespace when triggering from this app.
 # CLANG_GITHUB_PINECONE_APP_TYPE=github-clang
 # CLANG_GITHUB_PINECONE_NAMESPACE=github-clang
@@ -170,17 +164,18 @@ DATABASE_URL=postgres://user:password@localhost:5432/boost_dashboard
 # REPO_COUNT_LANGUAGES=C++,Python,Rust
 
 # =============================================================================
-# Boost Library Usage Dashboard (optional; for --publish)
+# Boost Library Usage Dashboard
 # =============================================================================
-# When set, run_boost_library_usage_dashboard --publish uses a persistent clone
-# at raw/boost_library_usage_dashboard/<owner>/<repo> (clone if missing, pull, copy, push).
+# Target repo for publishing (run_boost_library_usage_dashboard without --skip-publish).
+# Clone/pull/push uses GITHUB_TOKEN_WRITE (see GitHub tokens above).
 # BOOST_LIBRARY_USAGE_DASHBOARD_PUBLISH_OWNER=your-org
 # BOOST_LIBRARY_USAGE_DASHBOARD_PUBLISH_REPO=your-dashboard-repo
-# Token for clone/pull/push (defaults to GITHUB_TOKEN_WRITE if unset)
-# BOOST_LIBRARY_USAGE_DASHBOARD_PUBLISH_TOKEN=ghp_xxxx
-# Branch to publish to
 # BOOST_LIBRARY_USAGE_DASHBOARD_PUBLISH_BRANCH=main
 
+# Git commit author identity used when publishing (defaults shown)
+# GIT_AUTHOR_NAME=unknown
+# GIT_AUTHOR_EMAIL=unknown@noreply.github.com
+
 # =============================================================================
 # Workspace (optional; default: project_root/workspace)
 # =============================================================================
@@ -256,3 +251,15 @@ DATABASE_URL=postgres://user:password@localhost:5432/boost_dashboard
 
 # Path to context repository (where markdown files are exported)
 # DISCORD_CONTEXT_REPO_PATH=F:\boost\discord-cplusplus-together-context
+
+# =============================================================================
+# YouTube (cppa_youtube_script_tracker)
+# =============================================================================
+# YouTube Data API v3 key (console.cloud.google.com → APIs & Services → Credentials)
+# YOUTUBE_API_KEY=...
+
+# Pinecone namespace for YouTube video/transcript sync (default: youtube-scripts)
+# YOUTUBE_PINECONE_NAMESPACE=youtube-scripts
+
+# Earliest published_at to use when DB is empty (ISO 8601, e.g. 2015-01-01T00:00:00Z)
+# YOUTUBE_DEFAULT_PUBLISHED_AFTER=2015-01-01T00:00:00Z

.github/workflows/deploy-script/deploy.sh

@@ -60,4 +60,7 @@ until make health >/dev/null 2>&1; do
 done
 log "Stack is healthy."
 
+log "Sending startup notification..."
+DEPLOY_BRANCH="$BRANCH" make notify || log "WARNING: Startup notification failed (non-fatal)."
+
 log "Deploy completed."

.gitignore

@@ -52,3 +52,4 @@ discord_activity_tracker/tools/
 config/boost_collector_schedule.yaml
 # temp files
 temp/
+nul

Dockerfile

@@ -33,6 +33,10 @@ RUN chmod +x /app/docker-entrypoint.sh
 
 # Entrypoint runs as root, chowns mounted dirs, then exec's CMD as appuser via gosu
 RUN useradd --create-home appuser && chown -R appuser /app
+# Git 2.35+ blocks repos when directory owner != current user; bind mounts often
+# disagree (e.g. Docker Desktop on Windows). System config applies to root and appuser
+# (e.g. docker exec as root vs gosu appuser in entrypoint).
+RUN git config --system --add safe.directory '/app/workspace/*'
 ENTRYPOINT ["/app/docker-entrypoint.sh"]
 # Container starts as root so entrypoint can chown; CMD runs as appuser via gosu
 

Makefile

@@ -9,6 +9,7 @@
 SHELL := /bin/bash
 COMPOSE   := docker compose
 APP       := web
+BEAT      := celery_beat
 MANAGE    := $(COMPOSE) run --rm $(APP) python manage.py
 
 .DEFAULT_GOAL := help
@@ -32,6 +33,7 @@ help:
 	@echo "  Logs & status"
 	@echo "    ps             Show running containers"
 	@echo "    health         Verify DB, Redis, Selenium, and Celery containers"
+	@echo "    notify         Send Slack/Discord startup notification (celery_beat; optional DEPLOY_BRANCH)"
 	@echo "    logs           Follow logs for all services"
 	@echo "    logs-web       Follow logs for the web service"
 	@echo "    logs-worker    Follow logs for the Celery worker"
@@ -101,6 +103,10 @@ health:
 	$(COMPOSE) ps --status running celery_worker | grep -q celery_worker
 	$(COMPOSE) ps --status running celery_beat | grep -q celery_beat
 
+.PHONY: notify
+notify:
+	$(COMPOSE) exec -T -e DEPLOY_BRANCH="$(DEPLOY_BRANCH)" $(BEAT) python manage.py send_startup_notification
+
 .PHONY: logs
 logs:
 	$(COMPOSE) logs -f

boost_library_docs_tracker/fetcher.py

@@ -76,9 +76,9 @@ def download_source_zip(version: str, dest_dir: Path) -> Path:
     zip_name = f"boost_{normalized.replace('.', '_')}.zip"
     zip_path = dest_dir / zip_name
 
-    # if zip_path.exists():
-    #     logger.info("Source zip already present, skipping download: %s", zip_path)
-    #     return zip_path
+    if zip_path.exists():
+        logger.info("Source zip already present, skipping download: %s", zip_path)
+        return zip_path
 
     dest_dir.mkdir(parents=True, exist_ok=True)
     session = _get_session()
@@ -320,16 +320,26 @@ def crawl_library_pages(
 
         # Enqueue in-scope links
         soup = BeautifulSoup(resp.text, "lxml")
-        for a in soup.find_all("a", href=True):
-            href: str = a["href"]
-            abs_url = urljoin(final_url, href)
-            # Strip fragment
-            abs_url = abs_url.split("#")[0]
-            if (
-                abs_url not in visited
-                and abs_url.startswith(start_url)
-                and abs_url not in queue
-            ):
+        lib_segment = lib_key.split("/")[-1]
+        if not lib_segment:
+            logger.warning(
+                "Empty library key segment for lib_key=%r; skipping link discovery for %s",
+                lib_key,
+                final_url,
+            )
+        else:
+            for a in soup.find_all("a", href=True):
+                href: str = a["href"]
+                abs_url = urljoin(final_url, href)
+                # Strip fragment
+                abs_url = abs_url.split("#")[0]
+                if not abs_url.startswith(base_url):
+                    continue
+                # Stay within this library's doc subtree (path contains lib segment)
+                if lib_segment not in abs_url:
+                    continue
+                if abs_url in visited or abs_url in queue:
+                    continue
                 queue.append(abs_url)
 
     logger.debug(

boost_library_docs_tracker/html_to_md.py

@@ -9,14 +9,16 @@
 --------
 1. _preprocess_html   – remove Boost boilerplate from HTML before pandoc sees it
 2. _pandoc_convert    – HTML → GFM via pypandoc (CLI fallback)
-3. _postprocess_markdown – strip residual HTML artefacts and rejoin split lines
+3. _postprocess_markdown – strip residual HTML artefacts, rejoin split lines, then clean_text (unicode/line endings only)
 """
 
 import re
 import subprocess
 
 from bs4 import BeautifulSoup
 
+from core.utils.text_processing import clean_text
+
 try:
     import pypandoc
 except Exception:  # optional runtime dependency
@@ -299,4 +301,7 @@ def _postprocess_markdown(md: str) -> str:
     # 12. Collapse excessive blank lines to at most two
     md = _RE_EXCESS_BLANK.sub("\n\n", md)
 
-    return md.strip() + "\n"
+    # 13. Unicode / line-ending cleanup (no space collapsing — preserves markdown indent)
+    md = clean_text(md, remove_extra_spaces=False)
+
+    return md.rstrip() + "\n"