From 7cb400d210fcc006090b4800702d0200864b5172 Mon Sep 17 00:00:00 2001 From: sid-plt <218677312+sid-plt@users.noreply.github.com> Date: Thu, 27 Aug 2026 12:16:58 +0200 Subject: [PATCH] kicker-lineup-extraction --- README.md | 36 +- .../03_kicker_lineups.ipynb | 717 ++++++++++++++++++ project_paths.py | 3 + 3 files changed, 748 insertions(+), 8 deletions(-) create mode 100644 notebooks/05_predicted_lineups/03_kicker_lineups.ipynb diff --git a/README.md b/README.md index 4f6ddef..85e7257 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,12 @@ > Commit refreshed output files alongside the change that produced them so the > data history remains reproducible. -This project collects and derives Bundesliga fixture, odds, team, squad, and -player data from SofaScore, FotMob, Transfermarkt, KBStats, and Analyst. +This project supports a repeatable Bundesliga-to-Kickbase decision workflow: +collect fixtures, odds, team, squad, player, and predicted-lineup data; derive +comparable signals and expected-points scores; then build and save constrained +Kickbase squads. Its live sources include SofaScore, FotMob, Transfermarkt, +KBStats, RotoWire, LigaInsider, Kicker, and manually transcribed Kickbase +lineups. ## Project layout @@ -19,7 +23,10 @@ player data from SofaScore, FotMob, Transfermarkt, KBStats, and Analyst. high-rated-player analysis, KBStats average-points percentile filtering, and KBStats last-five-match-slot analysis. - `notebooks/05_predicted_lineups`: live predicted and confirmed lineup - collectors. + collectors from RotoWire, LigaInsider, and Kicker, plus compatible + source-specific outputs. +- `notebooks/06_score_creation`: score builders that combine source signals + into expected-points files for the optimizers. - `notebooks/07_squad_optimisation`: optimizer notebooks for Bundesliga Arena, KickbaseKIS Arena, All Limits Arena, and Kickbase.insider Arena, plus `manually_create_lineup.ipynb` for manual entry. @@ -30,6 +37,11 @@ player data from SofaScore, FotMob, Transfermarkt, KBStats, and Analyst. - `outputs/fotmob`: FotMob match IDs and odds. - `outputs/transfermarkt`: squad exports and diagnostic HTML. - `outputs/rotowire`: timestamped Bundesliga predicted-lineup snapshots. +- `outputs/ligainsider`: timestamped Bundesliga predicted-lineup snapshots, + including formation slots and candidate alternatives where supplied. +- `outputs/kicker`: timestamped Bundesliga predicted-lineup snapshots. Each + record preserves Kicker’s raw lineup text, formation, coach, bench, + unavailable players, and explanation alongside the common home/away schema. - `outputs/kickbase`: timestamped predicted-lineup snapshots transcribed from user-submitted Kickbase screenshots; player names remain source display names and are not resolved to canonical identities. @@ -80,8 +92,10 @@ python -m pytest ``` The existing `.venv` stays at the root because moving a virtual environment can -invalidate absolute paths stored inside it. The scraping notebooks expect a -compatible Chrome installation; their current configuration targets Chrome 150. +invalidate absolute paths stored inside it. The browser-backed scraping notebooks +expect a compatible Chrome installation; their current configuration targets +Chrome 150. If Kicker presents a browser-verification page, complete it manually +in the Chrome window when prompted; the notebook does not attempt to bypass it. ## Version-control conventions @@ -105,9 +119,12 @@ creating a timestamped history. statistics require `outputs/sofascore/reference/bundesliga_teams.json`. 5. Run the derived notebooks. The high-rated-player notebook automatically uses the newest valid timestamped file in `outputs/sofascore/team_form`. -6. Run `notebooks/05_predicted_lineups/01_rotowire_lineups.ipynb` whenever a - fresh RotoWire Bundesliga lineup snapshot is needed. -7. Create the required expected-points score file in `notebooks/06_score_creation`. +6. Run the required collectors in `notebooks/05_predicted_lineups`: RotoWire, + LigaInsider, Kicker, and/or the source-specific manual Kickbase workflow. + Their snapshots share the match/home/away/player envelope while retaining + source-specific details. +7. Create the required expected-points score file in + `notebooks/06_score_creation`. 8. Run an optimizer in `notebooks/07_squad_optimisation`, or run `manually_create_lineup.ipynb` to enter a formation, players, and captain interactively. A confirmed selection is stored in `outputs/selected_lineups`. @@ -128,6 +145,9 @@ SofaScore team-reference pipeline when their own inputs are available. | Team statistics | Team reference | `outputs/sofascore/team_stats` | | Transfermarkt squads | Live Transfermarkt pages | `outputs/transfermarkt/squads` | | RotoWire predicted lineups | Live RotoWire Bundesliga lineups page | `outputs/rotowire/predicted_lineups` | +| LigaInsider predicted lineups | Live LigaInsider team pages | `outputs/ligainsider/predicted_lineups` | +| Kicker predicted lineups | Live Kicker matchday and fixture pages | `outputs/kicker/predicted_lineups` | +| Kickbase screenshot lineups | User-submitted screenshots and fixture reference | `outputs/kickbase/predicted_lineups` | | KBStats players | Live KBStats API | `outputs/kbstats/players` | | Bundesliga snapshot | Team reference and live source data | `outputs/derived/bundesliga_snapshots` | | High-rated players | Latest team-form snapshot | `outputs/sofascore/high_rated_players` | diff --git a/notebooks/05_predicted_lineups/03_kicker_lineups.ipynb b/notebooks/05_predicted_lineups/03_kicker_lineups.ipynb new file mode 100644 index 0000000..2d96990 --- /dev/null +++ b/notebooks/05_predicted_lineups/03_kicker_lineups.ipynb @@ -0,0 +1,717 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "902fd53e", + "metadata": {}, + "source": [ + "# Kicker Bundesliga Predicted Lineups\n", + "\n", + "This notebook prompts for a Bundesliga matchday in the fixed 2026-27 season, collects every Kicker fixture's lineup page through one undetected Chrome 150 session, and writes a lossless predicted-lineup snapshot.\n", + "\n", + "Kicker's displayed lineup order is preserved. The first four paragraphs in each `kick__lineup-text` block are interpreted as starting XI, coach, bench, and unavailable players respectively; their original HTML and text are retained alongside the parsed fields." + ] + }, + { + "cell_type": "markdown", + "id": "8e00676e", + "metadata": {}, + "source": [ + "## 1. Resolve project and output paths" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "1c390f78", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Kicker lineup output directory: C:\\kickbase project\\outputs\\kicker\\predicted_lineups\n" + ] + } + ], + "source": [ + "import sys\n", + "from pathlib import Path\n", + "\n", + "\n", + "def locate_project_root() -> Path:\n", + " \"\"\"Find the repository when run from JupyterLab or VS Code.\"\"\"\n", + " starts = []\n", + " vscode_notebook = globals().get('__vsc_ipynb_file__')\n", + " if isinstance(vscode_notebook, str) and vscode_notebook.strip():\n", + " starts.append(Path(vscode_notebook).expanduser().resolve().parent)\n", + " starts.append(Path.cwd().resolve())\n", + " checked = set()\n", + " for start in starts:\n", + " for candidate in (start, *start.parents):\n", + " if candidate in checked:\n", + " continue\n", + " checked.add(candidate)\n", + " if (candidate / 'project_paths.py').is_file():\n", + " return candidate\n", + " raise FileNotFoundError(\n", + " 'Could not locate project_paths.py. Start Jupyter from the Kickbase project root.'\n", + " )\n", + "\n", + "\n", + "PROJECT_ROOT = locate_project_root()\n", + "if str(PROJECT_ROOT) not in sys.path:\n", + " sys.path.insert(0, str(PROJECT_ROOT))\n", + "\n", + "from project_paths import KICKER_PREDICTED_LINEUPS_DIR, ensure_directory\n", + "\n", + "output_directory = ensure_directory(KICKER_PREDICTED_LINEUPS_DIR)\n", + "print(f'Kicker lineup output directory: {output_directory}')" + ] + }, + { + "cell_type": "markdown", + "id": "5ca526fc", + "metadata": {}, + "source": [ + "## 2. Imports, source configuration, and matchday input" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "670f5fce", + "metadata": {}, + "outputs": [ + { + "name": "stdin", + "output_type": "stream", + "text": [ + "Bundesliga matchday (1-34): 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Kicker source URL: https://www.kicker.de/bundesliga/spieltag/2026-27/1\n" + ] + } + ], + "source": [ + "import json\n", + "import re\n", + "from datetime import datetime\n", + "from typing import Any\n", + "from urllib.parse import urljoin\n", + "\n", + "try:\n", + " import undetected_chromedriver as uc\n", + " from bs4 import BeautifulSoup\n", + " from selenium.common.exceptions import TimeoutException, WebDriverException\n", + " from selenium.webdriver.common.by import By\n", + " from selenium.webdriver.support.ui import WebDriverWait\n", + "except ImportError as exc:\n", + " raise ImportError(\n", + " 'Required packages are missing. Install them with: '\n", + " '%pip install beautifulsoup4 undetected-chromedriver selenium'\n", + " ) from exc\n", + "\n", + "SOURCE_NAME = 'Kicker'\n", + "SEASON = '2026-27'\n", + "SCHEDULE_URL_TEMPLATE = 'https://www.kicker.de/bundesliga/spieltag/{season}/{matchday}'\n", + "CHROME_MAJOR_VERSION = 150\n", + "PAGE_LOAD_TIMEOUT_SECONDS = 45\n", + "WAIT_TIMEOUT_SECONDS = 30\n", + "MIN_MATCHDAY = 1\n", + "MAX_MATCHDAY = 34\n", + "GAME_DATE_GROUP_SELECTOR = 'div.kick__v100-gameList.kick__module-margin'\n", + "GAME_ROW_SELECTOR = 'div.kick__v100-gameList__gameRow'\n", + "SCHEMA_LINK_SELECTOR = (\n", + " 'a.kick__v100-gameList__gameRow__stateCell__indicator'\n", + " '.kick__v100-gameList__gameRow__stateCell__indicator--schema'\n", + ")\n", + "LINEUP_BLOCK_SELECTOR = 'div.kick__lineup-text'\n", + "LINEUP_EXPLANATION_SELECTOR = '.kick__lineup-explanation'\n", + "CHALLENGE_MARKERS = (\n", + " \"verify that you're not a robot\",\n", + " 'verify you are human',\n", + " 'verify that you are human',\n", + " 'javascript is disabled',\n", + " 'access denied',\n", + ")\n", + "\n", + "\n", + "def prompt_matchday() -> int:\n", + " raw_matchday = input(f'Bundesliga matchday ({MIN_MATCHDAY}-{MAX_MATCHDAY}): ').strip()\n", + " try:\n", + " matchday = int(raw_matchday)\n", + " except ValueError as exc:\n", + " raise ValueError('Matchday must be a whole number.') from exc\n", + " if not MIN_MATCHDAY <= matchday <= MAX_MATCHDAY:\n", + " raise ValueError(f'Matchday must be between {MIN_MATCHDAY} and {MAX_MATCHDAY}.')\n", + " return matchday\n", + "\n", + "\n", + "matchday = prompt_matchday()\n", + "source_url = SCHEDULE_URL_TEMPLATE.format(season=SEASON, matchday=matchday)\n", + "print(f'Kicker source URL: {source_url}')" + ] + }, + { + "cell_type": "markdown", + "id": "c35a8030", + "metadata": {}, + "source": [ + "## 3. Parse Kicker schedule rows and lineup blocks" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "6874b8e2", + "metadata": {}, + "outputs": [], + "source": [ + "def clean_text(value: Any) -> str | None:\n", + " if value is None:\n", + " return None\n", + " cleaned = re.sub(r'\\s+', ' ', str(value).replace('\\xa0', ' ')).strip()\n", + " return cleaned or None\n", + "\n", + "\n", + "def linked_player(anchor: Any, fixture_url: str, formation_row: int | None = None, slot_index: int | None = None) -> dict[str, Any]:\n", + " displayed_name = clean_text(anchor.get_text(' ', strip=True))\n", + " href = clean_text(anchor.get('href'))\n", + " if displayed_name is None or href is None:\n", + " raise ValueError('A Kicker player link is missing its displayed name or URL.')\n", + " return {\n", + " 'full_name': None,\n", + " 'displayed_name': displayed_name,\n", + " 'position': None,\n", + " 'injury_status': None,\n", + " 'player_id': None,\n", + " 'player_url': urljoin(fixture_url, href),\n", + " 'formation_row': formation_row,\n", + " 'slot_index': slot_index,\n", + " 'starting_probability_rank': 1,\n", + " }\n", + "\n", + "\n", + "def parse_starting_xi(paragraph: Any | None, fixture_url: str) -> dict[str, Any]:\n", + " raw_text = clean_text(paragraph.get_text(' ', strip=True)) if paragraph else None\n", + " rows: list[list[dict[str, Any]]] = [[]]\n", + " if paragraph is not None:\n", + " for child in paragraph.children:\n", + " if getattr(child, 'name', None) == 'a':\n", + " rows[-1].append(linked_player(child, fixture_url))\n", + " elif getattr(child, 'name', None) is None and re.search(r'\\s[-–]\\s', str(child)):\n", + " rows.append([])\n", + " rows = [row for row in rows if row]\n", + " for formation_row, row in enumerate(rows, start=1):\n", + " for slot_index, player in enumerate(row, start=1):\n", + " player['formation_row'] = formation_row\n", + " player['slot_index'] = slot_index\n", + " line_counts = [len(row) for row in rows]\n", + " outfield_counts = line_counts[1:] if line_counts[:1] == [1] else line_counts\n", + " formation = '-'.join(str(count) for count in outfield_counts) or None\n", + " return {\n", + " 'raw_text': raw_text,\n", + " 'formation': formation,\n", + " 'formation_lines': outfield_counts,\n", + " 'formation_row_counts': line_counts,\n", + " 'players': [player for row in rows for player in row],\n", + " }\n", + "\n", + "\n", + "def parse_linked_list(paragraph: Any | None, fixture_url: str) -> list[dict[str, Any]]:\n", + " if paragraph is None:\n", + " return []\n", + " return [linked_player(anchor, fixture_url) for anchor in paragraph.select('a[href]')]\n", + "\n", + "\n", + "def parse_unavailable(paragraph: Any | None, fixture_url: str) -> dict[str, Any]:\n", + " raw_text = clean_text(paragraph.get_text(' ', strip=True)) if paragraph else None\n", + " players = []\n", + " for anchor in paragraph.select('a[href]') if paragraph is not None else []:\n", + " suffix = clean_text(anchor.next_sibling) or ''\n", + " reason_match = re.search(r'\\(([^()]*)\\)', suffix)\n", + " player = linked_player(anchor, fixture_url)\n", + " players.append({\n", + " 'raw_text': clean_text(f\"{player['displayed_name']}{suffix}\"),\n", + " 'name': player['displayed_name'],\n", + " 'reason': clean_text(reason_match.group(1)) if reason_match else None,\n", + " 'player_url': player['player_url'],\n", + " })\n", + " return {'raw_text': raw_text, 'players': players}\n", + "\n", + "\n", + "def parse_lineup_block(block: Any, lineup_index: int, fixture_url: str) -> dict[str, Any]:\n", + " paragraphs = []\n", + " for paragraph in block.find_all('p'):\n", + " paragraphs.append({\n", + " 'index': len(paragraphs) + 1,\n", + " 'outer_html': str(paragraph),\n", + " 'text': clean_text(paragraph.get_text(' ', strip=True)),\n", + " })\n", + " paragraph_tags = block.find_all('p')\n", + " starting_paragraph = paragraph_tags[0] if paragraph_tags else None\n", + " coach_paragraph = paragraph_tags[1] if len(paragraph_tags) > 1 else None\n", + " bench_paragraph = paragraph_tags[2] if len(paragraph_tags) > 2 else None\n", + " unavailable_paragraph = paragraph_tags[3] if len(paragraph_tags) > 3 else None\n", + " coach_players = parse_linked_list(coach_paragraph, fixture_url)\n", + " return {\n", + " 'lineup_index': lineup_index,\n", + " 'raw_html': str(block),\n", + " 'paragraphs': paragraphs,\n", + " 'starting_xi': parse_starting_xi(starting_paragraph, fixture_url),\n", + " 'coach': {\n", + " 'raw_text': clean_text(coach_paragraph.get_text(' ', strip=True)) if coach_paragraph else None,\n", + " 'name': coach_players[0]['displayed_name'] if coach_players else None,\n", + " 'coach_url': coach_players[0]['player_url'] if coach_players else None,\n", + " },\n", + " 'bench': {\n", + " 'raw_text': clean_text(bench_paragraph.get_text(' ', strip=True)) if bench_paragraph else None,\n", + " 'players': parse_linked_list(bench_paragraph, fixture_url),\n", + " },\n", + " 'unavailable': parse_unavailable(unavailable_paragraph, fixture_url),\n", + " }\n", + "\n", + "\n", + "def extract_fixture_links(schedule_html: str, schedule_url: str) -> list[dict[str, str]]:\n", + " soup = BeautifulSoup(schedule_html, 'html.parser')\n", + " date_groups = soup.select(GAME_DATE_GROUP_SELECTOR)\n", + " if not date_groups:\n", + " raise ValueError('No Kicker match-date groups were found on the schedule page.')\n", + " fixtures = []\n", + " seen_urls = set()\n", + " for date_group in date_groups:\n", + " for game_row in date_group.select(GAME_ROW_SELECTOR):\n", + " link = game_row.select_one(SCHEMA_LINK_SELECTOR)\n", + " href = clean_text(link.get('href') if link else None)\n", + " if href is None:\n", + " continue\n", + " fixture_url = urljoin(schedule_url, href)\n", + " if fixture_url in seen_urls:\n", + " continue\n", + " seen_urls.add(fixture_url)\n", + " fixtures.append({\n", + " 'fixture_url': fixture_url,\n", + " 'schedule_row_text': clean_text(game_row.get_text(' ', strip=True)) or '',\n", + " })\n", + " if not fixtures:\n", + " raise ValueError('No Kicker lineup schema links were found on the schedule page.')\n", + " return fixtures\n", + "\n", + "\n", + "def extract_team_name(block: Any) -> str | None:\n", + " logo = block.select_one('.kick__lineup-text__headline img[alt]')\n", + " if logo is not None:\n", + " return clean_text(logo.get('alt'))\n", + " headline = block.select_one('.kick__lineup-text__headline strong')\n", + " return clean_text(headline.get_text(' ', strip=True)) if headline else None\n", + "\n", + "\n", + "def extract_match_time(soup: Any, schedule_row_text: str) -> str:\n", + " time_element = soup.select_one('time[datetime]') or soup.select_one('time')\n", + " if time_element is not None:\n", + " return clean_text(time_element.get('datetime')) or clean_text(\n", + " time_element.get_text(' ', strip=True)\n", + " ) or schedule_row_text\n", + " schedule_time = re.search(r'\\b(?:Mo|Di|Mi|Do|Fr|Sa|So)\\.\\s*\\d{1,2}:\\d{2}\\b', schedule_row_text)\n", + " return clean_text(schedule_time.group(0)) if schedule_time else schedule_row_text\n", + "\n", + "\n", + "def make_kicker_players(starting_xi: dict[str, Any]) -> list[dict[str, Any]]:\n", + " return [dict(player) for player in starting_xi['players']]\n", + "\n", + "\n", + "def make_kicker_team(lineup: dict[str, Any], team_name: str | None, side: str) -> dict[str, Any]:\n", + " return {\n", + " 'team_name': team_name,\n", + " 'side': side,\n", + " 'lineup_status': 'Predicted Lineup',\n", + " 'players': make_kicker_players(lineup['starting_xi']),\n", + " 'kicker_details': {\n", + " key: lineup[key]\n", + " for key in ('lineup_index', 'raw_html', 'paragraphs', 'starting_xi', 'coach', 'bench', 'unavailable')\n", + " },\n", + " }\n", + "\n", + "\n", + "def parse_fixture_page(fixture_html: str, fixture: dict[str, str]) -> dict[str, Any] | None:\n", + " soup = BeautifulSoup(fixture_html, 'html.parser')\n", + " blocks = soup.select(LINEUP_BLOCK_SELECTOR)\n", + " if not blocks:\n", + " return None\n", + " if len(blocks) != 2:\n", + " raise ValueError(f'Expected two Kicker lineup blocks, found {len(blocks)}.')\n", + " home_lineup, away_lineup = [\n", + " parse_lineup_block(block, index, fixture['fixture_url'])\n", + " for index, block in enumerate(blocks, start=1)\n", + " ]\n", + " return {\n", + " 'match_time': extract_match_time(soup, fixture['schedule_row_text']),\n", + " 'home': make_kicker_team(home_lineup, extract_team_name(blocks[0]), 'home'),\n", + " 'away': make_kicker_team(away_lineup, extract_team_name(blocks[1]), 'away'),\n", + " 'kicker_details': {\n", + " **fixture,\n", + " 'lineup_block_count': len(blocks),\n", + " 'explanations': [\n", + " {\n", + " 'outer_html': str(explanation),\n", + " 'text': clean_text(explanation.get_text(' ', strip=True)),\n", + " }\n", + " for explanation in soup.select(LINEUP_EXPLANATION_SELECTOR)\n", + " ],\n", + " },\n", + " }\n", + "\n", + "\n", + "def page_contains_challenge(page_html: str) -> bool:\n", + " visible_text = BeautifulSoup(page_html, 'html.parser').get_text(' ', strip=True).casefold()\n", + " return any(marker in visible_text for marker in CHALLENGE_MARKERS)\n", + "\n", + "\n", + "def wait_for_required_elements(current_driver: Any, selector: str, page_label: str) -> None:\n", + " if page_contains_challenge(current_driver.page_source):\n", + " print(\n", + " f'Kicker is requesting a browser verification for the {page_label}. '\n", + " 'Complete it yourself in the Chrome window, then return here.'\n", + " )\n", + " input('Press Enter after the Kicker page has loaded normally: ')\n", + " try:\n", + " WebDriverWait(current_driver, WAIT_TIMEOUT_SECONDS).until(\n", + " lambda active_driver: active_driver.find_elements(By.CSS_SELECTOR, selector)\n", + " )\n", + " except TimeoutException as exc:\n", + " if page_contains_challenge(current_driver.page_source):\n", + " raise RuntimeError(\n", + " f'Kicker verification is still active for the {page_label}. '\n", + " 'Complete it manually, then rerun this cell.'\n", + " ) from exc\n", + " raise\n", + "\n", + "\n", + "def wait_for_document(current_driver: Any) -> bool:\n", + " return current_driver.execute_script('return document.readyState') == 'complete'\n" + ] + }, + { + "cell_type": "markdown", + "id": "2363f211", + "metadata": {}, + "source": [ + "## 4. Run local parser checks\n", + "\n", + "These checks cover date-group iteration, deduplication, the four documented paragraph roles, and a shortened block with optional paragraphs omitted." + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "970dd0cb", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Local Kicker parser checks passed.\n" + ] + } + ], + "source": [ + "schedule_fixture_html = '''\n", + "
\n", + "
Fixture A
\n", + "
\n", + "
\n", + "
Fixture B
\n", + "
Fixture A duplicate
\n", + "
\n", + "'''\n", + "fixture_links = extract_fixture_links(schedule_fixture_html, 'https://www.kicker.de/bundesliga/spieltag/2026-27/1')\n", + "assert [item['fixture_url'] for item in fixture_links] == [\n", + " 'https://www.kicker.de/fixture-a',\n", + " 'https://www.kicker.de/fixture-b',\n", + "]\n", + "lineup_fixture_html = '''\n", + "\n", + "
\n", + "
\"Home
\n", + "

Voraussichtliche Aufstellung
Goalkeeper - Right Back, Centre Back - Midfielder - Forward

\n", + "

Trainer
Sample Coach

\n", + "

Reservebank
Substitute One, Substitute Two

\n", + "

Es fehlen
Injured Player (muscle injury), Rested Player (rested)

\n", + "
\n", + "
\"Away

Voraussichtliche Aufstellung
Away Goalkeeper

\n", + "

Explanation kept verbatim.

\n", + "'''\n", + "parsed_fixture = parse_fixture_page(lineup_fixture_html, fixture_links[0])\n", + "assert parsed_fixture is not None\n", + "assert parsed_fixture['match_time'] == '2026-08-28T20:30+02:00'\n", + "assert parsed_fixture['home']['team_name'] == 'Home FC'\n", + "assert parsed_fixture['away']['team_name'] == 'Away FC'\n", + "assert parsed_fixture['home']['kicker_details']['starting_xi']['formation_lines'] == [2, 1, 1]\n", + "assert [player['displayed_name'] for player in parsed_fixture['home']['players']] == ['Goalkeeper', 'Right Back', 'Centre Back', 'Midfielder', 'Forward']\n", + "assert parsed_fixture['home']['players'][0]['formation_row'] == 1\n", + "assert parsed_fixture['home']['players'][1]['slot_index'] == 1\n", + "assert parsed_fixture['home']['players'][0]['player_url'] == 'https://www.kicker.de/goalkeeper/spieler'\n", + "assert parsed_fixture['home']['kicker_details']['coach']['name'] == 'Sample Coach'\n", + "assert [player['displayed_name'] for player in parsed_fixture['home']['kicker_details']['bench']['players']] == ['Substitute One', 'Substitute Two']\n", + "assert parsed_fixture['home']['kicker_details']['unavailable']['players'][0]['reason'] == 'muscle injury'\n", + "assert parsed_fixture['away']['players'][0]['formation_row'] == 1\n", + "assert 'kept verbatim' in parsed_fixture['kicker_details']['explanations'][0]['outer_html']\n", + "assert not page_contains_challenge('')\n", + "assert page_contains_challenge('
Verify that you are human
')\n", + "print('Local Kicker parser checks passed.')" + ] + }, + { + "cell_type": "markdown", + "id": "7ed76021", + "metadata": {}, + "source": [ + "## 5. Load the Kicker schedule and every fixture\n", + "\n", + "A fixture without lineup blocks is retained in `skipped_fixtures` with its reason. A challenge or unavailable schedule page stops the run before any output file is written." + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "b0786802", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found 9 unique Kicker fixture links.\n", + " 1. Parsed Bayern München vs VfB Stuttgart: https://www.kicker.de/bayern-gegen-stuttgart-2026-bundesliga-5226808/aufstellung\n", + " 2. Parsed SV Elversberg vs Bayer 04 Leverkusen: https://www.kicker.de/elversberg-gegen-leverkusen-2026-bundesliga-5226809/aufstellung\n", + " 3. Parsed RB Leipzig vs Bor. Mönchengladbach: https://www.kicker.de/leipzig-gegen-mgladbach-2026-bundesliga-5226812/aufstellung\n", + " 4. Parsed 1. FSV Mainz 05 vs SC Paderborn 07: https://www.kicker.de/mainz-gegen-paderborn-2026-bundesliga-5226815/aufstellung\n", + " 5. Parsed 1. FC Union Berlin vs Eintracht Frankfurt: https://www.kicker.de/union-gegen-frankfurt-2026-bundesliga-5226816/aufstellung\n", + " 6. Parsed 1. FC Köln vs TSG Hoffenheim: https://www.kicker.de/koeln-gegen-hoffenheim-2026-bundesliga-5226817/aufstellung\n", + " 7. Parsed Borussia Dortmund vs Hamburger SV: https://www.kicker.de/dortmund-gegen-hsv-2026-bundesliga-5226811/aufstellung\n", + " 8. Parsed SC Freiburg vs Werder Bremen: https://www.kicker.de/freiburg-gegen-bremen-2026-bundesliga-5226813/aufstellung\n", + " 9. Parsed FC Augsburg vs FC Schalke 04: https://www.kicker.de/augsburg-gegen-schalke-2026-bundesliga-5226814/aufstellung\n", + "Chrome driver closed.\n", + "Extracted fixtures: 9; skipped fixtures: 0\n" + ] + } + ], + "source": [ + "scrape_started_at = datetime.now().astimezone()\n", + "scrape_finished_at = None\n", + "driver = None\n", + "fixtures = []\n", + "matches = []\n", + "skipped_fixtures = []\n", + "\n", + "try:\n", + " driver = uc.Chrome(version_main=CHROME_MAJOR_VERSION)\n", + " driver.set_window_size(1920, 1080)\n", + " driver.set_page_load_timeout(PAGE_LOAD_TIMEOUT_SECONDS)\n", + " wait = WebDriverWait(driver, WAIT_TIMEOUT_SECONDS)\n", + "\n", + " driver.get(source_url)\n", + " wait.until(wait_for_document)\n", + " wait_for_required_elements(driver, GAME_DATE_GROUP_SELECTOR, 'schedule page')\n", + " fixtures = extract_fixture_links(driver.page_source, source_url)\n", + " print(f'Found {len(fixtures)} unique Kicker fixture links.')\n", + "\n", + " for fixture_number, fixture in enumerate(fixtures, start=1):\n", + " try:\n", + " driver.get(fixture['fixture_url'])\n", + " wait.until(wait_for_document)\n", + " wait_for_required_elements(driver, LINEUP_BLOCK_SELECTOR, 'fixture page')\n", + " parsed_fixture = parse_fixture_page(driver.page_source, fixture)\n", + " if parsed_fixture is None:\n", + " skipped_fixtures.append({**fixture, 'reason': 'No Kicker lineup blocks found.'})\n", + " else:\n", + " matches.append(parsed_fixture)\n", + " print(\n", + " f\"{fixture_number:>2}. Parsed {parsed_fixture['home']['team_name']} vs \"\n", + " f\"{parsed_fixture['away']['team_name']}: {fixture['fixture_url']}\"\n", + " )\n", + " except TimeoutException:\n", + " latest_html = driver.page_source if driver is not None else ''\n", + " reason = (\n", + " 'Kicker presented a bot/challenge page.'\n", + " if page_contains_challenge(latest_html)\n", + " else 'Timed out waiting for Kicker lineup blocks.'\n", + " )\n", + " skipped_fixtures.append({**fixture, 'reason': reason})\n", + " except (RuntimeError, ValueError, WebDriverException) as exc:\n", + " skipped_fixtures.append({**fixture, 'reason': str(exc)})\n", + "except TimeoutException as exc:\n", + " latest_html = driver.page_source if driver is not None else ''\n", + " if page_contains_challenge(latest_html):\n", + " raise RuntimeError('Kicker presented a bot/challenge page for the schedule URL.') from exc\n", + " raise RuntimeError('Timed out while loading the Kicker schedule page.') from exc\n", + "except WebDriverException as exc:\n", + " raise RuntimeError(\n", + " 'Could not load Kicker through undetected Chrome 150. Confirm that compatible Chrome 150 is installed.'\n", + " ) from exc\n", + "finally:\n", + " scrape_finished_at = datetime.now().astimezone()\n", + " if driver is not None:\n", + " try:\n", + " driver.quit()\n", + " driver.quit = lambda: None\n", + " print('Chrome driver closed.')\n", + " except Exception as shutdown_error:\n", + " print(f'Chrome shutdown warning: {type(shutdown_error).__name__}: {shutdown_error}')\n", + " finally:\n", + " driver = None\n", + "\n", + "if not fixtures:\n", + " raise RuntimeError('Kicker did not provide any fixture links; no JSON snapshot was written.')\n", + "print(f'Extracted fixtures: {len(matches)}; skipped fixtures: {len(skipped_fixtures)}')" + ] + }, + { + "cell_type": "markdown", + "id": "1d0baf56", + "metadata": {}, + "source": [ + "## 6. Validate and write the timestamped JSON snapshot" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "ccd086b0", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saved Kicker lineup snapshot: C:\\kickbase project\\outputs\\kicker\\predicted_lineups\\kicker_bundesliga_lineups_20260827_121116.json\n" + ] + } + ], + "source": [ + "team_count = sum(2 for _ in matches)\n", + "player_count = sum(\n", + " len(match[side]['players'])\n", + " for match in matches\n", + " for side in ('home', 'away')\n", + ")\n", + "lineup_block_count = sum(match['kicker_details']['lineup_block_count'] for match in matches)\n", + "output_data = {\n", + " 'metadata': {\n", + " 'source': SOURCE_NAME,\n", + " 'source_url': source_url,\n", + " 'season': SEASON,\n", + " 'matchday': matchday,\n", + " 'captured_at': scrape_started_at.isoformat(timespec='seconds'),\n", + " 'capture_finished_at': scrape_finished_at.isoformat(timespec='seconds'),\n", + " 'fixture_link_count': len(fixtures),\n", + " 'match_count': len(matches),\n", + " 'team_count': team_count,\n", + " 'player_count': player_count,\n", + " 'lineup_block_count': lineup_block_count,\n", + " 'skipped_fixture_count': len(skipped_fixtures),\n", + " 'skipped_fixtures': skipped_fixtures,\n", + " },\n", + " 'matches': matches,\n", + "}\n", + "\n", + "filename_timestamp = scrape_started_at.strftime('%Y%m%d_%H%M%S')\n", + "output_path = output_directory / f'kicker_bundesliga_lineups_{filename_timestamp}.json'\n", + "temporary_output_path = output_path.with_suffix('.json.tmp')\n", + "try:\n", + " with temporary_output_path.open('w', encoding='utf-8', newline='\\n') as file:\n", + " json.dump(output_data, file, ensure_ascii=False, indent=2)\n", + " file.write('\\n')\n", + " temporary_output_path.replace(output_path)\n", + "except OSError as exc:\n", + " raise OSError(f'Could not write Kicker snapshot {output_path}: {exc}') from exc\n", + "finally:\n", + " temporary_output_path.unlink(missing_ok=True)\n", + "\n", + "with output_path.open('r', encoding='utf-8') as file:\n", + " validated_output = json.load(file)\n", + "if validated_output['metadata']['matchday'] != matchday:\n", + " raise ValueError('Saved Kicker JSON has an unexpected matchday.')\n", + "if len(validated_output['matches']) != len(matches):\n", + " raise ValueError('Saved Kicker JSON has an unexpected match count.')\n", + "print(f'Saved Kicker lineup snapshot: {output_path.resolve()}')" + ] + }, + { + "cell_type": "markdown", + "id": "1d0f2143", + "metadata": {}, + "source": [ + "## 7. Report captured fixtures" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "59268b69", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 1. Fr. 20:30 | Bayern München vs VfB Stuttgart | https://www.kicker.de/bayern-gegen-stuttgart-2026-bundesliga-5226808/aufstellung\n", + " 2. Sa. 15:30 | SV Elversberg vs Bayer 04 Leverkusen | https://www.kicker.de/elversberg-gegen-leverkusen-2026-bundesliga-5226809/aufstellung\n", + " 3. Sa. 15:30 | RB Leipzig vs Bor. Mönchengladbach | https://www.kicker.de/leipzig-gegen-mgladbach-2026-bundesliga-5226812/aufstellung\n", + " 4. Sa. 15:30 | 1. FSV Mainz 05 vs SC Paderborn 07 | https://www.kicker.de/mainz-gegen-paderborn-2026-bundesliga-5226815/aufstellung\n", + " 5. Sa. 15:30 | 1. FC Union Berlin vs Eintracht Frankfurt | https://www.kicker.de/union-gegen-frankfurt-2026-bundesliga-5226816/aufstellung\n", + " 6. Sa. 15:30 | 1. FC Köln vs TSG Hoffenheim | https://www.kicker.de/koeln-gegen-hoffenheim-2026-bundesliga-5226817/aufstellung\n", + " 7. Sa. 18:30 | Borussia Dortmund vs Hamburger SV | https://www.kicker.de/dortmund-gegen-hsv-2026-bundesliga-5226811/aufstellung\n", + " 8. So. 15:30 | SC Freiburg vs Werder Bremen | https://www.kicker.de/freiburg-gegen-bremen-2026-bundesliga-5226813/aufstellung\n", + " 9. So. 17:30 | FC Augsburg vs FC Schalke 04 | https://www.kicker.de/augsburg-gegen-schalke-2026-bundesliga-5226814/aufstellung\n", + "\n", + "JSON output: C:\\kickbase project\\outputs\\kicker\\predicted_lineups\\kicker_bundesliga_lineups_20260827_121116.json\n" + ] + } + ], + "source": [ + "for match_number, match in enumerate(matches, start=1):\n", + " print(\n", + " f\"{match_number:>2}. {match['match_time']} | \"\n", + " f\"{match['home']['team_name']} vs {match['away']['team_name']} | \"\n", + " f\"{match['kicker_details']['fixture_url']}\"\n", + " )\n", + "\n", + "if skipped_fixtures:\n", + " print('\\nSkipped fixtures:')\n", + " for fixture in skipped_fixtures:\n", + " print(f\"- {fixture['fixture_url']}: {fixture['reason']}\")\n", + "\n", + "print(f'\\nJSON output: {output_path.resolve()}')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.14.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/project_paths.py b/project_paths.py index f433f77..3c843ee 100644 --- a/project_paths.py +++ b/project_paths.py @@ -44,6 +44,9 @@ LIGAINSIDER_OUTPUT_DIR = OUTPUTS_DIR / "ligainsider" LIGAINSIDER_PREDICTED_LINEUPS_DIR = LIGAINSIDER_OUTPUT_DIR / "predicted_lineups" +KICKER_OUTPUT_DIR = OUTPUTS_DIR / "kicker" +KICKER_PREDICTED_LINEUPS_DIR = KICKER_OUTPUT_DIR / "predicted_lineups" + KBSTATS_PLAYERS_DIR = OUTPUTS_DIR / "kbstats" / "players" EXPECTED_POINTS_DIR = OUTPUTS_DIR / "expected_points" OPTIMIZED_SQUAD_DIR = OUTPUTS_DIR / "optimized_squad"