Skip to content

Commit c271554

Browse files
hugovkhugovk
authored
Replace asyncio and httpx with urllib3 and ThreadPoolExecutor (#357)
1 parent 4e3feef commit c271554

3 files changed

Lines changed: 46 additions & 43 deletions

File tree

‎README.md‎

Lines changed: 25 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -45,32 +45,34 @@ Sphinx configuration in various branches:
4545
| 3.10 | sphinx==3.4.3 | needs_sphinx='3.2' |
4646
| 3.11 | sphinx~=7.2.0 | needs_sphinx='4.2' |
4747
| 3.12 | sphinx~=8.2.0 | needs_sphinx='8.2.0' |
48-
| 3.13 | sphinx~=8.2.0 | needs_sphinx='8.2.0' |
49-
| 3.14 | sphinx~=8.2.0 | needs_sphinx='8.2.0' |
50-
| 3.15 | sphinx~=8.2.0 | needs_sphinx='8.2.0' |
48+
| 3.13 | sphinx<9.0.0 | needs_sphinx='8.2.0' |
49+
| 3.14 | sphinx<9.0.0 | needs_sphinx='8.2.0' |
50+
| 3.15 | sphinx<9.0.0 | needs_sphinx='8.2.0' |
51+
| 3.16 | sphinx<9.0.0 | needs_sphinx='8.2.0' |
5152

5253
Sphinx build as seen on docs.python.org:
5354

54-
| version | el | en | es | fr | bn-in | id | it | ja | ko | pl | pt-br | ro | sv | tr | uk | zh-cn | zh-tw |
55-
|-----------|-------|-------|-------|-------|---------|-------|-------|-------|-------|-------|---------|-------|-------|-------|-------|---------|---------|
56-
| 2.6 | ø | 0.6.5 | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø |
57-
| 2.7 | ø | 2.3.1 | ø | 2.3.1 | ø | 2.3.1 | ø | 2.3.1 | 2.3.1 | ø | 2.3.1 | ø | ø | ø | ø | 2.3.1 | 2.3.1 |
58-
| 3.0 | ø | 0.6 | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø |
59-
| 3.1 | ø | 0.6.5 | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø |
60-
| 3.2 | ø | 1.0.7 | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø |
61-
| 3.3 | ø | 1.2 | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø |
62-
| 3.4 | ø | 1.2.3 | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø |
63-
| 3.5 | ø | 1.8.4 | 1.8.4 | 1.8.4 | ø | 1.8.4 | ø | 1.8.4 | 1.8.4 | 1.8.4 | 1.8.4 | ø | ø | ø | ø | 1.8.4 | 1.8.4 |
64-
| 3.6 | ø | 2.3.1 | 2.3.1 | 2.3.1 | ø | 2.3.1 | ø | 2.3.1 | 2.3.1 | 2.3.1 | 2.3.1 | ø | ø | ø | ø | 2.3.1 | 2.3.1 |
65-
| 3.7 | ø | 2.3.1 | 2.3.1 | 2.3.1 | ø | 2.3.1 | 2.3.1 | 2.3.1 | 2.3.1 | 2.3.1 | 2.3.1 | ø | ø | 2.3.1 | 2.3.1 | 2.3.1 | 2.3.1 |
66-
| 3.8 | ø | 2.4.4 | 2.4.4 | 2.4.4 | ø | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | ø | ø | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 |
67-
| 3.9 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | ø | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 |
68-
| 3.10 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | ø | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 |
69-
| 3.11 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | ø | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 |
70-
| 3.12 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | ø | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 |
71-
| 3.13 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 |
72-
| 3.14 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 |
73-
| 3.15 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 |
55+
| version | el | en | es | fr | bn-in | id | it | ja | ko | pl | pt-br | ro | ru | sv | tr | uk | zh-cn | zh-tw |
56+
|-----------|-------|-------|-------|-------|---------|-------|-------|-------|-------|-------|---------|-------|-------|-------|-------|-------|---------|---------|
57+
| 2.6 | ø | 0.6.5 | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø |
58+
| 2.7 | ø | 2.3.1 | ø | 2.3.1 | ø | 2.3.1 | ø | 2.3.1 | 2.3.1 | ø | 2.3.1 | ø | ø | ø | ø | ø | 2.3.1 | 2.3.1 |
59+
| 3.0 | ø | 0.6 | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø |
60+
| 3.1 | ø | 0.6.5 | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø |
61+
| 3.2 | ø | 1.0.7 | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø |
62+
| 3.3 | ø | 1.2 | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø |
63+
| 3.4 | ø | 1.2.3 | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø | ø |
64+
| 3.5 | ø | 1.8.4 | 1.8.4 | 1.8.4 | ø | 1.8.4 | ø | 1.8.4 | 1.8.4 | 1.8.4 | 1.8.4 | ø | ø | ø | ø | ø | 1.8.4 | 1.8.4 |
65+
| 3.6 | ø | 2.3.1 | 2.3.1 | 2.3.1 | ø | 2.3.1 | ø | 2.3.1 | 2.3.1 | 2.3.1 | 2.3.1 | ø | ø | ø | ø | ø | 2.3.1 | 2.3.1 |
66+
| 3.7 | ø | 2.3.1 | 2.3.1 | 2.3.1 | ø | 2.3.1 | 2.3.1 | 2.3.1 | 2.3.1 | 2.3.1 | 2.3.1 | ø | ø | ø | 2.3.1 | 2.3.1 | 2.3.1 | 2.3.1 |
67+
| 3.8 | ø | 2.4.4 | 2.4.4 | 2.4.4 | ø | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | ø | ø | ø | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 |
68+
| 3.9 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | ø | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 | 2.4.4 |
69+
| 3.10 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 | 3.4.3 |
70+
| 3.11 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 | 7.2.6 |
71+
| 3.12 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 |
72+
| 3.13 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 |
73+
| 3.14 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 |
74+
| 3.15 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 |
75+
| 3.16 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 | 8.2.3 |
7476
<!-- [[[end]]] -->
7577

7678
Install `tools_requirements.txt` then run `python check_versions.py

‎check_versions.py‎

Lines changed: 21 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,20 @@
11
#!/usr/bin/env python
22

33
import argparse
4-
import asyncio
4+
import concurrent.futures
55
import logging
66
import re
77
from pathlib import Path
88

99
import git
10-
import httpx
1110
import urllib3
1211
from tabulate import tabulate
1312

1413
import build_docs
1514

1615
logger = logging.getLogger(__name__)
17-
http = urllib3.PoolManager()
16+
MAX_WORKERS = 16
17+
http = urllib3.PoolManager(maxsize=MAX_WORKERS)
1818
VERSIONS = build_docs.parse_versions_from_peps_site(http)
1919
LANGUAGES = build_docs.parse_languages_from_config()
2020

@@ -87,33 +87,38 @@ def search_sphinx_versions_in_cpython(repo: git.Repo):
8787
print(tabulate(table, headers=headers, tablefmt="github", disable_numparse=True))
8888

8989

90-
async def get_version_in_prod(language: str, version: str) -> str:
90+
def get_version_in_prod(language: str, version: str) -> str:
9191
if language == "en":
9292
url = f"https://docs.python.org/{version}/"
9393
else:
9494
url = f"https://docs.python.org/{language}/{version}/"
95-
async with httpx.AsyncClient() as client:
96-
try:
97-
response = await client.get(url, timeout=5)
98-
except httpx.ConnectTimeout:
99-
return "(timeout)"
95+
try:
96+
response = http.request("GET", url, timeout=5)
97+
except urllib3.exceptions.HTTPError:
98+
return "(timeout)"
10099
# Python 2.6--3.7: sphinx.pocoo.org
101100
# from Python 3.8: www.sphinx-doc.org
102101
if created_using := re.search(
103-
r"(?:sphinx.pocoo.org|www.sphinx-doc.org).*?([0-9.]+[0-9])", response.text
102+
r"(?:sphinx.pocoo.org|www.sphinx-doc.org).*?([0-9.]+[0-9])",
103+
response.data.decode("utf-8", errors="replace"),
104104
):
105105
return created_using.group(1)
106106
return "ø"
107107

108108

109-
async def which_sphinx_is_used_in_production():
109+
def which_sphinx_is_used_in_production():
110+
with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
111+
futures = {
112+
(version.name, language.tag): executor.submit(
113+
get_version_in_prod, language.tag, version.name
114+
)
115+
for version in VERSIONS
116+
for language in LANGUAGES
117+
}
110118
table = [
111119
[
112120
version.name,
113-
*await asyncio.gather(*[
114-
get_version_in_prod(language.tag, version.name)
115-
for language in LANGUAGES
116-
]),
121+
*[futures[version.name, language.tag].result() for language in LANGUAGES],
117122
]
118123
for version in VERSIONS
119124
]
@@ -123,15 +128,12 @@ async def which_sphinx_is_used_in_production():
123128

124129
def check_versions(cpython_clone: str) -> None:
125130
logging.basicConfig(level=logging.INFO)
126-
logging.getLogger("charset_normalizer").setLevel(logging.WARNING)
127-
logging.getLogger("asyncio").setLevel(logging.WARNING)
128-
logging.getLogger("httpx").setLevel(logging.WARNING)
129131
repo = git.Repo(cpython_clone)
130132
print("Sphinx configuration in various branches:", end="\n\n")
131133
search_sphinx_versions_in_cpython(repo)
132134
print()
133135
print("Sphinx build as seen on docs.python.org:", end="\n\n")
134-
asyncio.run(which_sphinx_is_used_in_production())
136+
which_sphinx_is_used_in_production()
135137

136138

137139
def main():

‎tools_requirements.txt‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
11
GitPython
2-
httpx
32
tabulate
43
zc.lockfile

0 commit comments

Comments
 (0)