-
Notifications
You must be signed in to change notification settings - Fork 16
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
docs: fetch all external resources when building docs (#142)
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
- Loading branch information
1 parent
de1efa9
commit e0acfd8
Showing
20 changed files
with
217 additions
and
58 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
sphinx >=6.0,<7.0 | ||
myst-parser >=2.0.0, <3.0.0 | ||
nbsphinx >=0.8.5 | ||
ipython[notebook] | ||
pandoc >=1.0 | ||
docutils >=0.16 | ||
# https://github.com/jupyterlab/jupyterlab_pygments/issues/5 | ||
pygments >=2.4.1 | ||
sphinxcontrib-fulltoc >=1.0 | ||
sphinxcontrib-mockautodoc | ||
|
||
pt-lightning-sphinx-theme @ https://github.com/Lightning-AI/lightning_sphinx_theme/archive/master.zip | ||
sphinx-autodoc-typehints >=1.0 | ||
sphinx-paramlinks >=0.5.1 | ||
sphinx-togglebutton >=0.2 | ||
sphinx-copybutton >=0.3 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
coverage ==6.5.0 | ||
pytest ==7.2.0 | ||
pytest-cov ==4.0.0 | ||
pytest-timeout ==2.1.0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,16 +1 @@ | ||
sphinx >=6.0,<7.0 | ||
myst-parser >=2.0.0, <3.0.0 | ||
nbsphinx >=0.8.5 | ||
ipython[notebook] | ||
pandoc >=1.0 | ||
docutils >=0.16 | ||
# https://github.com/jupyterlab/jupyterlab_pygments/issues/5 | ||
pygments >=2.4.1 | ||
sphinxcontrib-fulltoc >=1.0 | ||
sphinxcontrib-mockautodoc | ||
|
||
pt-lightning-sphinx-theme @ https://github.com/Lightning-AI/lightning_sphinx_theme/archive/master.zip | ||
sphinx-autodoc-typehints >=1.0 | ||
sphinx-paramlinks >=0.5.1 | ||
sphinx-togglebutton >=0.2 | ||
sphinx-copybutton >=0.3 | ||
requests >=2.0.0 |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
from lightning_utilities.docs.retriever import fetch_external_assets |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
import glob | ||
import os | ||
import re | ||
|
||
|
||
def _transform_changelog(path_in: str, path_out: str) -> None: | ||
"""Adjust changelog titles so not to be duplicated. | ||
Args: | ||
path_in: input MD file | ||
path_out: output also MD file | ||
""" | ||
with open(path_in) as fp: | ||
chlog_lines = fp.readlines() | ||
# enrich short subsub-titles to be unique | ||
chlog_ver = "" | ||
for i, ln in enumerate(chlog_lines): | ||
if ln.startswith("## "): | ||
chlog_ver = ln[2:].split("-")[0].strip() | ||
elif ln.startswith("### "): | ||
ln = ln.replace("###", f"### {chlog_ver} -") | ||
chlog_lines[i] = ln | ||
with open(path_out, "w") as fp: | ||
fp.writelines(chlog_lines) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,99 @@ | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
import glob | ||
import logging | ||
import os | ||
import re | ||
from typing import List, Tuple | ||
|
||
import requests | ||
|
||
|
||
def _download_file(file_url: str, folder: str) -> str: | ||
"""Download a file from URL to a particular folder.""" | ||
fname = os.path.basename(file_url) | ||
file_path = os.path.join(folder, fname) | ||
if os.path.isfile(file_path): | ||
logging.warning(f'given file "{file_path}" already exists and will be overwritten with {file_url}') | ||
# see: https://stackoverflow.com/a/34957875 | ||
rq = requests.get(file_url) | ||
with open(file_path, "wb") as outfile: | ||
outfile.write(rq.content) | ||
return fname | ||
|
||
|
||
def _search_all_occurrences(list_files: List[str], pattern: str) -> List[str]: | ||
"""Search for all occurrences of specific patter in a collection of files. | ||
Args: | ||
list_files: list of files to be scanned | ||
pattern: pattern for search, reg. expression | ||
""" | ||
collected = [] | ||
for file_path in list_files: | ||
with open(file_path, encoding="UTF-8") as fo: | ||
body = fo.read() | ||
found = re.findall(pattern, body) | ||
collected += found | ||
return collected | ||
|
||
|
||
def _replace_remote_with_local( | ||
file_path: str, docs_folder: str, pairs_url_path: List[Tuple[str, str]], base_depth: int = 2 | ||
) -> None: | ||
"""Replace all URL with local files in a given file. | ||
Args: | ||
file_path: file for replacement | ||
docs_folder: the location of docs related to the project root | ||
pairs_url_path: pairs of URL and local file path to be swapped | ||
""" | ||
# drop the default/global path to the docs | ||
relt_path = os.path.dirname(file_path).replace(docs_folder, "") | ||
# filter the path starting with / as not empty folder names | ||
depth = len([p for p in relt_path.split(os.path.sep) if p]) | ||
with open(file_path, encoding="UTF-8") as fo: | ||
body = fo.read() | ||
for url, fpath in pairs_url_path: | ||
if depth: | ||
path_up = [".."] * depth | ||
fpath = os.path.join(*path_up, fpath) | ||
body = body.replace(url, fpath) | ||
with open(file_path, "w", encoding="UTF-8") as fw: | ||
fw.write(body) | ||
|
||
|
||
def fetch_external_assets( | ||
docs_folder: str = "docs/source", | ||
assets_folder: str = "fetched-s3-assets", | ||
file_pattern: str = "*.rst", | ||
retrieve_pattern: str = r"https?://[-a-zA-Z0-9_]+\.s3\.[-a-zA-Z0-9()_\\+.\\/=]+", | ||
) -> None: | ||
"""Search all URL in docs, download these files locally and replace online with local version. | ||
Args: | ||
docs_folder: the location of docs related to the project root | ||
assets_folder: a folder inside ``docs_folder`` to be created and saving online assets | ||
file_pattern: what kind of files shall be scanned | ||
retrieve_pattern: patter for reg. expression to search URL/S3 resources | ||
""" | ||
list_files = glob.glob(os.path.join(docs_folder, "**", file_pattern), recursive=True) | ||
if not list_files: | ||
logging.warning(f'no files were listed in folder "{docs_folder}" and pattern "{file_pattern}"') | ||
return | ||
|
||
urls = _search_all_occurrences(list_files, pattern=retrieve_pattern) | ||
if not urls: | ||
logging.info(f"no resources/assets were match in {docs_folder} for {retrieve_pattern}") | ||
return | ||
target_folder = os.path.join(docs_folder, assets_folder) | ||
os.makedirs(target_folder, exist_ok=True) | ||
pairs_url_file = [] | ||
for i, url in enumerate(set(urls)): | ||
logging.info(f" >> downloading ({i}/{len(urls)}): {url}") | ||
fname = _download_file(url, target_folder) | ||
pairs_url_file.append((url, os.path.join(assets_folder, fname))) | ||
|
||
for fpath in list_files: | ||
_replace_remote_with_local(fpath, docs_folder, pairs_url_file) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
import os | ||
|
||
_PATH_UNITTESTS = os.path.dirname(__file__) | ||
_PATH_ROOT = os.path.dirname(os.path.dirname(_PATH_UNITTESTS)) |
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
import os.path | ||
import shutil | ||
|
||
from unittests import _PATH_ROOT | ||
|
||
from lightning_utilities.docs import fetch_external_assets | ||
|
||
|
||
def test_retriever_s3(): | ||
path_docs = os.path.join(_PATH_ROOT, "docs", "source") | ||
path_index = os.path.join(path_docs, "index.rst") | ||
path_page = os.path.join(path_docs, "any", "extra", "page.rst") | ||
os.makedirs(os.path.dirname(path_page), exist_ok=True) | ||
shutil.copy(path_index, path_page) | ||
|
||
fetch_external_assets(docs_folder=path_docs) | ||
|
||
with open(path_index, encoding="UTF-8") as fo: | ||
body = fo.read() | ||
# that the image exists~ | ||
assert "Lightning.gif" in body | ||
# but it is not sourced from S3 | ||
assert ".s3." not in body | ||
|
||
with open(path_page, encoding="UTF-8") as fo: | ||
body = fo.read() | ||
# that the image exists~ | ||
assert "Lightning.gif" in body | ||
# check the proper depth | ||
assert os.path.sep.join(["..", ".."]) in body |