docs: add link checker utility (#8738)

* docs: add link checker utility

* add CI workflow to main

* fix some redirects

.circleci/real_config.yml

@@ -1208,6 +1208,22 @@ jobs:
           at: .
       - run: make -C docs upload-search-index
 
+  check-docs-links:
+    docker:
+      - image: <<pipeline.parameters.docker-image>>
+    steps:
+      - checkout
+      - setup-python-venv:
+          install-python: false
+          determined: true
+          model-hub: true
+          extra-requirements-file: "docs/requirements.txt"
+          executor: <<pipeline.parameters.docker-image>>
+      - attach_workspace:
+          at: .
+      - run:
+          command: make -C docs check-links
+
   publish-docs:
     docker:
       - image: hashicorp/terraform:light
@@ -2829,6 +2845,14 @@ workflows:
               only:
                 - main
 
+      - check-docs-links:
+          requires:
+            - build-docs
+          filters:
+            branches:
+              only:
+                - main
+
       - test-debian-packaging:
           requires:
             - package-and-push-system-local

docs/Makefile

@@ -69,6 +69,12 @@ pre-publish:
 .PHONY: publish-check
 	python3 redirects.py publish-check
 
+.PHONY: check-links
+check-links: attributions.rst
+	# any warnings will dump to stderr, continue regardless to parse output.txt
+	sphinx-build -b linkcheck . build -a || true
+	! grep -n broken build/output.txt
+
 .PHONY: publish
 publish: pre-publish publish-check
 	$(MAKE) -C deploy publish

docs/conf.py

@@ -149,5 +149,27 @@ def setup(app):
 # include that variant in the sitemap.
 sitemap_url_scheme = "latest/{link}"
 
+# ignore internal urls when running linkcheck
+# also ignore urls that have IP-checking measures in place that block aws
+# connections based on IP.
+linkcheck_ignore = [
+    r'^#',
+    r'^http://127.0.0.1',
+    r'^\.\./',
+    'https://www.hpe.com/us/en/hpe-machine-learning-development-environment.html'
+]
+
+linkcheck_timeout = 20
+
+# linkcheck gets confused by anchors sometimes
+linkcheck_anchors_ignore=[
+    'install-nvidia-device-plugin',
+    'tag/System/operation/SystemAuth',
+    'batch-size-related-parameters'
+]
+
+# Some pages block python requests. Set user-agent to appear as a browser.
+user_agent ="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36"
+
 with open(".redirects/redirects.json") as f:
     redirects = json.load(f)

docs/setup-cluster/on-prem/options/wsl.rst

@@ -25,18 +25,18 @@ master and agent running on the same machine within WSL.
 
 -  An Ubuntu or an Enterprise Linux WSL distribution installed from the Microsoft Store, such as:
 
-      -  `Ubuntu 22.04 LTS <https://www.microsoft.com/store/productId/9PDXGNCFSCZV>`_
-      -  `AlmaLinux 9 <https://www.microsoft.com/store/productId/9P5RWLM70SN9>`_
-      -  `Oracle Linux 9 <https://www.microsoft.com/store/productId/9MXQ65HLMC27>`_
-      -  `Pengwin Enterprise <https://www.microsoft.com/store/productId/9P70GX2HQNHN>`_
+      -  `Ubuntu 22.04 LTS <https://apps.microsoft.com/detail/9PDXGNCFSCZV>`_
+      -  `AlmaLinux 9 <https://apps.microsoft.com/detail/9P5RWLM70SN9>`_
+      -  `Oracle Linux 9 <https://apps.microsoft.com/detail/9mxq65hlmc27>`_
+      -  `Pengwin Enterprise <https://apps.microsoft.com/detail/9P70GX2HQNHN>`_
 
 -  `systemd enabled <https://learn.microsoft.com/en-us/windows/wsl/wsl-config#systemd-support>`_
    within your chosen WSL distribution.
 
 **Recommended**
 
 -  Windows 11 version 22H2, or later.
--  `Windows Terminal <https://www.microsoft.com/store/productId/9N0DX20HK701>`_.
+-  `Windows Terminal <https://apps.microsoft.com/detail/9N0DX20HK701>`_.
 
 .. _enable_systemd: