Re #7777 #7778: check requirements.txt for security advisory

There is a new make file,

    doc/Makefile

that defines these goals:

  - `check-requirements`:
    Check `requirements.txt` for security problems (CVEs) using `skjold`.
    This goal is intended for the "Users guide" CI.
    SKJOLD_GITHUB_API_TOKEN might have to be set if GITHUB_TOKEN is
    not in the environment, in order to access the GitHub GraphQL API.

  - `build-and-check-requirements`:
    Rebuild `requirements.txt` from `requirements.in` using `pip-compile`,
    and check with `check-requirements`.
    This goal is intended for manual invocation.

    It is invoked from the top Makefile via goal
    `users-guide-requirements`.

Alternatively, these goals could be coupled with the doc build
`make users-guide`.
However, since these goals require a couple of seconds to run, I think
it is annoying to call them on every build of the documentation.

.github/workflows/users-guide.yml

@@ -7,6 +7,9 @@ on:
     branches:
     - master
     paths:
+    - 'doc/Makefile'
+    - 'doc/pyproject.toml'
+    - 'doc/requirements.in'
     - 'doc/requirements.txt'
     - 'doc/*.inc'
     - 'doc/*.py'
@@ -15,6 +18,9 @@ on:
     - '.github/workflows/users-guide.yml'
   pull_request:
     paths:
+    - 'doc/Makefile'
+    - 'doc/pyproject.toml'
+    - 'doc/requirements.in'
     - 'doc/requirements.txt'
     - 'doc/*.inc'
     - 'doc/*.py'
@@ -25,6 +31,10 @@ on:
     types:
       - created
 
+defaults:
+  run:
+    shell: bash
+
 jobs:
   build:
     if: |
@@ -59,3 +69,7 @@ jobs:
       with:
         name: users-guide-html
         path: html/
+
+    - name: Check security of requirements.txt
+      run: |
+        make SKJOLD_GITHUB_API_TOKEN=${{ secrets.GITHUB_TOKEN }} -C doc check-requirements

.gitignore

-# trivial gitignore file
 .cabal-sandbox/
 cabal.sandbox.config
 cabal.project.local
@@ -15,7 +14,6 @@ dist-*
 register.sh
 ./cabal.config
 cabal-tests.log
-.python-sphinx-virtualenv/
 
 /Cabal/dist/
 /Cabal/tests/Setup
@@ -68,6 +66,8 @@ register.sh
 
 # python artifacts from documentation builds
 *.pyc
+.python-sphinx-virtualenv/
+/doc/.skjold_cache/
 
 # macOS folder metadata
 .DS_Store
@@ -76,4 +76,4 @@ register.sh
 bench.html
 
 # Emacs
-.projectile
+.projectile
\ No newline at end of file

Makefile

@@ -251,3 +251,12 @@ $(USERGUIDE_STAMP) : doc/*.rst
 .python-sphinx-virtualenv:
 	python3 -m venv .python-sphinx-virtualenv
 	(. ./.python-sphinx-virtualenv/bin/activate)
+
+# This goal is intended for manual invocation, always rebuilds.
+.PHONY: users-guide-requirements
+users-guide-requirements: doc/requirements.txt
+
+.PHONY: doc/requirements.txt
+doc/requirements.txt: .python-sphinx-virtualenv
+	. .python-sphinx-virtualenv/bin/activate \
+	  && make -C doc build-and-check-requirements

doc/Makefile

+# Build and safety-check requirements.txt
+
+# skjold needs a personal github access token.  This needs no permissions,
+# it is only required to query the GitHub GraphQL API v4.
+# See: https://pythonawesome.com/security-audit-python-project-dependencies-against-security-advisory-databases/
+# We attempt to get it from the environment variable GITHUB_TOKEN.
+# It can also be passed to this Makefile via either:
+#
+#   make GITHUB_TOKEN=... (build-and-)check-requirements
+#   make SKJOLD_GITHUB_API_TOKEN=... (build-and-)check-requirements
+#
+#
+SKJOLD_GITHUB_API_TOKEN=${GITHUB_TOKEN}
+
+.PHONY: build-and-check-requirements
+build-and-check-requirements: requirements.txt check-requirements
+
+# Always rebuild requirements.txt
+.PHONY: requirements.txt
+# requirements.txt is generated from requirements.in
+# via pip-compile included in the pip-tools package.
+# See https://modelpredict.com/wht-requirements-txt-is-not-enough
+requirements.txt: requirements.in
+	. ../.python-sphinx-virtualenv/bin/activate \
+	  && pip install pip-tools \
+	  && pip-compile requirements.in
+
+# Check requirements.txt for security violations via skjold,
+# configured in pyproject.toml.
+# See: https://pythonawesome.com/security-audit-python-project-dependencies-against-security-advisory-databases/
+.PHONY: check-requirements
+check-requirements:
+	@if [ "\'${SKJOLD_GITHUB_API_TOKEN}\'" == "\'\'" ] \
+	; then \
+	  echo "WARNING: Neither SKOLD_GITHUB_API_TOKEN nor GITHUB_TOKEN is set." \
+	; echo "Vulnerability check via skjold might fail when using the GitHub GraphQL API." \
+	; fi
+	. ../.python-sphinx-virtualenv/bin/activate \
+	  && pip install skjold \
+	  && skjold audit
+# NB: For portability, we use '.' (sh etc.) instead of 'source' (bash).
+
+# EOF

doc/pyproject.toml

+# https://pythonawesome.com/security-audit-python-project-dependencies-against-security-advisory-databases/
+[tool.skjold]
+sources       = ['github', 'gemnasium', 'pyup']
+report_only   = false
+         # ALT: true      # Report only, always exit with zero.
+report_format = 'cli'
+         # ALT: 'json'    # Output findings as `json`. Default is 'cli'.
+verbose       = true
+cache_dir     = '.skjold_cache'
+cache_expires = 43200     # Cache max. age. (43200 = 12hrs)
+ignore_file   = '.skjoldignore'

doc/requirements.in

 sphinx >= 3.1
 sphinx_rtd_theme
 sphinx-jsonschema
+# Pygments>=2.7.4 suggested by CVE-2021-20270 CVE-2021-27291
+Pygments >= 2.7.4

doc/requirements.txt

@@ -29,8 +29,10 @@ markupsafe==1.1.1
     # via jinja2
 packaging==20.9
     # via sphinx
-pygments==2.5.2
-    # via sphinx
+pygments==2.10.0
+    # via
+    #   -r requirements.in
+    #   sphinx
 pyparsing==2.4.7
     # via packaging
 pytz==2021.3