gh-127330: Update for OpenSSL 3.4 & document+improve the update process (GH-127331)

- Add `git describe` output to headers generated by `make_ssl_data.py`

  This info is more important than the date when the file was generated.
  It does mean that the tool now requires a Git checkout of OpenSSL,
  not for example a release tarball.

- Regenerate the older file to add the info.
  To the other older file, add a note about manual edits.

- Add notes on how to add a new OpenSSL version

- Add 3.4 error messages and multissl tests

.github/workflows/build.yml

@@ -250,7 +250,8 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-24.04]
-        openssl_ver: [3.0.15, 3.1.7, 3.2.3, 3.3.2]
+        openssl_ver: [3.0.15, 3.1.7, 3.2.3, 3.3.2, 3.4.0]
+        # See Tools/ssl/make_ssl_data.py for notes on adding a new version
     env:
       OPENSSL_VER: ${{ matrix.openssl_ver }}
       MULTISSL_DIR: ${{ github.workspace }}/multissl

Misc/NEWS.d/next/Library/2024-11-27-14-23-02.gh-issue-127331.9sNEC9.rst

@@ -0,0 +1 @@
+:mod:`ssl` can show descriptions for errors added in OpenSSL 3.4.

Modules/_ssl.c

@@ -120,8 +120,9 @@ static void _PySSLFixErrno(void) {
 #endif
 
 /* Include generated data (error codes) */
+/* See make_ssl_data.h for notes on adding a new version. */
 #if (OPENSSL_VERSION_NUMBER >= 0x30100000L)
-#include "_ssl_data_31.h"
+#include "_ssl_data_34.h"
 #elif (OPENSSL_VERSION_NUMBER >= 0x30000000L)
 #include "_ssl_data_300.h"
 #elif (OPENSSL_VERSION_NUMBER >= 0x10101000L)

Modules/_ssl_data_111.h

@@ -1,4 +1,6 @@
-/* File generated by Tools/ssl/make_ssl_data.py *//* Generated on 2023-06-01T02:58:04.081473 */
+/* File generated by Tools/ssl/make_ssl_data.py */
+/* Generated on 2024-11-27T12:48:46.194048+00:00 */
+/* Generated from Git commit OpenSSL_1_1_1w-0-ge04bd3433f */
 static struct py_ssl_library_code library_codes[] = {
 #ifdef ERR_LIB_ASN1
     {"ASN1", ERR_LIB_ASN1},

Modules/_ssl_data_300.h

@@ -1,4 +1,7 @@
-/* File generated by Tools/ssl/make_ssl_data.py *//* Generated on 2023-06-01T03:03:52.163218 */
+/* File generated by Tools/ssl/make_ssl_data.py */
+/* Generated on 2023-06-01T03:03:52.163218 */
+/* Manually edited to add definitions from 1.1.1 (GH-105174) */
+
 static struct py_ssl_library_code library_codes[] = {
 #ifdef ERR_LIB_ASN1
     {"ASN1", ERR_LIB_ASN1},

Tools/c-analyzer/cpython/_parser.py

@@ -70,9 +70,7 @@ def clean_lines(text):
 Python/thread_pthread_stubs.h
 
 # only huge constants (safe but parsing is slow)
-Modules/_ssl_data_31.h
-Modules/_ssl_data_300.h
-Modules/_ssl_data_111.h
+Modules/_ssl_data_*.h
 Modules/cjkcodecs/mappings_*.h
 Modules/unicodedata_db.h
 Modules/unicodename_db.h

Tools/ssl/make_ssl_data.py

@@ -5,9 +5,28 @@
 `library` and `reason` mnemonics to a more recent OpenSSL version.
 
 It takes two arguments:
-- the path to the OpenSSL source tree (e.g. git checkout)
+- the path to the OpenSSL git checkout
 - the path to the header file to be generated Modules/_ssl_data_{version}.h
 - error codes are version specific
+
+The OpenSSL git checkout should be at a specific tag, using commands like:
+    git tag --list 'openssl-*'
+    git switch --detach openssl-3.4.0
+
+
+After generating the definitions, compare the result with newest pre-existing file.
+You can use a command like:
+
+    git diff --no-index Modules/_ssl_data_31.h Modules/_ssl_data_34.h
+
+- If the new version *only* adds new definitions, remove the pre-existing file
+  and adjust the #include in _ssl.c to point to the new version.
+- If the new version removes or renumbers some definitions, keep both files and
+  add a new #include in _ssl.c.
+
+A newly supported OpenSSL version should also be added to:
+- Tools/ssl/multissltests.py
+- .github/workflows/build.yml
 """
 
 import argparse
@@ -15,6 +34,7 @@
 import operator
 import os
 import re
+import subprocess
 
 
 parser = argparse.ArgumentParser(
@@ -117,9 +137,17 @@ def main():
     # sort by libname, numeric error code
     args.reasons = sorted(reasons, key=operator.itemgetter(0, 3))
 
+    git_describe = subprocess.run(
+        ['git', 'describe', '--long', '--dirty'],
+        cwd=args.srcdir,
+        capture_output=True,
+        encoding='utf-8',
+        check=True,
+        )
     lines = [
-        "/* File generated by Tools/ssl/make_ssl_data.py */"
-        f"/* Generated on {datetime.datetime.utcnow().isoformat()} */"
+        "/* File generated by Tools/ssl/make_ssl_data.py */",
+        f"/* Generated on {datetime.datetime.now(datetime.UTC).isoformat()} */",
+        f"/* Generated from Git commit {git_describe.stdout.strip()} */",
     ]
     lines.extend(gen_library_codes(args))
     lines.append("")

Tools/ssl/multissltests.py

@@ -51,6 +51,8 @@
     "3.1.7",
     "3.2.3",
     "3.3.2",
+    "3.4.0",
+    # See make_ssl_data.py for notes on adding a new version.
 ]
 
 LIBRESSL_OLD_VERSIONS = [