gh-126623: Update libexpat to 2.6.4, make future updates easier (GH-126792)

Update libexpat to 2.6.4, make future updates easier.
2024-11-21 12:59:38 +01:00 · 2024-11-13 12:31:20 -06:00 · 2024-11-13 12:31:20 -06:00 · 3c99969094
commit 3c99969094
parent 8c9c6d3c12
7 changed files with 119 additions and 22 deletions
--- a/Misc/NEWS.d/next/Security/2024-11-13-11-09-12.gh-issue-126623.TO7NnR.rst
+++ b/Misc/NEWS.d/next/Security/2024-11-13-11-09-12.gh-issue-126623.TO7NnR.rst
@ -0,0 +1 @@
+Upgrade libexpat to 2.6.4
--- a/Misc/sbom.spdx.json
+++ b/Misc/sbom.spdx.json
@ -48,11 +48,11 @@
      "checksums": [
        {
          "algorithm": "SHA1",
-          "checksumValue": "6aaee1b194bea30f0a60d1cce71eada8b14d3526"
+          "checksumValue": "373cc00d87782a736970644d863ff2ebbd0e4886"
        },
        {
          "algorithm": "SHA256",
-          "checksumValue": "7bd4e53a8015534b5bbb58afe1a131b3989d3d4fca29bca685c44d34bcaa2555"
+          "checksumValue": "0f750bc336e510d14ac9a3e63fc2399f60f3f04f0061c426e86751ed5fba90e4"
        }
      ],
      "fileName": "Modules/expat/expat.h"
@ -62,11 +62,11 @@
      "checksums": [
        {
          "algorithm": "SHA1",
-          "checksumValue": "b70ce53fdc25ae482681ae2f6623c3c8edc9c1b7"
+          "checksumValue": "9e615c6e5c3ba00670f674a6b071bb855b0b563d"
        },
        {
          "algorithm": "SHA256",
-          "checksumValue": "86afb425ec9999eb4f1ec9ab2fb41c58c4aa5cb9bf934b8c94264670fc5a961d"
+          "checksumValue": "3d90a4b65c40a3f848c36100f4d73b933a015c7b7cd85c28e4331a6b845c1ad0"
        }
      ],
      "fileName": "Modules/expat/expat_external.h"
@ -128,18 +128,18 @@
      "fileName": "Modules/expat/nametab.h"
    },
    {
-      "SPDXID": "SPDXRef-FILE-Modules-expat-pyexpatns.h",
+      "SPDXID": "SPDXRef-FILE-Modules-expat-refresh.sh",
      "checksums": [
        {
          "algorithm": "SHA1",
-          "checksumValue": "f50c899172acd93fc539007bfb43315b83d407e4"
+          "checksumValue": "a9b0a33b8359cfe94b23972a1605daf8dcc605d9"
        },
        {
          "algorithm": "SHA256",
-          "checksumValue": "d571b8258cfaa067a20adef553e5fcedd6671ca4a8841483496de031bd904567"
+          "checksumValue": "19eb541460bc2ca8b87118acd3c048f6af77affbf8719ac29aa7b6c8d70f83fd"
        }
      ],
-      "fileName": "Modules/expat/pyexpatns.h"
+      "fileName": "Modules/expat/refresh.sh"
    },
    {
      "SPDXID": "SPDXRef-FILE-Modules-expat-siphash.h",
@ -188,11 +188,11 @@
      "checksums": [
        {
          "algorithm": "SHA1",
-          "checksumValue": "b2ec0ad170ccc21e63fbcfc8d7404cdd756eedd3"
+          "checksumValue": "3199fbd38b6fb158f73d5c8de6b6e6e3812ef803"
        },
        {
          "algorithm": "SHA256",
-          "checksumValue": "92159d4e17393e56ee85f47d9fb31348695a58589899aa01e7536cdc88f60b85"
+          "checksumValue": "c1518244dd5ea397e345d00e12cc45d42f43453ed208218559c981c97a0583e2"
        }
      ],
      "fileName": "Modules/expat/xmlparse.c"
@ -1749,7 +1749,7 @@
      "spdxElementId": "SPDXRef-PACKAGE-expat"
    },
    {
-      "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-pyexpatns.h",
+      "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-refresh.sh",
      "relationshipType": "CONTAINS",
      "spdxElementId": "SPDXRef-PACKAGE-expat"
    },
--- a/Modules/expat/expat.h
+++ b/Modules/expat/expat.h
@ -130,7 +130,9 @@ enum XML_Error {
  /* Added in 2.3.0. */
  XML_ERROR_NO_BUFFER,
  /* Added in 2.4.0. */
-  XML_ERROR_AMPLIFICATION_LIMIT_BREACH
+  XML_ERROR_AMPLIFICATION_LIMIT_BREACH,
+  /* Added in 2.6.4. */
+  XML_ERROR_NOT_STARTED,
 };

 enum XML_Content_Type {
@ -1066,7 +1068,7 @@ XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled);
 */
 #define XML_MAJOR_VERSION 2
 #define XML_MINOR_VERSION 6
-#define XML_MICRO_VERSION 3
+#define XML_MICRO_VERSION 4

 #ifdef __cplusplus
 }
--- a/Modules/expat/expat_external.h
+++ b/Modules/expat/expat_external.h
@ -40,6 +40,10 @@
 #ifndef Expat_External_INCLUDED
 #define Expat_External_INCLUDED 1

+/* Namespace external symbols to allow multiple libexpat version to
+   co-exist. */
+#include "pyexpatns.h"
+
 /* External API definitions */

 /* Expat tries very hard to make the API boundary very specifically
@ -64,11 +68,6 @@
   compiled with the cdecl calling convention as the default since
   system headers may assume the cdecl convention.
 */
-
-/* Namespace external symbols to allow multiple libexpat version to
-   co-exist. */
-#include "pyexpatns.h"
-
 #ifndef XMLCALL
 #  if defined(_MSC_VER)
 #    define XMLCALL __cdecl
--- a/Modules/expat/refresh.sh
+++ b/Modules/expat/refresh.sh
@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+#
+# Use this script to update libexpat
+
+set -e
+set -o pipefail
+
+if [[ "${BASH_VERSINFO[0]}" -lt 4 ]]; then
+  echo "A bash version >= 4 required. Got: $BASH_VERSION" >&2
+  exit 1
+fi
+
+# Update this when updating to a new version after verifying that the changes
+# the update brings in are good. These values are used for verifying the SBOM, too.
+expected_libexpat_tag="R_2_6_4"
+expected_libexpat_version="2.6.4"
+expected_libexpat_sha256="fd03b7172b3bd7427a3e7a812063f74754f24542429b634e0db6511b53fb2278"
+
+expat_dir="$(realpath "$(dirname -- "${BASH_SOURCE[0]}")")"
+cd ${expat_dir}
+
+# Step 1: download and copy files
+curl --location "https://github.com/libexpat/libexpat/releases/download/${expected_libexpat_tag}/expat-${expected_libexpat_version}.tar.gz" > libexpat.tar.gz
+echo "${expected_libexpat_sha256} libexpat.tar.gz" | sha256sum --check
+
+# Step 2: Pull files from the libexpat distribution
+declare -a lib_files
+lib_files=(
+  ascii.h
+  asciitab.h
+  expat.h
+  expat_external.h
+  iasciitab.h
+  internal.h
+  latin1tab.h
+  nametab.h
+  siphash.h
+  utf8tab.h
+  winconfig.h
+  xmlparse.c
+  xmlrole.c
+  xmlrole.h
+  xmltok.c
+  xmltok.h
+  xmltok_impl.c
+  xmltok_impl.h
+  xmltok_ns.c
+)
+for f in "${lib_files[@]}"; do
+  tar xzvf libexpat.tar.gz "expat-${expected_libexpat_version}/lib/${f}" --strip-components 2
+done
+rm libexpat.tar.gz
+
+# Step 3: Add the namespacing include to expat_external.h
+sed -i 's/#define Expat_External_INCLUDED 1/&\n\n\/* Namespace external symbols to allow multiple libexpat version to\n   co-exist. \*\/\n#include "pyexpatns.h"/' expat_external.h
+
+echo "Updated; verify all is okay using git diff and git status."
--- a/Modules/expat/xmlparse.c
+++ b/Modules/expat/xmlparse.c
@ -1,4 +1,4 @@
-/* ba4cdf9bdb534f355a9def4c9e25d20ee8e72f95b0a4d930be52e563f5080196 (2.6.3+)
+/* c5625880f4bf417c1463deee4eb92d86ff413f802048621c57e25fe483eb59e4 (2.6.4+)
                            __  __            _
                         ___\ \/ /_ __   __ _| |_
                        / _ \\  /| '_ \ / _` | __|
@ -40,6 +40,7 @@
   Copyright (c) 2023      Owain Davies <owaind@bath.edu>
   Copyright (c) 2023-2024 Sony Corporation / Snild Dolkow <snild@sony.com>
   Copyright (c) 2024      Berkay Eren Ürün <berkay.ueruen@siemens.com>
+   Copyright (c) 2024      Hanno Böck <hanno@gentoo.org>
   Licensed under the MIT license:

   Permission is  hereby granted,  free of charge,  to any  person obtaining
@ -2234,6 +2235,9 @@ XML_StopParser(XML_Parser parser, XML_Bool resumable) {
  if (parser == NULL)
    return XML_STATUS_ERROR;
  switch (parser->m_parsingStatus.parsing) {
+  case XML_INITIALIZED:
+    parser->m_errorCode = XML_ERROR_NOT_STARTED;
+    return XML_STATUS_ERROR;
  case XML_SUSPENDED:
    if (resumable) {
      parser->m_errorCode = XML_ERROR_SUSPENDED;
@ -2244,7 +2248,7 @@ XML_StopParser(XML_Parser parser, XML_Bool resumable) {
  case XML_FINISHED:
    parser->m_errorCode = XML_ERROR_FINISHED;
    return XML_STATUS_ERROR;
-  default:
+  case XML_PARSING:
    if (resumable) {
 #ifdef XML_DTD
      if (parser->m_isParamEntity) {
@ -2255,6 +2259,9 @@ XML_StopParser(XML_Parser parser, XML_Bool resumable) {
      parser->m_parsingStatus.parsing = XML_SUSPENDED;
    } else
      parser->m_parsingStatus.parsing = XML_FINISHED;
+    break;
+  default:
+    assert(0);
  }
  return XML_STATUS_OK;
 }
@ -2519,6 +2526,9 @@ XML_ErrorString(enum XML_Error code) {
  case XML_ERROR_AMPLIFICATION_LIMIT_BREACH:
    return XML_L(
        "limit on input amplification factor (from DTD and entities) breached");
+  /* Added in 2.6.4. */
+  case XML_ERROR_NOT_STARTED:
+    return XML_L("parser not started");
  }
  return NULL;
 }
@ -7856,7 +7866,7 @@ accountingReportDiff(XML_Parser rootParser,
  assert(! rootParser->m_parentParser);

  fprintf(stderr,
-          " (+" EXPAT_FMT_PTRDIFF_T("6") " bytes %s|%d, xmlparse.c:%d) %*s\"",
+          " (+" EXPAT_FMT_PTRDIFF_T("6") " bytes %s|%u, xmlparse.c:%d) %*s\"",
          bytesMore, (account == XML_ACCOUNT_DIRECT) ? "DIR" : "EXP",
          levelsAwayFromRootParser, source_line, 10, "");

@ -7969,7 +7979,7 @@ entityTrackingReportStats(XML_Parser rootParser, ENTITY *entity,

  fprintf(
      stderr,
-      "expat: Entities(%p): Count %9d, depth %2d/%2d %*s%s%s; %s length %d (xmlparse.c:%d)\n",
+      "expat: Entities(%p): Count %9u, depth %2u/%2u %*s%s%s; %s length %d (xmlparse.c:%d)\n",
      (void *)rootParser, rootParser->m_entity_stats.countEverOpened,
      rootParser->m_entity_stats.currentDepth,
      rootParser->m_entity_stats.maximumDepthSeen,
--- a/Tools/build/generate_sbom.py
+++ b/Tools/build/generate_sbom.py
@ -59,6 +59,8 @@ PACKAGE_TO_FILES = {
        include=["Modules/expat/**"],
        exclude=[
            "Modules/expat/expat_config.h",
+            "Modules/expat/pyexpatns.h",
+            "Modules/_hacl/refresh.sh",
        ]
    ),
    "macholib": PackageFiles(
@ -218,6 +220,32 @@ def check_sbom_packages(sbom_data: dict[str, typing.Any]) -> None:
                "HACL* SBOM version doesn't match value in 'Modules/_hacl/refresh.sh'"
            )

+        # libexpat specifies its expected rev in a refresh script.
+        if package["name"] == "libexpat":
+            libexpat_refresh_sh = (CPYTHON_ROOT_DIR / "Modules/expat/refresh.sh").read_text()
+            libexpat_expected_version_match = re.search(
+                r"expected_libexpat_version=\"([0-9]+\.[0-9]+\.[0-9]+)\"",
+                libexpat_refresh_sh
+            )
+            libexpat_expected_sha256_match = re.search(
+                r"expected_libexpat_sha256=\"[a-f0-9]{40}\"",
+                libexpat_refresh_sh
+            )
+            libexpat_expected_version = libexpat_expected_version_match and libexpat_expected_version_match.group(1)
+            libexpat_expected_sha256 = libexpat_expected_sha256_match and libexpat_expected_sha256_match.group(1)
+
+            error_if(
+                libexpat_expected_version != version,
+                "libexpat SBOM version doesn't match value in 'Modules/expat/refresh.sh'"
+            )
+            error_if(
+                package["checksums"] != [{
+                    "algorithm": "SHA256",
+                    "checksumValue": libexpat_expected_sha256
+                }],
+                "libexpat SBOM checksum doesn't match value in 'Modules/expat/refresh.sh'"
+            )
+
        # License must be on the approved list for SPDX.
        license_concluded = package["licenseConcluded"]
        error_if(