[CI][REFACTOR] Decouple data.py from Jenkins script and docker images

ci/jenkins/data.py

@@ -15,6 +15,19 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+"""Bundle registry for CI artifact stashing.
+
+Single source of truth for the file lists uploaded to / downloaded from S3 by
+``ci/scripts/jenkins/s3.py``. This module deliberately carries nothing else —
+docker image tags live in ``ci/jenkins/docker-images.ini`` and Jinja-template
+metadata (image platforms, AWS endpoints) lives in ``ci/jenkins/generate.py``.
+
+CLI: ``python3 ci/jenkins/data.py <bundle> [<bundle> ...]`` resolves bundle
+names to their file paths (one per line; exit 1 on unknown name). Used by
+``s3.py`` at Jenkins runtime and by any external caller that needs
+data-driven artifact lists.
+"""
+
 import sys
 
 files_to_stash = {
@@ -24,15 +37,16 @@
     "hexagon_api": [
         "build/hexagon_api_output",
     ],
-    # This library is produced with HIDE_PRIVATE_SYMBOLS=ON
-    "tvm_allvisible": ["build/libtvm_allvisible.so"],
     # runtime files
     "tvm_runtime": ["build/libtvm_runtime.so", "build/config.cmake"],
-    # compiler files
+    # compiler files (libtvm_allvisible is the HIDE_PRIVATE_SYMBOLS=ON
+    # variant cpptest links against; bundled here so every consumer of
+    # tvm_lib gets it without having to remember a second bundle name).
     "tvm_lib": [
         "build/libtvm.so",
         "build/libtvm_runtime.so",
         "build/lib/libtvm_ffi.so",
+        "build/libtvm_allvisible.so",
         "build/config.cmake",
     ],
     # gpu related compiler files
@@ -43,46 +57,12 @@
 }
 
 
-# AWS info
-aws_default_region = "us-west-2"
-aws_ecr_url = "dkr.ecr." + aws_default_region + ".amazonaws.com"
-
-# Docker Images
-docker_images = {
-    "ci_arm": {
-        "tag": "tlcpack/ci-arm:20251130-061900-c429a2b1",
-        "platform": "ARM",
-    },
-    "ci_cpu": {
-        "tag": "tlcpack/ci-cpu:20251130-061900-c429a2b1",
-        "platform": "CPU",
-    },
-    "ci_gpu": {
-        "tag": "tlcpack/ci-gpu:20251130-061900-c429a2b1",
-        "platform": "GPU",
-    },
-    "ci_lint": {
-        "tag": "tlcpack/ci-lint:20251130-061900-c429a2b1",
-        "platform": "CPU",
-    },
-    "ci_wasm": {
-        "tag": "tlcpack/ci-wasm:20251130-061900-c429a2b1",
-        "platform": "CPU",
-    },
-}
-
-data = {
-    "images": [{"name": k, "platform": v["platform"]} for k, v in docker_images.items()],
-    "aws_default_region": aws_default_region,
-    "aws_ecr_url": aws_ecr_url,
-    **{k: v["tag"] for k, v in docker_images.items()},
-    **files_to_stash,
-}
-
 if __name__ == "__main__":
-    # This is used in docker/dev_common.sh to look up image tags
-    name = sys.argv[1]
-    if name in docker_images:
-        print(docker_images[name]["tag"])
-    else:
-        exit(1)
+    paths = []
+    for name in sys.argv[1:]:
+        if name not in files_to_stash:
+            print(f"unknown bundle: {name}", file=sys.stderr)
+            sys.exit(1)
+        paths.extend(files_to_stash[name])
+    for p in paths:
+        print(p)

ci/jenkins/docker-images.ini

@@ -19,7 +19,7 @@
 [jenkins]
 ci_tag: 20260301-134651-63f099ad
 ci_arm: tlcpack/ci-arm:%(ci_tag)s
-ci_cpu: tlcpack/ci_cpu:%(ci_tag)s
+ci_cpu: tlcpack/ci-cpu:%(ci_tag)s
 ci_gpu: tlcpack/ci-gpu:%(ci_tag)s
 ci_lint: tlcpack/ci-lint:%(ci_tag)s
 ci_wasm: tlcpack/ci-wasm:%(ci_tag)s

ci/jenkins/generate.py

@@ -16,6 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 import argparse
+import configparser
 import datetime
 import difflib
 import re
@@ -24,12 +25,43 @@
 from pathlib import Path
 
 import jinja2
-from data import data
 
 REPO_ROOT = Path(__file__).resolve().parent.parent.parent
 JENKINS_DIR = REPO_ROOT / "ci" / "jenkins"
 TEMPLATES_DIR = JENKINS_DIR / "templates"
 GENERATED_DIR = JENKINS_DIR / "generated"
+DOCKER_IMAGES_INI = JENKINS_DIR / "docker-images.ini"
+
+# Platform mapping for the CI image registry. The ini section names are the
+# image names; this dict pins their Jenkins agent label. Keep in sync with
+# the [jenkins] section of docker-images.ini when adding/removing images.
+_IMAGE_PLATFORMS = {
+    "ci_arm": "ARM",
+    "ci_cpu": "CPU",
+    "ci_gpu": "GPU",
+    "ci_lint": "CPU",
+    "ci_wasm": "CPU",
+}
+
+
+def _build_render_context() -> dict:
+    """Build the Jinja render context: image metadata + AWS endpoints."""
+    config = configparser.ConfigParser()
+    config.read(DOCKER_IMAGES_INI)
+    images = [
+        {"name": name, "platform": platform}
+        for name, platform in _IMAGE_PLATFORMS.items()
+        if config.has_option("jenkins", name)
+    ]
+    aws_default_region = "us-west-2"
+    return {
+        "images": images,
+        "aws_default_region": aws_default_region,
+        "aws_ecr_url": f"dkr.ecr.{aws_default_region}.amazonaws.com",
+    }
+
+
+data = _build_render_context()
 
 
 class Change:

ci/jenkins/templates/arm_jenkinsfile.groovy.j2

@@ -31,5 +31,5 @@
   )
   cmake_build(ci_arm, 'build')
   make_cpp_tests(ci_arm, 'build')
-  {{ m.upload_artifacts(tag='arm', filenames=tvm_lib + cpptest) }}
+  {{ m.upload_artifacts(tag='arm', bundles=["tvm_lib", "cpptest"]) }}
 {% endcall %}

ci/jenkins/templates/cpu_jenkinsfile.groovy.j2

@@ -31,7 +31,7 @@
   )
   cmake_build(ci_cpu, 'build')
   make_cpp_tests(ci_cpu, 'build')
-  {{ m.upload_artifacts(tag='cpu', filenames=tvm_lib + tvm_allvisible + cpptest) }}
+  {{ m.upload_artifacts(tag='cpu', bundles=["tvm_lib", "cpptest"]) }}
 {% endcall %}
 
 {% set test_method_names = [] %}

ci/jenkins/templates/gpu_jenkinsfile.groovy.j2

@@ -27,13 +27,13 @@
 ) %}
   sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build"
   cmake_build("${ci_gpu} --no-gpu", 'build')
-  {{ m.upload_artifacts(tag='gpu', filenames=tvm_lib + tvm_allvisible + tvm_lib_gpu_extra) }}
+  {{ m.upload_artifacts(tag='gpu', bundles=["tvm_lib", "tvm_lib_gpu_extra"]) }}
 
   // compiler test
   sh "rm -rf build"
   sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu_other.sh build"
   cmake_build("${ci_gpu} --no-gpu", 'build')
-  {{ m.upload_artifacts(tag='gpu2', filenames=tvm_lib) }}
+  {{ m.upload_artifacts(tag='gpu2', bundles=["tvm_lib"]) }}
 {% endcall %}
 
 {% set test_method_names = [] %}

ci/jenkins/templates/utils/macros.j2

@@ -166,12 +166,19 @@ test()
   },
 {% endmacro %}
 
-{% macro upload_artifacts(action, tag, filenames) %}
+{% macro upload_artifacts(tag, bundles=none, filenames=none) %}
+{% if bundles is not none %}
+sh(
+      script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/{{ tag }}{% for b in bundles %} --bundle {{ b }}{% endfor %}",
+      label: 'Upload artifacts to S3',
+    )
+{% elif filenames is not none %}
 {% set items = ' '.join(filenames) %}
 sh(
       script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/{{ tag }} --items {{ items }}",
       label: 'Upload artifacts to S3',
     )
+{% endif %}
 {% endmacro %}
 
 {% macro download_artifacts(tag) %}

ci/scripts/jenkins/determine_docker_images.py

@@ -70,11 +70,28 @@ def image_exists(spec: str) -> bool:
         return False
 
 
+def lookup_image_tag(name: str) -> str:
+    """Resolve image ``name`` (e.g. ``ci_cpu``) to its tag string from the ini.
+
+    Pure ini read — no Docker Hub query, no tlcpackstaging fallback. Used by
+    ``docker/dev_common.sh`` for local-dev image-name shortcuts where the
+    full Hub-existence check is unnecessary.
+    """
+    config = configparser.ConfigParser()
+    config.read(IMAGE_TAGS_FILE)
+    return config.get("jenkins", name)
+
+
 if __name__ == "__main__":
     init_log()
     parser = argparse.ArgumentParser(
         description="Writes out Docker images names to be used to .docker-image-names/"
     )
+    parser.add_argument(
+        "--lookup-only",
+        metavar="NAME",
+        help="Print the tag for NAME from the ini and exit (no Docker Hub query, no fallback).",
+    )
     parser.add_argument(
         "--testing-docker-data",
         help="(testing only) JSON data to mock response from Docker Hub API",
@@ -89,6 +106,11 @@ def image_exists(spec: str) -> bool:
         help="(testing only) Folder to write image names to",
     )
     args, other = parser.parse_known_args()
+
+    if args.lookup_only:
+        print(lookup_image_tag(args.lookup_only))
+        raise SystemExit(0)
+
     name_dir = Path(args.base_dir)
 
     if args.testing_images_data:

ci/scripts/jenkins/open_docker_update_pr.py

@@ -17,6 +17,7 @@
 # under the License.
 
 import argparse
+import configparser
 import datetime
 import json
 import logging
@@ -32,7 +33,7 @@
 from should_rebuild_docker import docker_api
 
 JENKINS_DIR = REPO_ROOT / "ci" / "jenkins"
-IMAGES_FILE = JENKINS_DIR / "data.py"
+IMAGES_FILE = JENKINS_DIR / "docker-images.ini"
 GENERATE_SCRIPT = JENKINS_DIR / "generate.py"
 GITHUB_TOKEN = os.environ["GITHUB_TOKEN"]
 BRANCH = "nightly-docker-update"
@@ -127,33 +128,41 @@ def latest_tlcpackstaging_image(source: str) -> str | None:
     remote = git(["config", "--get", f"remote.{args.remote}.url"])
     user, repo = parse_remote(remote)
 
-    # Read the existing images from the Jenkinsfile
+    # Read the existing images from ci/jenkins/docker-images.ini.
+    # The ini has a single ``[jenkins]`` section with a shared ``ci_tag`` key
+    # and one ``ci_<name>: tlcpack/ci-<name>:%(ci_tag)s`` entry per image.
+    # Resolve each image to its full spec (with interpolation applied) and
+    # check against Docker Hub for newer tags.
     logging.info(f"Reading {IMAGES_FILE}")
+    config = configparser.ConfigParser()
+    config.read(IMAGES_FILE)
     with open(IMAGES_FILE) as f:
-        content = f.readlines()
+        content = f.read()
 
-    # Build a new Jenkinsfile with the latest images from tlcpack or tlcpackstaging
     replacements = {}
-
-    for line in content:
-        m = re.match(r'"tag": "(.*)",', line.strip())
-        if m is not None:
-            image_spec = m.groups()[0]
-            logging.info(f"Found match on line {line.strip()}")
-            new_image = latest_tlcpackstaging_image(image_spec)
-            if new_image is None:
-                logging.info("No new image found")
-            else:
-                logging.info(f"Using new image {new_image}")
-                new_line = f'        "tag": "{new_image}",\n'
-                replacements[line] = new_line
+    for key in config.options("jenkins"):
+        if key == "ci_tag":
+            continue
+        image_spec = config.get("jenkins", key)
+        logging.info(f"Found {key} = {image_spec}")
+        new_image = latest_tlcpackstaging_image(image_spec)
+        if new_image is None:
+            logging.info("No new image found")
+            continue
+        logging.info(f"Using new image {new_image}")
+        # Rewrite the ``ci_<name>:`` line with the resolved tag (breaking
+        # the ``%(ci_tag)s`` interpolation for that single entry) so the
+        # update is unambiguous and doesn't disturb other images that share
+        # the old tag.
+        old_line_re = re.compile(rf"^{re.escape(key)}\s*[:=].*$", re.MULTILINE)
+        new_line = f"{key}: {new_image}"
+        replacements[old_line_re] = new_line
 
     # Re-generate the Jenkinsfiles
     command = f"python3 {shlex.quote(str(GENERATE_SCRIPT))}"
 
-    content = "".join(content)
-    for old_line, new_line in replacements.items():
-        content = content.replace(old_line, new_line)
+    for old_line_re, new_line in replacements.items():
+        content = old_line_re.sub(new_line, content)
 
     print(f"Updated to:\n{content}")