download-artifacts-from-pipeline.py

#!/usr/bin/env python3

from pathlib import Path
import argparse
import json
import logging
import subprocess
import gitlab
import os

logging.basicConfig(level=logging.INFO)

def strip_prefix(s, prefix):
    if s.startswith(prefix):
        return s[len(prefix):]
    else:
        return None

def job_triple(job_name):
    bindists = {
            "build-linux-alpine: [i386, alpine3_12, 9.0.2]": "i386-linux-alpine3_12",
            "build-linux-alpine: [x86_64, alpine3_12]": "x86_64-linux-alpine3_12",
            "build-linux: [aarch64, aarch64-linux, deb10]": "aarch64-linux-deb10",
            "build-linux: [aarch64, aarch64-linux, deb11]": "aarch64-linux-deb11",
            "build-linux: [i386, x86_64-linux, deb9]": "i386-linux-deb9",
            "build-linux: [x86_64, x86_64-linux, centos7]": "x86_64-linux-centos7",
            "build-linux: [x86_64, x86_64-linux, deb10]": "x86_64-linux-deb10",
            "build-linux: [x86_64, x86_64-linux, deb11]": "x86_64-linux-deb11",
            "build-linux: [x86_64, x86_64-linux, deb9]": "x86_64-linux-deb9",
            "build-linux: [x86_64, x86_64-linux, fedora33]": "x86_64-linux-fedora33",
            "build-linux: [x86_64, x86_64-linux, rocky8]": "x86_64-linux-rocky8",
            "build-linux: [x86_64, x86_64-linux, ubuntu18_04]": "x86_64-linux-ubuntu18_04",
            "build-linux: [x86_64, x86_64-linux, ubuntu20_04]": "x86_64-linux-ubuntu20_04",
            "build-x86_64-windows": "x86_64-windows",
            "build-aarch64-darwin": "aarch64-darwin",
            "build-x86_64-darwin": "x86_64-darwin",
    }

    if job_name in bindists:
        return bindists[job_name]
    else:
        return None

class UnhandledJobException(Exception):
    # Raised when there is a release job in the pipeline but we don"t explicitly handle it.
    def __init__(self, name):
        self.message = f"{name} is a release job but not downloaded"
        super().__init__(self.message)

def fetch_artifacts(release: str, pipeline_id: int,
                    dest_dir: Path, gl: gitlab.Gitlab):
    dest_dir.mkdir(exist_ok=True)
    # Write the pipeline id into output directory
    with open(f"{dest_dir}/metadata.json", "w") as out: json.dump({ "pipeline_id": pipeline_id }, out)

    proj = gl.projects.get("haskell/cabal")
    pipeline = proj.pipelines.get(pipeline_id)
    tmpdir = Path("fetch-gitlab")
    tmpdir.mkdir(exist_ok=True)
    for pipeline_job in pipeline.jobs.list(all=True):
        if len(pipeline_job.artifacts) == 0:
            logging.info(f"job {pipeline_job.name} ({pipeline_job.id}) has no artifacts")
            continue

        job = proj.jobs.get(pipeline_job.id)
        triple = job_triple(job.name)
        if triple is None:
            if job.name.startswith("build"):
                raise(UnhandledJobException(job.name))
            logging.info(f"ignoring {job.name}")
            continue

        try:
            destdir = tmpdir / job.name
            zip_name = Path(f"{tmpdir}/{job.name}.zip")
            if not zip_name.exists() or zip_name.stat().st_size == 0:
                logging.info(f"downloading archive {zip_name} for job {job.name} (job {job.id})...")
                with open(zip_name, "wb") as f:
                    job.artifacts(streamed=True, action=f.write)

            if zip_name.stat().st_size == 0:
                logging.info(f"artifact archive for job {job.name} (job {job.id}) is empty")
                continue

            subprocess.run(["unzip", "-bo", zip_name, "-d", destdir])

            if job.name == "build-x86_64-windows":
                bindist_files = list(destdir.glob("out/cabal-install*.zip"))
                bindist = bindist_files[0]
                dest = dest_dir / bindist.name
                final_destination = Path("../../release") / Path(bindist.name)
                bindist.rename(final_destination)
                logging.info(f"Extracted {job.name} to {final_destination}")
            else:
                bindist_files = list(destdir.glob("out/cabal-install*.tar.xz"))
                logging.info(f"bindist_files: {bindist_files}")
                bindist = bindist_files[0]
                logging.info(f"Bindist: {bindist}")
                dest = dest_dir / bindist.name
                final_destination = Path("../../release") / Path(bindist.name)
                bindist.rename(final_destination)

        except Exception as e:
            logging.error(f"Error fetching job {job.name}: \"{e}\"")
            pass

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--pipeline", "-p", required=True, type=int, help="pipeline id")
    parser.add_argument("--release", "-r", required=True, type=str, help="release name")
    parser.add_argument("--output", "-o", type=Path, default=Path.cwd(), help="output directory")
    parser.add_argument("--profile", "-P", default="haskell",
                        help="python-gitlab.cfg profile name")
    args = parser.parse_args()
    gl = gitlab.Gitlab.from_config(args.profile)
    fetch_artifacts(args.release, args.pipeline,
                    dest_dir=args.output, gl=gl)

if __name__ == "__main__":
    main()