Lookup job id when looking for artifacts from upstream pipeline

Unfortunately the interface [1] we were using previously will not
provide an artifact until a pipeline has successfully finished on the
requested branch.

This partially reverts commit 00db7a0d.

[1] https://docs.gitlab.com/ee/api/job_artifacts.html#download-a-single-artifact-file-from-specific-tag-or-branch

.gitlab-ci.yml

@@ -27,6 +27,10 @@ variables:
   # Default GHC bindist
   GHC_TARBALL: "https://gitlab.haskell.org/api/v4/projects/1/jobs/artifacts/master/raw/ghc-x86_64-fedora27-linux.tar.xz?job=validate-x86_64-linux-fedora27"
 
+  # Default this to ghc/ghc> to make it more convenient to run from the web
+  # interface.
+  UPSTREAM_PROJECT_ID: 1
+
   # CPUS is set by the runner, as usual.
 
   # EXTRA_HC_OPTS are passed to via --ghc-options to GHC during the package
@@ -42,6 +46,10 @@ variables:
   # These are set by the "upstream" pipeline for `build-pipeline` pipelines:
   #
   #   UPSTREAM_PROJECT_PATH: The path of the upstream project (e.g. `ghc/ghc`)
+  #   UPSTREAM_PIPELINE_ID: The ID of the upstream pipeline
+  #
+  # Instead of UPSTREAM_PIPELINE_ID you can also pass:
+
   #   UPSTREAM_COMMIT_SHA: The ref or commit SHA of the GHC build to be tested
   #
 
@@ -52,11 +60,22 @@ build-pipeline:
   before_script:
     - |
       if [ -n "$UPSTREAM_COMMIT_SHA" ]; then
+        # N.B. We can't use this if the upstream pipeline might be in-progress
+        # since the below URL cannot provide an artifact until a pipeline has
+        # run to completion on the requested branch. This is in general
+        # not the case for GHC pipelines. Consequently, in this case we will
+        # usually rather provide UPSTREAM_PIPELINE_ID.
         echo "Pulling binary distribution from commit $UPSTREAM_COMMIT_SHA of project $UPSTREAM_PROJECT_PATH..."
         GHC_TARBALL="https://gitlab.haskell.org/$UPSTREAM_PROJECT_PATH/-/jobs/artifacts/$UPSTREAM_COMMIT_SHA/raw/ghc-x86_64-fedora27-linux.tar.xz?job=validate-x86_64-linux-fedora27"
+      elif [ -n "$UPSTREAM_PIPELINE_ID" ]; then
+        job_name="validate-x86_64-linux-fedora27"
+        job_id=$(nix run -f ci/default.nix \
+          -c find-job $UPSTREAM_PROJECT_ID $UPSTREAM_PIPELINE_ID $job_name)
+        echo "Pulling ${job_name} binary distribution from Pipeline $UPSTREAM_PIPELINE_ID (job $job_id)..."
+        GHC_TARBALL="https://gitlab.haskell.org/$UPSTREAM_PROJECT_ID/jobs/$job_id/artifacts/ghc-x86_64-fedora27-linux.tar.xz"
       fi
   rules:
-    - if: '$UPSTREAM_COMMIT_SHA'
+    - if: '$UPSTREAM_COMMIT_SHA || $UPSTREAM_PIPELINE_ID'
       when: always
     - when: never
 
@@ -67,7 +86,7 @@ build-master:
     GHC_TARBALL: "https://gitlab.haskell.org/api/v4/projects/1/jobs/artifacts/master/raw/ghc-x86_64-fedora27-linux.tar.xz?job=validate-x86_64-linux-fedora27"
     EXTRA_HC_OPTS: "-dcore-lint -ddump-timings"
   rules:
-    - if: '$UPSTREAM_COMMIT_SHA'
+    - if: '$UPSTREAM_COMMIT_SHA || $UPSTREAM_PIPELINE_ID'
       when: never
     - when: always
 
@@ -78,7 +97,7 @@ build-9.0:
     GHC_TARBALL: "https://gitlab.haskell.org/api/v4/projects/1/jobs/artifacts/ghc-9.0/raw/ghc-x86_64-fedora27-linux.tar.xz?job=validate-x86_64-linux-fedora27"
     EXTRA_HC_OPTS: "-dcore-lint"
   rules:
-    - if: '$UPSTREAM_COMMIT_SHA'
+    - if: '$UPSTREAM_COMMIT_SHA || $UPSTREAM_PIPELINE_ID'
       when: never
     - when: always
 
@@ -89,7 +108,7 @@ build-9.2:
     GHC_TARBALL: "https://gitlab.haskell.org/api/v4/projects/1/jobs/artifacts/ghc-9.2/raw/ghc-x86_64-fedora27-linux.tar.xz?job=validate-x86_64-linux-fedora27"
     EXTRA_HC_OPTS: "-dcore-lint"
   rules:
-    - if: '$UPSTREAM_COMMIT_SHA'
+    - if: '$UPSTREAM_COMMIT_SHA || $UPSTREAM_PIPELINE_ID'
       when: never
     - when: always
 

ci/default.nix

@@ -67,6 +67,9 @@ let
             --set USE_NIX 1 \
             --set CI_CONFIG ${./config.sh}
 
+        makeWrapper ${./find-job.sh} $out/bin/find-job \
+            --prefix PATH : ${stdenv.lib.makeBinPath deps}:$out/bin
+
         makeWrapper ${xz}/bin/xz $out/bin/xz
         makeWrapper ${curl}/bin/curl $out/bin/curl
       '';

ci/find-job.sh

+#!/usr/bin/env bash
+
+set -e
+
+project_id=$1
+pipeline_id=$2
+job_name=$3
+
+# Access token is a protected environment variable in the head.hackage project and
+# is necessary for this query to succeed. Sadly job tokens only seem to
+# give us access to the project being built.
+curl \
+  --silent --show-error \
+  -H "Private-Token: $ACCESS_TOKEN" \
+  "https://gitlab.haskell.org/api/v4/projects/$project_id/pipelines/$pipeline_id/jobs?scope[]=success" \
+  > resp.json
+
+job_id=$(jq ". | map(select(.name == \"$job_name\")) | .[0].id" < resp.json)
+if [ "$job_id" = "null" ]; then
+  echo "Error finding job $job_name for $pipeline_id in project $project_id:" >&2
+  cat resp.json >&2
+  rm resp.json
+  exit 1
+else
+  rm resp.json
+  echo -n "$job_id"
+fi