Insert job's web_url into ci_failure table; improve search for pull_image

gitlab-bot/ghc_perf_import_service/__init__.py

@@ -51,8 +51,7 @@ class GHCPerfWebhookServer(WebhookServer):
         t = threading.Thread( \
                 target=self._process_spurious_failures, \
                 args=(event['build_id'], \
-                        event['build_started_at'], \
-                        event['repository']['homepage']))
+                        event['project_id']))
         t.start()
 
         proj_id = event['project_id']
@@ -74,7 +73,7 @@ class GHCPerfWebhookServer(WebhookServer):
 
         else:
             n = event['build_name']
-            logging.info(f'Skipping {project.name}/{n} job')
+            logging.info(f'Skipping further processing of {project.name}/{n} job')
 
     def _process_nofib_job(self, job: ProjectJob, tmp_dir: Path):
         logging.info(f'Processing nofib job {job.id}...')
@@ -161,11 +160,14 @@ class GHCPerfWebhookServer(WebhookServer):
         with ZipFile(archive_path) as archive:
             archive.extractall(path=out_dir)
 
-    def _process_spurious_failures(self, job_id: int, date: datetime, project_url: str) -> None:
+    def _process_spurious_failures(self, job_id: int, project_id: int) -> None:
         """
         Adds a record of known spurious failures to the ci_failure table.
         """
 
+        project = self.gl.projects.get(project_id)
+        job = project.jobs.get(job_id)
+
         # Get trace
 
         ## It's not enough to get artifacts, because they aren't created for jobs
@@ -174,7 +176,7 @@ class GHCPerfWebhookServer(WebhookServer):
         ## large logs (and all our logs are large). We have to go through the
         ## user API, not the machine API.
 
-        raw_url = f'{project_url}/-/jobs/{job_id}/raw'
+        raw_url = f'{job.web_url}/raw'
         resp = requests.get(raw_url, headers={'user-agent': GHCPerfWebhookServer.user_agent})
         if re.search('users/sign_in$', resp.url):
             logging.error(f'{raw_url} got redirected to login -- probably a 404')
@@ -190,8 +192,7 @@ class GHCPerfWebhookServer(WebhookServer):
         if grep(b'Cannot connect to the Docker daemon at unix:///var/run/docker.sock'):
             joblog('docker failure')
             failures.append('docker')
-        if grep(b'Error response from daemon: \w+ "https://registry.gitlab.haskell.org') \
-                or grep(b'Error response from daemon: manifest for registry.gitlab.haskell.org'):
+        if grep(b'failed to pull image "registry.gitlab.haskell.org'):
             joblog('image pull failure')
             failures.append('pull_image')
         if grep(b'Failed to connect to gitlab.haskell.org'):
@@ -213,10 +214,10 @@ class GHCPerfWebhookServer(WebhookServer):
 
         with psycopg2.connect(self.conn_string) as db:
             cur = db.cursor()
-            values = map(lambda f: (job_id, f, date), failures)
+            values = map(lambda f: (job_id, f, job.created_at, job.web_url), failures)
             cur.executemany('''
-                insert into ci_failure (job_id, type, job_date)
-                values (%s, %s, %s)
+                insert into ci_failure (job_id, type, job_date, web_url)
+                values (%s, %s, %s, %s)
                 on conflict do nothing
                 ''', values)
 
@@ -261,9 +262,7 @@ def main() -> None:
     elif args.failcheck is not None:
         if args.failcheck_project is None:
             raise Exception('--failcheck-project must be specified with --failcheck')
-        project = gl.projects.get(args.failcheck_project)
-        job = project.jobs.get(args.failcheck)
-        server._process_spurious_failures(args.failcheck, job.started_at, project.web_url)
+        server._process_spurious_failures(args.failcheck, args.failcheck_project)
 
     else:
         server.run()