Commit f8ac1a59 authored by Ben Gamari's avatar Ben Gamari 🐢

dhallify dockerfiles

This is a major refactoring and consolidation of the Docker image
generation logic. It eliminates a great deal of repetition, resolves
several inconsistencies, and makes the Docker images significantly more
maintainable.
parent 3ec3d603
Pipeline #24257 passed with stages
in 1 minute and 44 seconds
image: docker:19.03
stages:
- lint
- prepare
- build
services:
- docker:19.03-dind
variables:
CONTAINER_IMAGE_BASE: registry.gitlab.haskell.org/$CI_PROJECT_PATH
DOCKER_DRIVER: overlay2
DOCKER_TLS_CERTDIR: "/certs"
lint:
stage: lint
image: hadolint/hadolint:latest-debian
# Generate the CI configuration
generate:
stage: prepare
image: debian:buster
variables:
DHALL_URL: https://github.com/dhall-lang/dhall-haskell/releases/download/1.34.0/dhall-1.34.0-x86_64-linux.tar.bz2
tags:
- x86_64-linux
script:
- find . -name "Dockerfile" -print0 | xargs -0 -n1 hadolint
.build:
- apt-get update
- apt-get install -y curl tar bzip2
- curl -L $DHALL_URL | tar -jx
- ./bin/dhall text --file=gitlab-pipeline.dhall > gitlab-pipeline.yaml
artifacts:
paths:
- gitlab-pipeline.yaml
# Spawn a child pipeline to lint and build the Dockerfiles
build-images:
stage: build
before_script:
- docker info
script:
- |
CONTAINER_IMAGE="$CONTAINER_IMAGE_BASE/$IMAGE"
docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
docker pull $CONTAINER_IMAGE:latest || true
docker build --cache-from $CONTAINER_IMAGE:latest --tag $CONTAINER_IMAGE:$CI_COMMIT_SHA --tag $CONTAINER_IMAGE:latest $IMAGE
if [ -z "$CI_MERGE_REQUEST_ID" ]; then
docker push $CONTAINER_IMAGE:$CI_COMMIT_SHA
docker push $CONTAINER_IMAGE:latest
fi
build-linters:
extends: .build
variables:
IMAGE: linters
tags:
- docker
- x86_64-linux
build-aarch64-linux-deb10:
extends: .build
variables:
IMAGE: aarch64-linux-deb10
allow_failure: true
tags:
- docker
- aarch64-linux
build-armv7-linux-deb10:
extends: .build
variables:
IMAGE: armv7-linux-deb10
allow_failure: true
tags:
- docker
- armv7-linux
build-i386-linux-deb9:
extends: .build
variables:
IMAGE: i386-linux-deb9
tags:
- docker
- x86_64-linux
build-x86_64-linux-centos7:
extends: .build
variables:
IMAGE: x86_64-linux-centos7
tags:
- docker
- x86_64-linux
build-x86_64-linux-alpine:
extends: .build
variables:
IMAGE: x86_64-linux-alpine
tags:
- docker
- x86_64-linux
build-x86_64-linux-deb8:
extends: .build
variables:
IMAGE: x86_64-linux-deb8
tags:
- docker
- x86_64-linux
build-x86_64-linux-deb9:
extends: .build
variables:
IMAGE: x86_64-linux-deb9
tags:
- docker
- x86_64-linux
build-x86_64-linux-deb10:
extends: .build
variables:
IMAGE: x86_64-linux-deb10
tags:
- docker
- x86_64-linux
build-x86_64-linux-fedora27:
extends: .build
variables:
IMAGE: x86_64-linux-fedora27
tags:
- docker
- x86_64-linux
build-x86_64-linux-ubuntu2004:
extends: .build
variables:
IMAGE: x86_64-linux-ubuntu2004
tags:
- docker
- x86_64-linux
needs:
- generate
trigger:
include:
- artifact: gitlab-pipeline.yaml
job: generate
strategy: depend
\ No newline at end of file
--| A description of a Docker image
let
CF = ./deps/Containerfile.dhall
let
--| The name a of a Docker image
ImageName = Text
let
type: Type =
{ name: ImageName
, runnerTags: List Text
, jobStage: Text
, needs: List ImageName
, image: CF.Type
}
in
{ Type = type
, default =
{ needs = [] : List ImageName
, jobStage = "build"
}
, ImageName = ImageName
}
\ No newline at end of file
......@@ -8,3 +8,37 @@ Registry.
See the [GHC
Wiki](https://gitlab.haskell.org/ghc/ghc/wikis/continuous-integration-configuration)
for details.
CI Pipeline
-----------
The Dockerfiles are defined via [Dhall](https://dhall-lang.org/) expressions.
In particular, each major class of Linux distributions has its definitions defined in
a file in the `images/` directory. Logic for managing various shared toolchain
components (e.g. GHC, `cabal-install`, and LLVM) is contained in `components/`.
The CI build proceeds as follows:
* the pipeline in `.gitlab-ci.yml` uses `gitlab-pipeline.dhall` to generate
`gitlab-pipeline.yaml` which defines a child pipeline describing the jobs
needed to build each of the defined images.
* the `generate` job in `gitlab-pipeline.yaml` generates Dockerfiles
using `dockerfiles.dhall`. These are propagated as artifacts to the jobs
that follow.
* each of the jobs in the `build` and `build-derived` stages derived in
`gitlab-pipeline.yaml` build a Docker image from one of the Dockerfiles
generated by the `generate` job.
Building images manually
------------------------
To build images outside of CI (e.g. for testing) simply generate the
`Dockerfile`s from `dockerfiles.dhall` using the `dhall to-directory-tree`
command:
```bash
$ dhall to-directory-tree --file=dockerfiles.dhall --output=dockerfiles
```
This will produce a directory, `dockerfiles/`, which contains one directory
per Docker image.
FROM arm64v8/debian:buster
ENV LANG C.UTF-8
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
# Core build utilities
RUN apt-get update \
&& apt-get install --no-install-recommends -qy zlib1g-dev libtinfo-dev libsqlite3-0 libsqlite3-dev \
ca-certificates g++ git make automake autoconf gcc \
perl python3 texinfo xz-utils lbzip2 bzip2 patch openssh-client sudo time \
jq wget curl locales libnuma-dev \
# For LLVM
libtinfo5 \
# Documentation tools
python3-sphinx texlive-xetex texlive-latex-extra texlive-binaries texlive-fonts-recommended lmodern texlive-generic-extra \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
# Boot LLVM
ENV BOOT_LLVM_DIR /opt/llvm-bootstrap
ENV BOOT_LLVM_VERSION 7.0.0
ENV PATH /usr/local/bin:$PATH
RUN curl -L https://releases.llvm.org/$BOOT_LLVM_VERSION/clang+llvm-$BOOT_LLVM_VERSION-aarch64-linux-gnu.tar.xz | tar -xJC . && \
mkdir $BOOT_LLVM_DIR && \
cp -R clang+llvm*/* $BOOT_LLVM_DIR && \
rm -R clang+llvm* && \
$BOOT_LLVM_DIR/bin/llc --version
# GHC
ENV GHC_VERSION 8.8.3
RUN curl -L https://downloads.haskell.org/~ghc/$GHC_VERSION/ghc-$GHC_VERSION-aarch64-deb9-linux.tar.xz | tar -xJ
WORKDIR /ghc-$GHC_VERSION
RUN ./configure --prefix=/usr/local LLC=$BOOT_LLVM_DIR/bin/llc OPT=$BOOT_LLVM_DIR/bin/opt && \
make install
WORKDIR /
RUN rm -Rf ghc-*
ENV GHC /usr/local/bin/ghc
RUN $ghc --version
# LLVM
ENV LLVM_DIR /opt/llvm
ENV LLVM_VERSION 10.0.0
ENV LLC $LLVM_DIR/bin/llc
ENV OPT $LLVM_DIR/bin/opt
RUN curl -L https://github.com/llvm/llvm-project/releases/download/llvmorg-10.0.0/clang+llvm-10.0.0-aarch64-linux-gnu.tar.xz \
| tar -xJC . && \
mkdir $LLVM_DIR && \
cp -R clang+llvm*/* $LLVM_DIR && \
rm -R clang+llvm* && \
llc --version
# Cabal
ENV CABAL_VERSION 3.2.0.0
RUN curl -L http://home.smart-cactus.org/~ben/ghc/cabal-install-$CABAL_VERSION-aarch64-debian9-linux.tar.xz | tar -Jx && \
mv cabal /usr/local/bin/cabal
ENV CABAL /usr/local/bin/cabal
# Create a normal user.
RUN adduser ghc --gecos "GHC builds" --disabled-password
RUN echo "ghc ALL = NOPASSWD : ALL" > /etc/sudoers.d/ghc
USER ghc
WORKDIR /home/ghc/
# Build Haskell tools
RUN $CABAL v2-update && \
$CABAL install hscolour happy alex \
--constraint 'happy ^>= 1.20.0' \
--enable-static --install-method=copy \
--installdir=/usr/local/bin
ENV HAPPY /usr/local/bin/happy
ENV ALEX /usr/local/bin/alex
ENV HSCOLOUR /usr/local/bin/hscolour
CMD ["bash"]
FROM arm32v7/debian:buster
ENV LANG C.UTF-8
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
# Core build utilities
RUN apt-get update \
&& apt-get install --no-install-recommends -qy zlib1g-dev libtinfo-dev libgmp10 libgmp-dev \
libsqlite3-0 libsqlite3-dev \
ca-certificates g++ git make automake autoconf gcc \
perl python3 texinfo xz-utils lbzip2 bzip2 patch openssh-client sudo time \
jq wget curl locales \
# For LLVM
libtinfo5 \
# Documentation tools
python3-sphinx texlive-xetex texlive-latex-extra texlive-binaries texlive-fonts-recommended lmodern texlive-generic-extra \
cabal-install \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
# Boot LLVM
ENV BOOT_LLVM_DIR /opt/llvm-bootstrap
ENV BOOT_LLVM_VERSION 7.0.0
ENV PATH /usr/local/bin:$PATH
RUN curl -L https://releases.llvm.org/$BOOT_LLVM_VERSION/clang+llvm-$BOOT_LLVM_VERSION-armv7a-linux-gnueabihf.tar.xz | tar -xJC . && \
mkdir $BOOT_LLVM_DIR && \
cp -R clang+llvm*/* $BOOT_LLVM_DIR && \
rm -R clang+llvm* && \
$BOOT_LLVM_DIR/bin/llc --version
# GHC
ENV GHC_VERSION 8.8.1
RUN curl -L https://downloads.haskell.org/ghc/$GHC_VERSION/ghc-$GHC_VERSION-armv7l-deb9-linux.tar.xz | tar -xJ
WORKDIR /ghc-$GHC_VERSION
RUN ./configure --prefix=/usr/local LLC=$BOOT_LLVM_DIR/bin/llc OPT=$BOOT_LLVM_DIR/bin/opt && \
make install
WORKDIR /
RUN rm -Rf ghc-*
ENV GHC /usr/local/bin/ghc
RUN $GHC --version
# LLVM
ENV LLVM_DIR /opt/llvm
ENV LLVM_VERSION 10.0.0
ENV LLC $LLVM_DIR/bin/llc
ENV OPT $LLVM_DIR/bin/opt
RUN curl -L https://github.com/llvm/llvm-project/releases/download/llvmorg-10.0.0/clang+llvm-10.0.0-armv7a-linux-gnueabihf.tar.xz \
| tar -xJC . && \
mkdir $LLVM_DIR && \
cp -R clang+llvm*/* $LLVM_DIR && \
rm -R clang+llvm* && \
llc --version
# Cabal
ENV CABAL_VERSION 3.2.0.0
RUN cabal update && cabal install "cabal-install==$CABAL_VERSION" && \
mv "$HOME/.cabal/bin/cabal" /usr/local/bin/cabal
ENV CABAL /usr/local/bin/cabal
# Create a normal user.
RUN adduser ghc --gecos "GHC builds" --disabled-password
RUN echo "ghc ALL = NOPASSWD : ALL" > /etc/sudoers.d/ghc
USER ghc
WORKDIR /home/ghc/
# Build Haskell tools
RUN $CABAL v2-update && \
$CABAL install hscolour happy alex \
--constraint 'happy ^>= 1.20.0' \
--enable-static --install-method=copy \
--installdir=/usr/local/bin
ENV HAPPY /usr/local/bin/happy
ENV ALEX /usr/local/bin/alex
ENV HSCOLOUR /usr/local/bin/hscolour
CMD ["bash"]
let
CF = ../deps/Containerfile.dhall
let
setEnv: Text -> CF.Type =
\(path: Text) ->
CF.env (toMap { CABAL = path })
let
installFromBindist: Text -> CF.Type =
\(bindistUrl: Text) ->
let
path: Text = "/usr/local/bin/cabal"
in
CF.run "install cabal"
[ "curl -L ${bindistUrl} | tar -Jx"
, "mv cabal ${path}"
, "${path} --version"
]
# setEnv path
let
installFromSource: Text -> CF.Type =
\(version: Text) ->
let
path: Text = "/usr/local/bin/cabal"
in
CF.run "install cabal"
[ "cabal update"
, "cabal install cabal-install==${version}"
, "mv $HOME/.cabal/bin/cabal ${path}"
, "${path} --version"
]
# setEnv path
let
type: Type =
< FromBindist : Text
| FromSource : Text
| FromDistribution : Text
>
let fromUpstreamBindist =
\(opts: { triple: Text, version: Text }) ->
type.FromBindist "https://downloads.haskell.org/cabal/cabal-install-${opts.version}/cabal-install-${opts.version}-${opts.triple}.tar.xz"
let
install: type -> CF.Type =
\(src: type) ->
merge
{ FromBindist = installFromBindist
, FromSource = installFromSource
, FromDistribution = setEnv
} src
in
{ Type = type
, install = install
, fromUpstreamBindist = fromUpstreamBindist
}
\ No newline at end of file
let
Prelude = ../deps/Prelude.dhall
let
CF = ../deps/Containerfile.dhall
let
BindistSpec: Type =
{ version : Text
, triple : Text
}
let
Options =
{ Type =
{ bindist : BindistSpec
, destDir : Text
, configureOpts : List Text
}
, default =
{ configureOpts = [] : List Text
}
}
let
install: Options.Type -> CF.Type =
\(opts: Options.Type) ->
let
version = opts.bindist.version
let
bindistUrl: Text = "https://downloads.haskell.org/~ghc/${version}/ghc-${version}-${opts.bindist.triple}.tar.xz"
in
CF.run "fetch GHC" [ "curl -L ${bindistUrl} | tar -Jx -C /tmp" ]
# CF.run "configure bindist"
[ "cd /tmp/ghc-${version}*"
, "./configure ${Prelude.Text.concatSep " " opts.configureOpts} --prefix=${opts.destDir}"
, "make install"
]
# CF.run "remove temporary directory" [ "rm -Rf /tmp/ghc-${version}-*" ]
# CF.run "test GHC" [ "${opts.destDir}/bin/ghc --version" ]
in
{ Options = Options
, BindistSpec = BindistSpec
, install = install
}
\ No newline at end of file
let
CF = ../deps/Containerfile.dhall
let
build: CF.Type =
CF.run "build haskell tools"
[ "$CABAL user-config update"
, "$CABAL v2-update"
, "$CABAL v2-install hscolour happy alex --constraint='happy ^>= 1.20' --with-compiler=$GHC --enable-static --install-method=copy --installdir=$HOME/toolchain"
]
# CF.env (toMap
{ HAPPY = "/home/ghc/toolchain/happy"
, ALEX = "/home/ghc/toolchain/alex"
, HSCOLOUR = "/home/ghc/toolchain/hscolour"
})
in
{
build = build
}
\ No newline at end of file
let
Prelude = ../deps/Prelude.dhall
let
CF = ../deps/Containerfile.dhall
let
BindistSpec: Type =
{ version : Text
, triple : Text
}
let
Options =
{ Type =
{ bindist : BindistSpec
, destDir : Text
}
, default = {}
}
let
install: Options.Type -> CF.Type =
\(opts: Options.Type) ->
let
url: Text = "https://github.com/llvm/llvm-project/releases/download/llvmorg-${opts.bindist.version}/clang+llvm-${opts.bindist.version}-${opts.bindist.triple}.tar.xz"
in
CF.run "install LLVM for bootstrap GHC"
[ "curl -L ${url} | tar -xJC ."
, "mkdir ${opts.destDir}"
, "cp -R clang+llvm*/* ${opts.destDir}"
, "rm -R clang+llvm*"
, "${opts.destDir}/bin/llc --version"
]
let
maybeInstallTo: Text -> Optional BindistSpec -> CF.Type =
\(destDir: Text) -> \(opts: Optional BindistSpec) ->
merge
{ Some = \(tv: BindistSpec) -> install { bindist = tv, destDir = destDir }
, None = [] : CF.Type
} opts
let
setEnv: Text -> CF.Type =
\(destDir: Text) ->
CF.env (toMap
{ LLC = "${destDir}/bin/llc"
, OPT = "${destDir}/bin/opt"
})
in
{ Options = Options
, BindistSpec = BindistSpec
, install = install
, maybeInstallTo = maybeInstallTo
, setEnv = setEnv
}
--https://raw.githubusercontent.com/softwarefactory-project/dhall-containerfile/e3ea2f2e02bd979b4255dfbc5d7086dbbfcbc6c2/package.dhall
https://raw.githubusercontent.com/bgamari/dhall-containerfile/2b690b52bfc0177f14e0500567ea31af56ef2726/package.dhall
https://raw.githubusercontent.com/bgamari/dhall-gitlab-ci/84c72592054cc1399fc018093d3516e4116a4d51/package.dhall
\ No newline at end of file
https://prelude.dhall-lang.org/package.dhall
let
Prelude = ./deps/Prelude.dhall
let
Map = Prelude.Map
let
CF = ./deps/Containerfile.dhall
let
Image = ./Image.dhall
let
images: List Image.Type = ./images.dhall
let
DockerfileDir: Type = { Dockerfile : Text }
let
toDockerfileDir: Image.Type -> Map.Entry Text DockerfileDir =
\(cf: Image.Type) -> Map.keyValue DockerfileDir cf.name { Dockerfile = CF.render cf.image }
in Prelude.List.map Image.Type (Map.Entry Text DockerfileDir) toDockerfileDir images
\ No newline at end of file
-- Defines the child GitLab CI pipeline which builds our Docker images.
let
Prelude = ./deps/Prelude.dhall
let
Map = Prelude.Map
let
CF = ./deps/Containerfile.dhall
let
Image = ./Image.dhall
let
GitLab = ./deps/GitLabCi.dhall
let
images: List Image.Type = ./images.dhall
let
mkJobEntry: Text -> GitLab.Job.Type -> Prelude.Map.Entry Text GitLab.Job.Type =
Prelude.Map.keyValue GitLab.Job.Type
let
toJob: Image.Type -> Prelude.Map.Entry Text GitLab.Job.Type =
\(image: Image.Type) ->
let
docker_base_url: Text = "registry.gitlab.haskell.org/$CI_PROJECT_PATH"
let
imageName: Text = "${docker_base_url}/${image.name}"
let job: GitLab.Job.Type =
GitLab.Job::
{ image = Some "docker:19.03"
, stage = Some image.jobStage
, tags = Some (image.runnerTags # [ "docker" ])
, variables = toMap
{ DOCKER_DRIVER = "overlay2"
, DOCKER_TLS_CERTDIR = "/certs"
}
, needs = image.needs # [ "generate-dockerfiles" ]
, dependencies = [ "generate-dockerfiles" ]
, services = Some [ "docker:19.03-dind" ]
, before_script = Some [ "docker info" ]
, script =
[ "docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY"
, "docker pull ${imageName}:latest || true"
, "docker build --cache-from ${imageName}:latest --tag ${imageName}:$CI_COMMIT_SHA --tag ${imageName}:latest dockerfiles/${image.name}"
, ''
if [ -z "$CI_MERGE_REQUEST_ID" ]; then
docker push ${imageName}:$CI_COMMIT_SHA
docker push ${imageName}:latest
fi
''
]
}
in mkJobEntry image.name job
let
dockerfilesJob =
let
dhallUrl = "https://github.com/dhall-lang/dhall-haskell/releases/download/1.34.0/dhall-1.34.0-x86_64-linux.tar.bz2"
in GitLab.Job::
{ stage = Some "prepare"
, image = Some "debian:buster"
, tags = Some [ "x86_64-linux" ]
, script = [
, "mkdir -p dockerfiles"
, "apt-get update"
, "apt-get install -y curl tar bzip2"
, "curl -L ${dhallUrl} | tar -jx"
, "./bin/dhall to-directory-tree --file=dockerfiles.dhall --output=dockerfiles"
]
, artifacts =
Some (GitLab.ArtifactsSpec:: { paths = [ "dockerfiles" ] })
}
let
lintJob =