Skip to content

Commit

Permalink
Build with docker and sh rather than nix (#10)
Browse files Browse the repository at this point in the history
After this change, all linux packages will be built inside docker
containers and macos packages will be built with shell scripts that run
directly in the github action. The main reason is that it allows us to
use opam for package management rather than nix, which means that newly
released packages become available the instant they are released, rather
than needing to wait for new versions to land in nix's package repo.

Another issue that this solves is that some macos builds of
ocaml-lsp-server don't work properly when built with nix (read more
[here](https://discuss.ocaml.org/t/corrupted-compiled-interface-after-dune-build/14919)).

Signed-off-by: Stephen Sherratt <[email protected]>
  • Loading branch information
gridbugs authored Nov 21, 2024
1 parent 04d9297 commit bded525
Show file tree
Hide file tree
Showing 18 changed files with 260 additions and 620 deletions.
170 changes: 98 additions & 72 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,142 +5,168 @@ on:
- '*'

jobs:
ocamlformat:
name: Build ocamlformat binaries
ocaml-lsp-server-build-script:
name: ocaml-lsp-server binaries via build.sh
permissions:
contents: write
strategy:
matrix:
os: [ubuntu-latest, macos-13, macos-14]
ocaml: ["5.2.0"]
os: [macos-14, macos-13]
ocaml: ["5.2.1", "5.2.0", "5.1.1"]
include:
- os: ubuntu-latest
version: "0.26.2"
linking: static
target_triple: x86_64-unknown-linux-musl
- os: macos-13
version: "0.26.2"
linking: dynamic
target_triple: x86_64-apple-darwin
- os: macos-14
version: "0.26.2"
linking: dynamic
version: "1.19.0"
ocaml: "5.2.1"
target_triple: aarch64-apple-darwin
- os: macos-14
version: "1.19.0"
ocaml: "5.2.0"
target_triple: aarch64-apple-darwin
- os: macos-14
version: "1.18.0"
ocaml: "5.1.1"
target_triple: aarch64-apple-darwin
- os: macos-13
version: "1.19.0"
ocaml: "5.2.1"
target_triple: x86_64-apple-darwin
- os: macos-13
version: "1.19.0"
ocaml: "5.2.0"
target_triple: x86_64-apple-darwin
- os: macos-13
version: "1.18.0"
ocaml: "5.1.1"
target_triple: x86_64-apple-darwin

runs-on: ${{ matrix.os }}

steps:
- name: Set archive environment variables
run: |
echo "FLAKE_NAME=./flakes/ocamlformat/#ocamlformat_$(echo ${{ matrix.version }} | tr . _)_ocaml_$(echo ${{ matrix.ocaml }} | tr . _)_${{ matrix.linking }}" >> $GITHUB_ENV
echo "ARCHIVE_NAME=ocamlformat.${{ matrix.version }}+binary-ocaml-${{ matrix.ocaml }}-built-${{ github.ref_name }}-${{ matrix.target_triple }}" >> $GITHUB_ENV
- name: Set-up OCaml
uses: ocaml/setup-ocaml@v3
with:
ocaml-compiler: 5
- uses: actions/checkout@v4
- uses: cachix/install-nix-action@v22
- name: Build ocamlformat
- name: Build ocaml-lsp-server
run: |
nix build $FLAKE_NAME
mkdir -p $ARCHIVE_NAME
cp -r result/* $ARCHIVE_NAME
cp -r flakes/ocamlformat/static/* $ARCHIVE_NAME
find $ARCHIVE_NAME -type d -exec chmod +w {} +
tar --format=posix -cvf $ARCHIVE_NAME.tar $ARCHIVE_NAME
gzip -9 $ARCHIVE_NAME.tar
TAG=${{ github.ref_name }} TARGET=${{ matrix.target_triple }} OCAML_VERSION=${{ matrix.ocaml }} OCAML_LSP_SERVER_VERSION=${{ matrix.version }} OUTPUT=$(pwd)/ocaml-lsp-server.${{ matrix.version }}+binary-ocaml-${{ matrix.ocaml }}-built-${{ github.ref_name }}-${{ matrix.target_triple }}.tar.gz build-scripts/ocaml-lsp-server/build.sh
- uses: ncipollo/release-action@v1
with:
allowUpdates: true
artifacts: "*.tar.gz"

ocaml-lsp-server:
name: Build ocaml-lsp-server binaries

ocaml-lsp-server-dockerfile:
name: ocaml-lsp-server binaries via docker
permissions:
contents: write
strategy:
matrix:
os: [ubuntu-latest, macos-13]
ocaml: ["5.2.0", "5.1.1"]
os: [ubuntu-latest]
ocaml: ["5.2.1", "5.2.0", "5.1.1"]
include:
- os: ubuntu-latest
version: "1.19.0"
ocaml: "5.2.0"
linking: static
ocaml: "5.2.1"
target_triple: x86_64-unknown-linux-musl
- os: macos-13
- os: ubuntu-latest
version: "1.19.0"
ocaml: "5.2.0"
linking: dynamic
target_triple: x86_64-apple-darwin
target_triple: x86_64-unknown-linux-musl
- os: ubuntu-latest
version: "1.18.0"
ocaml: "5.1.1"
linking: static
target_triple: x86_64-unknown-linux-musl
- os: macos-13
version: "1.18.0"
ocaml: "5.1.1"
linking: dynamic
target_triple: x86_64-apple-darwin

runs-on: ${{ matrix.os }}

steps:
- name: Set archive environment variables
run: |
echo "FLAKE_NAME=./flakes/ocaml-lsp-server/#ocaml_lsp_server_$(echo ${{ matrix.version }} | tr . _)_ocaml_$(echo ${{ matrix.ocaml }} | tr . _)_${{ matrix.linking }}" >> $GITHUB_ENV
echo "ARCHIVE_NAME=ocaml-lsp-server.${{ matrix.version }}+binary-ocaml-${{ matrix.ocaml }}-built-${{ github.ref_name }}-${{ matrix.target_triple }}" >> $GITHUB_ENV
- uses: actions/checkout@v4
- uses: cachix/install-nix-action@v22
- name: Build ocaml-lsp-server
- name: Build ocamlformat
run: |
nix build $FLAKE_NAME
mkdir -p $ARCHIVE_NAME
cp -r result/* $ARCHIVE_NAME
cp -r flakes/ocaml-lsp-server/static/* $ARCHIVE_NAME
find $ARCHIVE_NAME -type d -exec chmod +w {} +
tar --format=posix -cvf $ARCHIVE_NAME.tar $ARCHIVE_NAME
gzip -9 $ARCHIVE_NAME.tar
docker buildx build build-scripts/ocaml-lsp-server --target=export --output type=local,dest=$(pwd)/out/ --build-arg TAG=${{ github.ref_name }} --build-arg TARGET=${{ matrix.target_triple }} --build-arg OCAML_VERSION=${{ matrix.ocaml }} --build-arg OCAML_LSP_SERVER_VERSION=${{ matrix.version }}
- uses: ncipollo/release-action@v1
with:
allowUpdates: true
artifacts: "*.tar.gz"
artifacts: "out/*.tar.gz"


ocaml-lsp-server-build-script:
name: Build ocaml-lsp-server binaries from the build script (rather than the flake)
ocamlformat-build-script:
name: ocamlformat binaries via build.sh
permissions:
contents: write
strategy:
matrix:
os: [macos-14]
ocaml: ["5.2.0", "5.1.1"]
os: [macos-14, macos-13]
ocaml: ["5.2.1", "5.1.1"]
include:
- os: ubuntu-latest
version: "0.26.2"
ocaml: "5.2.1"
target_triple: x86_64-unknown-linux-musl
- os: macos-13
version: "0.26.2"
ocaml: "5.2.1"
target_triple: x86_64-apple-darwin
- os: macos-14
version: "1.19.0"
ocaml: "5.2.0"
linking: dynamic
version: "0.26.2"
ocaml: "5.2.1"
target_triple: aarch64-apple-darwin
- os: ubuntu-latest
version: "0.26.1"
ocaml: "5.1.1"
target_triple: x86_64-unknown-linux-musl
- os: macos-13
version: "0.26.1"
ocaml: "5.1.1"
target_triple: x86_64-apple-darwin
- os: macos-14
version: "1.18.0"
version: "0.26.1"
ocaml: "5.1.1"
linking: dynamic
target_triple: aarch64-apple-darwin

runs-on: ${{ matrix.os }}

steps:
- name: Set script name
run: |
echo "BUILD_SCRIPT=./build-scripts/ocaml-lsp-server.${{ matrix.version }}-ocaml.${{ matrix.ocaml }}.sh" >> $GITHUB_ENV
- name: Set-up OCaml
uses: ocaml/setup-ocaml@v3
with:
ocaml-compiler: 5
- uses: actions/checkout@v4
- name: Build ocaml-lsp-server
- name: Build ocamlformat
run: |
$BUILD_SCRIPT ${{ github.ref_name }} ${{ matrix.target_triple }} $(pwd)
TAG=${{ github.ref_name }} TARGET=${{ matrix.target_triple }} OCAML_VERSION=${{ matrix.ocaml }} OCAMLFORMAT_VERSION=${{ matrix.version }} OUTPUT=$(pwd)/ocamlformat.${{ matrix.version }}+binary-ocaml-${{ matrix.ocaml }}-built-${{ github.ref_name }}-${{ matrix.target_triple }}.tar.gz build-scripts/ocamlformat/build.sh
- uses: ncipollo/release-action@v1
with:
allowUpdates: true
artifacts: "*.tar.gz"

ocamlformat-dockerfile:
name: ocamlformat binaries via docker
permissions:
contents: write
strategy:
matrix:
os: [ubuntu-latest]
ocaml: ["5.2.1", "5.1.1"]
include:
- os: ubuntu-latest
version: "0.26.2"
ocaml: "5.2.1"
target_triple: x86_64-unknown-linux-musl
- os: ubuntu-latest
version: "0.26.1"
ocaml: "5.1.1"
target_triple: x86_64-unknown-linux-musl

runs-on: ${{ matrix.os }}

steps:
- uses: actions/checkout@v4
- name: Build ocamlformat
run: |
docker buildx build build-scripts/ocamlformat --target=export --output type=local,dest=$(pwd)/out/ --build-arg TAG=${{ github.ref_name }} --build-arg TARGET=${{ matrix.target_triple }} --build-arg OCAML_VERSION=${{ matrix.ocaml }} --build-arg OCAMLFORMAT_VERSION=${{ matrix.version }}
- uses: ncipollo/release-action@v1
with:
allowUpdates: true
artifacts: "out/*.tar.gz"
9 changes: 1 addition & 8 deletions build-scripts/README.md
Original file line number Diff line number Diff line change
@@ -1,10 +1,3 @@
# Build Scripts

## Why not just use nix?

It's necessary to produce binaries which are statically-linked with muslc as
they can be used on all distros of linux (including distros that don't link
with the gnu linker such as alpine and nixos). While it's possible to also
produce MacOS binaries with the same nix derivations, we found that aarch64
binaries for MacOS don't work correctly when built with nix due to [this
issue](https://discuss.ocaml.org/t/corrupted-compiled-interface-after-dune-build/14919).
Shell scripts and dockerfiles for building packages.
1 change: 1 addition & 0 deletions build-scripts/ocaml-lsp-server/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
out
49 changes: 49 additions & 0 deletions build-scripts/ocaml-lsp-server/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
FROM alpine:3.20.3 AS build

ARG TAG
ARG TARGET
ARG OCAML_VERSION
ARG OCAML_LSP_SERVER_VERSION

ENV PACKAGE=ocaml-lsp-server
ENV ARCHIVE_NAME="$PACKAGE.$OCAML_LSP_SERVER_VERSION+binary-ocaml-$OCAML_VERSION-built-$TAG-$TARGET"

RUN apk update && apk add \
clang \
make \
opam \
bash \
rsync \
gzip \
;

ENV USERNAME=user
ENV HOME=/home/$USERNAME
RUN adduser -D $USERNAME
USER $USERNAME
WORKDIR $HOME

RUN opam init --disable-sandboxing --auto-setup --bare

# Build ocamllsp
RUN mkdir switch
WORKDIR switch
RUN opam switch create . "ocaml.$OCAML_VERSION"
RUN opam install -y --deps-only "$PACKAGE.$OCAML_LSP_SERVER_VERSION"
RUN opam source "$PACKAGE.$OCAML_LSP_SERVER_VERSION" --dir=app
WORKDIR app
COPY statically-link.diff ./statically-link.diff
RUN patch -p1 < statically-link.diff
RUN opam install -y "./$PACKAGE.opam"

# Make the package
RUN mkdir -p "$ARCHIVE_NAME/bin"
RUN cp ../_opam/bin/ocamllsp "$ARCHIVE_NAME/bin"
COPY static/README.md "$ARCHIVE_NAME/README.md"
RUN mkdir -p out
RUN tar -cvf "out/$ARCHIVE_NAME.tar" "$ARCHIVE_NAME"
RUN gzip -9 "out/$ARCHIVE_NAME.tar"


FROM scratch AS export
COPY --from=build /home/user/switch/app/out /
28 changes: 28 additions & 0 deletions build-scripts/ocaml-lsp-server/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
# Build scripts for ocaml-lsp-server

The shell scripts build dynamically-linked binaries of ocamllsp, suitable for
macOS (provided they were built on macOS of course). The dockerfiles build
statically-linked binaries suitable for linux, regardless of distro.

## Dockerfile

Creates a package archive containing a statically-linked ocamllsp binary in the
`out` directory (which will be created if necessary).

Example usage:
```
$ docker buildx build . --target=export --output type=local,dest=$(pwd)/out/ --build-arg TAG=2024-11-21.0 --build-arg TARGET=x86_64-unknown-linux-musl --build-arg OCAML_VERSION=5.1.1 --build-arg OCAML_LSP_SERVER_VERSION=1.18.0
```

It will put the package archive in the `out` directory.

## build.sh

Creates a package archive containing a dynamically-linked ocamllsp binary at
the absolute path specified by `$OUTPUT`.

Example usage:
```
$ TAG=2024-11-21.0 TARGET=aarch64-apple-darwin OCAML_VERSION=5.1.1 OCAML_LSP_SERVER_VERSION=1.18.0 OUTPUT=$(pwd)/ocaml-lsp-server.1.18.0+binary-ocaml-5.1.1-built-2024-11-21.0-aarch64-apple-darwin.tar.gz ./build.sh
```
Original file line number Diff line number Diff line change
@@ -1,13 +1,6 @@
#!/bin/sh
set -eu

TAG=$1
TARGET=$2
OUTPUT=$3

OCAML_VERSION=5.1.1
OCAML_LSP_SERVER_VERSION=1.18.0

TMP_DIR="$(mktemp -d -t ocaml-binary-packages-build.XXXXXXXXXX)"
trap 'rm -rf "$TMP_DIR"' EXIT

Expand Down
File renamed without changes.
File renamed without changes.
1 change: 1 addition & 0 deletions build-scripts/ocamlformat/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
out
Loading

0 comments on commit bded525

Please sign in to comment.