cache build stages

.dockerignore

@@ -1,2 +1,3 @@
 # each stage does its own yarn install
-node_modules
+**/node_modules/
+**/.git

.github/workflows/ci.yml

@@ -59,10 +59,12 @@ jobs:
         # The .ts scripts depend upon this
       - run: npm install --global tsx
 
+      - name: build use images
+        run: ./buildUseImages.ts
+
       - name: build test images
-        run: |
-          docker info
-          ./buildTestImages.ts
+        run: ./buildTestImages.ts
+
       - name: run test images
         run: ./runTestImages.ts
 
@@ -78,6 +80,9 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:main
+          cache-to: type=inline
+
       - name: notify on failure
         if: failure() && github.event_name != 'pull_request'
         uses: ./.github/actions/notify-status

buildTestImages.ts

@@ -2,7 +2,7 @@
 
 import { parseArgs } from 'node:util';
 import { execSync } from 'node:child_process';
-import { imageNameForProposalTest, readProposals } from './common';
+import { imageNameForProposal, readProposals } from './common';
 import { refreshDockerfile } from './makeDockerfile';
 
 refreshDockerfile();
@@ -25,7 +25,7 @@ for (const proposal of proposals) {
   if (!dry) {
     console.log(`\nBuilding test image for proposal ${proposal.proposalName}`);
   }
-  const { name, target } = imageNameForProposalTest(proposal);
+  const { name, target } = imageNameForProposal(proposal, 'test');
   // 'load' to ensure the images are output to the Docker client. Seems to be necessary
   // for the CI docker/build-push-action to re-use the cached stages.
   const cmd = `docker buildx build --load --tag ${name} --target ${target} .`;

buildUseImages.ts

@@ -0,0 +1,36 @@
+#!/usr/bin/env tsx
+
+import { parseArgs } from 'node:util';
+import { execSync } from 'node:child_process';
+import { imageNameForProposal, readProposals } from './common';
+import { refreshDockerfile } from './makeDockerfile';
+
+refreshDockerfile();
+
+const options = {
+  match: { short: 'm', type: 'string' },
+  dry: { type: 'boolean' },
+} as const;
+const { values } = parseArgs({ options });
+
+const { match, dry } = values;
+
+const allProposals = readProposals();
+
+const proposals = match
+  ? allProposals.filter(p => p.proposalName.includes(match))
+  : allProposals;
+
+for (const proposal of proposals) {
+  if (!dry) {
+    console.log(`\nBuilding use image for proposal ${proposal.proposalName}`);
+  }
+  const { name, target } = imageNameForProposal(proposal, 'use');
+  // 'load' to ensure the images are output to the Docker client. Seems to be necessary
+  // for the CI docker/build-push-action to re-use the cached stages.
+  const cmd = `docker buildx build --load --tag ${name} --target ${target} .`;
+  console.log(cmd);
+  if (!dry) {
+    execSync(cmd, { stdio: 'inherit' });
+  }
+}

common.ts

@@ -44,8 +44,11 @@ export function readProposals(): ProposalInfo[] {
   return proposalPaths.map(readInfo);
 }
 
-export function imageNameForProposalTest(proposal) {
-  const target = `test-${proposal.proposalName}`;
+export function imageNameForProposal(
+  proposal: ProposalCommon,
+  stage: 'use' | 'test',
+) {
+  const target = `${stage}-${proposal.proposalName}`;
   return {
     name: `${repository}:${target}`,
     target,

runTestImages.ts

@@ -2,7 +2,7 @@
 
 import { parseArgs } from 'node:util';
 import { execSync } from 'node:child_process';
-import { imageNameForProposalTest, readProposals } from './common';
+import { imageNameForProposal, readProposals } from './common';
 
 const options = {
   match: { short: 'm', type: 'string' },
@@ -19,7 +19,7 @@ const proposals = match
 
 for (const proposal of proposals) {
   console.log(`Running test image for proposal ${proposal.proposalName}`);
-  const { name } = imageNameForProposalTest(proposal);
+  const { name } = imageNameForProposal(proposal, 'test');
   // 'rm' to remove the container when it exits
   const cmd = `docker run --rm ${name}`;
   execSync(cmd, { stdio: 'inherit' });

upgrade-test-scripts/run_eval.sh

@@ -6,8 +6,6 @@ set -e
 
 source ./env_setup.sh
 
-export SLOGFILE=slog.slog
-
 PROPOSAL_PATH=$1
 
 startAgd

upgrade-test-scripts/run_test.sh

@@ -6,8 +6,6 @@ set -e
 
 source ./env_setup.sh
 
-export SLOGFILE=slog.slog
-
 PROPOSAL_PATH=$1
 
 startAgd

upgrade-test-scripts/run_use.sh

@@ -23,8 +23,6 @@ fi
 
 source ./env_setup.sh
 
-export SLOGFILE=slog.slog
-
 echo "Starting agd in the background."
 startAgd
 

upgrade-test-scripts/start_agd.sh

@@ -6,7 +6,5 @@ grep -qF 'printKeys' /root/.bashrc || echo "printKeys" >>/root/.bashrc
 
 source ./env_setup.sh
 
-export SLOGFILE=slog.slog
-
 # don't use startAgd() because that backgrounds
 agd start --log_level warn "$@"

upgrade-test-scripts/start_to_to.sh

@@ -6,8 +6,6 @@ grep -qF 'printKeys' /root/.bashrc || echo "printKeys" >>/root/.bashrc
 
 source ./env_setup.sh
 
-export SLOGFILE=slog.slog
-
 startAgd
 
 if [[ -z "${UPGRADE_TO}" ]]; then