From 9dc01167c35e8e55064cd224c09ce67bf627b684 Mon Sep 17 00:00:00 2001 From: Frank Halasz Date: Fri, 11 Feb 2022 21:27:28 -0800 Subject: [PATCH] Complete revamp of the buildRelease and buildDocker workflows for Medley. Also adding the buildReleaseInclDocker composite workflow. (#674) --- .github/workflows/buildDocker.yml | 293 ++++++++++++++----- .github/workflows/buildLoadup.yml | 201 +++++++++++-- .github/workflows/buildReleaseInclDocker.yml | 36 +++ Dockerfile | 64 ++-- 4 files changed, 468 insertions(+), 126 deletions(-) create mode 100644 .github/workflows/buildReleaseInclDocker.yml diff --git a/.github/workflows/buildDocker.yml b/.github/workflows/buildDocker.yml index 8a89e766..deeab470 100644 --- a/.github/workflows/buildDocker.yml +++ b/.github/workflows/buildDocker.yml @@ -1,112 +1,245 @@ -# based on https://blog.oddbit.com/post/2020-09-25-building-multi-architecture-im/ ---- -# Interlisp workflow to build Docker Image that support multiple architectures -name: Build Medley Docker image +#******************************************************************************* +# buidDocker.yml +# +# Workflow to build and push a multiplatform (amd64, arm64 & arm7) Linux Docker +# image for Medley. This workflow uses the latest Maiko docker image and the +# latest Medley release on github. +# +# This workflow contains a sentry that causes it to skip the build (as identified +# by its commit SHA) if its already been done. Setting the "force" input to true +# will bypass this sentry, +# +# Updated 2022-01-18 by Frank Halasz from on earlier buildDocker.yml +# +# Copyright 2022 by Interlisp.org +# +# ****************************************************************************** -# Run this workflow on demand +name: 'Build/Push Docker Image' + +# Run this workflow on ... on: workflow_dispatch: + force: + description: "Force build even if build already successfully completed for this commit" + type: choice + options: + - 'false' + - 'true' + + workflow_call: + outputs: + successful: + description: "'True' if medley docker build completed successully" + value: ${{ jobs.complete.outputs.build_successful }} + inputs: + force: + description: "Force build even if build already successfully completed for this commit" + required: false + type: string + default: 'false' + secrets: + DOCKER_USERNAME: + required: true + DOCKER_PASSWORD: + required: true + +defaults: + run: + shell: bash + -# Jobs that compose this workflow jobs: - # Job to build the docker image - docker: + +###################################################################################### + + # Regularize the inputs so they can be referenced the same way whether they are + # the result of a workflow_dispatch or a workflow_call + + inputs: runs-on: ubuntu-latest + outputs: + force: ${{ steps.force.outputs.force }} steps: - # Checkout the branch - - name: Checkout + - id: force + run: > + if [ '${{ toJSON(inputs) }}' = 'null' ]; + then echo ::set-output name=force::'${{ github.event.inputs.force }}'; echo "workflow_dispatch"; + else echo ::set-output name=force::'${{ inputs.force }}'; echo "workflow_call"; + fi + + + +###################################################################################### + + # Use sentry-action to determine if this release has already been built + # based on the latest commit to the repo + + sentry: + needs: inputs + runs-on: ubuntu-latest + outputs: + release_not_built: ${{ steps.check.outputs.release_not_built }} + + steps: + # Checkout the actions for this repo owner + - name: Checkout Actions + uses: actions/checkout@v2 + with: + repository: ${{ github.repository_owner }}/.github + path: ./Actions_${{ github.sha }} + - run: mv ./Actions_${{ github.sha }}/actions ../actions && rm -rf ./Actions_${{ github.sha }} + + # Check if build already run for this commit + - name: Build already completed? + id: check + continue-on-error: true + uses: ./../actions/check-sentry-action + with: + tag: "docker" + +###################################################################################### + + # + # Build and push the medley docker image + # + + build_and-push: + + runs-on: ubuntu-latest + + needs: [inputs, sentry] + if: | + needs.sentry.outputs.release_not_built == 'true' + || needs.inputs.outputs.force == 'true' + + steps: + # Checkout latest commit + - name: Checkout Medley uses: actions/checkout@v2 - # Get the Medley Release Information - - name: Get Medley Release Information - id: medley_version - uses: abatilo/release-info-action@v1.3.0 - with: - owner: Interlisp - repo: medley - - # Get the Maiko Release Information - - name: Get Maiko Release Information - id: maiko_version - uses: abatilo/release-info-action@v1.3.0 - with: - owner: Interlisp - repo: maiko - - # Setup needed environment variables - - name: Prepare - id: prep + # Set repo env variables + - name: Set repo/docker env variables + id: repo_env run: | - DOCKERHUB_ACCOUNT=interlisp - DOCKER_IMAGE=${DOCKERHUB_ACCOUNT}/${GITHUB_REPOSITORY#*/} - VERSION=latest - MAIKO_RELEASE=${{ steps.maiko_version.outputs.latest_tag }} - MEDLEY_RELEASE=${{ steps.medley_version.outputs.latest_tag }} + REPO_NAME=${GITHUB_REPOSITORY#*/} + echo "REPO_NAME=${REPO_NAME}" >> ${GITHUB_ENV} + echo ::set-output name=repo_name::${REPO_NAME} + DOCKER_OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]') + echo "DOCKER_OWNER=${DOCKER_OWNER}" >> ${GITHUB_ENV} + echo ::set-output name=docker_owner::${DOCKER_OWNER} - TAGS="${DOCKER_IMAGE}:${MEDLEY_RELEASE},${DOCKER_IMAGE}:${VERSION},${DOCKER_IMAGE}:${MAIKO_RELEASE}" + # Get tag of latest Medley release. + - name: Get Medley Release Information + id: release_info + uses: abatilo/release-info-action@v1.3.0 + with: + owner: ${{ github.repository_owner }} + repo: medley - # Set output parameters. - echo ::set-output name=tags::${TAGS} + # Get asset tars from latest Medley release + - name: Download Release Assets + uses: robinraju/release-downloader@v1.2 + with: + repository: ${{ github.repository_owner }}/medley + token: ${{ secrets.GITHUB_TOKEN }} + latest: true + fileName: "*" + out-file-path: "release_tars" + + # Get Maiko release information about latest Maiko Docker Image + - name: Get info from latest Maiko image + id: maiko_setup + run: | + docker pull ${DOCKER_OWNER}/maiko:latest + MAIKO_RELEASE=$(docker run --entrypoint /bin/bash ${DOCKER_OWNER}/maiko:latest -c "echo \${MAIKO_RELEASE}") + echo "MAIKO_RELEASE=${MAIKO_RELEASE}" >> ${GITHUB_ENV} + echo ::set-output name=maiko_release::${MAIKO_RELEASE} + + # Setup environment variables + - name: Setup Environment Variables + id: setup_env + run: | + RELEASE_TAG=${{ steps.release_info.outputs.latest_tag }} + DOCKER_IMAGE=${DOCKER_OWNER}/${REPO_NAME} + DOCKER_TAGS="${DOCKER_IMAGE}:latest,${DOCKER_IMAGE}:${RELEASE_TAG#*-}_${MAIKO_RELEASE#*-}" + echo ::set-output name=docker_tags::${DOCKER_TAGS} echo ::set-output name=docker_image::${DOCKER_IMAGE} echo ::set-output name=build_time::$(date -u +'%Y-%m-%dT%H:%M:%SZ') - echo ::set-output name=version::${VERSION} - echo ::set-output name=maiko_release::${MAIKO_RELEASE} - echo ::set-output name=medley_release::${MEDLEY_RELEASE} + echo ::set-output name=release_tag::${RELEASE_TAG} + echo "release_tag=${RELEASE_TAG}" >> ${GITHUB_ENV} - # Download Medley Release Assets - - name: Download Release Assets - uses: robinraju/release-downloader@v1.2 - with: - repository: Interlisp/medley - token: ${{ secrets.GITHUB_TOKEN }} - latest: true - fileName: "*" - - # Download Maiko Release Assets - - name: Download Release Assets - uses: robinraju/release-downloader@v1.2 - with: - repository: Interlisp/maiko - token: ${{ secrets.GITHUB_TOKEN }} - latest: true - fileName: "*" - - # Setup Docker Machine Emulation environment + # Setup the Docker Machine Emulation environment. - name: Set up QEMU uses: docker/setup-qemu-action@master with: - platforms: all + platforms: linux/amd64,linux/arm64,linux/arm/v7 - # Setup Docker Buildx function + # Setup the Docker Buildx funtion - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@master - # Login to DockerHub - required to store the image + # Login into DockerHub - required to store the created image - name: Login to DockerHub - if: github.event_name != 'pull_request' uses: docker/login-action@v1 with: - username: ${{ secrets.DOCKER_USERNAME }} + username: ${{ steps.repo_env.outputs.docker_owner }} password: ${{ secrets.DOCKER_PASSWORD }} - # Start the Docker Build using the Dockerfilein the repository - - name: Build + # Do the Docker Build using the Dockerfile in the repository + # checked out and the release tars just downloaded. + # Push the result to Docker Hub + - name: Build Docker Image for Push to Docker Hub uses: docker/build-push-action@v2 with: builder: ${{ steps.buildx.outputs.name }} - context: . - file: ./Dockerfile - # Platforms - # linux/amd64 -- Standard x86_64 - # linux/arm64 -- Apple M1 - # linux/arm/v7 -- Raspberry pi - platforms: linux/amd64,linux/arm64,linux/arm/v7 - # Push the created image - push: true - # tags to assign to the Docker image - tags: ${{ steps.prep.outputs.tags }} build-args: | - medley_release=${{steps.prep.outputs.medley_release}} - maiko_release=${{steps.prep.outputs.maiko_release}} - build_date=${{steps.prep.outputs.build_time}} + BUILD_DATE=${{ steps.setup_env.outputs.build_time }} + RELEASE_TAG=${{ steps.setup_env.outputs.release_tag }} + MAIKO_RELEASE=${{ steps.setup_env.outputs.maiko_release }} + DOCKER_OWNER=${{ steps.repo_env.outputs.docker_owner }} + REPO_OWNER=${{ github.repository_owner }} + context: ./release_tars + file: ./Dockerfile + platforms: linux/amd64,linux/arm64,linux/arm/v7 + # Push the result to DockerHub + push: true + tags: ${{ steps.setup_env.outputs.docker_tags }} + +###################################################################################### + + # Use set-sentry-action to determine set the sentry that says this release has + # been successfully built + + complete: + + runs-on: ubuntu-latest + + outputs: + build_successful: ${{ steps.output.outputs.build_successful }} + + needs: [inputs, sentry, build_and-push] + + steps: + # Checkout the actions for this repo owner + - name: Checkout Actions + uses: actions/checkout@v2 + with: + repository: ${{ github.repository_owner }}/.github + path: ./Actions_${{ github.sha }} + - run: mv ./Actions_${{ github.sha }}/actions ../actions && rm -rf ./Actions_${{ github.sha }} + + # Set sentry + - name: Set flag that build for this commit has been completed + id: set + uses: ./../actions/set-sentry-action + with: + tag: "docker" + + - name: Output + id: output + run: | + echo ::set-output name=build_successful::'true' + +###################################################################################### \ No newline at end of file diff --git a/.github/workflows/buildLoadup.yml b/.github/workflows/buildLoadup.yml index 32781959..3caceedb 100644 --- a/.github/workflows/buildLoadup.yml +++ b/.github/workflows/buildLoadup.yml @@ -1,46 +1,150 @@ -# Interlisp workflow to build Medley release -name: Build Medley Release +#******************************************************************************* +# buidLoadup.yml +# +# Interlisp workflow to build Medley release and push it to github. This workflow +# is platform independent - but runs on Linux/amd64. +# +# This workflow contains a sentry that causes it to skip the build (as identified +# by its commit SHA) if its already been done. Setting the "force" input to true +# will bypass this sentry, +# +# 2022-01-17 Frank Halasz based on an earlier version of buildLoadup for Medley. +# +# Copyright 2022 by Interlisp.org +# +# ****************************************************************************** -# Run this workflow on push to master +name: Build/Push Medley Release + +# Run this workflow on ... on: workflow_dispatch: inputs: - tag: - description: 'Release Tag' + force: + description: "Force build even if build already successfully completed for this commit" + type: choice + options: + - 'false' + - 'true' -# Jobs that compose this workflow + workflow_call: + outputs: + successful: + description: "'True' if medley build completed successully" + value: ${{ jobs.complete.outputs.build_successful }} + inputs: + force: + description: "Force build even if build already successfully completed for this commit" + required: false + type: string + default: 'false' + +defaults: + run: + shell: bash + + jobs: - # Build Loadup - loadup: + +###################################################################################### + + # Regularize the inputs so they can be referenced the same way whether they are + # the result of a workflow_dispatch or a workflow_call + + inputs: runs-on: ubuntu-latest + outputs: + force: ${{ steps.force.outputs.force }} steps: - - name: Set release tag if currently undefined - if: ${{ github.event.inputs.tag == null }} - run: | - echo "tag=medley-`date +%y%m%d`" >> $GITHUB_ENV + - id: force + run: > + if [ '${{ toJSON(inputs) }}' = 'null' ]; + then echo ::set-output name=force::'${{ github.event.inputs.force }}'; echo "workflow_dispatch"; + else echo ::set-output name=force::'${{ inputs.force }}'; echo "workflow_call"; + fi + - - name: Set release tag to input value - if: ${{ github.event.inputs.tag != null }} - run: | - echo "tag=${{ github.event.inputs.tag }}" >> $GITHUB_ENV +###################################################################################### + + # Use sentry-action to determine if this release has already been built + # based on the latest commit to the repo + + sentry: + needs: inputs + runs-on: ubuntu-latest + outputs: + release_not_built: ${{ steps.check.outputs.release_not_built }} + + steps: + # Checkout the actions for this repo owner + - name: Checkout Actions + uses: actions/checkout@v2 + with: + repository: ${{ github.repository_owner }}/.github + path: ./Actions_${{ github.sha }} + - run: mv ./Actions_${{ github.sha }}/actions ../actions && rm -rf ./Actions_${{ github.sha }} + + # Check if build already run for this commit + - name: Build already completed? + id: check + continue-on-error: true + uses: ./../actions/check-sentry-action + with: + tag: "loadup" + +###################################################################################### + + # Do the loadup + # + + loadup: + + runs-on: ubuntu-latest + + needs: [inputs, sentry] + if: | + needs.sentry.outputs.release_not_built == 'true' + || needs.inputs.outputs.force == 'true' + + steps: + # Checkout the actions for this repo owner + - name: Checkout Actions + uses: actions/checkout@v2 + with: + repository: ${{ github.repository_owner }}/.github + path: ./Actions_${{ github.sha }} + - run: mv ./Actions_${{ github.sha }}/actions ../actions && rm -rf ./Actions_${{ github.sha }} + + # Checkout latest commit - name: Checkout Medley uses: actions/checkout@v2 + # Setup release tag + - name: Setup Release Tag + id: tag + uses: ./../actions/release-tag-action + + # Setup environment variables + - name: Setup Environment Variables + id: setup_env + run: | + echo ::set-output name=build_time::$(date -u +'%Y-%m-%dT%H:%M:%SZ') + # Get Maiko release information, retrieves the name of the latest # release. Used to download the correct Maiko release - name: Get Maiko Release Information id: latest_version uses: abatilo/release-info-action@v1.3.0 with: - owner: Interlisp + owner: ${{ github.repository_owner }} repo: maiko # Download Maiko Release Assets - name: Download Release Assets uses: robinraju/release-downloader@v1.2 with: - repository: Interlisp/maiko + repository: ${{ github.repository_owner }}/maiko token: ${{ secrets.GITHUB_TOKEN }} latest: true fileName: "${{ steps.latest_version.outputs.latest_tag }}-linux.x86_64.tgz" @@ -49,7 +153,7 @@ jobs: run: | tar -xvzf "${{ steps.latest_version.outputs.latest_tag }}-linux.x86_64.tgz" - - name: install vnc + - name: Install vnc run: sudo apt-get update && sudo apt-get install -y tightvncserver - name: Build Loadout @@ -59,23 +163,25 @@ jobs: PATH="$PWD/maiko:$PATH" scripts/loadup-all.sh - - name: Build release tar get libs + - name: Build loadups release tar run: | cp -p tmp/full.sysout tmp/lisp.sysout tmp/*.dribble tmp/whereis.hash loadups/ cp -p tmp/exports.all tmp/RDSYS tmp/RDSYS.LCOM library/ cd .. - tar cfz medley/tmp/$tag-loadups.tgz \ + tar cfz medley/tmp/${release_tag}-loadups.tgz \ medley/loadups/lisp.sysout \ medley/loadups/full.sysout \ medley/loadups/whereis.hash \ medley/library/exports.all \ medley/library/RDSYS/ \ medley/library/RDSYS.LCOM + env: + release_tag: ${{ steps.tag.outputs.release_tag }} - - name: tar part 2 + - name: Build runtime release tar run: | cd .. - tar cfz medley/tmp/$tag-runtime.tgz \ + tar cfz medley/tmp/${release_tag}-runtime.tgz \ --exclude "*~" --exclude "*#*" \ medley/docs/dinfo \ medley/docs/Documentation\ Tools \ @@ -91,16 +197,57 @@ jobs: medley/fonts/other \ medley/sources/ \ medley/internal/library + env: + release_tag: ${{ steps.tag.outputs.release_tag }} - name: Release notes run: | - sed s/'$tag'/$tag/g < release-notes.md > tmp/release-notes.md + sed s/'$tag'/${{ steps.tag.outputs.release_tag }}/g < release-notes.md > tmp/release-notes.md - - name: push the release + - name: Push the release uses: ncipollo/release-action@v1.8.10 with: - artifacts: tmp/${{ env.tag }}-loadups.tgz,tmp/${{ env.tag }}-runtime.tgz - tag: ${{ env.tag }} + artifacts: tmp/${{ env.release_tag }}-loadups.tgz,tmp/${{ env.release_tag }}-runtime.tgz + tag: ${{ env.release_tag }} draft: true bodyfile: tmp/release-notes.md token: ${{ secrets.GITHUB_TOKEN }} + env: + release_tag: ${{ steps.tag.outputs.release_tag }} + +###################################################################################### + + # Use set-sentry-action to determine set the sentry that says this release has + # been successfully built + + complete: + + runs-on: ubuntu-latest + + outputs: + build_successful: ${{ steps.output.outputs.build_successful }} + + needs: [inputs, sentry, loadup] + + steps: + # Checkout the actions for this repo owner + - name: Checkout Actions + uses: actions/checkout@v2 + with: + repository: ${{ github.repository_owner }}/.github + path: ./Actions_${{ github.sha }} + - run: mv ./Actions_${{ github.sha }}/actions ../actions && rm -rf ./Actions_${{ github.sha }} + + # Set sentry + - name: Set flag that build for this commit has been completed + id: set + uses: ./../actions/set-sentry-action + with: + tag: "loadup" + + - name: Output + id: output + run: | + echo ::set-output name=build_successful::'true' + +###################################################################################### \ No newline at end of file diff --git a/.github/workflows/buildReleaseInclDocker.yml b/.github/workflows/buildReleaseInclDocker.yml new file mode 100644 index 00000000..e4350ed0 --- /dev/null +++ b/.github/workflows/buildReleaseInclDocker.yml @@ -0,0 +1,36 @@ +#******************************************************************************* +# buidReleaseInclDocker.yml +# +# Interlisp webflow to build a Medley release and push it to github. +# And to build a multiplatform Docker image for the release and push it to Docker Hub. +# +# This workflow just calls two reuseable workflows to the two task: +# buildLoadup.yml and buildDocker.yml +# +# 2022-01-18 Frank Halasz +# +# Copyright 2022 by Interlisp.org +# +# ****************************************************************************** + + +name: "Build/Push Release & Docker" + +# Run this workflow on ... +on: + workflow_dispatch: + + +# Jobs that compose this workflow +jobs: + # Build Loadup + do_release: + uses: ./.github/workflows/buildLoadup.yml + + # Build Docker Image + do_docker: + needs: do_release + uses: ./.github/workflows/buildDocker.yml + secrets: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} diff --git a/Dockerfile b/Dockerfile index 3b9a2c72..6754a2e4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,25 +1,51 @@ -FROM ubuntu:focal -ARG build_date -ARG medley_release -ARG maiko_release -LABEL name="Medley" -# LABEL tags=${tags} -LABEL description="The Medley Interlisp environment" -LABEL url="https://github.com/Interlisp/medley" -LABEL build-time=$build_date -ENV BUILD_DATE=$build_date -ENV MEDLEY_RELEASE=$medley_release -ENV MAIKO_RELEASE=$maiko_release +#******************************************************************************* +# +# Dockerfile to build Medley image from latest Maiko image +# plus latest release tars from github +# +# Copyright 2022 by Interlisp.org +# +# ****************************************************************************** +ARG DOCKER_OWNER=interlisp + +FROM ${DOCKER_OWNER}/maiko:latest + +# Add tightvnc server to the image RUN apt-get update && apt-get install -y tightvncserver +# Handle ARGs, ENV variables, and LABELs +ARG BUILD_DATE=unknown +ARG RELEASE_TAG=unknown +ARG MAIKO_RELEASE=unknown +ARG REPO_OWNER=Interlisp +LABEL name="Medley" +LABEL description="The Medley Interlisp environment" +LABEL url="https://github.com/${REPO_OWNER}/medley" +LABEL build-time=$BUILD_DATE +LABEL release_tag=$RELEASE_TAG +LABEL maiko_release=$MAIKO_RELEASE + +ENV MEDLEY_BUILD_DATE=$BUILD_DATE +ENV MEDLEY_RELEASE=$RELEASE_TAG + +ARG INSTALL_LOCATION=/usr/local/interlisp +ENV INSTALL_LOCATION=${INSTALL_LOCATION} + +# Copy over the release tars +RUN mkdir -p ${INSTALL_LOCATION} +ADD ./*.tgz ${INSTALL_LOCATION} + +# Create a run_medley script in /usr/local/bin +RUN mkdir -p /usr/local/bin && \ + echo "#!/bin/bash" > /usr/local/bin/run-medley && \ + echo "cd ${INSTALL_LOCATION}" >> /usr/local/bin/run-medley && \ + echo './run-medley "$@"' >> /usr/local/bin/run-medley && \ + chmod ugo+x /usr/local/bin/run-medley + +# "Finalize" image EXPOSE 5900 - -# Copy and uncompress loadup and required source files. -ADD *.tgz /home - -WORKDIR /home/medley - RUN adduser --disabled-password --gecos "" medley USER medley -ENTRYPOINT USER=medley Xvnc -geometry 1280x720 :0 & DISPLAY=:0 PATH="/app/maiko:$PATH" ./run-medley -full -g 1280x720 -sc 1280x720 +WORKDIR /home/medley +ENTRYPOINT USER=medley Xvnc -geometry 1280x720 :0 & DISPLAY=:0 ${INSTALL_LOCATION}/medley/run-medley -full -g 1280x720 -sc 1280x720