Installers for Linux: workflow changes and more to support standard Linux installations (#1058)

* Adding LANG environment variable to docker image; adding MAIKO_ and MEDLEY_INSTALLDIR environment variables; Changing /usr/local/bin/run-medley to a symbolic link instead of a shell script * Added draft input to all workflows, so that can create draft releases as well as regular releases * Update buildDocker.yml to handle deprecation of set-output and to update versions of actions to handle node 12 to node 16 transition. * Added scripts and updated github workflows to support creation of deb installers for Linux and WSL * Fix minor bug in buildLoadup.yml * First pass implementation of deb installer * Fixing wget of vncviewer in build_deb.sh * Fix typo in buildLoadup.yml in call to build_deb.sh * Multiple small fixes to medley.sh from debugging. Change postinst script and how its created in build_deb. Add postrm script in build_deb. * Reworking vnc portion of Medley.sh - including removing dependency on startx and xinit * Misc fixes to medley_vnc.sh script; fix creation of postinst and postrm in build_deb.sh * Cleaning up window geometry amd screen size in medley.sh * Created apps.sysout loadup with rooms, notecards, clos on top of full.sysout; added plumbing for -apps flag to run-medley to run this syout; created a new init file for this sysout that calls MEDLEYDIR-INIT; all of this is based on online.sysout * Create UNIXUTILS file in library with ShellWhich function - linux which command equivalent. Also move ShellCommand from UNIXPRINT to UNIXUTILS. * Adding UNIXUTILS to LOADUP-FULL so it gets included in full.sysout * Change of names from open(er) to browse(r). Refine the browse(r) functions a bit * Minor bug fixes * Update Apps.ShowDoc to new ShellBrowsefunction * Adding apps support into the .github builds; adding xdg-utils as dependecy in debs * fixing bug as to where notecards is checked out in BuildLoadup. Needs to be before loadups so app.sysout can be built * Added defaulting to Interlisp exec tomedley.sh and APPS-INIT. Works only in apps.sysout. Added wlsu package to wsl debs since wlsview is not always installed by defailt. Fixed Notefiles directories issues in Apps.Init. Made medley.sh compute medleydir based on where the script is located. Can now work for /usr/lcal/interlisp as well as local directories. * Added -id - feature to medley.sh so id can be directory mae. Removed extraneous set -x commands in medley.sh from debugging. In build_deb.sh changed compression to xz for deb files since debian does not support the zstd compression that ubuntu uses. * For wsl deb files, make sure wslu package is not 4.0 - which is bad. Change how we choose an open port and open display in medley_vnc.sh. Add notecards download to build_deb.sh. Fix type in medley.sh * Add (FILES UNIXUTILS) to UNIXPRINTCOMS so that ShellCommand is loaded in case only UNIXPRINT is loaded. For backward compatibility. * Moved medley.sh and associates to script/medley dir; fixed up args to medley.sh; added usage and --help to medley.sh * Add comprehensive tar files to releases to match deb files for local installs; add --id -- arg to medley.sh * Remove remaining reference to usr/local/interlisp to ensure local install works * Fix bug in buildLoadup - couldn't file install tars * Add medley symbolic linkto loadups, so it comes thru to local install tars * Fix up error messaging in medley.sh scripts * Created man page for medley and added it throughout build up, installers, etc. * Add support for a downloads page on OIO, including creating said page while building a release * Fix full_release_tag in downloads section of buildLoadup.yml * Misc fixups on downloads page * Adding online man page stored on oio static server. * Fix minor bug in man installation in deb file
2026-01-25 12:05:41 +00:00 · 2023-01-30 22:19:07 -08:00
parent 0c9b539bc4
commit f2ef7cc8f6
45 changed files with 2521 additions and 204 deletions
--- a/.github/workflows/buildDocker.yml
+++ b/.github/workflows/buildDocker.yml
@@ -21,6 +21,12 @@ name: 'Build/Push Docker Image'
 on:
  workflow_dispatch:
    inputs: 
+      draft:
+        description: "Mark this as a draft release"
+        type: choice
+        options:
+        - 'false'
+        - 'true'
      force:
        description: "Force build even if build already successfully completed for this commit"
        type: choice
@@ -34,6 +40,11 @@ on:
        description: "'True' if medley docker build completed successully"
        value: ${{ jobs.complete.outputs.build_successful }}
    inputs:
+      draft:
+        description: "Mark this as a draft release"
+        required: false
+        type: string
+        default: 'false'
      force:
        description: "Force build even if build already successfully completed for this commit"
        required: false
@@ -60,13 +71,20 @@ jobs:
  inputs:
    runs-on: ubuntu-latest
    outputs:
-      force: ${{ steps.force.outputs.force }}
+      draft: ${{ steps.one.outputs.draft }}
+      force: ${{ steps.one.outputs.force }}
    steps:
-      - id: force
+      - id: one
        run: >
          if [ '${{ toJSON(inputs) }}' = 'null'  ];
-          then echo ::set-output name=force::'${{ github.event.inputs.force }}'; echo "workflow_dispatch";
-          else echo ::set-output name=force::'${{ inputs.force }}'; echo "workflow_call";
+          then
+              echo "workflow_dispatch";
+              echo "draft=${{ github.event.inputs.draft }}" >> $GITHUB_OUTPUT;
+              echo "force=${{ github.event.inputs.force }}" >> $GITHUB_OUTPUT;
+          else
+              echo "workflow_call";
+              echo "draft=${{ inputs.draft }}" >> $GITHUB_OUTPUT;
+              echo "force=${{ inputs.force }}" >> $GITHUB_OUTPUT;
          fi
        

@@ -85,7 +103,7 @@ jobs:
    steps: 
      # Checkout the actions for this repo owner
      - name: Checkout Actions
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
        with:
          repository: ${{ github.repository_owner }}/.github
          path: ./Actions_${{ github.sha }}
@@ -117,7 +135,7 @@ jobs:
    steps:
      # Checkout latest commit
      - name: Checkout Medley
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3

      # Set repo env variables
      - name: Set repo/docker env variables
@@ -125,22 +143,22 @@ jobs:
        run: |
          REPO_NAME=${GITHUB_REPOSITORY#*/}
          echo "REPO_NAME=${REPO_NAME}" >> ${GITHUB_ENV}
-          echo ::set-output name=repo_name::${REPO_NAME}
+          echo "repo_name=${REPO_NAME}" >> ${GITHUB_OUTPUT}
          DOCKER_NAMESPACE=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
          echo "DOCKER_NAMESPACE=${DOCKER_NAMESPACE}" >> ${GITHUB_ENV}
-          echo ::set-output name=docker_namespace::${DOCKER_NAMESPACE}
+          echo "docker_namespace=${DOCKER_NAMESPACE}" >> ${GITHUB_OUTPUT}

      # Get tag of latest Medley release.
      - name: Get Medley Release Information
        id: release_info
-        uses: abatilo/release-info-action@v1.3.0
+        uses: abatilo/release-info-action@v1.3.2
        with:
          owner: ${{ github.repository_owner }}
          repo: medley
          
      # Get asset tars from latest Medley release
      - name: Download Release Assets
-        uses: robinraju/release-downloader@v1.2
+        uses: robinraju/release-downloader@v1.7
        with:
          repository: ${{ github.repository_owner }}/medley
          token: ${{ secrets.GITHUB_TOKEN }}
@@ -155,7 +173,7 @@ jobs:
          docker pull ${DOCKER_NAMESPACE}/maiko:latest
          MAIKO_RELEASE=$(docker run --entrypoint /bin/bash ${DOCKER_NAMESPACE}/maiko:latest -c "echo \${MAIKO_RELEASE}")
          echo "MAIKO_RELEASE=${MAIKO_RELEASE}" >> ${GITHUB_ENV}
-          echo ::set-output name=maiko_release::${MAIKO_RELEASE}
+          echo "maiko_release=${MAIKO_RELEASE}" >> ${GITHUB_OUTPUT}

      # Setup environment variables
      - name: Setup Environment Variables
@@ -163,11 +181,14 @@ jobs:
        run: |
          RELEASE_TAG=${{ steps.release_info.outputs.latest_tag }}
          DOCKER_IMAGE=${DOCKER_NAMESPACE}/${REPO_NAME}
-          DOCKER_TAGS="${DOCKER_IMAGE}:latest,${DOCKER_IMAGE}:${RELEASE_TAG#*-}_${MAIKO_RELEASE#*-}"
-          echo ::set-output name=docker_tags::${DOCKER_TAGS}
-          echo ::set-output name=docker_image::${DOCKER_IMAGE}
-          echo ::set-output name=build_time::$(date -u +'%Y-%m-%dT%H:%M:%SZ')
-          echo ::set-output name=release_tag::${RELEASE_TAG}
+          if [ "${{ needs.inputs.outputs.draft }}" = "false" ];
+              then DOCKER_TAGS="${DOCKER_IMAGE}:latest,${DOCKER_IMAGE}:${RELEASE_TAG#*-}_${MAIKO_RELEASE#*-}"
+              else DOCKER_TAGS="${DOCKER_IMAGE}:draft"
+          fi
+          echo "docker_tags=${DOCKER_TAGS}" >> ${GITHUB_OUTPUT}
+          echo "docker_image=${DOCKER_IMAGE}" >> ${GITHUB_OUTPUT}
+          echo "build_time=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> ${GITHUB_OUTPUT}
+          echo "release_tag=${RELEASE_TAG}" >> ${GITHUB_OUTPUT}
          echo "release_tag=${RELEASE_TAG}" >> ${GITHUB_ENV}

      # Setup the Docker Machine Emulation environment.  
@@ -183,7 +204,7 @@ jobs:

      # Login into DockerHub - required to store the created image
      - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
@@ -192,7 +213,7 @@ jobs:
      # checked out and the release tars just downloaded.
      # Push the result to Docker Hub
      - name: Build Docker Image for Push to Docker Hub
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
        with:
          builder: ${{ steps.buildx.outputs.name }}
          build-args: |
@@ -225,7 +246,7 @@ jobs:
    steps: 
      # Checkout the actions for this repo owner
      - name: Checkout Actions
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
        with:
          repository: ${{ github.repository_owner }}/.github
          path: ./Actions_${{ github.sha }}
@@ -241,6 +262,6 @@ jobs:
      - name: Output
        id: output
        run: |
-          echo ::set-output name=build_successful::'true'
+          echo "build_successful='true'" >> ${GITHUB_OUTPUT}
          
 ######################################################################################
--- a/.github/workflows/buildLoadup.yml
+++ b/.github/workflows/buildLoadup.yml
@@ -10,7 +10,7 @@
 #
 # 2022-01-17 Frank Halasz based on an earlier version of buildLoadup for Medley.
 #
-# Copyright 2022 by Interlisp.org
+# Copyright 2022-2023 by Interlisp.org
 #
 # ******************************************************************************

@@ -20,6 +20,12 @@ name: Build/Push Medley Release
 on:
  workflow_dispatch:
    inputs:
+      draft:
+        description: "Mark this as a draft release"
+        type: choice
+        options:
+        - 'false'
+        - 'true'
      force:
        description: "Force build even if build already successfully completed for this commit"
        type: choice
@@ -33,6 +39,11 @@ on:
        description: "'True' if medley build completed successully"
        value: ${{ jobs.complete.outputs.build_successful }}
    inputs:
+      draft:
+        description: "Mark this as a draft release"
+        required: false
+        type: string
+        default: 'false'
      force:
        description: "Force build even if build already successfully completed for this commit"
        required: false
@@ -54,13 +65,20 @@ jobs:
  inputs:
    runs-on: ubuntu-latest
    outputs:
-      force: ${{ steps.force.outputs.force }}
+      draft: ${{ steps.one.outputs.draft }}
+      force: ${{ steps.one.outputs.force }}
    steps:
-      - id: force
+      - id: one
        run: >
          if [ '${{ toJSON(inputs) }}' = 'null'  ];
-          then echo "force=${{ github.event.inputs.force }}" >> $GITHUB_OUTPUT; echo "workflow_dispatch";
-          else echo "force=${{ inputs.force }}" >> $GITHUB_OUTPUT; echo "workflow_call";
+          then
+              echo "workflow_dispatch";
+              echo "draft=${{ github.event.inputs.draft }}" >> $GITHUB_OUTPUT;
+              echo "force=${{ github.event.inputs.force }}" >> $GITHUB_OUTPUT;
+          else
+              echo "workflow_call";
+              echo "draft=${{ inputs.draft }}" >> $GITHUB_OUTPUT;
+              echo "force=${{ inputs.force }}" >> $GITHUB_OUTPUT;
          fi
        

@@ -129,12 +147,20 @@ jobs:
      - name: Setup Environment Variables
        id: setup_env
        run: |
-          echo "build_time=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
+          echo "build_time=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> ${GITHUB_OUTPUT}
+          echo "TARBALL_DIR=installers/deb/tmp/tarballs" >>${GITHUB_ENV}
+          echo "DEBS_DIR=installers/deb/debs" >>${GITHUB_ENV}
+          echo "TARS_DIR=installers/deb/tars" >>${GITHUB_ENV}
+          echo "MEDLEY_RELEASE_TAG=${RELEASE_TAG}" >>${GITHUB_ENV}
+
+      # Setup some needed dirs in workspace
+      - name: Create work dirs
+        run: mkdir -p ${TARBALL_DIR}

      # Get Maiko release information, retrieves the name of the latest
      # release.  Used to download the correct Maiko release
      - name: Get Maiko Release Information
-        id: latest_version
+        id: maiko
        uses: abatilo/release-info-action@v1.3.2
        with:
          owner: ${{ github.repository_owner }}
@@ -147,42 +173,59 @@ jobs:
          repository: ${{ github.repository_owner }}/maiko
          token: ${{ secrets.GITHUB_TOKEN }}
          latest: true
-          fileName: "${{ steps.latest_version.outputs.latest_tag }}-linux.x86_64.tgz" 
+          out-file-path: ${{ env.TARBALL_DIR }}
+          fileName: "${{ steps.maiko.outputs.latest_tag }}-linux.*.tgz" 

-      - name: Untar Maiko Release
+      - name: Untar Maiko Release for use in loadup
        run: |
-          tar -xvzf "${{ steps.latest_version.outputs.latest_tag }}-linux.x86_64.tgz"
+          tar -xzf "${TARBALL_DIR}/${{ steps.maiko.outputs.latest_tag }}-linux.x86_64.tgz"

+      # Checkout Notecards and tar it in the tarballsdir
+      - name: Checkout Notecards
+        uses: actions/checkout@v3
+        with:
+          repository: ${{ github.repository_owner }}/notecards
+          path: ./notecards
+      - run: mv ./notecards ../notecards
+      - name: Tar notecards into tarball dir
+        run: |
+          cd ..
+          tar cfz medley/${TARBALL_DIR}/notecards.tgz notecards
+
+      #
      - name: Install vnc
        run: sudo apt-get update && sudo apt-get install -y tightvncserver

-      - name: Build Loadout
+      - name: Build Loadup sysouts and databases
        run: |
          Xvnc -geometry 1280x720 :0 &
          export DISPLAY=":0"
          PATH="$PWD/maiko:$PATH" 
-          scripts/loadup-all.sh
+          scripts/loadup-all.sh -apps

      - name: Build loadups release tar
        run: |
          cd ..
-          tar cfz medley/tmp/${RELEASE_TAG}-loadups.tgz        \
+          tar cfz medley/${TARBALL_DIR}/${MEDLEY_RELEASE_TAG}-loadups.tgz        \
            medley/loadups/lisp.sysout               \
            medley/loadups/full.sysout               \
+            medley/loadups/apps.sysout               \
            medley/loadups/whereis.hash              \
            medley/library/exports.all
      
      - name: Build runtime release tar
        run: |
          cd ..
-          tar cfz medley/tmp/${RELEASE_TAG}-runtime.tgz          \
+          tar cfz medley/${TARBALL_DIR}/${MEDLEY_RELEASE_TAG}-runtime.tgz          \
                     --exclude "*~" --exclude "*#*"              \
                     --exclude exports.all                       \
                     medley/clos                                 \
                     medley/docs/dinfo                           \
+                     medley/docs/man-page/medley.1.gz            \
                     medley/doctools                             \
                     medley/greetfiles                           \
                     medley/rooms                                \
+                     medley/medley                               \
                     medley/run-medley                           \
                     medley/scripts                              \
                     medley/fonts/displayfonts                   \
@@ -194,25 +237,71 @@ jobs:
                     medley/sources                              \
                     medley/internal

+
+      - name: Build .deb files for 3 architectures
+        run: |
+          cd installers/deb
+          ./build_deb.sh
+
      - name: Delete existing release with same tag (if any)
        uses: cb80/delrel@latest
        with:
-            tag: ${{ env.RELEASE_TAG }} 
+            tag: ${{ env.MEDLEY_RELEASE_TAG }} 
        continue-on-error: true

      - name: Push the release
+        id: push_release
        uses: ncipollo/release-action@v1
        with: 
          allowUpdates: true
          artifacts:
-            tmp/${{ env.RELEASE_TAG }}-loadups.tgz,
-            tmp/${{ env.RELEASE_TAG }}-runtime.tgz
-          tag: ${{ env.RELEASE_TAG }}
-          draft: false
+            ${{ env.TARBALL_DIR }}/${{ env.MEDLEY_RELEASE_TAG }}-loadups.tgz,
+            ${{ env.TARBALL_DIR }}/${{ env.MEDLEY_RELEASE_TAG }}-runtime.tgz,
+            ${{ env.DEBS_DIR }}/*.deb,
+            ${{ env.TARS_DIR }}/*.tgz
+          tag: ${{ env.MEDLEY_RELEASE_TAG }}
+          draft: ${{ needs.inputs.outputs.draft }}
          prerelease: false
          generateReleaseNotes: true
          token: ${{ secrets.GITHUB_TOKEN }}

+      - name: Update the downloads page and the man page to the OIO satic page host
+        run: |
+            maiko_release_tag="${{ steps.maiko.outputs.latest_tag }}"
+            medley_short_release_tag="${MEDLEY_RELEASE_TAG#medley-}"
+            full_release_filename="${MEDLEY_RELEASE_TAG/medley/medley-full}_${maiko_release_tag#maiko-}"
+            # Need info about where github stores assets because draft releases are not tagged
+            release_url="${{ steps.push_release.outputs.html_url }}"
+            github_subdir="$( echo "${release_url}" | sed -e "s#^.*/\([^/]\+\)\$#\1#g" )"
+            #
+            local_template="installers/downloads_page/medley_downloads.html"
+            local_filename="medley_downloads.html"
+            if [ "${{ needs.inputs.outputs.draft }}" = "true" ];
+            then
+                remote_filename="draft_downloads"
+            else
+                remote_filename="${local_filename%.html}"
+            fi            
+            remote_filepath="/srv/oio/static/${remote_filename}"
+            sed \
+               -e "s/@@@FULL.RELEASE.FILENAME@@@/${full_release_filename}/g" \
+               -e "s/@@@GITHUB.SUBDIR@@@/${github_subdir}/g" \
+               -e "s/@@@MEDLEY.SHORT.RELEASE.TAG@@@/${medley_short_release_tag}/g" \
+               < "${local_template}" > "${local_filename}"
+            local_manpath="docs/man-page/man_medley.html"
+            remote_manpath="/srv/oio/static/man_medley.html"
+            echo "-rm ${remote_filepath}.oldold" > batch
+            echo "-rename ${remote_filepath}.old ${remote_filepath}.oldold" >> batch
+            echo "-rename ${remote_filepath}.html ${remote_filepath}.old" >> batch
+            echo "-put ${local_filename} ${remote_filepath}.html" >> batch
+            echo "-put ${local_manpath} ${remote_manpath}" >> batch
+            eval $(ssh-agent)
+            ssh-add - <<< "${SSH_KEY}"
+            sftp -o StrictHostKeyChecking=no -b batch ubuntu@online.interlisp.org
+        env:
+            SSH_KEY: ${{ secrets.OIO_SSH_KEY }}
+
+
 ######################################################################################

  # Use set-sentry-action to determine set the sentry that says this release has
--- a/.github/workflows/buildReleaseInclDocker.yml
+++ b/.github/workflows/buildReleaseInclDocker.yml
@@ -19,18 +19,96 @@ name: "Build/Push Release & Docker"
 # Run this workflow on ...
 on:
  workflow_dispatch:
-  
+    inputs:
+      draft:
+        description: "Mark this as a draft release"
+        type: choice
+        options:
+        - 'false'
+        - 'true'
+      force:
+        description: "Force build even if build already successfully completed for this commit"
+        type: choice
+        options:
+        - 'false'
+        - 'true'
+
+  workflow_call:
+    outputs:
+      successful:
+        description: "'True' if medley build completed successully"
+        value: ${{ jobs.complete.outputs.build_successful }}
+    inputs:
+      draft:
+        description: "Mark this as a draft release"
+        required: false
+        type: string
+        default: 'false'
+      force:
+        description: "Force build even if build already successfully completed for this commit"
+        required: false
+        type: string
+        default: 'false'
+
+defaults:
+  run:
+    shell: bash
+    

 # Jobs that compose this workflow
 jobs:
+
+
+######################################################################################
+
+  # Regularize the inputs so they can be referenced the same way whether they are
+  # the result of a workflow_dispatch or a workflow_call
+
+  inputs:
+    runs-on: ubuntu-latest
+    outputs:
+      draft: ${{ steps.one.outputs.draft }}
+      force: ${{ steps.one.outputs.force }}
+    steps:
+      - id: one
+        run: >
+          if [ '${{ toJSON(inputs) }}' = 'null'  ];
+          then
+              echo "workflow_dispatch";
+              echo "draft=${{ github.event.inputs.draft }}" >> $GITHUB_OUTPUT;
+              echo "force=${{ github.event.inputs.force }}" >> $GITHUB_OUTPUT;
+          else
+              echo "workflow_call";
+              echo "draft=${{ inputs.draft }}" >> $GITHUB_OUTPUT;
+              echo "force=${{ inputs.force }}" >> $GITHUB_OUTPUT;
+          fi
+        
+
+######################################################################################
+
+
  # Build Loadup
  do_release:
+    needs: inputs
    uses: ./.github/workflows/buildLoadup.yml
+    with:
+        draft: ${{ needs.inputs.outputs.draft }}
+        force: ${{ needs.inputs.outputs.force }}
    
+
+######################################################################################
+
  # Build Docker Image
  do_docker:
-    needs: do_release
+    needs: [inputs, do_release]
    uses: ./.github/workflows/buildDocker.yml
+    with:
+        draft: ${{ needs.inputs.outputs.draft }}
+        force: ${{ needs.inputs.outputs.force }}
    secrets:
-      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
-      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+        DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+
+
+######################################################################################
+