Automate HCFILES workflow (also add "up one level" button to index.html pages) (#1784)

* First pass at workflow for doing HCFILES on each release

* Finish doHCFILES workflow

* Fix delete of gh-page branch in DoHCFILES workflow

* Redo doHCFILES workflow for files.interlisp.org; add indexing to do_hcfiles script

* Fiddling with workflow names so that I can test doHCFILES.yml on a branch

* in doHCFILES workflow fix use of GH_TOKEN

* Fix typo in doHCFILES workflow

* Debugging doHCFILES workflow

* Fix multiple bugs in do_hcfiles script; fixed multiple bugs in doHCFILES workflow

* Debugging move

* in do_hcfiles.sh add back in Tedit file stoHCFILES run

* Clean up do_hcfiles.sh a bit

* Add debugging code to doHCFILES workflow

* In MAKE-INDEX-HTMLS, add code to ensure that the original case of the files/directory names are preserved since (DIRECTORY) seems to return names ia all-caps, always

* Debugging doHCFILES

* Fiddling with debugging code in doHCFILES workflow

* Add MEDLEY-INIT-VARS to cm file in do_hcfiles.sh

* Undo effect of merging fgh_hcfiles-updates into fgh_hcfiles-workflow.  fgh_hcfiles-update will be abadoned

* Add up button to index.html files in MAKE-INDEX-HTMLS

* Update MAKE-INDEX-HTMLS to include an up-on-level button in index.html files.  Move fio files to medley instead of source.  Streamline doHCFILES workflow

* Debugging

* In MAKE-INDEX-HTMLS, make sure that the up-one button does not appear in the top-level index.html

* In doHCFILES workflow, add difference between development(draft) and production; add doHCFILES workflow into buildReleraseInclDocker workflow

* Update MAKE-INDEX-HTMLS with new onclick script to handle directories properly

* Fix typo in buildRelease workflow

* Polishing up do_hcfiles.sh

* Return buildDocker.yml to original state after using it to test doHCFILES.yml

.github/workflows/buildReleaseInclDocker.yml

@@ -110,6 +110,16 @@ jobs:
         force: ${{ needs.inputs.outputs.force }}
     secrets: inherit
 
+######################################################################################
+
+  # Run HCFILES and push to files.interlisp.org
+  do_HCFILES:
+    needs: [inputs, do_release]
+    uses: ./.github/workflows/doHCFILES.yml
+    with:
+        draft: ${{ needs.inputs.outputs.draft }}
+    secrets: inherit
+
 ######################################################################################
 
   # Kickoff workflow in online repo to build and deploy Medley docker image to oio

.github/workflows/doHCFILES.yml

@@ -0,0 +1,139 @@
+#*******************************************************************************
+# doHCFILES.yml
+#
+# Interlisp workflow to run HCFILES.  HCFILES prints out PDF files for all of the
+# files in the Medley directory and posts them on files.interlisp.org.
+#  
+# This workflow is designed to be kickjed off by the buildReleaseInclDocker
+# workflow running in the Medley repo, once the release has been completed successfully
+#
+# Copyright 2024 by Interlisp.org
+#
+# ******************************************************************************
+
+name: Run HCFILES
+
+# Run this workflow on ...
+on:
+  workflow_dispatch:
+    inputs:
+      draft:
+        description: "Mark this as a draft release"
+        type: choice
+        options:
+        - 'false'
+        - 'true'
+
+  workflow_call:
+    inputs:
+      draft:
+        description: "Mark this as a draft release"
+        required: false
+        type: string
+        default: 'false'
+    secrets:
+      OIO_SSH_KEY:
+        required: true
+      MU_TOKEN:
+        required: true
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+
+  run_HCFILES:
+
+    runs-on: ubuntu-latest
+
+    steps:
+
+      - name: Checkout Medley repo
+        uses: actions/checkout@v4
+
+      - name: Checkout notecards
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ github.repository_owner }}/notecards
+          path: ./notecards
+
+      - name: Checkout loops
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ github.repository_owner }}/loops
+          path: ./loops
+
+      - name: Checkout test
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ github.repository_owner }}/test
+          path: ./test
+
+      - name: Cleanup .git for notecards, loops, test
+        run: rm -rf ./notecards/.git ./loops/.git ./test/.git
+
+      - name: Download Maiko
+        run: |
+          gh release download --output /tmp/maiko.tgz \
+                              --repo ${{ github.repository_owner }}/maiko \
+                              --pattern '*-linux.x86_64.tgz'
+          tar -xzf /tmp/maiko.tgz
+        env:
+          GH_TOKEN: ${{ secrets.MU_TOKEN }}
+
+      - name: Install vnc & ghostscript (ps2pdf)
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y tightvncserver
+          sudo apt-get install -y ghostscript
+
+      - name: Build apps.sysout
+        run: |
+          Xvnc -geometry 1280x720 :0 &
+          export DISPLAY=":0"
+          scripts/loadup-all.sh -apps
+
+      - name: Run HCFILES
+        run: |
+          export DISPLAY=":0"
+          scripts/do_hcfiles.sh
+
+      - name: Push Medley files (including created pdf files) to files.interlisp.org
+        run: |
+            # create a tar file of all of the directories to be pushed
+            tarfile=/tmp/source-$$.tgz
+            tar -c -z -f ${tarfile} --exclude=.git .
+            # set up ssh identity
+            eval $(ssh-agent)
+            ssh-add - <<< "${SSH_KEY}"
+            # set destination directory on files.interlisp.org
+            if [ "${{ inputs.draft }}" = "true" ]
+            then
+              dest=/srv/oio/files/development/medley
+            else
+              dest=/srv/oio/files/production/medley
+            fi
+            # Push tar file up to files.interlisp.org
+            batchfile=/tmp/batch-$$
+            echo "-put ${tarfile} ${dest}.tgz" > ${batchfile}
+            sftp -o StrictHostKeyChecking=no -b ${batchfile} ubuntu@files.interlisp.org
+            # now tar is up, untar it and juggle backups
+            scriptfile=/tmp/script-$$
+            # create script file to do the work
+            cat > ${scriptfile} <<EOF
+            rm -rf ${dest}.new
+            mkdir -p ${dest}.new
+            tar -C ${dest}.new -x -z -f ${dest}.tgz
+            rm -f ${dest}.tgz
+            rm -rf ${dest}.oldold
+            if [ -e ${dest}.old ]; then mv ${dest}.old ${dest}.oldold; fi
+            if [ -e ${dest} ]; then mv ${dest} ${dest}.old; fi
+            mv ${dest}.new ${dest}
+            EOF
+            # execute the script file via ssh
+            ssh -aTxo BatchMode=yes ubuntu@files.interlisp.org /bin/sh -s < ${scriptfile}
+        env:
+            SSH_KEY: ${{ secrets.OIO_SSH_KEY }}
+
+

internal/MEDLEY-UTILS

@@ -1,12 +1,10 @@
 (DEFINE-FILE-INFO PACKAGE "INTERLISP" READTABLE "INTERLISP" BASE 10)
 
-
(FILECREATED "30-Jun-2024 08:28:44" {DSK}<Users>larry>il>medley>internal>MEDLEY-UTILS.;2
 28720  
+
(FILECREATED "14-Jul-2024 12:51:12" {DSK}<home>frank>il>medley>internal>MEDLEY-UTILS.;16
 30093  
 
-      :EDIT-BY "lmm"
+      :CHANGES-TO (FNS MAKE-INDEX-HTMLS)
 
-      :CHANGES-TO (FNS HCFILES)
-
-      :PREVIOUS-DATE "29-Apr-2024 16:25:20" {DSK}<Users>larry>il>medley>internal>MEDLEY-UTILS.;1)
+      :PREVIOUS-DATE "13-Jul-2024 23:39:43" {DSK}<home>frank>il>medley>internal>MEDLEY-UTILS.;14)
 
 
 (PRETTYCOMPRINT MEDLEY-UTILSCOMS)
@@ -126,13 +124,16 @@
            "Welcome to Fuller sysout"])
 
 (MAKE-INDEX-HTMLS

-  [LAMBDA (BASE TOP)
                                         (* ; "Edited 29-Apr-2024 14:18 by lmm")


+  [LAMBDA (BASE TOP LEVEL)
                                   (* ; "Edited 29-Apr-2024 14:18 by lmm")


                                                              (* ; "Edited 26-Apr-2024 16:15 by lmm")


                                                              (* ; "Edited 20-Apr-2024 12:34 by lmm")


                                                              (* ; "Edited 13-Apr-2024 21:18 by lmm")

     [OR BASE (SETQ BASE (PSEUDOFILENAME (MEDLEYDIR]
     (OR (DIRECTORYNAMEP BASE)
         (ERROR BASE "not a directory name"))
+    (OR (AND (NUMBERP LEVEL)
+             (IGREATERP LEVEL 0))
+        (SETQ LEVEL 1))
     (LET* ((SUBDIRS NIL)
            (DEST (PACKFILENAME 'NAME "index" 'EXTENSION "html" 'VERSION NIL 'BODY BASE))
            (PSEUDOHOST (EQ (NTHCHAR BASE (CL:1- 0))
@@ -140,9 +141,20 @@
            SLASHED SHORTNAME)
           (CL:WITH-OPEN-FILE
            (S DEST :DIRECTION :OUTPUT :IF-EXISTS :NEW-VERSION :IF-DOES-NOT-EXIST :CREATE)
-           (CL:FORMAT S "<HTML><HEAD><TITLE>Index page for ~a</TITLE></HEAD>~%%" (SETQ SLASHED
-                                                                                  (SLASHIT BASE)))
+           (CL:FORMAT S "<HTML>~%%<HEAD>~%%")
+           (CL:FORMAT S "<TITLE>Index page for ~a</TITLE>~%%" (SETQ SLASHED (SLASHIT BASE)))
+           (CL:FORMAT S "<SCRIPT>~%%")
+           (CL:FORMAT S "    function up_onclick(){~%%")
+           (CL:FORMAT S "        var newLoc = location.href.replace(/\/index.html$/i, %"%");~%%")
+           (CL:FORMAT S "        location = newLoc.replace(/\/[^\/]+\/?$/, %"%");~%%")
+           (CL:FORMAT S "     }~%%")
+           (CL:FORMAT S "</SCRIPT>~%%")
+           (CL:FORMAT S "</HEAD>~%%")
            (CL:FORMAT S "<BODY><H1>Index page for ~a</H1>~%%" SLASHED)
+           (CL:UNLESS (EQ LEVEL 1)
+                  (CL:FORMAT S 
+      "<DIV>~%%<BUTTON TYPE=%"BUTTON%" ONCLICK=%"up_onclick()%">Go up one level</BUTTON>~%%</DIV>~%%"
+                         ))
            (CL:FORMAT S "<P>This is an index of the files just to link them in.~%%<UL>~%%")
            (FOR
 FULLNAME IN
 (DIRECTORY (CONCAT BASE "*.*;"))
               DO
 (IF
 (EQ (NTHCHAR FULLNAME -1)
@@ -183,7 +195,8 @@
 
                    ELSE
 (CL:FORMAT S "<LI><A HREF=%"~a%">~a</A></LI>~%%" SHORTNAME SHORTNAME)))
            (CL:FORMAT S "</UL></BODY></HTML>~%%"))
-          (NCONC SUBDIRS (FOR
 D IN
 SUBDIRS join
 (MAKE-INDEX-HTMLS
 D (OR TOP BASE])
+          (NCONC SUBDIRS (FOR
 D IN
 SUBDIRS join
 (MAKE-INDEX-HTMLS
 D (OR TOP BASE)
+                                                       (ADD1 LEVEL])
 
 (MEDLEY-FIX-LINKS

   [LAMBDA (UNIXPATH)
                                       (* ; "Edited 18-Jan-2021 12:01 by larry")

@@ -336,13 +349,16 @@
                          (PRINTOUT T "DONE" T))])
 
 (MAKE-INDEX-HTMLS

-  [LAMBDA (BASE TOP)
                                         (* ; "Edited 29-Apr-2024 14:18 by lmm")


+  [LAMBDA (BASE TOP LEVEL)
                                   (* ; "Edited 29-Apr-2024 14:18 by lmm")


                                                              (* ; "Edited 26-Apr-2024 16:15 by lmm")


                                                              (* ; "Edited 20-Apr-2024 12:34 by lmm")


                                                              (* ; "Edited 13-Apr-2024 21:18 by lmm")

     [OR BASE (SETQ BASE (PSEUDOFILENAME (MEDLEYDIR]
     (OR (DIRECTORYNAMEP BASE)
         (ERROR BASE "not a directory name"))
+    (OR (AND (NUMBERP LEVEL)
+             (IGREATERP LEVEL 0))
+        (SETQ LEVEL 1))
     (LET* ((SUBDIRS NIL)
            (DEST (PACKFILENAME 'NAME "index" 'EXTENSION "html" 'VERSION NIL 'BODY BASE))
            (PSEUDOHOST (EQ (NTHCHAR BASE (CL:1- 0))
@@ -350,9 +366,20 @@
            SLASHED SHORTNAME)
           (CL:WITH-OPEN-FILE
            (S DEST :DIRECTION :OUTPUT :IF-EXISTS :NEW-VERSION :IF-DOES-NOT-EXIST :CREATE)
-           (CL:FORMAT S "<HTML><HEAD><TITLE>Index page for ~a</TITLE></HEAD>~%%" (SETQ SLASHED
-                                                                                  (SLASHIT BASE)))
+           (CL:FORMAT S "<HTML>~%%<HEAD>~%%")
+           (CL:FORMAT S "<TITLE>Index page for ~a</TITLE>~%%" (SETQ SLASHED (SLASHIT BASE)))
+           (CL:FORMAT S "<SCRIPT>~%%")
+           (CL:FORMAT S "    function up_onclick(){~%%")
+           (CL:FORMAT S "        var newLoc = location.href.replace(/\/index.html$/i, %"%");~%%")
+           (CL:FORMAT S "        location = newLoc.replace(/\/[^\/]+\/?$/, %"%");~%%")
+           (CL:FORMAT S "     }~%%")
+           (CL:FORMAT S "</SCRIPT>~%%")
+           (CL:FORMAT S "</HEAD>~%%")
            (CL:FORMAT S "<BODY><H1>Index page for ~a</H1>~%%" SLASHED)
+           (CL:UNLESS (EQ LEVEL 1)
+                  (CL:FORMAT S 
+      "<DIV>~%%<BUTTON TYPE=%"BUTTON%" ONCLICK=%"up_onclick()%">Go up one level</BUTTON>~%%</DIV>~%%"
+                         ))
            (CL:FORMAT S "<P>This is an index of the files just to link them in.~%%<UL>~%%")
            (FOR
 FULLNAME IN
 (DIRECTORY (CONCAT BASE "*.*;"))
               DO
 (IF
 (EQ (NTHCHAR FULLNAME -1)
@@ -393,7 +420,8 @@
 
                    ELSE
 (CL:FORMAT S "<LI><A HREF=%"~a%">~a</A></LI>~%%" SHORTNAME SHORTNAME)))
            (CL:FORMAT S "</UL></BODY></HTML>~%%"))
-          (NCONC SUBDIRS (FOR
 D IN
 SUBDIRS join
 (MAKE-INDEX-HTMLS
 D (OR TOP BASE])
+          (NCONC SUBDIRS (FOR
 D IN
 SUBDIRS join
 (MAKE-INDEX-HTMLS
 D (OR TOP BASE)
+                                                       (ADD1 LEVEL])
 )
 
 (PUTPROPS MEDLEY-UTILS
 FILETYPE
 :COMPILE-FILE)
@@ -500,9 +528,9 @@
 (ADDTOVAR LAMA
 )
 )
 (DECLARE%: DONTCOPY
-  (FILEMAP (NIL (1099 11665 (GATHER-INFO 1109 . 6491) (MAKE-FULLER-DB 6493 . 7270) (MAKE-INDEX-HTMLS 
-7272 . 11034) (MEDLEY-FIX-LINKS 11036 . 11429) (MEDLEY-FIX-DATES 11431 . 11663)) (12844 15632 (
-MAKE-EXPORTS-ALL 12854 . 13913) (MAKE-WHEREIS-HASH 13915 . 15104) (MAKE-WHEREIS-LOOPS 15106 . 15630)) 
-(15633 23672 (HCFILES 15643 . 19906) (MAKE-INDEX-HTMLS 19908 . 23670)) (23922 28534 (RECOMPILE-ONE 
-23932 . 25829) (RECMPL 25831 . 26434) (COMPILE-SETUP 26436 . 27060) (REMAKEFILES 27062 . 28532)))))
+  (FILEMAP (NIL (1086 12345 (GATHER-INFO 1096 . 6478) (MAKE-FULLER-DB 6480 . 7257) (MAKE-INDEX-HTMLS 
+7259 . 11714) (MEDLEY-FIX-LINKS 11716 . 12109) (MEDLEY-FIX-DATES 12111 . 12343)) (13524 16312 (
+MAKE-EXPORTS-ALL 13534 . 14593) (MAKE-WHEREIS-HASH 14595 . 15784) (MAKE-WHEREIS-LOOPS 15786 . 16310)) 
+(16313 25045 (HCFILES 16323 . 20586) (MAKE-INDEX-HTMLS 20588 . 25043)) (25295 29907 (RECOMPILE-ONE 
+25305 . 27202) (RECMPL 27204 . 27807) (COMPILE-SETUP 27809 . 28433) (REMAKEFILES 28435 . 29905)))))
 STOP

scripts/do_hcfiles.sh

@@ -0,0 +1,141 @@
+#!/bin/sh
+
+main() {
+        MEDLEYDIR=$(cd "${SCRIPTDIR}/.." && pwd)
+        export MEDLEYDIR
+        logindir=/tmp/hcfiles-$$
+        mkdir -p "${logindir}"
+        cmfile=${logindir}/hcfiles.cm
+
+	cat >"${cmfile}" <<-"EOF"
+	"
+
+	(PROGN
+          (IL:MEDLEY-INIT-VARS 'IL:GREET)
+	  (IL:FILESLOAD MEDLEY-UTILS PDFSTREAM GITFNS))
+          (IL:DRIBBLE '{DSK}${logindir}/hcfiles.dribble)
+          (IL:SETQ IL:*UPPER-CASE-FILE-NAMES* NIL)
+          (IL:SETQ IL:NO-HELP NIL)
+          (IL:ADVISE 'IL:UNSAFE.TO.MODIFY :BEFORE '(RETURN NIL))
+          (IL:ADVISE 'IL:HELP :BEFORE '(IL:COND (IL:NO-HELP (IL:ERROR IL:MESS1 IL:MESS2 T))))
+          (IL:LET ((IL:NO-HELP T)) (DECLARE (SPECIAL IL:NO-HELP)) (IL:HCFILES))
+          (IL:MAKE-INDEX-HTMLS)
+          (IL:DRIBBLE)
+          (IL:LOGOUT T)
+	)
+
+	"
+	EOF
+
+
+        /bin/sh "${MEDLEYDIR}/scripts/medley/medley.command"     \
+             --config -                                          \
+             --id hcfiles_+                                      \
+             --geometry 1024x768                                 \
+             --noscroll                                          \
+             --logindir "${logindir}"                            \
+             --greet "${cmfile}"                                 \
+             --apps
+
+}
+
+# shellcheck disable=SC2164,SC2034
+if [ -z "${SCRIPTDIR}" ]
+then
+	#
+	#
+	# Some functions to determine what directory this script is being executed from
+	#
+	#
+	get_abs_filename() {
+	  # $1 : relative filename
+	  echo "$(cd "$(dirname "$1")" && pwd)/$(basename "$1")"
+	}
+
+	# This function taken from
+	# https://stackoverflow.com/questions/29832037/how-to-get-script-directory-in-posix-sh
+	rreadlink() (
+
+	  # Execute this function in a *subshell* to localize variables and the effect of `cd`.
+
+	  target=$1
+	  fname=
+	  targetDir=
+	  CDPATH=
+
+	  # Try to make the execution environment as predictable as possible:
+	  # All commands below are invoked via `command`, so we must make sure that `command`
+	  # itself is not redefined as an alias or shell function.
+	  # (Note that command is too inconsistent across shells, so we don't use it.)
+	  # `command` is a *builtin* in bash, dash, ksh, zsh, and some platforms do not even have
+	  # an external utility version of it (e.g, Ubuntu).
+	  # `command` bypasses aliases and shell functions and also finds builtins 
+	  # in bash, dash, and ksh. In zsh, option POSIX_BUILTINS must be turned on for that
+	  # to happen.
+	  { \unalias command; \unset -f command; } >/dev/null 2>&1
+	  [ -n "$ZSH_VERSION" ] && options[POSIX_BUILTINS]=on # make zsh find *builtins* with `command` too.
+
+	  while :; do # Resolve potential symlinks until the ultimate target is found.
+	      [ -L "$target" ] || [ -e "$target" ] || { command printf '%s\n' "ERROR: '$target' does not exist." >&2; return 1; }
+	      command cd "$(command dirname -- "$target")" # Change to target dir; necessary for correct resolution of target path.
+	      fname=$(command basename -- "$target") # Extract filename.
+	      [ "$fname" = '/' ] && fname='' # !! curiously, `basename /` returns '/'
+	      if [ -L "$fname" ]; then
+	        # Extract [next] target path, which may be defined
+	        # *relative* to the symlink's own directory.
+	        # Note: We parse `ls -l` output to find the symlink target
+	        #       which is the only POSIX-compliant, albeit somewhat fragile, way.
+	        target=$(command ls -l "$fname")
+	        target=${target#* -> }
+	        continue # Resolve [next] symlink target.
+	      fi
+	      break # Ultimate target reached.
+	  done
+	  targetDir=$(command pwd -P) # Get canonical dir. path
+	  # Output the ultimate target's canonical path.
+	  # Note that we manually resolve paths ending in /. and /.. to make sure we have a normalized path.
+	  if [ "$fname" = '.' ]; then
+	    command printf '%s\n' "${targetDir%/}"
+	  elif  [ "$fname" = '..' ]; then
+	    # Caveat: something like /var/.. will resolve to /private (assuming /var@ -> /private/var), i.e. the '..' is applied
+	    # AFTER canonicalization.
+	    command printf '%s\n' "$(command dirname -- "${targetDir}")"
+	  else
+	    command printf '%s\n' "${targetDir%/}/$fname"
+	  fi
+	)
+
+	get_script_dir() {
+
+	    # call this with $0 (from main script) as its (only) parameter
+	    # if you need to preserve cwd, run this is a subshell since
+	    # it can change cwd
+
+	    # set -x
+
+	    local_SCRIPT_PATH="$( get_abs_filename "$1" )";
+
+	    while [ -h "$local_SCRIPT_PATH" ];
+	    do
+	        cd "$( dirname -- "$local_SCRIPT_PATH"; )";
+	        local_SCRIPT_PATH="$( rreadlink "$local_SCRIPT_PATH" )";
+	    done
+
+	    cd "$( dirname -- "$local_SCRIPT_PATH"; )" > '/dev/null';
+	    local_SCRIPT_PATH="$( pwd; )";
+
+	    # set +x
+
+	    echo "${local_SCRIPT_PATH}"
+	}
+
+	# end of script directory functions
+	###############################################################################
+
+        # figure out the script dir
+        SCRIPTDIR="$(get_script_dir "$0")"
+	export SCRIPTDIR
+
+fi
+
+main "$@"