diff --git a/tools/bin/github_md2html b/tools/bin/github_md2html new file mode 100755 index 00000000..4f28063f --- /dev/null +++ b/tools/bin/github_md2html @@ -0,0 +1,255 @@ +#!/usr/bin/perl -w +# $Id: github_md2html 837 2017-01-02 19:23:34Z mueller $ +# +# Copyright 2016-2017 by Walter F.J. Mueller +# +# This program is free software; you may redistribute and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 2, or at your option any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY, without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for complete details. +# +# Revision History: +# Date Rev Version Comment +# 2017-01-02 837 1.1 add -standalone and -trace; add rate wait +# 2016-12-17 823 1.0 Initial version +# + +use 5.14.0; # require Perl 5.14 or higher +use strict; # require strict checking +use Getopt::Long; +use LWP::UserAgent; +use JSON::XS; + +my %opts = (); + +GetOptions(\%opts, + "context:s", "force", "standalone", "trace", + "dump", "help") || exit 1; + +my $url_ghapi_md = "https://api.github.com/markdown"; +my $url_ghapi_mdraw = "https://api.github.com/markdown/raw"; +my $url_css_ghmd = "https://wfjm.github.io/css/github-markdown.css"; +my $url_css_ghmdbody = "https://wfjm.github.io/css/github-markdown-body.css"; + +sub print_help; +sub do_md2html; + +autoflush STDOUT 1 if (-p STDOUT); # autoflush if output into pipe +autoflush STDOUT 1 if (-t STDOUT); # autoflush if output into term + +if (exists $opts{help}) { + print_help; + exit 0; +} + +my @flist; +foreach my $arg (@ARGV) { + if (! -e $arg) { + print STDERR "github_md2html-E: file or directory '$arg' not found\n"; + } elsif (-f $arg) { + push @flist, $arg; + } elsif (-d $arg) { + open (FFILE, "find $arg -name '*.md' -type f | sort |") + or die "Failed to run 'find $arg': $!"; + while () { + chomp; + push @flist, $_; + } + close FFILE; + } elsif (-l $arg) { + print STDERR "github_md2html-W: symlink '$arg' ignored\n"; + } else { + print STDERR "github_md2html-E: '$arg' not file or directory, ignored\n"; + } +} + +unless (scalar @flist) { + print STDERR "github_md2html-E: no files specified of found\n"; + print_help; + exit 1; +} + +foreach my $file (@flist) { + do_md2html($file); +} + +#------------------------------------------------------------------------------- + +sub do_request { + my ($ifile,$ua,$req) = @_; + for (my $nretry=0; $nretry<10; $nretry++) { + my $res = $ua->request($req); + + if (exists $opts{dump}) { + print "------------------------------------------------------\n"; + print "response for $ifile\n"; + print $res->as_string,"\n"; + print "------------------------------------------------------\n"; + } + + return $res if $res->is_success; + + my $rate_limit = $res->header('X-RateLimit-Limit'); + my $rate_remain = $res->header('X-RateLimit-Remaining'); + my $rate_reset = $res->header('X-RateLimit-Reset'); + if ($res->status_line eq '403 Forbidden' && + defined $rate_limit && defined $rate_remain && defined $rate_reset && + $rate_remain == 0) { + my $twait = $rate_reset - time; + my $twait_min = $twait / 60; + my $twait_sec = $twait % 60; + printf "github_md2html-I: rate limit %d/hr reached, wait %2dm%02ds\n", + $rate_limit, + $twait_min, $twait_sec; + sleep $twait+1; + } else { + print STDERR "github_md2html-E: api error:'" . $res->status_line . "'\n"; + print STDERR "response for $ifile\n"; + print STDERR $res->as_string,"\n"; + return undef; + } + } + print STDERR "github_md2html-E: retry limit reached\n"; + return undef; +} + +#------------------------------------------------------------------------------- + +sub do_md2html { + my ($ifile) = @_; + my $ofile = "$ifile.html"; + my $doit = exists $opts{force} || (not -e $ofile); + + unless ($doit) { + # -M returns - in fractional days + # --> thus file age in days relative to now + $doit = -M $ofile > -M $ifile; # output older than input + } + + unless ($doit) { + print "$ifile: ok\n"; + return; + } + + # get file path and name + my $ifile_path = '.'; + my $ifile_name = $ifile; + if ($ifile =~ m|^(.+)/(.+)|) { + $ifile_path = $1; + $ifile_name = $2; + } + + # read input file + my $idata; + { + local $/; # slurp file ... + open IFILE, $ifile or die "file open read failed"; + $idata = ; + close IFILE; + } + + # prepare request + my $ua = LWP::UserAgent->new; + my $req; + + # with context --> use markdown api (json based) + if (exists $opts{context}) { + $req = HTTP::Request->new('POST', $url_ghapi_md); + my %apireq = ('mode' => 'gfm', + 'context' => $opts{context}, + 'text' => $idata + ); + my $apireq_json = encode_json(\%apireq); + $req->content($apireq_json); + + # no context --> use markdown/raw api + } else { + $req = HTTP::Request->new('POST', $url_ghapi_mdraw); + $req->content_type('text/x-markdown'); + $req->content($idata); + } + + if (exists $opts{dump}) { + print "------------------------------------------------------\n"; + print "request for $ifile\n"; + print $req->as_string,"\n"; + } + + my $res = do_request($ifile, $ua, $req); + return unless defined $res; + + my $html = $res->decoded_content; + + if (exists $opts{standalone}) { + $html =~ s{$ofile" or die "file open write failed"; + print OFILE '',"\n"; + print OFILE '',"\n"; + print OFILE '',"\n"; + print OFILE ' ',"\n"; + print OFILE ' ',"\n"; + print OFILE ' ',"\n"; + print OFILE ' ',"\n"; + print OFILE '',"\n"; + print OFILE '',"\n"; + if ($opts{standalone} && $ifile_name eq 'README.md') { + print OFILE '

Directory README.',"\n"; + print OFILE 'To local directory listing.

',"\n"; + } + print OFILE $html,"\n"; + print OFILE '',"\n"; + print OFILE '',"\n"; + close OFILE; + + my $rate_limit = $res->header('X-RateLimit-Limit'); + my $rate_remain = $res->header('X-RateLimit-Remaining'); + my $rate_reset = $res->header('X-RateLimit-Reset'); + my $time_reset = $rate_reset - time; + my $reset_min = $time_reset / 60; + my $reset_sec = $time_reset % 60; + + printf "%s: done, rate %d of %d for %2dm%02ds\n", + $ifile, $rate_remain, $rate_limit, $reset_min, $reset_sec; +} + +#------------------------------------------------------------------------------- + +sub print_help { + print "usage: github_md2html [opts] files...\n"; + print " --force update all (default: check timestamps)\n"; + print " --standalone modify links for local browser usage\n"; + print " --trace trace link mapping in standalone mode\n"; + print " --context c uses context and markdown api (default: raw api)\n"; + print " --dump print HTTP request and response\n"; + print " --help this message\n"; +} diff --git a/tools/man/man1/github_md2html.1 b/tools/man/man1/github_md2html.1 new file mode 100644 index 00000000..45638861 --- /dev/null +++ b/tools/man/man1/github_md2html.1 @@ -0,0 +1,77 @@ +.\" -*- nroff -*- +.\" $Id: github_md2html.1 837 2017-01-02 19:23:34Z mueller $ +.\" +.\" Copyright 2017- by Walter F.J. Mueller +.\" +.\" ------------------------------------------------------------------ +. +.TH GITHUB_MD2HTML 1 2017-01-02 "Retro Project" "Retro Project Manual" +.\" ------------------------------------------------------------------ +.SH NAME +github_md2html \- convert markdown to html with GitHub API +.\" ------------------------------------------------------------------ +.SH SYNOPSIS +. +.SY github_md2html +.OP OPTIONS +.I FILE... +.YS +. +.\" ------------------------------------------------------------------ +.SH DESCRIPTION +Converts markdown files to html using the GitHub converter API. +\fIFILE\fP can either be a file name or a directory name (e.g. '.'). +If it's a directory, the whole sub-tree will be scanned and all files +with an extension of \fI.md\fP will be converted. +The created html files have the extension \fI.md.html\fP. + +Unless the \fB-force\fP option is given the script checks whether the +\fI.md.html\fP +file already exists and converts only when the markdown file is newer than +the html file. +. +.\" ------------------------------------------------------------------ +.SH OPTIONS +. +.\" ---------------------------------------------- +.IP "\fB\-force\fR" +re-create all files, even when they exist and are up-to-date. +. +.\" ---------------------------------------------- +.IP "\fB\-standalone\fR" +modify local links for usage with local browser. All links pointing to a +local \fI.md\fP file will be redirected to the \fI.md.html\fP file, and +all links pointing to a local directory will be redirected to a +\fIREADME.md.html\fP in case the directory README exists. +This mode is is useful when one wants to inspect the files directly +with a browser. +. +.\" ---------------------------------------------- +.IP "\fB\-trace\fR" +trace link mapping in \fI-standalone\fP mode. +. +.\" ---------------------------------------------- +.IP "\fB\-context \fIcont\fR" +defines the GitHub repository context. +. +.\" ---------------------------------------------- +.IP "\fB\-dump\fR" +print HTTP request and response. +. +.\" ---------------------------------------------- +.IP "\fB\-help\fR" +print help text. +. +.\" ------------------------------------------------------------------ +.SH EXIT STATUS +In case of an error an exit status 1 is returned. +. +. +.\" ------------------------------------------------------------------ +.SH EXAMPLES +.IP "\fBgithub_md2html -s .\fR" 4 +will convert all \fI.md\fP files in the current sub-tree in standalone mode. + +.\" ------------------------------------------------------------------ +.SH AUTHOR +Walter F.J. Mueller