From b207b8ea485f87d0faf5e7c81bfbf69052ab259a Mon Sep 17 00:00:00 2001 From: Stephen Gaito Date: Fri, 29 Nov 2019 09:11:32 +0000 Subject: [PATCH] refactored to provide build stage headers and improved the upload function --- buildScripts/buildFontforge | 6 + buildScripts/buildPdf2htmlEX | 6 + buildScripts/buildPdf2htmlEXClang | 6 + buildScripts/buildPoppler | 6 + buildScripts/createAppImage | 6 + buildScripts/createDockerImage | 6 + buildScripts/getBuildTools | 7 ++ buildScripts/getDevLibraries | 7 ++ buildScripts/getFontforge | 8 ++ buildScripts/getPdf2htmlEX | 8 ++ buildScripts/getPoppler | 8 ++ buildScripts/installFontforge | 8 ++ buildScripts/installPdf2htmlEX | 8 ++ buildScripts/installPoppler | 7 ++ buildScripts/uploadRelease | 23 +++- buildScripts/uploadReleaseDSL | 194 +++++++++++++++++++++++++++++ buildScripts/uploadReleaseTool | 196 ------------------------------ 17 files changed, 308 insertions(+), 202 deletions(-) create mode 100755 buildScripts/uploadReleaseDSL delete mode 100755 buildScripts/uploadReleaseTool diff --git a/buildScripts/buildFontforge b/buildScripts/buildFontforge index 8639556..f2d5823 100755 --- a/buildScripts/buildFontforge +++ b/buildScripts/buildFontforge @@ -4,6 +4,12 @@ source buildScripts/reSourceVersionEnvs +echo "" +echo "-------------------------------------------------------------------" +echo "BUILDING FontForge" +echo "-------------------------------------------------------------------" +echo "" + # This is a work-around to fix TravisCI's declaration of PYTHON_CFLAGS # which is then honoured exactly as is by the fontforge bootstrap and # configure steps. diff --git a/buildScripts/buildPdf2htmlEX b/buildScripts/buildPdf2htmlEX index efbc54d..1d49382 100755 --- a/buildScripts/buildPdf2htmlEX +++ b/buildScripts/buildPdf2htmlEX @@ -2,6 +2,12 @@ # This bash script builds pdf2htmlEX +echo "" +echo "-------------------------------------------------------------------" +echo "BUILDING pdf2htmlEX (using gcc)" +echo "-------------------------------------------------------------------" +echo "" + cd pdf2htmlEX mkdir build cd build diff --git a/buildScripts/buildPdf2htmlEXClang b/buildScripts/buildPdf2htmlEXClang index 6d0b7ee..3ea3a3f 100755 --- a/buildScripts/buildPdf2htmlEXClang +++ b/buildScripts/buildPdf2htmlEXClang @@ -2,6 +2,12 @@ # This bash script builds pdf2htmlEX using Clang (if it is installed) +echo "" +echo "-------------------------------------------------------------------" +echo "BUILDING pdf2htmlEX (using CLang)" +echo "-------------------------------------------------------------------" +echo "" + cd pdf2htmlEX mkdir build cd build diff --git a/buildScripts/buildPoppler b/buildScripts/buildPoppler index 711fbd3..52df3e9 100755 --- a/buildScripts/buildPoppler +++ b/buildScripts/buildPoppler @@ -2,6 +2,12 @@ # This bash script builds the latest poppler +echo "" +echo "-------------------------------------------------------------------" +echo "BUILDING Poppler" +echo "-------------------------------------------------------------------" +echo "" + cd poppler mkdir build cd build diff --git a/buildScripts/createAppImage b/buildScripts/createAppImage index 399d862..5321e1e 100755 --- a/buildScripts/createAppImage +++ b/buildScripts/createAppImage @@ -4,6 +4,12 @@ source ./buildScripts/reSourceVersionEnvs +echo "" +echo "-------------------------------------------------------------------" +echo "CREATING pdf2htmlEX AppImage" +echo "-------------------------------------------------------------------" +echo "" + # For appimage output plugin # export VERSION="$PDF2HTMLEX_BRANCH-$BUILD_TIME" diff --git a/buildScripts/createDockerImage b/buildScripts/createDockerImage index ce6f169..8945c71 100755 --- a/buildScripts/createDockerImage +++ b/buildScripts/createDockerImage @@ -2,6 +2,12 @@ # This bash script creates a docker image from an existing pdf2htmlEX +echo "" +echo "-------------------------------------------------------------------" +echo "CREATING pdf2htmlEX Docker Image" +echo "-------------------------------------------------------------------" +echo "" + if [ -x "$(which docker)" ]; then source buildScripts/reSourceVersionEnvs diff --git a/buildScripts/getBuildTools b/buildScripts/getBuildTools index 91156df..92e42fd 100755 --- a/buildScripts/getBuildTools +++ b/buildScripts/getBuildTools @@ -5,6 +5,13 @@ # set the bash environment variable 'UNATTENDED' to '--assume-yes' for # unattended use (for example in the .travis.yml script) +echo "" +echo "-------------------------------------------------------------------" +echo "INSTALLING Build Tools (using APT)" +echo " (UNATTENDED: [$UNATTENDED])" +echo "-------------------------------------------------------------------" +echo "" + sudo apt-get update sudo apt-get $UNATTENDED install \ aptitude \ diff --git a/buildScripts/getDevLibraries b/buildScripts/getDevLibraries index 6631536..37b80f5 100755 --- a/buildScripts/getDevLibraries +++ b/buildScripts/getDevLibraries @@ -6,6 +6,13 @@ # set the bash environment variable 'UNATTENDED' to '--assume-yes' for # unattended use (for example in the .travis.yml script) +echo "" +echo "-------------------------------------------------------------------" +echo "INSTALLING development libraries (using APT)" +echo " (UNATTENDED: [$UNATTENDED])" +echo "-------------------------------------------------------------------" +echo "" + sudo apt-get update sudo apt-get $UNATTENDED install \ libcairo-dev \ diff --git a/buildScripts/getFontforge b/buildScripts/getFontforge index 402aee9..1ad7bab 100755 --- a/buildScripts/getFontforge +++ b/buildScripts/getFontforge @@ -2,6 +2,14 @@ # This bash script gets and unpacks the latest fontforge AppImage +echo "" +echo "-------------------------------------------------------------------" +echo "GETTING FontForge sources (using wget)" +echo " (FONTFORGE_VERSION: [$FONTFORGE_VERSION])" +echo "-------------------------------------------------------------------" +echo "" + + source buildScripts/reSourceVersionEnvs FONTFORGE_SRC=$FONTFORGE_VERSION.tar.gz diff --git a/buildScripts/getPdf2htmlEX b/buildScripts/getPdf2htmlEX index 2232ad1..5c988d6 100755 --- a/buildScripts/getPdf2htmlEX +++ b/buildScripts/getPdf2htmlEX @@ -5,6 +5,14 @@ PDF2HTMLEX_BRANCH=newBuildSystem +echo "" +echo "-------------------------------------------------------------------" +echo "GETTING pdf2htmlEX sources (using wget)" +echo " (PDF2HTMLEX_BRANCH: [$PDF2HTMLEX_BRANCH])" +echo "-------------------------------------------------------------------" +echo "" + + wget https://codeload.github.com/stephengaito/pdf2htmlEX/zip/$PDF2HTMLEX_BRANCH mv $PDF2HTMLEX_BRANCH $PDF2HTMLEX_BRANCH.zip diff --git a/buildScripts/getPoppler b/buildScripts/getPoppler index a6888d7..ca25e1c 100755 --- a/buildScripts/getPoppler +++ b/buildScripts/getPoppler @@ -4,6 +4,14 @@ source buildScripts/reSourceVersionEnvs +echo "" +echo "-------------------------------------------------------------------" +echo "GETTING Poppler source code (using wget)" +echo " (POPPLER_VERSION: [$POPPLER_VERSION])" +echo "-------------------------------------------------------------------" +echo "" + + echo "Getting poppler version: $POPPLER_VERSION" rm -rf $POPPLER_VERSION.tar.xz diff --git a/buildScripts/installFontforge b/buildScripts/installFontforge index 77c2058..1ea43f9 100755 --- a/buildScripts/installFontforge +++ b/buildScripts/installFontforge @@ -1,5 +1,13 @@ #!/bin/bash +echo "" +echo "-------------------------------------------------------------------" +echo "INSTALLING FontForge locally" +echo " (UNATTENDED: [$UNATTENDED])" +echo "-------------------------------------------------------------------" +echo "" + + if [ -z "$UNATTENDED" ] ; then echo "WARNING: this may over-write any existing" echo "FontForge version you have installed on your system." diff --git a/buildScripts/installPdf2htmlEX b/buildScripts/installPdf2htmlEX index 4193c6b..c3b861e 100755 --- a/buildScripts/installPdf2htmlEX +++ b/buildScripts/installPdf2htmlEX @@ -1,5 +1,13 @@ #!/bin/bash +echo "" +echo "-------------------------------------------------------------------" +echo "INSTALLING pdf2htmlEX locally" +echo " (UNATTENDED: [$UNATTENDED])" +echo "-------------------------------------------------------------------" +echo "" + + if [ -z "$UNATTENDED" ] ; then echo "This installation assumes you have 'sudo' privileges" diff --git a/buildScripts/installPoppler b/buildScripts/installPoppler index ec97f26..2ae5b15 100755 --- a/buildScripts/installPoppler +++ b/buildScripts/installPoppler @@ -2,6 +2,13 @@ # This bash script installs Poppler into the local system +echo "" +echo "-------------------------------------------------------------------" +echo "INSTALLING Poppler locally" +echo " (UNATTENDED: [$UNATTENDED])" +echo "-------------------------------------------------------------------" +echo "" + if [ -z "$UNATTENDED" ] ; then echo "WARNING: this may over-write any existing" diff --git a/buildScripts/uploadRelease b/buildScripts/uploadRelease index 4b52d3c..fefc3c3 100755 --- a/buildScripts/uploadRelease +++ b/buildScripts/uploadRelease @@ -11,6 +11,14 @@ # source ./buildScripts/reSourceVersionEnvs +source ./buildScripts/uploadReleaseDSL + +echo "" +echo "-------------------------------------------------------------------" +echo "UPLOADING pdf2htmlEX AppImage and Docker Images" +echo "-------------------------------------------------------------------" +echo "" + ################################## # upload github release artefacts @@ -58,16 +66,19 @@ echo "machine uploads.github.com" >> $HOME/.netrc echo " login $GITHUB_USERNAME" >> $HOME/.netrc echo " password $GITHUB_TOKEN" >> $HOME/.netrc -echo $BUILD_TIME > buildTime +echo $BUILD_TIME > buildTime echo $APPIMAGE_NAME > appImageName -echo $DOCKER_NAME > dockerImageName +echo $DOCKER_NAME > dockerImageName -echo ../buildScripts/uploadReleaseTool "$TRAVIS_REPO_SLUG" "continuous" "Latest release" \ - -- $APPIMAGE_NAME appImageName dockerImageName buildTime +deleteReleaseByTag "$TRAVIS_REPO_SLUG" "continuous" -../buildScripts/uploadReleaseTool "$TRAVIS_REPO_SLUG" "continuous" "Latest release" \ - -- $APPIMAGE_NAME appImageName dockerImageName buildTime +createNewRelease "$TRAVIS_REPO_SLUG" "continuous" "Latest release" \ + ../buildScripts/uploadReleaseMessage +uploadAnAsset $upload_url appImageName "text/plain" +uploadAnAsset $upload_url buildTime "text/plain" +uploadAnAsset $upload_url dockerImageName "text/plain" +uploadAnAsset $upload_url $APPIMAGE_NAME "application/zip" ################################## # push docker image diff --git a/buildScripts/uploadReleaseDSL b/buildScripts/uploadReleaseDSL new file mode 100755 index 0000000..3d36db6 --- /dev/null +++ b/buildScripts/uploadReleaseDSL @@ -0,0 +1,194 @@ +#!/usr/bin/env bash + +# This bash script is a collection of functions to interact with the GitHub +# release and git/tag APIs. It essentially provides a bash based DSL for +# uploading GitHub release artefacts. + +# NOTE-1: These functions make use of your ".netrc" file to authenticate +# with GitHub. In order to use these functions, make sure you have **both** +# "api.github.com" and "upload.github.com" in this file. For example: +# +# machine api.github.com +# login foca +# password +# machine uploads.github.com +# login foca +# password +# +# You can generate the required access token at +# https://github.com/settings/tokens +# Make sure this access token has access to the "repo" scope. +# +# NOTE-2: These functions require the "jq" library to parse the JSON +# returned by the GitHub APIs. + +############################# +function deleteReleaseByTag { + + [ -n "$1" ] || (echo "deleteReleaseByTag: missing repository"; exit 1); + [ -n "$2" ] || (echo "deleteReleaseByTag: missing releaseTag"; exit 1); + + REPO=$1 + releaseTag=$2 + + echo "" + echo "deleting the release tagged $releaseTag" + echo " from the repository $REPO" + echo "" + + echo "looking for an existing '$releaseTag' release in the repo $REPO" + + response=$( + curl --fail \ + --netrc \ + --silent \ + --location \ + --request "GET" \ + "https://api.github.com/repos/${REPO}/releases" + ) + + releaseID=$(echo $response | jq '.[] | select(.tag_name == "$releaseTag") | .id') + + if [ -n "$releaseID" ] ; then + + echo "deleting an existing '$releaseTag' release in the repo $REPO" + response=$( + curl --fail \ + --netrc \ + --silent \ + --location \ + --request "DELETE" \ + "https://api.github.com/repos/${REPO}/releases/$releaseID" + ) + + fi + + echo "looking for an existing '$releaseTag' git/tag in the repo $REPO" + + response=$( + curl --fail \ + --netrc \ + --silent \ + --location \ + --request "GET" \ + "https://api.github.com/repos/${REPO}/git/matching-refs/tags/$releaseTag" + ) + + tagURL=$(echo $response | jq '.[].url') + tagURL="${tagURL%\"}" + tagURL="${tagURL#\"}" + + if [ -n "$tagURL" ]; then + echo "deleting an existing '$releaseTag' git/tag in the repo $REPO" + response=$( + curl --fail \ + --netrc \ + --silent \ + --location \ + --request "DELETE" \ + $tagURL + ) + fi +} + +########################### +function createNewRelease { +# returns: upload_url (envVar) + + [ -n "$1" ] || (echo "createNewRelease: missing repository"; exit 1); + [ -n "$2" ] || (echo "createNewRelease: missing releaseTag"; exit 1); + [ -n "$3" ] || (echo "createNewRelease: missing release name"; exit 1); + [ -n "$4" ] || (echo "createNewRelease: missing release description filename"; exit 1); + + REPO=$1 + TAG=$2 + NAME=$3 + BODY=$(cat $4) + + echo "" + echo "Creating a new release in the repository $REPO" + echo " with tag $TAG" + echo " and name $NAME" + echo " using the message in the file $4" + echo "--------------------------------------------------------------" + echo $BODY + echo "--------------------------------------------------------------" + + payload=$( + jq --null-input \ + --arg tag "$TAG" \ + --arg name "$NAME" \ + --arg body "$BODY" \ + '{ tag_name: $tag, name: $name, body: $body, draft: false, prerelease: true }' + ) + + response=$( + curl --fail \ + --netrc \ + --silent \ + --location \ + --data "$payload" \ + "https://api.github.com/repos/${REPO}/releases" + ) + + upload_url="$(echo "$response" | jq -r .upload_url | sed -e "s/{?name,label}//")" +} + +######################## +function uploadAnAsset { + + [ -n "$1" ] || (echo "uploadAnAsset: missing upload url"; exit 1); + [ -n "$2" ] || (echo "uploadAnAsset: missing file name"; exit 1); + [ -n "$3" ] || (echo "uploadAnAsset: missing file type"; exit 1); + + upload_url=$1 + file=$2 + fileType=$3 + + echo "" + echo "uploading the asset $file" + echo " of type $fileType" + + curl --netrc \ + --silent \ + --header "Content-Type:$fileType" \ + --data-binary "@$file" \ + "$upload_url?name=$(basename "$file")" + echo "" +} + +# The above has been heavily modified into a collection of bash functions +# by Stephen Gaito working on the pdf2htmlEX project. +# +# It has been based upon the code in: +# https://gist.github.com/foca/38d82e93e32610f5241709f8d5720156 +# on 2019-11-28 +# +# The relevant GitHub API can be found at: +# https://developer.github.com/v3/git/refs/ +# https://developer.github.com/v3/repos/releases/ +# +# The original code has been used under the following (MIT-like) license: + +# Copyright (c) 2016 Nicolas Sanguinetti +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. diff --git a/buildScripts/uploadReleaseTool b/buildScripts/uploadReleaseTool deleted file mode 100755 index f60d8a9..0000000 --- a/buildScripts/uploadReleaseTool +++ /dev/null @@ -1,196 +0,0 @@ -#!/usr/bin/env bash - -set -e -[ -z "$DEBUG" ] || set -x; - -usage() { - echo "$0 [] [-- ...]" >&2; -} - -if [ "$1" = "-h" -o "$1" = "--help" ]; then - usage - cat >&2 <\`: ":user/:name" of the repository. For example, "foca/mpp". - * \`\`: Name of the tag for this release. For example, "v1.0.0". - * \`\`: Optional suffix for the release name. - -You can pass a list of files to upload as release assets by giving them after a -\`--\` argument. - -If you supply text on \`STDIN\` it will be used as the release notes. - -EXAMPLES: - - $ $0 foca/mpp v1.0.0 -- pkg/*.tar.gz - - Creates a release named "mpp v1.0.0" and adds any tar.gz file in - \`./pkg\` as an asset. - - $ $0 foca/mpp v1.0.1 "Bugfixes" -- pkg/*.tar.gz < notes.md - - Creates a release named "mpp v1.0.1: Bugfixes", adds any tar.gz - file in \`./pkg\` as an asset, and uses the contents of \`notes.md\` - as the release notes. - -NOTE: - -This uses your \`.netrc\` file to authenticate with GitHub. In order to run the -script, make sure you have **both** \`api.github.com\` and \`upload.github.com\` in -this file. For example: - -machine api.github.com - login foca - password -machine uploads.github.com - login foca - password - -Generate this access token at https://github.com/settings/tokens and make sure -it has access to the \`"repo"\` scope. -EOS - exit 1; -fi - -[ -n "$2" ] || (usage; exit 1); - -REPO="$1" -echo "Repository: [$REPO]" -shift - -TAG="$1" -echo " Tag: [$TAG]" -shift - -NAME="$(basename "$REPO") ${TAG}" -if [ -n "$1" -a "$1" != "--" ]; then - NAME="${NAME}: $1"; - echo " Name: [$NAME]" - shift -fi - -BODY="$(cat ../buildScripts/uploadReleaseMessage)" - -if [ "$1" = "--" -a "$#" -ge "2" ]; then - shift - ASSETS="$@" -fi - -echo "looking for an existing 'continuous' release" - -response=$( - curl --fail \ - --netrc \ - --silent \ - --location \ - --request "GET" \ - "https://api.github.com/repos/${REPO}/releases" -) - -releaseID=$(echo $response | jq '.[] | select(.tag_name == "continuous") | .id') - -if [ -n "$releaseID" ] ; then - - echo "deleting an existing 'continuous' release" - response=$( - curl --fail \ - --netrc \ - --silent \ - --location \ - --request "DELETE" \ - "https://api.github.com/repos/${REPO}/releases/$releaseID" - ) - -fi - -echo "looking for an existing 'continuous' tag" - -response=$( - curl --fail \ - --netrc \ - --silent \ - --location \ - --request "GET" \ - "https://api.github.com/repos/${REPO}/git/matching-refs/tags/continuous" -) - -tagURL=$(echo $response | jq '.[].url') -tagURL="${tagURL%\"}" -tagURL="${tagURL#\"}" - -if [ -n "$tagURL" ]; then - echo "deleting an existing 'continuous' tag" - response=$( - curl --fail \ - --netrc \ - --silent \ - --location \ - --request "DELETE" \ - $tagURL - ) -fi - -payload=$( - jq --null-input \ - --arg tag "$TAG" \ - --arg name "$NAME" \ - --arg body "$BODY" \ - '{ tag_name: $tag, name: $name, body: $body, draft: false, prerelease: true }' -) - -response=$( - curl --fail \ - --netrc \ - --silent \ - --location \ - --data "$payload" \ - "https://api.github.com/repos/${REPO}/releases" -) - -upload_url="$(echo "$response" | jq -r .upload_url | sed -e "s/{?name,label}//")" - -for file in $ASSETS; do - echo "" - echo "uploading $file" - curl --netrc \ - --silent \ - --header "Content-Type:application/gzip" \ - --data-binary "@$file" \ - "$upload_url?name=$(basename "$file")" - echo "" -done - -# The above was taken from: -# https://gist.github.com/foca/38d82e93e32610f5241709f8d5720156 -# on 2019-11-28 -# and has been altered to delete the "continuous" tag/release -# using: https://developer.github.com/v3/git/refs/ -# and: https://developer.github.com/v3/repos/releases/ -# as well as using the ./buildScripts/uploadReleaseMessage -# by Stephen Gaito -# it has been used under the following (MIT-like) license: - -# Copyright (c) 2016 Nicolas Sanguinetti -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, -# copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following -# conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE.