diff --git a/.travis.yml b/.travis.yml index 24e0ff6..9189c38 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ jobs: include: - os: linux dist: bionic - env: DOCKER_FROM="ubuntu:18.04" + env: CONTAINER_FROM="ubuntu:18.04" services: - docker addons: diff --git a/buildScripts/Readme.md b/buildScripts/Readme.md index 33fae62..4031cb5 100644 --- a/buildScripts/Readme.md +++ b/buildScripts/Readme.md @@ -52,16 +52,18 @@ For most users, you probably really want to simply download one of the (It will not currently work on MacOS or Alpine based machines). -- [Docker](https://www.docker.com/) Image from the [`pdf2htmlEX` Docker +- [OCI](https://opencontainers.org/) Image from the [`pdf2htmlEX` Docker hub](https://hub.docker.com/orgs/pdf2htmlex/repositories). - This will work on any machine with Docker installed. + This will work on any machine with an OCI Container system (such as + Docker, Podman, CRI-O, Kubernetes) installed. (Note: that *advanced* use of `pdf2htmlEX` requires careful attention to the configuration of various tools, such as fontconfig, iconv and your locally available fonts use by the poppler and fontforge libraries. The - docker images created by the pdf2htmlEX team might not be as well - configured for *your needs* as a docker created and configured by you) + OCI container images created by the pdf2htmlEX team might not be as well + configured for *your needs* as an OCI container created and configured + by you) ### Building yourself @@ -163,7 +165,7 @@ Typically, most users, will run one of the following "top-level" scripts: 1. AppImage - 2. Docker image + 2. OCI Container image 3. Debian archive @@ -172,7 +174,7 @@ Typically, most users, will run one of the following "top-level" scripts: 1. Alpine tar file - 2. Docker image + 2. OCI Container image 3. **`runTests`** @@ -229,12 +231,12 @@ Typically, most users, will run one of the following "top-level" scripts: `pdf2htmlEX`, installs it and `poppler-data` into a Debian archive (`*.deb`). -- **`createDockerAlpineImageFromTarFile`**: Installs the Alpine tar file +- **`createContainerAlpineImageFromTarFile`**: Installs the Alpine tar file archive of `pdf2htmlEX` created by `createAlpineTarFile` into an Alpine - Docker image. + Container. -- **`createDockerUbuntuImageFromDeb`**: Installs the Debian archive of - `pdf2htmlEX` created by `createDebianPackage` into a Docker image. +- **`createContainerUbuntuImageFromDeb`**: Installs the Debian archive of + `pdf2htmlEX` created by `createDebianPackage` into a Container. - **`getBuildToolsAlpine`**: Locally `apk` installs all development *tools* required to build `pdf2htmlEX`. @@ -283,13 +285,13 @@ Typically, most users, will run one of the following "top-level" scripts: `pdf2htmlEX/pdf2htmlEX/test` directory. See the `pdf2htmlEX/pdf2htmlEx/test` directory's Readme file for details. -- **`uploadDockerImage`**: Upload the `pdf2htmlEX` Docker image to +- **`uploadContainerImage`**: Upload the `pdf2htmlEX` Container image to Docker hub repository associated to the docker hub users specified in - the `DOCKER_USERNAME` environement variable. + the `DOCKER_HUB_USERNAME` environement variable. - Unless the `DOCKER_USERNAME` and `DOCKER_PASSWORD` environment variables - are pre-defined, this script will prompt the user for the respective - values. + Unless the `DOCKER_HUB_USERNAME` and `DOCKER_HUB_PASSWORD` environment + variables are pre-defined, this script will prompt the user for the + respective values. - **`uploadGitHubRelease`**: Upload the `pdf2htmlEX` artefacts (AppImage, Debian archive, test results, etc) to the *continuous* section of the diff --git a/buildScripts/buildInstallLocallyAlpine b/buildScripts/buildInstallLocallyAlpine index 5a535bc..27a9cd8 100755 --- a/buildScripts/buildInstallLocallyAlpine +++ b/buildScripts/buildInstallLocallyAlpine @@ -1,7 +1,7 @@ #!/bin/sh # This shell script builds the complete pdf2htmlEX application LOCALLY -# (It does not create the AppImage or Docker images) +# (It does not create the AppImage or Container images) # Adjust the following two environment variables to suit your needs # diff --git a/buildScripts/buildInstallLocallyApt b/buildScripts/buildInstallLocallyApt index a26afc4..6afd497 100755 --- a/buildScripts/buildInstallLocallyApt +++ b/buildScripts/buildInstallLocallyApt @@ -1,7 +1,7 @@ #!/bin/sh # This shell script builds the complete pdf2htmlEX application LOCALLY -# (It does not create the AppImage or Docker images) +# (It does not create the AppImage or Container images) # Adjust the following two environment variables to suit your needs # diff --git a/buildScripts/buildInstallLocallyDnf b/buildScripts/buildInstallLocallyDnf new file mode 100755 index 0000000..970062d --- /dev/null +++ b/buildScripts/buildInstallLocallyDnf @@ -0,0 +1,49 @@ +#!/bin/sh + +# This shell script builds the complete pdf2htmlEX application LOCALLY +# (It does not create the AppImage or Container images) + +# Adjust the following two environment variables to suit your needs +# +export UNATTENDED="-y --setopt=install_weak_deps=False" +export MAKE_PARALLEL="-j $(nproc)" + +# choose one of the following... +# +# export PDF2HTMLEX_BRANCH="<>" +export PDF2HTMLEX_BRANCH="$(git rev-parse --abbrev-ref HEAD)" + +# The following environment variable determines where the poppler, +# poppler-data, fontforge and pdf2htmlEX packages are installed. +# CHANGE IT TO SUIT YOUR NEEDS: +# +export PDF2HTMLEX_PREFIX=/usr/local + +# Ensure all Apt packages are installed with no user interaction +# +export DEBIAN_FRONTEND=noninteractive + +set -ev + +################ +# do the build + +./buildScripts/versionEnvs + +./buildScripts/reportEnvs + +./buildScripts/getBuildToolsDnf + +./buildScripts/getDevLibrariesDnf + +./buildScripts/getPoppler + +./buildScripts/buildPoppler + +./buildScripts/getFontforge + +./buildScripts/buildFontforge + +./buildScripts/buildPdf2htmlEX + +./buildScripts/installPdf2htmlEX diff --git a/buildScripts/createAlpineTarFile b/buildScripts/createAlpineTarFile index da425f6..fe57736 100755 --- a/buildScripts/createAlpineTarFile +++ b/buildScripts/createAlpineTarFile @@ -1,12 +1,13 @@ #!/bin/sh -# This shell script creates a docker image from an existing pdf2htmlEX +# This shell script creates a tar file which can use used to create an OCI +# container from an existing pdf2htmlEX. # -# This is the part which can run *inside* a docker container. +# This is the part which can run *inside* an OCI container. echo "" echo "-------------------------------------------------------------------" -echo "CREATING pdf2htmlEX Docker Image (run inside a docker container)" +echo "CREATING pdf2htmlEX Container Image (run inside an OCI container)" echo " (based on Alpine linux $BUILD_DIST)" echo "-------------------------------------------------------------------" echo "" diff --git a/buildScripts/createAppImage b/buildScripts/createAppImage index 1a72152..10809cc 100755 --- a/buildScripts/createAppImage +++ b/buildScripts/createAppImage @@ -53,15 +53,15 @@ cd ../imageBuild LINUX_DEPLOY_APP_IMAGE=linuxdeploy-$MACHINE_ARCH.AppImage -wget https://github.com/linuxdeploy/linuxdeploy/releases/download/continuous/$LINUX_DEPLOY_APP_IMAGE +wget $LINUX_DEPLOY_URL/$LINUX_DEPLOY_APP_IMAGE chmod a+x $LINUX_DEPLOY_APP_IMAGE -if [ -f /.dockerenv ] ; then - # WE ARE INSIDE A DOCKER CONTAINER... +if [ ! -w /dev/fuse ] ; then + # We are in an environment in which FUSE is not useable... # # We explicilty extract the appimage to a squashfs to allow it to be used - # inside Docker containers + # even if FUSE is not available. # ./$LINUX_DEPLOY_APP_IMAGE --appimage-extract # diff --git a/buildScripts/createContainerAlpineImageFromTarFile b/buildScripts/createContainerAlpineImageFromTarFile new file mode 100755 index 0000000..2a6a804 --- /dev/null +++ b/buildScripts/createContainerAlpineImageFromTarFile @@ -0,0 +1,77 @@ +#!/bin/sh + +# This shell script creates a container image from an existing pdf2htmlEX +# +# This is the part which must be run *outside* of any OCI container. + +echo "" +echo "-------------------------------------------------------------------" +echo "CREATING pdf2htmlEX Container Image (run outside any OCI container)" +echo " (based on Alpine linux $BUILD_DIST)" +echo "-------------------------------------------------------------------" +echo "" + +# Collect everything that will be needed... + +# source buildScripts/reSourceVersionEnvs +. buildScripts/reSourceVersionEnvs + +set -ev + +mkdir -p imageBuild/containerDir + +cd imageBuild/containerDir + +cp ../$ALPINE_NAME.* . + +if [ -z "$CONTAINER_FROM" ]; then + echo "" + read -p "Enter the container image for the 'from' base: " CONTAINER_FROM + echo "" + if [ -z "$CONTAINER_FROM" ]; then + echo "CONTAINER_FROM not set... so we can not build the container image" + exit 1 + fi +fi + +if [ -z "$DOCKER_HUB_USERNAME" ]; then + echo "" + read -p "Enter a docker hub username: " DOCKER_HUB_USERNAME + echo "" + if [ -z "$DOCKER_HUB_USERNAME" ]; then + echo "DOCKER_HUB_USERNAME not set... so we can not build the container image" + exit 1 + fi +fi + +if [ -x "$(which podman)" ]; then + alias docker=podman +fi + +export CONTAINER_NAME="$DOCKER_HUB_USERNAME/pdf2htmlex:$PDF2HTMLEX_NAME" + +echo "export CONTAINER_FROM=\"$CONTAINER_FROM\"" >> ../../buildScripts/reSourceVersionEnvs +echo "export DOCKER_HUB_USERNAME=\"$DOCKER_HUB_USERNAME\"" >> ../../buildScripts/reSourceVersionEnvs +echo "export CONTAINER_NAME=\"$CONTAINER_NAME\"" >> ../../buildScripts/reSourceVersionEnvs + +cat > Dockerfile <> ../../buildScripts/reSourceVersionEnvs +echo "export DOCKER_HUB_USERNAME=\"$DOCKER_HUB_USERNAME\"" >> ../../buildScripts/reSourceVersionEnvs +echo "export CONTAINER_NAME=\"$CONTAINER_NAME\"" >> ../../buildScripts/reSourceVersionEnvs + +cat > Dockerfile <> ../../buildScripts/reSourceVersionEnvs -echo "export DOCKER_USERNAME=\"$DOCKER_USERNAME\"" >> ../../buildScripts/reSourceVersionEnvs -echo "export DOCKER_NAME=\"$DOCKER_NAME\"" >> ../../buildScripts/reSourceVersionEnvs - -cat > Dockerfile <> ../../buildScripts/reSourceVersionEnvs -echo "export DOCKER_USERNAME=\"$DOCKER_USERNAME\"" >> ../../buildScripts/reSourceVersionEnvs -echo "export DOCKER_NAME=\"$DOCKER_NAME\"" >> ../../buildScripts/reSourceVersionEnvs - -cat > Dockerfile <> $HOME/.netrc echo " password $GITHUB_TOKEN" >> $HOME/.netrc echo $APPIMAGE_NAME > appImageName.txt -echo $DOCKER_NAME > dockerImageName.txt +echo $CONTAINER_NAME > containerImageName.txt echo $DPKG_NAME > debianArchiveName.txt echo $TEST_RESULTS_NAME > testResultsName.txt @@ -81,12 +81,12 @@ deleteReleaseByTag "$TRAVIS_REPO_SLUG" "continuous-$BUILD_DIST" createNewRelease "$TRAVIS_REPO_SLUG" "continuous-$BUILD_DIST" "Latest $BUILD_DIST release" \ ../buildScripts/uploadGitHubReleaseMessage -uploadAnAsset $upload_url "appImageName.txt" "text/plain" -uploadAnAsset $upload_url "testResultsName.txt" "text/plain" -uploadAnAsset $upload_url "buildInfo.sh" "text/plain" -uploadAnAsset $upload_url "dockerImageName.txt" "text/plain" -uploadAnAsset $upload_url "debianArchiveName.txt" "text/plain" -uploadAnAsset $upload_url $APPIMAGE_NAME "application/zip" -uploadAnAsset $upload_url $TEST_RESULTS_NAME "application/zip" -uploadAnAsset $upload_url $DPKG_NAME "application/x-debian-package" +uploadAnAsset $upload_url "appImageName.txt" "text/plain" +uploadAnAsset $upload_url "testResultsName.txt" "text/plain" +uploadAnAsset $upload_url "buildInfo.sh" "text/plain" +uploadAnAsset $upload_url "containerImageName.txt" "text/plain" +uploadAnAsset $upload_url "debianArchiveName.txt" "text/plain" +uploadAnAsset $upload_url $APPIMAGE_NAME "application/zip" +uploadAnAsset $upload_url $TEST_RESULTS_NAME "application/zip" +uploadAnAsset $upload_url $DPKG_NAME "application/x-debian-package" diff --git a/buildScripts/uploadGitHubReleaseMessage b/buildScripts/uploadGitHubReleaseMessage index 0070e94..0b1bccc 100644 --- a/buildScripts/uploadGitHubReleaseMessage +++ b/buildScripts/uploadGitHubReleaseMessage @@ -18,7 +18,7 @@ We release three types of binary objects: While the Debian archive is built on a specific Ubuntu release, it can usually be used on any more recent Ubuntu releases. -3. You can alternatively get pdf2htmlEX Docker images from Docker Hub in +3. You can alternatively get pdf2htmlEX Container images from Docker Hub in either of the repositories: pdf2htmlEX/pdf2htmlEX (offical release) diff --git a/buildScripts/uploadImages b/buildScripts/uploadImages index 28760a0..f0e9224 100755 --- a/buildScripts/uploadImages +++ b/buildScripts/uploadImages @@ -1,6 +1,6 @@ #!/bin/sh -# This shell script uploads already existing pdf2htmlEX AppImage and Docker +# This shell script uploads already existing pdf2htmlEX AppImage and Container # Images ################ @@ -8,10 +8,10 @@ set -ev -./buildScripts/reportEnvs || { echo 'reportEnvs FAILED' ; exit 1 ; } +./buildScripts/reportEnvs || { echo 'reportEnvs FAILED' ; exit 1 ; } -./buildScripts/uploadGitHubRelease || { echo 'uploadGitHubRelease FAILED' ; exit 1 ; } +./buildScripts/uploadGitHubRelease || { echo 'uploadGitHubRelease FAILED' ; exit 1 ; } -./buildScripts/uploadDockerImage || { echo 'uploadDockerImage FAILED' ; exit 1 ; } +./buildScripts/uploadContainerImage || { echo 'uploadContainerImage FAILED' ; exit 1 ; } diff --git a/buildScripts/versionEnvs b/buildScripts/versionEnvs index 53a58ac..914d064 100755 --- a/buildScripts/versionEnvs +++ b/buildScripts/versionEnvs @@ -4,9 +4,9 @@ # versions # see: https://poppler.freedesktop.org/releases.html -# current working: 0.83.0 +# current working: 0.89.0 -export PDF2HTMLEX_VERSION=0.18.8.rc1 +export PDF2HTMLEX_VERSION=0.18.8.rc2 export POPPLER_VERSION=poppler-0.89.0 #export POPPLER_VERSION=poppler-0.88.0 @@ -29,6 +29,21 @@ export FONTFORGE_VERSION=20200314 #export FONTFORGE_VERSION=20190317 #export FONTFORGE_VERSION=20170731 +################################################################## +# Specify a working version of linuxDeploy to build the appImage +# +# see: https://github.com/linuxdeploy/linuxdeploy/issues/142#issuecomment-669910999 +# +# The following is the most up to date version of linuxdeploy: +# +#export LINUX_DEPLOY_URL=https://github.com/linuxdeploy/linuxdeploy/releases/download/continuous +# +# The following is a recent known to work version of linuxdeploy from 2020/Aug/05 +# +export LINUX_DEPLOY_URL=https://artifacts.assassinate-you.net/artifactory/list/linuxdeploy/travis-456 + +################################################################### + if [ -z "$PDF2HTMLEX_BRANCH" ]; then export PDF2HTMLEX_BRANCH="$(git rev-parse --abbrev-ref HEAD)" if [ -z "$PDF2HTMLEX_BRANCH" ]; then @@ -42,16 +57,22 @@ if [ -z "$PDF2HTMLEX_BRANCH" ]; then fi fi -if test -r /etc/lsb-release ; then - cat /etc/lsb-release - # source /etc/lsb-release - . /etc/lsb-release - export BUILD_DIST=$DISTRIB_CODENAME - export BUILD_OS=$DISTRIB_ID -elif test -r /etc/alpine-release ; then - cat /etc/alpine-release - export BUILD_OS=alpine - export BUILD_DIST=$(cat /etc/alpine-release) +# Following @ViliusSutkus89's suggestion in +# PR https://github.com/pdf2htmlEX/pdf2htmlEX/pull/77 +# We have replaced the use of lsb-release/alpine-release +# with the freedesktop.org's os-release based system to +# obtain the distribution release information.... +# +# See: https://www.freedesktop.org/software/systemd/man/os-release.html +# +if test -r /etc/os-release ; then + cat /etc/os-release + export BUILD_OS=$(grep '^ID=' /etc/os-release | cut -d'=' -f2) + export BUILD_DIST=$(grep '^VERSION_ID=' /etc/os-release | cut -d'=' -f2) +elif test -r /usr/lib/os-release ; then + cat /usr/lib/os-release + export BUILD_OS=$(grep '^ID=' /usr/lib/os-release | cut -d'=' -f2) + export BUILD_DIST=$(grep '^VERSION_ID=' /usr/lib/os-release | cut -d'=' -f2) else echo "FAILURE: could not determine release" exit -1 @@ -76,3 +97,4 @@ echo "export BUILD_DIST=\"$BUILD_DIST\"" >> buildScripts/reSourc echo "export BUILD_DATE=\"$BUILD_DATE\"" >> buildScripts/reSourceVersionEnvs echo "export BUILD_TIME=\"$BUILD_TIME\"" >> buildScripts/reSourceVersionEnvs echo "export PDF2HTMLEX_PREFIX=\"$PDF2HTMLEX_PREFIX\"" >> buildScripts/reSourceVersionEnvs +echo "export LINUX_DEPLOY_URL=\"$LINUX_DEPLOY_URL\"" >> buildScripts/reSourceVersionEnvs diff --git a/pdf2htmlEX/test/installAutomaticTestSoftwareDnf b/pdf2htmlEX/test/installAutomaticTestSoftwareDnf new file mode 100755 index 0000000..f1e87ed --- /dev/null +++ b/pdf2htmlEX/test/installAutomaticTestSoftwareDnf @@ -0,0 +1,39 @@ +#!/bin/sh + +set -ev + +# This shell script installs all local software required to run the +# pdf2htmlEX tests + +export DEBIAN_FRONTEND=noninteractive + +# Start by making sure all required apt packages exist +# +sudo dnf -y --setopt=install_weak_deps=False install \ + wget \ + diffutils \ + zip \ + python3 \ + python3-pip \ + xorg-x11-server-Xvfb \ + firefox + +# Now get the geckodriver for firefox (as required by selenium) +# +oldPWD=$(pwd) +cd /tmp +# +wget https://github.com/mozilla/geckodriver/releases/download/v0.26.0/geckodriver-v0.26.0-linux64.tar.gz +# +tar xvf geckodriver-v0.26.0-linux64.tar.gz +# +sudo mv geckodriver /usr/local/bin +# +cd $oldPWD + +# Now make sure all python packages exist (install into the local user's +# PyPI archive) +# +pip3 install \ + selenium \ + Pillow