From d2e869f05c12711849c6f7ed86e9444ff205c6e2 Mon Sep 17 00:00:00 2001 From: Stephen Gaito Date: Wed, 3 Jun 2020 12:17:44 +0000 Subject: [PATCH] updated local tests and added bash scripts to help automate testing. --- .gitattributes | 1 + .gitignore | 24 ++--- buildScripts/buildInstallLocally | 4 + buildScripts/travisLinuxDoItAll | 4 + pdf2htmlEX/test/README.md | 90 +++++++++++++++++++ pdf2htmlEX/test/browser_tests.py | 22 +++-- .../fontfile3_opentype.html | 48 +++++----- .../geneve_1564/geneve_1564.html | 22 ++--- ...ackground_with_page_rotation_issue402.html | 11 +-- .../browser_tests/with_form/with_form.html | 30 +++---- pdf2htmlEX/test/compareTestImages | 31 +++++++ pdf2htmlEX/test/installAutomaticTestSoftware | 33 +++++++ pdf2htmlEX/test/installManualTestSoftware | 13 +++ pdf2htmlEX/test/regenerateTest | 48 ++++++++++ pdf2htmlEX/test/runLocalBrowserTests | 51 +++++++++++ pdf2htmlEX/test/runLocalTests | 5 ++ pdf2htmlEX/test/start_xvfb.sh | 2 - pdf2htmlEX/test/test.py.in | 2 +- pdf2htmlEX/test/test_local_browser.py | 62 +++++++------ pdf2htmlEX/test/test_output.py | 6 +- 20 files changed, 405 insertions(+), 104 deletions(-) create mode 100644 .gitattributes create mode 100755 pdf2htmlEX/test/compareTestImages create mode 100755 pdf2htmlEX/test/installAutomaticTestSoftware create mode 100755 pdf2htmlEX/test/installManualTestSoftware create mode 100755 pdf2htmlEX/test/regenerateTest create mode 100755 pdf2htmlEX/test/runLocalBrowserTests create mode 100755 pdf2htmlEX/test/runLocalTests delete mode 100755 pdf2htmlEX/test/start_xvfb.sh diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..b451356 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +test export-ignore diff --git a/.gitignore b/.gitignore index c855e82..13e1fba 100644 --- a/.gitignore +++ b/.gitignore @@ -14,19 +14,19 @@ CTestTestfile.cmake gmon.out install_manifest.txt Makefile -pdf2htmlEX -pdf2htmlEX.1 +pdf2htmlEX/build +pdf2htmlEX/pdf2htmlEX.1 *.pyc -share/base.css -share/base.min.css -share/fancy.css -share/fancy.min.css -share/pdf2htmlEX.js -share/pdf2htmlEX.min.js -src/pdf2htmlEX-config.h -src/util/css_const.h -test export-ignore +pdf2htmlEX/share/base.css +pdf2htmlEX/share/base.min.css +pdf2htmlEX/share/fancy.css +pdf2htmlEX/share/fancy.min.css +pdf2htmlEX/share/pdf2htmlEX.js +pdf2htmlEX/share/pdf2htmlEX.min.js +pdf2htmlEX/src/pdf2htmlEX-config.h +pdf2htmlEX/src/util/css_const.h Testing/* DartConfiguration.tcl -test/test.py +pdf2htmlEX/test/test.py +pdf2htmlEX/test/geckodriver.log *.swp diff --git a/buildScripts/buildInstallLocally b/buildScripts/buildInstallLocally index 7091d2c..11c0d69 100755 --- a/buildScripts/buildInstallLocally +++ b/buildScripts/buildInstallLocally @@ -19,6 +19,10 @@ export PDF2HTMLEX_BRANCH="$(git rev-parse --abbrev-ref HEAD)" # export PDF2HTMLEX_PREFIX=/usr/local +# Ensure all Apt packages are installed with no user interaction +# +export DEBIAN_FRONTEND=noninteractive + set -ev ################ diff --git a/buildScripts/travisLinuxDoItAll b/buildScripts/travisLinuxDoItAll index 3953b28..07bf766 100755 --- a/buildScripts/travisLinuxDoItAll +++ b/buildScripts/travisLinuxDoItAll @@ -11,6 +11,10 @@ export PDF2HTMLEX_BRANCH="$(git rev-parse --abbrev-ref HEAD)" export PDF2HTMLEX_PREFIX=/usr/local +# Ensure all Apt packages are installed with no user interaction +# +export DEBIAN_FRONTEND=noninteractive + ################ # do the build diff --git a/pdf2htmlEX/test/README.md b/pdf2htmlEX/test/README.md index 9370d8f..7b8f708 100644 --- a/pdf2htmlEX/test/README.md +++ b/pdf2htmlEX/test/README.md @@ -1,3 +1,93 @@ +# pdf2htmlEX tests + +This directory contains a collection of python3 unittests of the output of +pdf2htmlEX. + +The graphical output of pdf2htmlEX can be tested both locally and remotely +using [Selenium](https://www.selenium.dev/) and the [Pillow Python Imaging +Library](https://python-pillow.org/). + +The browser tests use Selenium to take a screenshot of a FireFox browser's +rendering of the pdf2htmlEX output for a given pdf file and compares that +image to an image of the previously saved reference html. + +## Tests which are currently failing: + +- **browser_tests/text_visibility** At the moment clipping has been broken +and needs to be fixed. Rerun `runLocalBrowserTests` and use the +`compareTestImages` for the `test_visibility` test to see the problem. + +## Running tests + +There are three bash scripts which automate the running of a given +collection of tests: + +1. **runLocalTests** runs a simple collection of tests which do not +require Selenium or a browser. + +``` + ./runLocalTests +``` + +2. **runLocalBrowserTests** runs a more complex collection of tests which +*require* Selenium, a FireFox browser, as well as a 'virtual frame buffer' +(Xvfb) to be installed. + +``` + ./runLocalBrowserTests +``` + +3. **runRemoteBrowserTests** runs the same complex collection of tests as +run by `runLocalBrowserTests` but this time using 'Sauce Connect'. (At the +moment this is not fully implemented or (re)tested) + +``` + ./runRemoteBrowserTests +``` + +In order to run these tests, you *must* have the correct testing software +installed locally. To do this you can run the command: + +``` + ./installAutomaticTestSoftware +``` + +## Understanding browser test failures + +If any of the automatic browser tests *fail* then you might want to +manually view the PNG images for a given test using the command: + +``` + ./compareTestImages <> +``` + +This command opens the three PNG images associated with a given failed +test so that you can manually compare the new output (`*.out.png`), the +reference output (`*.ref.png`) and an image of the 'difference' between +the two images (`*.diff.png`). To pass, the 'difference' image must be +*completely* black. + +Usually it will be obvious that the newer version of pdf2htmlEX has only +slightly moved various image elements. Any such tests can be made to pass +by updating the reference html using the tool: + +``` + ./regenerateTestHtml <> +``` + +This command will regenerate the reference html for the specifed test. + +All of these manual comparison tools require additional software which can +be installed using the command: + +``` + ./installManualTestSoftware +``` + +--- + +## OLD README contents: + ### Dependencies - python2 and packages diff --git a/pdf2htmlEX/test/browser_tests.py b/pdf2htmlEX/test/browser_tests.py index c1880a9..dc0066a 100644 --- a/pdf2htmlEX/test/browser_tests.py +++ b/pdf2htmlEX/test/browser_tests.py @@ -5,6 +5,7 @@ import subprocess import shutil import unittest +#from selenium.common.exceptions import WebDriverException from PIL import Image, ImageChops from test import Common @@ -30,7 +31,7 @@ class BrowserTests(Common): def run_test_case(self, filename, args=[], page_must_load=True): basefilename, extension = os.path.splitext(filename) - self.assertEquals(extension.lower(), '.pdf', 'Input file is not PDF') + self.assertEqual(extension.lower(), '.pdf', 'Input file is not PDF') htmlfilename = basefilename + '.html' ref_htmlfolder = os.path.join(self.TEST_DATA_DIR, basefilename) @@ -52,21 +53,28 @@ class BrowserTests(Common): pngfilename_out = os.path.join(self.PNGDIR, basefilename + '.out.png') self.generate_image(out_htmlfilename, pngfilename_out) - out_img = Image.open(pngfilename_out) + out_img = Image.open(pngfilename_out).convert('RGB') pngfilename_ref = os.path.join(self.PNGDIR, basefilename + '.ref.png') self.generate_image(ref_htmlfilename, pngfilename_ref, page_must_load=page_must_load) - ref_img = Image.open(pngfilename_ref) + ref_img = Image.open(pngfilename_ref).convert('RGB') diff_img = ImageChops.difference(ref_img, out_img); + # ALWAYS save the diff image so we can manually check the diff + # see: (http://stackoverflow.com/questions/15721484): + diff_file_name = os.path.join(self.PNGDIR, basefilename + '.diff.png') + diff_img.convert('RGB').save(diff_file_name) + diff_bbox = diff_img.getbbox() - if diff_bbox is not None: + print("\nTesting at: [", basefilename, "]") + + if diff_bbox is None: + print(" passed") + else: + print(" diff bounding box: ", diff_bbox, " should be None") diff_size = (diff_bbox[2] - diff_bbox[0]) * (diff_bbox[3] - diff_bbox[1]) img_size = ref_img.size[0] * ref_img.size[1] - # save the diff image (http://stackoverflow.com/questions/15721484): - diff_file_name = os.path.join(self.PNGDIR, basefilename + '.diff.png') - diff_img.convert('RGB').save(diff_file_name) self.fail(('PNG files %s and %s differ by at most %d pixels, '+ '(%f%% of %d pixels in total), difference: %s') % (pngfilename_out, pngfilename_ref, diff --git a/pdf2htmlEX/test/browser_tests/fontfile3_opentype/fontfile3_opentype.html b/pdf2htmlEX/test/browser_tests/fontfile3_opentype/fontfile3_opentype.html index 6fee4fd..65e7097 100644 --- a/pdf2htmlEX/test/browser_tests/fontfile3_opentype/fontfile3_opentype.html +++ b/pdf2htmlEX/test/browser_tests/fontfile3_opentype/fontfile3_opentype.html @@ -19,9 +19,9 @@