Implement a proper regression testsuite

.github/workflows/tests.yml

@@ -0,0 +1,68 @@
+name: ffDesign Testsuite
+
+on:
+  push:
+    branches:
+      - main
+      - wip
+  pull_request:
+
+permissions: read-all
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        freecad: ["1.0.2", "1.1.1"]
+
+    defaults:
+      run:
+        shell: bash -el {0}
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v7
+
+      - name: Cache FreeCAD executables
+        id: cache-freecad
+        uses: actions/cache@v6
+        with:
+          path: FreeCAD.AppImage
+          key: freecad-cache-v1-${{ matrix.freecad }}
+
+      - name: Download FreeCAD 1.1.1 AppImage
+        if: steps.cache-freecad.outputs.cache-hit != 'true' && matrix.freecad == '1.1.1'
+        run: |
+          curl -L -o FreeCAD.AppImage https://github.com/FreeCAD/FreeCAD/releases/download/1.1.1/FreeCAD_1.1.1-Linux-x86_64-py311.AppImage
+          chmod +x FreeCAD.AppImage
+
+      - name: Download FreeCAD 1.0.2 AppImage
+        if: steps.cache-freecad.outputs.cache-hit != 'true' && matrix.freecad == '1.0.2'
+        run: |
+          curl -L -o FreeCAD.AppImage https://github.com/FreeCAD/FreeCAD/releases/download/1.0.2/FreeCAD_1.0.2-conda-Linux-x86_64-py311.AppImage
+          chmod +x FreeCAD.AppImage
+
+      - name: Install virtual display
+        run: sudo apt-get update && sudo apt-get install --yes xvfb
+
+      - name: Install addon
+        run: |
+          mkdir -p "$HOME/.local/share/FreeCAD/Mod"
+          ln -s "$GITHUB_WORKSPACE" "$HOME/.local/share/FreeCAD/Mod/FusedFilamentDesign"
+
+      - name: Download test documents
+        run: |
+          git clone https://github.com/Rahix/ffDesign_TestData.git "$GITHUB_WORKSPACE/ffDesign_TestData"
+
+      - name: Show FreeCAD version
+        if: matrix.freecad == '1.1.1' # FreeCAD 1.0 hangs on a dialog window...
+        run: |
+          xvfb-run -a ./FreeCAD.AppImage -c -v --verbose
+
+      - name: Run FreeCAD tests
+        run: |
+          set -o pipefail
+          xvfb-run -a ./FreeCAD.AppImage -t ffDesign_Tests 2>&1 | tee freecad-tests.log
+          ! grep -q "Traceback (most recent call last)" freecad-tests.log

.gitignore

@@ -1 +1,2 @@
 __pycache__/
+/ffDesign_TestData/

Documentation/Testing.md

@@ -0,0 +1,66 @@
+## Running the ffDesign Testsuite
+FusedFilamentDesign has a small testsuite to check correct behavior of the
+commands. This testsuite uses a lot of prepared FreeCAD documents which are
+stored separately in the [ffDesign_TestData] repository.
+
+### Test Architecture
+The tests are mainly regression tests where a prepared body is modified by one
+of ffDesign's commands and then the resulting shape is compared against a
+reference.
+
+This comparison is done by performing an XOR of the Body shapes and checking
+that the remaining shape is empty.
+
+Each test document for such regression tests contains one Body named
+`BodyUnderTest` (the prepared one) and one Body named `BodyExpected` (the
+expected final shape).
+
+### Downloading test documents
+Before running the testsuite, you must download the test documents from
+[ffDesign_TestData].
+
+The ffDesign_TestData repository uses [git-lfs] for storing the FreeCAD
+documents. If you do not yet have git-lfs installed, download it from the
+project homepage: <https://git-lfs.com/>
+
+Then run this command to enable git-lfs for your user account:
+
+```bash
+git lfs install
+```
+
+You are now ready to download the ffDesign_TestData. Navigate to the addon
+directory of FusedFilamentDesign and clone the test data:
+
+```bash
+# May differ for your system
+cd ~/.FreeCAD/Mod/FusedFilamentDesign/
+
+git clone https://github.com/Rahix/ffDesign_TestData
+```
+
+You are now ready to run the testsuite:
+
+### Running the testsuite
+Run the full testsuite using the following command:
+
+```bash
+FreeCAD -t ffDesign_Tests
+```
+
+FreeCAD will show failing tests in the terminal output.
+
+### Troubleshooting failing tests
+If you want to see why a specific test failed, you can instruct FreeCAD to stay
+open and tell the testsuite to not discard the test document. **This only works
+when running a single test at a time!**
+
+```bash
+FFDESIGN_KEEP_TEST_DOC=1 FreeCAD -r ffDesign_Tests.HoleLocations.test_construction_and_non_defining
+```
+
+(Note the `-r` instead of `-t`)
+
+
+[ffDesign_TestData]: https://github.com/Rahix/ffDesign_TestData
+[git-lfs]: https://git-lfs.com/

InitGui.py

@@ -59,3 +59,6 @@ class ffDesignWB:
 
 
 register_pd_toolbar()
+
+if "ffDesign_Tests" not in FreeCAD.__unit_test__:
+    FreeCAD.__unit_test__.append("ffDesign_Tests")

ffDesign_Tests.py

@@ -0,0 +1,240 @@
+import abc
+import unittest
+import pathlib
+import os
+
+import FreeCAD as App
+import FreeCADGui as Gui
+import BOPTools.SplitFeatures
+
+from PySide import QtCore
+
+import ffDesign_Utils as Utils
+
+import ffDesign_CounterboreBridges
+import ffDesign_RibThreads
+import ffDesign_RoofBridge
+
+TEST_DOCUMENTS_PATH = "ffDesign_TestData"
+
+
+def get_test_dir() -> pathlib.Path:
+    test_documents_repo = pathlib.Path(__file__).parent / TEST_DOCUMENTS_PATH
+
+    if not test_documents_repo.is_dir():
+        Utils.Log.error(f"""\
+Test documents repository {test_documents_repo.name} does not exist! Maybe you need to download it first?
+
+Full path: {test_documents_repo}
+
+Please read the documentation of {Utils.Log.addon} to understand how to prepare for running this testsuite.""")
+        raise Exception(f"{Utils.Log.addon} test documents directory is missing")
+
+    # Depending on the FreeCAD version, we select the relevant subdirectory.
+    subdir_name = "FreeCAD_1.0"
+    if Utils.check_freecad_version(min_version=[1, 1, 0]):
+        subdir_name = "FreeCAD_1.1"
+
+    return test_documents_repo / subdir_name
+
+
+class ffDesignTestCase(unittest.TestCase, abc.ABC):
+    test_document = None  # Name of the test document for this testcase
+
+    def setUp(self):
+        # Skip all the ffDesign dialogs during test runs
+        Utils.SKIP_ALL_DIALOGS = True
+
+        test_document_path = get_test_dir() / self.test_document
+        if not test_document_path.exists():
+            Utils.Log.error(f"""\
+Test document {test_document_path.name} is missing.
+
+Do you have the latest version of the test documents repository?
+
+Full path: {test_document_path}
+
+Please read the documentation of {Utils.Log.addon} to understand how to prepare for running this testsuite.""")
+            raise Exception(f"{test_document_path.name} is missing")
+
+        self.doc = App.openDocument(str(test_document_path))
+
+    def tearDown(self):
+        # Re-enable ffDesign dialogs after the test completes
+        Utils.SKIP_ALL_DIALOGS = False
+
+        # You can use FFDESIGN_KEEP_TEST_DOC=1 to keep the test document open to inspect it.
+        # Only makes sense when running tests with `-r` instead of `-t`.
+        # You should probably only use this when running a single test case.
+        if os.environ.get("FFDESIGN_KEEP_TEST_DOC", default="0") == "0":
+            App.closeDocument(self.doc.Name)
+
+    def prepare_regression_test(self):
+        self.body = self.doc.getObjectsByLabel("BodyUnderTest")[0]
+        self.body_expected = self.doc.getObjectsByLabel("BodyExpected")[0]
+
+        Gui.ActiveDocument.ActiveView.setActiveObject("pdbody", self.body)
+
+    def assert_expected_body(self):
+        # Move the body_expected to the origin where the other body should already be
+        self.body_expected.Placement = App.Placement(App.Vector(0, 0, 0), App.Rotation(App.Vector(0, 0, 1), 0))
+
+        # Run a Part XOR operation between the two bodies to find differences
+        xor_body = BOPTools.SplitFeatures.makeXOR(name="Comparison_XOR")
+        xor_body.Objects = [self.body, self.body_expected]
+        xor_body.Proxy.execute(xor_body)
+
+        self.body_expected.ViewObject.hide()
+        self.body.ViewObject.Transparency = 70
+        # Make the xor_body bright red
+        xor_body.ViewObject.ShapeAppearance = [App.Material(DiffuseColor=(255, 0, 0, 255))]
+
+        # Check that the resulting shape is utterly empty
+        self.assertEqual(xor_body.Shape.Volume, 0, "Detected differences between resulting and expected shape")
+        self.assertEqual(len(xor_body.Shape.Vertexes), 0, "Detected differences between resulting and expected shape")
+
+
+class ffDesignTestCase_Fc11(ffDesignTestCase):
+    """A testcase that only runs on FreeCAD >1.1."""
+
+    def setUp(self):
+        if not Utils.check_freecad_version(min_version=[1, 1, 0]):
+            raise unittest.SkipTest("This testcase only works on FreeCAD >=1.1")
+
+        super().setUp()
+
+
+class HoleLocations(ffDesignTestCase_Fc11):
+    test_document = "HoleLocations.FCStd"
+
+    @unittest.expectedFailure
+    def test_plausibility(self):
+        self.prepare_regression_test()
+        self.assert_expected_body()
+
+    def test_construction_and_non_defining(self):
+        """
+        Test that hole locations are correctly inferred from a sketch with both
+        construction circles and non-defining external geometry.
+        """
+        self.prepare_regression_test()
+        ffDesign_CounterboreBridges.CounterboreBridgesCommand().Activated()
+        self.assert_expected_body()
+
+
+class RibThreads(ffDesignTestCase):
+    test_document = "TapHoles.FCStd"
+
+    def test_generate_rib_threads(self):
+        """
+        Smoke tests that we can generate rib threads for the predefined metric M4 tap holes.
+        """
+        self.prepare_regression_test()
+
+        # TODO: Use the command rather than directly generating using the addon function
+        hole = self.body.Tip
+        Utils.assert_hole(hole)
+
+        ffDesign_RibThreads.verify_rib_thread_suitability(hole)
+
+        dialog = ffDesign_RibThreads.RibThreadsTaskPanel(self.body, hole)
+        Gui.Control.showDialog(dialog)
+        dialog.accept()
+
+        # The file expects an additional rotation of the rib threads to be present
+        varset = self.body.getObject("Hole_RibThread_Settings")
+        varset.Rotation = "90deg"
+        App.ActiveDocument.recompute()
+
+        self.assert_expected_body()
+
+
+class RibThreads_Fc11(ffDesignTestCase_Fc11):
+    test_document = "TapHoles.FCStd"
+
+    def thread_test(self, thread_type: str, thread_size: str):
+        # Prep for the next test
+        App.closeDocument(self.doc.Name)
+        self.setUp()
+
+        try:
+            self.prepare_regression_test()
+
+            hole = self.body.Tip
+            Utils.assert_hole(hole)
+
+            hole.ThreadType = thread_type
+            hole.ThreadSize = thread_size
+            App.ActiveDocument.recompute()
+
+            dialog = ffDesign_RibThreads.RibThreadsTaskPanel(self.body, hole)
+            Gui.Control.showDialog(dialog)
+            dialog.accept()
+        except Exception as e:
+            Utils.Log.warning(f"Test failure during thread {thread_type} {thread_size}")
+            raise e from None
+
+    def test_known_metric(self):
+        for thread in ffDesign_RibThreads.RIB_PARAMETERS["ISOMetricProfile"]:
+            if "x" not in thread or thread == "M6x1":
+                # This is one of the legacy thread names, skip
+                continue
+
+            self.thread_test("ISOMetricProfile", thread)
+
+    def test_known_unc(self):
+        for thread in ffDesign_RibThreads.RIB_PARAMETERS["UNC"]:
+            self.thread_test("UNC", thread)
+
+
+class RoofBridges(ffDesignTestCase):
+    test_document = "RoofBridges.FCStd"
+
+    def apply_roof_bridge_for_hole(
+        self,
+        hole_name: str,
+        *,
+        double_sided=None,
+        include_counterbore=None,
+        clearance=None,
+        angle_60=False,
+        ignore_missing=False,
+    ):
+        hole = self.body.getObject(hole_name)
+        Utils.assert_hole(hole)
+
+        dialog = ffDesign_RoofBridge.RoofBridgeTaskPanel(self.body, hole)
+        Gui.Control.showDialog(dialog)
+
+        if double_sided is not None:
+            if double_sided:
+                dialog.form.DoubleSided.setCheckState(QtCore.Qt.CheckState.Checked)
+            else:
+                dialog.form.DoubleSided.setCheckState(QtCore.Qt.CheckState.Unchecked)
+
+        if include_counterbore is not None:
+            if include_counterbore:
+                dialog.form.DoCounterbore.setCheckState(QtCore.Qt.CheckState.Checked)
+            else:
+                dialog.form.DoCounterbore.setCheckState(QtCore.Qt.CheckState.Unchecked)
+
+        if clearance is not None:
+            dialog.form.BridgeClearance.setProperty("rawValue", clearance)
+
+        if angle_60:
+            dialog.form.Angle60.toggle()
+
+        dialog.accept()
+
+    def test_roof_bridges(self):
+        self.prepare_regression_test()
+
+        self.apply_roof_bridge_for_hole("Hole")
+        self.apply_roof_bridge_for_hole("Hole001", double_sided=True, angle_60=True)
+        self.apply_roof_bridge_for_hole("Hole002", double_sided=True)
+        self.apply_roof_bridge_for_hole("Hole003", include_counterbore=False, clearance=0.6)
+
+        self.body.getObject("Hole002").RoofBridgeRotation = "0deg"
+        App.ActiveDocument.recompute()
+
+        self.assert_expected_body()