diff --git a/test/units/testsuite-74.pstore.sh b/test/units/testsuite-74.pstore.sh
index 1b475a3f2a..9be8066e8e 100755
--- a/test/units/testsuite-74.pstore.sh
+++ b/test/units/testsuite-74.pstore.sh
@@ -125,6 +125,17 @@ start_pstore() {
     journalctl --sync
 }
 
+at_exit() {
+    set +e
+
+    mountpoint -q /sys/fs/pstore && umount /sys/fs/pstore
+    rm -fr /var/lib/systemd/pstore/*
+    rm -f /run/systemd/system/systemd-pstore.service.d/99-StartLimitInterval.conf
+    rm -f /run/systemd/pstore.conf.d/99-test.conf
+}
+
+trap at_exit EXIT
+
 # To avoid having to depend on the VM providing the pstore, let's simulate
 # it using a simple bind mount
 PSTORE_DIR="$(mktemp -d)"