diff --git a/Makefile b/Makefile
index 4b41a20d..24ead901 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,8 @@
 PACKAGE_NAME=osbuild
 VERSION=1
 
+.PHONY: sdist tarball srpm rpm copy-rpms-to-test check-working-directory vagrant-test vagrant-test-keep-running
+
 sdist:
 	python3 setup.py sdist
 	find `pwd`/dist -name '*.tar.gz' -printf '%f\n' -exec mv {} . \;
@@ -29,11 +31,23 @@ rpm: $(PACKAGE_NAME).spec tarball
 	rm -r "`pwd`/rpmbuild"
 	rm -r "`pwd`/build"
 
-copy-rpms-to-test:
+copy-rpms-to-test: rpm
 	- rm test/testing-rpms/*.rpm
 	find `pwd`/output -name '*.rpm' -printf '%f\n' -exec cp {} test/testing-rpms/ \;
 
-vagrant-test: rpm copy-rpms-to-test
+check-working-directory:
+	@if [ "`git status --porcelain --untracked-files=no | wc -l`" != "0" ]; then \
+	  echo "Uncommited changes, refusing (Use git add . && git commit or git stash to clean your working directory)."; \
+	  exit 1; \
+	fi
+
+vagrant-test: check-working-directory copy-rpms-to-test
+	- $(MAKE) -C test destroy
+	- $(MAKE) -C test up
+	$(MAKE) -C test run-tests-remotely
+	- $(MAKE) -C test destroy
+
+vagrant-test-keep-running: check-working-directory copy-rpms-to-test
 	- $(MAKE) -C test up
 	- $(MAKE) -C test install-deps
 	$(MAKE) -C test run-tests-remotely
diff --git a/test/Makefile b/test/Makefile
index 73bdbe0d..84e0fb7f 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -1,6 +1,9 @@
 up:
 	vagrant up
 
+destroy:
+	vagrant destroy -f
+
 install-deps:
 	vagrant provision
 
diff --git a/test/Vagrantfile b/test/Vagrantfile
index 392f9474..e3ab32ff 100644
--- a/test/Vagrantfile
+++ b/test/Vagrantfile
@@ -2,6 +2,8 @@
 # vi: set ft=ruby :
 
 $script = <<-SCRIPT
+packages=$(find /vagrant/testing-rpms -name '*.noarch.rpm')
+sudo dnf remove -y $(basename -a ${packages[@]} | sed 's/-[0-9].*.rpm$//') || true
 sudo dnf install qemu-system-x86 /vagrant/testing-rpms/*.noarch.rpm -y
 SCRIPT