diff --git a/.gitignore b/.gitignore
index 5d953a8d..6b4b4b0e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
 *.rpm
 *.tar.gz
 *.egg-info
+output
diff --git a/Makefile b/Makefile
index 92b056b8..4b41a20d 100644
--- a/Makefile
+++ b/Makefile
@@ -15,6 +15,7 @@ srpm: $(PACKAGE_NAME).spec tarball
 	  $(PACKAGE_NAME).spec
 
 rpm: $(PACKAGE_NAME).spec tarball
+	- rm -r "`pwd`/output"
 	mkdir -p "`pwd`/output"
 	mkdir -p "`pwd`/rpmbuild"
 	/usr/bin/rpmbuild -bb \
@@ -25,13 +26,12 @@ rpm: $(PACKAGE_NAME).spec tarball
 	  --define "_rpmdir `pwd`/output" \
 	  --define "_buildrootdir `pwd`/build" \
 	  $(PACKAGE_NAME).spec
-	find `pwd`/output -name '*.rpm' -printf '%f\n' -exec mv {} . \;
 	rm -r "`pwd`/rpmbuild"
-	rm -r "`pwd`/output"
 	rm -r "`pwd`/build"
 
 copy-rpms-to-test:
-	cp *.rpm test/
+	- rm test/testing-rpms/*.rpm
+	find `pwd`/output -name '*.rpm' -printf '%f\n' -exec cp {} test/testing-rpms/ \;
 
 vagrant-test: rpm copy-rpms-to-test
 	- $(MAKE) -C test up
diff --git a/test/README.md b/test/README.md
new file mode 100644
index 00000000..25a5f734
--- /dev/null
+++ b/test/README.md
@@ -0,0 +1,23 @@
+# Setup
+
+To run the tests in vagrant virtual machine, please follow this tutorial:
+https://developer.fedoraproject.org/tools/vagrant/vagrant-libvirt.html
+
+(run also `sudo systemctl start libvirtd`)
+
+# Using Vagrant
+
+To start a Vagrant box by hand, run `vagrant up` in this directory. To stop and remove all volumes run `vagrant destroy` again in this directory.
+
+# Troubleshooting
+
+In case you accidentally deleted `.vagrant` directory, you can use some of these commands in order to get rid of running instance:
+```
+$ virsh list # this should display test_default
+$ virsh managedsave-remove test_default
+$ virsh undefine test_default
+# or using vagrant cli tool
+$ vagrant global-status
+$ vagrant destroy <id>
+$ vagrant global-status --prune
+```
diff --git a/test/Vagrantfile b/test/Vagrantfile
index 51cb5ee3..392f9474 100644
--- a/test/Vagrantfile
+++ b/test/Vagrantfile
@@ -2,7 +2,7 @@
 # vi: set ft=ruby :
 
 $script = <<-SCRIPT
-sudo dnf install qemu-system-x86 /vagrant/*.noarch.rpm -y
+sudo dnf install qemu-system-x86 /vagrant/testing-rpms/*.noarch.rpm -y
 SCRIPT
 
 Vagrant.configure("2") do |config|
diff --git a/test/testing-rpms/README.md b/test/testing-rpms/README.md
new file mode 100644
index 00000000..61834dca
--- /dev/null
+++ b/test/testing-rpms/README.md
@@ -0,0 +1,3 @@
+# Warning
+
+Don't use this folder. RPM files stored here are automatically managed from the Makefile.