diff --git a/bin/setup.sh b/bin/setup.sh
index 1b482c9e..3ca41a29 100755
--- a/bin/setup.sh
+++ b/bin/setup.sh
@@ -165,7 +165,7 @@ if ! (python3 --version && python3 -m pip --version && python3 -m django --versi
     exit 1
 fi
 
-if ! (python3 -m django --version && which -a archivebox); then
+if ! (python3 -m django --version && python3 -m pip show archivebox && which -a archivebox); then
     echo "[X] Django and ArchiveBox were not found after installing!"
     echo "    Check to see if a previous step failed."
     echo