diff --git a/Makefile b/Makefile
index 17d93c5..6a1429b 100644
--- a/Makefile
+++ b/Makefile
@@ -23,7 +23,7 @@ load:
 	mkdir -p $@
 
 gem_dir := $(shell readlink -f ../gems)
-gems := $(shell bundle show --paths | xargs -I {} sh -c 'test -d {}/ext && basename {}')
+gems := $(shell bundle show --paths | xargs -I {} sh -c 'test -f {}/ext/*/extconf.rb && basename {}')
 gems_musl := $(patsubst %,$(gem_dir)/%-x86_64-linux-musl.gem,$(gems))
 
 $(gem_dir)/%-x86_64-linux-musl.gem: