diff --git a/package/asterisk/asterisk.mk b/package/asterisk/asterisk.mk
index 82ca2ba739..de3a0d5291 100644
--- a/package/asterisk/asterisk.mk
+++ b/package/asterisk/asterisk.mk
@@ -84,7 +84,6 @@ ASTERISK_CONF_OPTS = \
 	--without-postgres \
 	--without-pjproject \
 	--without-popt \
-	--without-portaudio \
 	--without-pri \
 	--without-pwlib \
 	--without-radius \
@@ -173,6 +172,13 @@ else
 ASTERISK_CONF_OPTS += --without-opus
 endif
 
+ifeq ($(BR2_PACKAGE_PORTAUDIO),y)
+ASTERISK_DEPENDENCIES += portaudio
+ASTERISK_CONF_OPTS += --with-portaudio
+else
+ASTERISK_CONF_OPTS += --without-portaudio
+endif
+
 ifeq ($(BR2_PACKAGE_OPENSSL),y)
 ASTERISK_DEPENDENCIES += openssl
 ASTERISK_CONF_OPTS += --with-ssl