diff --git a/configure.ac b/configure.ac
index 45dc219..473a396 100644
--- a/configure.ac
+++ b/configure.ac
@@ -10,6 +10,9 @@ AC_PREREQ([2.69])
 AC_INIT([cglm], [0.6.1], [info@recp.me])
 AM_INIT_AUTOMAKE([-Wall -Werror foreign subdir-objects serial-tests])
 
+# Don't use the default cflags (-O2 -g), we set ours manually in Makefile.am.
+: ${CFLAGS=""}
+
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_SRCDIR([src/])
 AC_CONFIG_HEADERS([config.h])