diff --git a/configure.ac b/configure.ac
index 91918e7d0..cf8904fdd 100644
--- a/configure.ac
+++ b/configure.ac
@@ -485,16 +485,8 @@ AM_COND_IF([OPTIMIZE],
 	CXXFLAGS+=" -fbranch-target-load-optimize"
 	CXXFLAGS+=" -frerun-loop-opt"
 	CXXFLAGS+=" -fdevirtualize-at-ltrans"
-
-	dnl Note these flags are unreliable with LTO
-	CXXFLAGS+=" -g"
-	CXXFLAGS+=" -ggdb"
 ], [
 	optimize="no"
-	CXXFLAGS+=" -g"
-	CXXFLAGS+=" -ggdb"
-	CXXFLAGS+=" -fvar-tracking"
-	CXXFLAGS+=" -fvar-tracking-assignments"
 	CXXFLAGS+=" -falign-functions"
 ])
 
@@ -502,6 +494,18 @@ dnl
 dnl Compiler intrumentation
 dnl
 
+AM_COND_IF([DEBUG],
+[
+	CXXFLAGS+=" -g"
+	CXXFLAGS+=" -ggdb"
+	CXXFLAGS+=" -fvar-tracking"
+	CXXFLAGS+=" -fvar-tracking-assignments"
+], [
+	dnl Note these flags are unreliable with LTO
+	CXXFLAGS+=" -g"
+	CXXFLAGS+=" -ggdb"
+])
+
 dnl  Creates an ELF section in the output which contains a list of
 dnl  the command line arguments used for compilation.
 CXXFLAGS+=" -frecord-gcc-switches"