diff --git a/pkgs/tools/misc/fontforge/default.nix b/pkgs/tools/misc/fontforge/default.nix
index 388f603873ff..ba1154ab72d5 100644
--- a/pkgs/tools/misc/fontforge/default.nix
+++ b/pkgs/tools/misc/fontforge/default.nix
@@ -35,7 +35,8 @@ stdenv.mkDerivation rec {
 
   configureFlags =
     lib.optionals (!withPython) [ "--disable-python-scripting" "--disable-python-extension" ]
-    ++ lib.optional withGTK "--enable-gtk2-use";
+    ++ lib.optional withGTK "--enable-gtk2-use"
+    ++ lib.optional (!withGTK) "--without-x";
 
   preConfigure = ''
     cp -r "${gnulib}" ./gnulib