diff --git a/nixos/modules/programs/ssh.nix b/nixos/modules/programs/ssh.nix
index cc835081c9f9..b6fd9868f98f 100644
--- a/nixos/modules/programs/ssh.nix
+++ b/nixos/modules/programs/ssh.nix
@@ -56,7 +56,6 @@ in
 
       setXAuthLocation = mkOption {
         type = types.bool;
-        default = config.services.xserver.enable;
         description = ''
           Whether to set the path to <command>xauth</command> for X11-forwarded connections.
           This causes a dependency on X11 packages.
@@ -165,6 +164,9 @@ in
 
   config = {
 
+    programs.ssh.setXAuthLocation =
+      mkDefault (config.services.xserver.enable || config.programs.ssh.forwardX11);
+
     assertions =
       [ { assertion = cfg.forwardX11 -> cfg.setXAuthLocation;
           message = "cannot enable X11 forwarding without setting XAuth location";
diff --git a/nixos/modules/services/networking/ssh/sshd.nix b/nixos/modules/services/networking/ssh/sshd.nix
index f900ef494abf..ef186a20f932 100644
--- a/nixos/modules/services/networking/ssh/sshd.nix
+++ b/nixos/modules/services/networking/ssh/sshd.nix
@@ -227,6 +227,8 @@ in
 
   config = mkIf cfg.enable {
 
+    programs.ssh.setXAuthLocation = mkForce cfg.forwardX11;
+
     users.extraUsers.sshd =
       { isSystemUser = true;
         description = "SSH privilege separation user";