diff --git a/pkgs/top-level/make-tarball.nix b/pkgs/top-level/make-tarball.nix
index 7352d3d90940..ccc1d1d519cb 100644
--- a/pkgs/top-level/make-tarball.nix
+++ b/pkgs/top-level/make-tarball.nix
@@ -34,13 +34,20 @@ pkgs.releaseTools.sourceTarball {
   checkPhase = ''
     echo "generating packages.json"
 
-    packages=$out/packages.json.br
-
     (
       echo -n '{"version":2,"packages":'
       NIX_STATE_DIR=$TMPDIR NIX_PATH= nix-env -f $src -qa --meta --json --show-trace --arg config 'import ${./packages-config.nix}'
       echo -n '}'
-    ) | sed "s|$src/||g" | jq -c | brotli -9 > $packages
+    ) | sed "s|$src/||g" | jq -c > packages.json
+
+    # Arbitrary number. The index has ~115k packages as of April 2024.
+    if [ $(jq -r '.packages | length' < packages.json) -lt 100000 ]; then
+      echo "ERROR: not enough packages in the search index, bailing out!"
+      exit 1
+    fi
+
+    packages=$out/packages.json.br
+    brotli -9 < packages.json > $packages
 
     mkdir -p $out/nix-support
     echo "file json-br $packages" >> $out/nix-support/hydra-build-products