diff --git a/pkgs/development/python-modules/evaluate/default.nix b/pkgs/development/python-modules/evaluate/default.nix
new file mode 100644
index 000000000000..1ada1243bf72
--- /dev/null
+++ b/pkgs/development/python-modules/evaluate/default.nix
@@ -0,0 +1,70 @@
+{ lib
+, buildPythonPackage
+, fetchFromGitHub
+, pythonOlder
+, pythonRelaxDepsHook
+, pytestCheckHook
+, datasets
+, dill
+, fsspec
+, huggingface-hub
+, importlib-metadata
+, multiprocess
+, numpy
+, packaging
+, pandas
+, requests
+, responses
+, tqdm
+, xxhash
+}:
+
+buildPythonPackage rec {
+  pname = "evaluate";
+  version = "0.4.0";
+  format = "setuptools";
+
+  disabled = pythonOlder "3.7";
+
+  src = fetchFromGitHub {
+    owner = "huggingface";
+    repo = pname;
+    rev = "refs/tags/v${version}";
+    hash = "sha256-O3W2m12R94iY3F7xgkIiiIyqI6vqiZPXn4jAqEDjVCw=";
+  };
+
+  nativeBuildInputs = [ pythonRelaxDepsHook ];
+  pythonRelaxDeps = [ "responses" ];
+
+  propagatedBuildInputs = [
+    datasets
+    numpy
+    dill
+    pandas
+    requests
+    tqdm
+    xxhash
+    multiprocess
+    fsspec
+    huggingface-hub
+    packaging
+    responses
+  ] ++ lib.optionals (pythonOlder "3.8") [
+    importlib-metadata
+  ];
+
+  # most tests require internet access.
+  doCheck = false;
+
+  pythonImportsCheck = [
+    "evaluate"
+  ];
+
+  meta = with lib; {
+    homepage = "https://huggingface.co/docs/evaluate/index";
+    description = "Easily evaluate machine learning models and datasets";
+    changelog = "https://github.com/huggingface/evaluate/releases/tag/v${version}";
+    license = licenses.asl20;
+    maintainers = with maintainers; [ bcdarwin ];
+  };
+}
diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix
index 1f62eb218e23..21e75c132962 100644
--- a/pkgs/top-level/python-packages.nix
+++ b/pkgs/top-level/python-packages.nix
@@ -3167,6 +3167,8 @@ self: super: with self; {
 
   et_xmlfile = callPackage ../development/python-modules/et_xmlfile { };
 
+  evaluate = callPackage ../development/python-modules/evaluate { };
+
   ev3dev2 = callPackage ../development/python-modules/ev3dev2 { };
 
   evdev = callPackage ../development/python-modules/evdev { };