Merge pull request #255998 from zeuner/tensorflow-2-13

tensorflow: 2.11.1 -> 2.13.0
2023-10-27 12:39:11 +02:00 · 2023-10-27 12:39:11 +02:00 · c05811cfed
commit c05811cfed
parent de4db3f41c b44bcba394
8 changed files with 751 additions and 38 deletions
--- a/pkgs/development/python-modules/tensorflow/absl_py_argparse_flags.patch
+++ b/pkgs/development/python-modules/tensorflow/absl_py_argparse_flags.patch
@ -0,0 +1,11 @@
+diff -Naurd x/third_party/systemlibs/absl_py.absl.flags.BUILD y/third_party/systemlibs/absl_py.absl.flags.BUILD
+--- x/third_party/systemlibs/absl_py.absl.flags.BUILD	2023-09-20 06:02:15.079683729 +0000
+++ y/third_party/systemlibs/absl_py.absl.flags.BUILD	2023-09-20 06:04:58.491897303 +0000
+@@ -5,3 +5,7 @@
+ py_library(
+     name = "flags",
+ )
+
+py_library(
+    name = "argparse_flags",
+)
--- a/pkgs/development/python-modules/tensorflow/absl_to_std.patch
+++ b/pkgs/development/python-modules/tensorflow/absl_to_std.patch
@ -0,0 +1,389 @@
+diff -Naurd x/tensorflow/c/experimental/stream_executor/stream_executor.cc y/tensorflow/c/experimental/stream_executor/stream_executor.cc
+--- x/tensorflow/c/experimental/stream_executor/stream_executor.cc	2023-10-19 14:25:48.648646416 +0000
+++ y/tensorflow/c/experimental/stream_executor/stream_executor.cc	2023-10-19 14:29:11.700743574 +0000
+@@ -23,6 +23,7 @@
+ 
+ #include <string>
+ #include <utility>
+#include <optional>
+ 
+ #include "absl/functional/any_invocable.h"
+ #include "tensorflow/c/c_api_macros.h"
+@@ -275,17 +276,17 @@
+     stream_executor_->unified_memory_deallocate(&device_, mem);
+   }
+ 
+-  absl::optional<AllocatorStats> GetAllocatorStats() override {
+  std::optional<AllocatorStats> GetAllocatorStats() override {
+     SP_AllocatorStats c_stats{SP_ALLOCATORSTATS_STRUCT_SIZE};
+     TF_Bool has_stats =
+         stream_executor_->get_allocator_stats(&device_, &c_stats);
+     if (!has_stats) {
+-      return absl::nullopt;
+      return std::nullopt;
+     }
+     tsl::Status status = ValidateSPAllocatorStats(c_stats);
+     if (!status.ok()) {
+       LOG(ERROR) << status.message();
+-      return absl::nullopt;
+      return std::nullopt;
+     }
+     ::stream_executor::AllocatorStats stats;
+     stats.num_allocs = c_stats.num_allocs;
+diff -Naurd x/tensorflow/c/experimental/stream_executor/stream_executor_test.cc y/tensorflow/c/experimental/stream_executor/stream_executor_test.cc
+--- x/tensorflow/c/experimental/stream_executor/stream_executor_test.cc	2023-10-19 14:25:48.648646416 +0000
+++ y/tensorflow/c/experimental/stream_executor/stream_executor_test.cc	2023-10-19 14:29:11.700743574 +0000
+@@ -15,6 +15,7 @@
+ #include "tensorflow/c/experimental/stream_executor/stream_executor.h"
+ 
+ #include <utility>
+#include <optional>
+ 
+ #include "tensorflow/c/experimental/stream_executor/stream_executor_internal.h"
+ #include "tensorflow/c/experimental/stream_executor/stream_executor_test_util.h"
+@@ -239,7 +240,7 @@
+   };
+ 
+   StreamExecutor* executor = GetExecutor(0);
+-  absl::optional<AllocatorStats> optional_stats = executor->GetAllocatorStats();
+  std::optional<AllocatorStats> optional_stats = executor->GetAllocatorStats();
+   ASSERT_TRUE(optional_stats.has_value());
+   AllocatorStats stats = optional_stats.value();
+   ASSERT_EQ(stats.bytes_in_use, 123);
+diff -Naurd x/tensorflow/compiler/xla/stream_executor/allocator_stats.h y/tensorflow/compiler/xla/stream_executor/allocator_stats.h
+--- x/tensorflow/compiler/xla/stream_executor/allocator_stats.h	2023-10-19 14:25:55.064649379 +0000
+++ y/tensorflow/compiler/xla/stream_executor/allocator_stats.h	2023-10-19 14:29:11.700743574 +0000
+@@ -17,6 +17,7 @@
+ #define TENSORFLOW_COMPILER_XLA_STREAM_EXECUTOR_ALLOCATOR_STATS_H_
+ 
+ #include <string>
+#include <optional>
+ 
+ #include "absl/types/optional.h"
+ #include "tensorflow/compiler/xla/stream_executor/platform/port.h"
+diff -Naurd x/tensorflow/core/common_runtime/mkl_cpu_allocator.h y/tensorflow/core/common_runtime/mkl_cpu_allocator.h
+--- x/tensorflow/core/common_runtime/mkl_cpu_allocator.h	2023-10-19 14:25:58.996651199 +0000
+++ y/tensorflow/core/common_runtime/mkl_cpu_allocator.h	2023-10-19 14:29:11.700743574 +0000
+@@ -22,6 +22,7 @@
+ #ifdef INTEL_MKL
+ 
+ #include <cstdlib>
+#include <optional>
+ 
+ #include "tensorflow/core/common_runtime/bfc_allocator.h"
+ #include "tensorflow/core/common_runtime/pool_allocator.h"
+@@ -80,7 +81,7 @@
+     port::AlignedFree(ptr);
+   }
+ 
+-  absl::optional<AllocatorStats> GetStats() override {
+  std::optional<AllocatorStats> GetStats() override {
+     mutex_lock l(mutex_);
+     return stats_;
+   }
+@@ -242,7 +243,7 @@
+       large_size_allocator_->DeallocateRaw(ptr);
+     }
+   }
+-  absl::optional<AllocatorStats> GetStats() override {
+  std::optional<AllocatorStats> GetStats() override {
+     auto s_stats = small_size_allocator_->GetStats();
+     auto l_stats = large_size_allocator_->GetStats();
+ 
+diff -Naurd x/tensorflow/core/common_runtime/pluggable_device/pluggable_device_simple_allocator.cc y/tensorflow/core/common_runtime/pluggable_device/pluggable_device_simple_allocator.cc
+--- x/tensorflow/core/common_runtime/pluggable_device/pluggable_device_simple_allocator.cc	2023-10-19 14:25:59.236651310 +0000
+++ y/tensorflow/core/common_runtime/pluggable_device/pluggable_device_simple_allocator.cc	2023-10-19 14:29:11.700743574 +0000
+@@ -32,7 +32,7 @@
+   return sub_allocator_->Free(ptr, 0);
+ }
+ 
+-absl::optional<AllocatorStats> PluggableDeviceSimpleAllocator::GetStats() {
+std::optional<AllocatorStats> PluggableDeviceSimpleAllocator::GetStats() {
+   AllocatorStats stats_;
+   stats_.num_allocs = 0;
+   stats_.peak_bytes_in_use = 0;
+diff -Naurd x/tensorflow/core/common_runtime/pluggable_device/pluggable_device_simple_allocator.h y/tensorflow/core/common_runtime/pluggable_device/pluggable_device_simple_allocator.h
+--- x/tensorflow/core/common_runtime/pluggable_device/pluggable_device_simple_allocator.h	2023-10-19 14:25:59.236651310 +0000
+++ y/tensorflow/core/common_runtime/pluggable_device/pluggable_device_simple_allocator.h	2023-10-19 14:29:11.700743574 +0000
+@@ -19,6 +19,7 @@
+ #include <string>
+ #include <unordered_map>
+ #include <vector>
+#include <optional>
+ 
+ #include "tensorflow/core/common_runtime/device/device_mem_allocator.h"
+ #include "tensorflow/core/platform/thread_annotations.h"
+@@ -37,7 +38,7 @@
+ 
+   bool TracksAllocationSizes() const override { return false; }
+   string Name() override { return "Simple allocator"; }
+-  absl::optional<AllocatorStats> GetStats() override;
+  std::optional<AllocatorStats> GetStats() override;
+ 
+   AllocatorMemoryType GetMemoryType() const override {
+     return sub_allocator_->GetMemoryType();
+diff -Naurd x/tensorflow/core/common_runtime/process_state.h y/tensorflow/core/common_runtime/process_state.h
+--- x/tensorflow/core/common_runtime/process_state.h	2023-10-19 14:25:59.076651236 +0000
+++ y/tensorflow/core/common_runtime/process_state.h	2023-10-19 14:29:11.704743576 +0000
+@@ -20,6 +20,7 @@
+ #include <map>
+ #include <unordered_map>
+ #include <vector>
+#include <optional>
+ 
+ #include "tensorflow/core/framework/allocator.h"
+ #include "tensorflow/core/framework/allocator_registry.h"
+@@ -144,7 +145,7 @@
+   size_t AllocatedSize(const void* p) const override {
+     return a_->AllocatedSize(p);
+   }
+-  absl::optional<AllocatorStats> GetStats() override { return a_->GetStats(); }
+  std::optional<AllocatorStats> GetStats() override { return a_->GetStats(); }
+   bool ClearStats() override { return a_->ClearStats(); }
+ 
+   AllocatorMemoryType GetMemoryType() const override {
+diff -Naurd x/tensorflow/core/common_runtime/step_stats_collector.cc y/tensorflow/core/common_runtime/step_stats_collector.cc
+--- x/tensorflow/core/common_runtime/step_stats_collector.cc	2023-10-19 14:25:59.112651253 +0000
+++ y/tensorflow/core/common_runtime/step_stats_collector.cc	2023-10-19 14:29:11.704743576 +0000
+@@ -15,6 +15,7 @@
+ #include "tensorflow/core/common_runtime/step_stats_collector.h"
+ 
+ #include <memory>
+#include <optional>
+ 
+ #include "tensorflow/core/common_runtime/costmodel_manager.h"
+ #include "tensorflow/core/framework/allocation_description.pb.h"
+@@ -175,7 +176,7 @@
+   memory->set_peak_bytes(std::get<1>(sizes));
+   memory->set_live_bytes(std::get<2>(sizes));
+ 
+-  absl::optional<AllocatorStats> stats = allocator->GetStats();
+  std::optional<AllocatorStats> stats = allocator->GetStats();
+   if (stats) {
+     memory->set_allocator_bytes_in_use(stats->bytes_in_use);
+   }
+diff -Naurd x/tensorflow/core/framework/allocator_test.cc y/tensorflow/core/framework/allocator_test.cc
+--- x/tensorflow/core/framework/allocator_test.cc	2023-10-19 14:25:59.524651443 +0000
+++ y/tensorflow/core/framework/allocator_test.cc	2023-10-19 14:29:11.704743576 +0000
+@@ -17,6 +17,7 @@
+ 
+ #include <algorithm>
+ #include <vector>
+#include <optional>
+ 
+ #include "tensorflow/core/framework/typed_allocator.h"
+ #include "tensorflow/core/platform/logging.h"
+@@ -33,7 +34,7 @@
+ 
+ static void CheckStats(Allocator* a, int64_t num_allocs, int64_t bytes_in_use,
+                        int64_t peak_bytes_in_use, int64_t largest_alloc_size) {
+-  absl::optional<AllocatorStats> stats = a->GetStats();
+  std::optional<AllocatorStats> stats = a->GetStats();
+   EXPECT_TRUE(stats);
+   if (!stats) {
+     return;
+@@ -255,7 +256,7 @@
+   EXPECT_EQ(e0.Name(), "MemoryAllocation")
+       << "XSpace: " << xspace.DebugString();
+   {
+-    absl::optional<std::string> bytes_allocated, peak_bytes_in_use,
+    std::optional<std::string> bytes_allocated, peak_bytes_in_use,
+         requested_bytes, allocation_bytes;
+     e0.ForEachStat([&](const ::tensorflow::profiler::XStatVisitor& stat) {
+       LOG(ERROR) << "STAT " << stat.Name() << ": " << stat.ToString();
+@@ -282,7 +283,7 @@
+   EXPECT_EQ(e1.Name(), "MemoryDeallocation")
+       << "XSpace: " << xspace.DebugString();
+   {
+-    absl::optional<std::string> bytes_allocated, peak_bytes_in_use,
+    std::optional<std::string> bytes_allocated, peak_bytes_in_use,
+         allocation_bytes;
+     e1.ForEachStat([&](const ::tensorflow::profiler::XStatVisitor& stat) {
+       if (stat.Name() == "bytes_allocated") {
+diff -Naurd x/tensorflow/core/framework/tracking_allocator_test.cc y/tensorflow/core/framework/tracking_allocator_test.cc
+--- x/tensorflow/core/framework/tracking_allocator_test.cc	2023-10-19 14:25:59.700651525 +0000
+++ y/tensorflow/core/framework/tracking_allocator_test.cc	2023-10-19 14:29:11.704743576 +0000
+@@ -16,6 +16,7 @@
+ #include "tensorflow/core/framework/tracking_allocator.h"
+ 
+ #include <unordered_map>
+#include <optional>
+ 
+ #include "tensorflow/core/framework/allocator.h"
+ #include "tensorflow/core/platform/logging.h"
+@@ -44,7 +45,7 @@
+     EXPECT_NE(size_map_.end(), iter);
+     return iter->second;
+   }
+-  absl::optional<AllocatorStats> GetStats() override { return absl::nullopt; }
+  std::optional<AllocatorStats> GetStats() override { return std::nullopt; }
+ 
+  private:
+   std::unordered_map<const void*, size_t> size_map_;
+@@ -58,7 +59,7 @@
+   }
+   void DeallocateRaw(void* ptr) override {}
+   bool TracksAllocationSizes() const override { return true; }
+-  absl::optional<AllocatorStats> GetStats() override { return absl::nullopt; }
+  std::optional<AllocatorStats> GetStats() override { return std::nullopt; }
+ };
+ 
+ TEST(TrackingAllocatorTest, SimpleNoTracking) {
+diff -Naurd x/tensorflow/core/grappler/clusters/single_machine.cc y/tensorflow/core/grappler/clusters/single_machine.cc
+--- x/tensorflow/core/grappler/clusters/single_machine.cc	2023-10-19 14:25:59.964651648 +0000
+++ y/tensorflow/core/grappler/clusters/single_machine.cc	2023-10-19 14:29:11.704743576 +0000
+@@ -17,6 +17,7 @@
+ 
+ #include <atomic>
+ #include <memory>
+#include <optional>
+ 
+ #include "tensorflow/cc/training/queue_runner.h"
+ #include "tensorflow/core/common_runtime/device.h"
+@@ -230,7 +231,7 @@
+       return Status(absl::StatusCode::kInvalidArgument,
+                     "Tracking allocation is not enabled.");
+     }
+-    absl::optional<AllocatorStats> stats = allocator->GetStats();
+    std::optional<AllocatorStats> stats = allocator->GetStats();
+     (*device_peak_memory)[device->name()] =
+         (stats ? stats->peak_bytes_in_use : 0);
+   }
+diff -Naurd x/tensorflow/core/kernels/stack.cc y/tensorflow/core/kernels/stack.cc
+--- x/tensorflow/core/kernels/stack.cc	2023-10-19 14:26:01.668652437 +0000
+++ y/tensorflow/core/kernels/stack.cc	2023-10-19 14:29:11.704743576 +0000
+@@ -18,6 +18,7 @@
+ #include <limits.h>
+ #include <atomic>
+ #include <vector>
+#include <optional>
+ 
+ #include "tensorflow/core/common_runtime/device.h"
+ #include "tensorflow/core/framework/device_base.h"
+@@ -245,7 +246,7 @@
+     DeviceContext* device_ctxt = ctx->op_device_context();
+     auto device = static_cast<tensorflow::Device*>(ctx->device());
+     Allocator* allocator = device->GetAllocator(alloc_attrs);
+-    absl::optional<AllocatorStats> stats = allocator->GetStats();
+    std::optional<AllocatorStats> stats = allocator->GetStats();
+     if (stats && *stats->bytes_limit &&
+         stats->bytes_in_use > (*stats->bytes_limit * kOccupancy)) {
+       // Asynchronously copy the tensor from GPU to CPU memory.
+diff -Naurd x/tensorflow/python/tfe_wrapper.cc y/tensorflow/python/tfe_wrapper.cc
+--- x/tensorflow/python/tfe_wrapper.cc	2023-10-19 14:26:10.716656639 +0000
+++ y/tensorflow/python/tfe_wrapper.cc	2023-10-19 14:29:11.708743578 +0000
+@@ -14,6 +14,7 @@
+ ==============================================================================*/
+ 
+ #include <memory>
+#include <optional>
+ 
+ #include "Python.h"
+ #include "absl/strings/match.h"
+@@ -691,7 +692,7 @@
+     tensorflow::AllocatorAttributes attrs;
+     tensorflow::Allocator* allocator = matched_device->GetAllocator(attrs);
+ 
+-    if (absl::optional<tensorflow::AllocatorStats> stats =
+    if (std::optional<tensorflow::AllocatorStats> stats =
+             allocator->GetStats()) {
+       return std::map<std::string, int64_t>{{"current", stats->bytes_in_use},
+                                             {"peak", stats->peak_bytes_in_use}};
+diff -Naurd x/tensorflow/tsl/framework/allocator.h y/tensorflow/tsl/framework/allocator.h
+--- x/tensorflow/tsl/framework/allocator.h	2023-10-19 14:26:15.884659044 +0000
+++ y/tensorflow/tsl/framework/allocator.h	2023-10-19 14:29:11.708743578 +0000
+@@ -216,7 +216,7 @@
+   }
+ 
+   // Fills in 'stats' with statistics collected by this allocator.
+-  virtual absl::optional<AllocatorStats> GetStats() { return absl::nullopt; }
+  virtual std::optional<AllocatorStats> GetStats() { return std::nullopt; }
+ 
+   // If implemented, clears the internal stats except for the `in_use` fields
+   // and sets the `peak_bytes_in_use` to be equal to the `bytes_in_use`. Returns
+diff -Naurd x/tensorflow/tsl/framework/bfc_allocator.cc y/tensorflow/tsl/framework/bfc_allocator.cc
+--- x/tensorflow/tsl/framework/bfc_allocator.cc	2023-10-19 14:26:15.900659052 +0000
+++ y/tensorflow/tsl/framework/bfc_allocator.cc	2023-10-19 14:29:11.708743578 +0000
+@@ -1205,7 +1205,7 @@
+   return md;
+ }
+ 
+-absl::optional<AllocatorStats> BFCAllocator::GetStats() {
+std::optional<AllocatorStats> BFCAllocator::GetStats() {
+   mutex_lock l(lock_);
+   return stats_;
+ }
+diff -Naurd x/tensorflow/tsl/framework/bfc_allocator.h y/tensorflow/tsl/framework/bfc_allocator.h
+--- x/tensorflow/tsl/framework/bfc_allocator.h	2023-10-19 14:26:15.900659052 +0000
+++ y/tensorflow/tsl/framework/bfc_allocator.h	2023-10-19 14:29:11.708743578 +0000
+@@ -22,6 +22,7 @@
+ #include <string>
+ #include <unordered_map>
+ #include <vector>
+#include <optional>
+ 
+ #include "absl/container/flat_hash_set.h"
+ #include "tensorflow/tsl/framework/allocator.h"
+@@ -93,7 +94,7 @@
+ 
+   int64_t AllocationId(const void* ptr) const override;
+ 
+-  absl::optional<AllocatorStats> GetStats() override;
+  std::optional<AllocatorStats> GetStats() override;
+ 
+   bool ClearStats() override;
+ 
+diff -Naurd x/tensorflow/tsl/framework/cpu_allocator_impl.cc y/tensorflow/tsl/framework/cpu_allocator_impl.cc
+--- x/tensorflow/tsl/framework/cpu_allocator_impl.cc	2023-10-19 14:26:15.928659065 +0000
+++ y/tensorflow/tsl/framework/cpu_allocator_impl.cc	2023-10-19 14:29:11.708743578 +0000
+@@ -15,6 +15,7 @@
+ 
+ #include <algorithm>
+ #include <atomic>
+#include <optional>
+ 
+ #include "tensorflow/tsl/framework/allocator.h"
+ #include "tensorflow/tsl/framework/allocator_registry.h"
+@@ -145,8 +146,8 @@
+         /*level=*/tsl::profiler::TraceMeLevel::kInfo);
+   }
+ 
+-  absl::optional<AllocatorStats> GetStats() override {
+-    if (!cpu_allocator_collect_stats) return absl::nullopt;
+  std::optional<AllocatorStats> GetStats() override {
+    if (!cpu_allocator_collect_stats) return std::nullopt;
+     mutex_lock l(mu_);
+     return stats_;
+   }
+diff -Naurd x/tensorflow/tsl/framework/tracking_allocator.cc y/tensorflow/tsl/framework/tracking_allocator.cc
+--- x/tensorflow/tsl/framework/tracking_allocator.cc	2023-10-19 14:26:15.968659084 +0000
+++ y/tensorflow/tsl/framework/tracking_allocator.cc	2023-10-19 14:29:11.708743578 +0000
+@@ -152,7 +152,7 @@
+   }
+ }
+ 
+-absl::optional<AllocatorStats> TrackingAllocator::GetStats() {
+std::optional<AllocatorStats> TrackingAllocator::GetStats() {
+   return allocator_->GetStats();
+ }
+ 
+diff -Naurd x/tensorflow/tsl/framework/tracking_allocator.h y/tensorflow/tsl/framework/tracking_allocator.h
+--- x/tensorflow/tsl/framework/tracking_allocator.h	2023-10-19 14:26:15.968659084 +0000
+++ y/tensorflow/tsl/framework/tracking_allocator.h	2023-10-19 14:29:11.712743580 +0000
+@@ -17,6 +17,7 @@
+ #define TENSORFLOW_TSL_FRAMEWORK_TRACKING_ALLOCATOR_H_
+ 
+ #include <unordered_map>
+#include <optional>
+ 
+ #include "tensorflow/tsl/framework/allocator.h"
+ #include "tensorflow/tsl/lib/gtl/inlined_vector.h"
+@@ -66,7 +67,7 @@
+   size_t RequestedSize(const void* ptr) const override;
+   size_t AllocatedSize(const void* ptr) const override;
+   int64_t AllocationId(const void* ptr) const override;
+-  absl::optional<AllocatorStats> GetStats() override;
+  std::optional<AllocatorStats> GetStats() override;
+   bool ClearStats() override;
+ 
+   AllocatorMemoryType GetMemoryType() const override {
--- a/pkgs/development/python-modules/tensorflow/com_google_absl_add_log.patch
+++ b/pkgs/development/python-modules/tensorflow/com_google_absl_add_log.patch
@ -0,0 +1,166 @@
+diff -Naurd x/third_party/absl/system.absl.base.BUILD y/third_party/absl/system.absl.base.BUILD
+--- x/third_party/absl/system.absl.base.BUILD	2023-09-17 09:12:05.499753364 +0000
+++ y/third_party/absl/system.absl.base.BUILD	2023-09-17 09:16:01.200082822 +0000
+@@ -22,7 +22,12 @@
+ 
+ cc_library(
+     name = "raw_logging_internal",
+-    linkopts = ["-labsl_raw_logging_internal"],
+    linkopts = [
+        "-labsl_raw_logging_internal",
+        "-labsl_log_internal_conditions",
+        "-labsl_log_internal_message",
+        "-labsl_log_internal_nullguard",
+    ],
+     visibility = [
+         "//absl:__subpackages__",
+     ],
+diff -Naurd x/third_party/absl/system.absl.log.BUILD y/third_party/absl/system.absl.log.BUILD
+--- x/third_party/absl/system.absl.log.BUILD	1970-01-01 00:00:00.000000000 +0000
+++ y/third_party/absl/system.absl.log.BUILD	2023-09-17 09:12:11.795762177 +0000
+@@ -0,0 +1,134 @@
+load("@rules_cc//cc:defs.bzl", "cc_library")
+
+package(default_visibility = ["//visibility:public"])
+
+cc_library(
+    name = "absl_check",
+    deps = [
+    ],
+)
+
+cc_library(
+    name = "absl_log",
+    deps = [
+    ],
+)
+
+cc_library(
+    name = "check",
+    deps = [
+    ],
+)
+
+cc_library(
+    name = "die_if_null",
+    deps = [
+        ":log",
+        "//absl/base:config",
+        "//absl/base:core_headers",
+        "//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "flags",
+    deps = [
+        ":globals",
+        "//absl/base:config",
+        "//absl/base:core_headers",
+        "//absl/base:log_severity",
+        "//absl/flags:flag",
+        "//absl/flags:marshalling",
+        "//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "globals",
+    deps = [
+        "//absl/base:atomic_hook",
+        "//absl/base:config",
+        "//absl/base:core_headers",
+        "//absl/base:log_severity",
+        "//absl/hash",
+        "//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "initialize",
+    deps = [
+        ":globals",
+        "//absl/base:config",
+        "//absl/time",
+    ],
+)
+
+cc_library(
+    name = "log",
+    deps = [
+    ],
+)
+
+cc_library(
+    name = "log_entry",
+    deps = [
+        "//absl/base:config",
+        "//absl/base:core_headers",
+        "//absl/base:log_severity",
+        "//absl/strings",
+        "//absl/time",
+        "//absl/types:span",
+    ],
+)
+
+cc_library(
+    name = "log_sink",
+    deps = [
+        ":log_entry",
+        "//absl/base:config",
+    ],
+)
+
+cc_library(
+    name = "log_sink_registry",
+    deps = [
+        ":log_sink",
+        "//absl/base:config",
+    ],
+)
+
+cc_library(
+    name = "log_streamer",
+    deps = [
+        ":absl_log",
+        "//absl/base:config",
+        "//absl/base:log_severity",
+        "//absl/strings",
+        "//absl/strings:internal",
+        "//absl/types:optional",
+        "//absl/utility",
+    ],
+)
+
+cc_library(
+    name = "scoped_mock_log",
+    deps = [
+        ":log_entry",
+        ":log_sink",
+        ":log_sink_registry",
+        "//absl/base:config",
+        "//absl/base:log_severity",
+        "//absl/base:raw_logging_internal",
+        "//absl/strings",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
+cc_library(
+    name = "structured",
+    deps = [
+        "//absl/base:config",
+        "//absl/strings",
+    ],
+)
+diff -Naurd x/third_party/absl/workspace.bzl y/third_party/absl/workspace.bzl
+--- x/third_party/absl/workspace.bzl	2023-09-17 09:12:05.499753364 +0000
+++ y/third_party/absl/workspace.bzl	2023-09-17 09:12:11.795762177 +0000
+@@ -20,6 +20,7 @@
+         "flags",
+         "functional",
+         "hash",
+        "log",
+         "memory",
+         "meta",
+         "numeric",
--- a/pkgs/development/python-modules/tensorflow/default.nix
+++ b/pkgs/development/python-modules/tensorflow/default.nix
@ -1,18 +1,18 @@
-{ stdenv, bazel_5, buildBazelPackage, isPy3k, lib, fetchFromGitHub, symlinkJoin
-, addOpenGLRunpath, fetchpatch
+{ stdenv, bazel_5, buildBazelPackage, lib, fetchFromGitHub, symlinkJoin
+, addOpenGLRunpath, fetchpatch, fetchzip, linkFarm
 # Python deps
 , buildPythonPackage, pythonOlder, python
 # Python libraries
-, numpy, tensorboard, absl-py
-, packaging, setuptools, wheel, keras, keras-preprocessing, google-pasta
+, numpy, tensorboard, abseil-cpp, absl-py
+, packaging, setuptools, wheel, keras-preprocessing, google-pasta
 , opt-einsum, astunparse, h5py
 , termcolor, grpcio, six, wrapt, protobuf-python, tensorflow-estimator-bin
 , dill, flatbuffers-python, portpicker, tblib, typing-extensions
 # Common deps
-, git, pybind11, which, binutils, glibcLocales, cython, perl, coreutils
+, git, pybind11, which, binutils, glibcLocales, cython, perl
 # Common libraries
 , jemalloc, mpi, gast, grpc, sqlite, boringssl, jsoncpp, nsync
-, curl, snappy, flatbuffers-core, lmdb-core, icu, double-conversion, libpng, libjpeg_turbo, giflib, protobuf-core
+, curl, snappy, flatbuffers-core, icu, double-conversion, libpng, libjpeg_turbo, giflib, protobuf-core
 # Upstream by default includes cuda support since tensorflow 1.15. We could do
 # that in nix as well. It would make some things easier and less confusing, but
 # it would also make the default tensorflow package unfree. See
@ -53,7 +53,18 @@ let
    if cudaSupport then cudaPackages.backendStdenv
    else if originalStdenv.isDarwin then llvmPackages_11.stdenv
    else originalStdenv;
-  inherit (cudaPackages) cudatoolkit cudnn nccl;
+  inherit (cudaPackages) cudatoolkit nccl;
+  # use compatible cuDNN (https://www.tensorflow.org/install/source#gpu)
+  # cudaPackages.cudnn led to this:
+  # https://github.com/tensorflow/tensorflow/issues/60398
+  cudnn = cudaPackages.cudnn_8_6;
+  gentoo-patches = fetchzip {
+    url = "https://dev.gentoo.org/~perfinion/patches/tensorflow-patches-2.12.0.tar.bz2";
+    hash = "sha256-SCRX/5/zML7LmKEPJkcM5Tebez9vv/gmE4xhT/jyqWs=";
+  };
+  protobuf-extra = linkFarm "protobuf-extra" [
+    { name = "include"; path = protobuf-core.src; }
+  ];
 in

 assert cudaSupport -> cudatoolkit != null
@ -99,7 +110,7 @@ let

  tfFeature = x: if x then "1" else "0";

-  version = "2.11.1";
+  version = "2.13.0";
  variant = lib.optionalString cudaSupport "-gpu";
  pname = "tensorflow${variant}";

@ -208,14 +219,14 @@ let
      owner = "tensorflow";
      repo = "tensorflow";
      rev = "refs/tags/v${version}";
-      hash = "sha256-q59cUW6613byHk4LGl+sefO5czLSWxOrSyLbJ1pkNEY=";
+      hash = "sha256-Rq5pAVmxlWBVnph20fkAwbfy+iuBNlfFy14poDPd5h0=";
    };

    # On update, it can be useful to steal the changes from gentoo
    # https://gitweb.gentoo.org/repo/gentoo.git/tree/sci-libs/tensorflow

    nativeBuildInputs = [
-      which pythonEnv cython perl protobuf-core
+      which pythonEnv cython perl protobuf-core protobuf-extra
    ] ++ lib.optional cudaSupport addOpenGLRunpath;

    buildInputs = [
@ -225,6 +236,7 @@ let
      git

      # libs taken from system through the TF_SYS_LIBS mechanism
+      abseil-cpp
      boringssl
      curl
      double-conversion
@ -236,7 +248,6 @@ let
      jsoncpp
      libjpeg_turbo
      libpng
-      lmdb-core
      (pybind11.overridePythonAttrs (_: { inherit stdenv; }))
      snappy
      sqlite
@ -265,6 +276,7 @@ let
      "astor_archive"
      "astunparse_archive"
      "boringssl"
+      "com_google_absl"
      # Not packaged in nixpkgs
      # "com_github_googleapis_googleapis"
      # "com_github_googlecloudplatform_google_cloud_cpp"
@ -284,7 +296,6 @@ let
      "icu"
      "jsoncpp_git"
      "libjpeg_turbo"
-      "lmdb"
      "nasm"
      "opt_einsum_archive"
      "org_sqlite"
@ -328,6 +339,25 @@ let
    GCC_HOST_COMPILER_PREFIX = lib.optionalString cudaSupport "${cudatoolkit_cc_joined}/bin";
    GCC_HOST_COMPILER_PATH = lib.optionalString cudaSupport "${cudatoolkit_cc_joined}/bin/cc";

+    patches = [
+      "${gentoo-patches}/0002-systemlib-Latest-absl-LTS-has-split-cord-libs.patch"
+      "${gentoo-patches}/0005-systemlib-Updates-for-Abseil-20220623-LTS.patch"
+      "${gentoo-patches}/0007-systemlibs-Add-well_known_types_py_pb2-target.patch"
+      # https://github.com/conda-forge/tensorflow-feedstock/pull/329/commits/0a63c5a962451b4da99a9948323d8b3ed462f461
+      (fetchpatch {
+        name = "fix-layout-proto-duplicate-loading.patch";
+        url = "https://raw.githubusercontent.com/conda-forge/tensorflow-feedstock/0a63c5a962451b4da99a9948323d8b3ed462f461/recipe/patches/0001-Omit-linking-to-layout_proto_cc-if-protobuf-linkage-.patch";
+        hash = "sha256-/7buV6DinKnrgfqbe7KKSh9rCebeQdXv2Uj+Xg/083w=";
+      })
+      ./com_google_absl_add_log.patch
+      ./absl_py_argparse_flags.patch
+      ./protobuf_python.patch
+      ./pybind11_protobuf_python_runtime_dep.patch
+      ./pybind11_protobuf_newer_version.patch
+    ] ++ lib.optionals (stdenv.hostPlatform.system == "aarch64-darwin") [
+      ./absl_to_std.patch
+    ];
+
    postPatch = ''
      # bazel 3.3 should work just as well as bazel 3.1
      rm -f .bazelversion
@ -395,18 +425,30 @@ let
    fetchAttrs = {
      sha256 = {
      x86_64-linux = if cudaSupport
-        then "sha256-lURiR0Ra4kynDXyfuONG+A7CpxnAsfKzIdFTExKzp1o="
-        else "sha256-lDvRgj+UlaneRGZOO9UVCb6uyxcbRJfUhABf/sgKPi0=";
-      aarch64-linux = "sha256-z2d45fqHz5HW+qkv3fR9hMg3sEwUzJfxF54vng85bHk=";
-      x86_64-darwin = "sha256-AAvuz8o6ZRkaSYMgaep74lDDQcxOupDCX4vRaK/jnCU=";
-      aarch64-darwin = "sha256-kexRSvfQqb92ZRuUqAO070RnUUBidAqghiA7Y8do9vc=";
+        then "sha256-5VFMNHeLrUxW5RTr6EhT3pay9nWJ5JkZTGirDds5QkU="
+        else "sha256-KzgWV69Btr84FdwQ5JI2nQEsqiPg1/+TWdbw5bmxXOE=";
+      aarch64-linux = "sha256-9btXrNHqd720oXTPDhSmFidv5iaZRLjCVX8opmrMjXk=";
+      x86_64-darwin = "sha256-gqb03kB0z2pZQ6m1fyRp1/Nbt8AVVHWpOJSeZNCLc4w=";
+      aarch64-darwin = "sha256-WdgAaFZU+ePwWkVBhLzjlNT7ELfGHOTaMdafcAMD5yo=";
      }.${stdenv.hostPlatform.system} or (throw "unsupported system ${stdenv.hostPlatform.system}");
    };

    buildAttrs = {
      outputs = [ "out" "python" ];

+      # need to rebuild schemas since we use a different flatbuffers version
      preBuild = ''
+        (cd tensorflow/lite/schema;${flatbuffers-core}/bin/flatc --gen-object-api -c schema.fbs)
+        (cd tensorflow/lite/schema;${flatbuffers-core}/bin/flatc --gen-object-api -c conversion_metadata.fbs)
+        (cd tensorflow/lite/acceleration/configuration;${flatbuffers-core}/bin/flatc -o configuration.fbs --proto configuration.proto)
+        sed -i s,tflite.proto,tflite,g tensorflow/lite/acceleration/configuration/configuration.fbs/configuration.fbs
+        (cd tensorflow/lite/acceleration/configuration;${flatbuffers-core}/bin/flatc --gen-compare --gen-object-api -c configuration.fbs/configuration.fbs)
+        cp -r tensorflow/lite/acceleration/configuration/configuration.fbs tensorflow/lite/experimental/acceleration/configuration
+        (cd tensorflow/lite/experimental/acceleration/configuration;${flatbuffers-core}/bin/flatc -c configuration.fbs/configuration.fbs)
+        (cd tensorflow/lite/delegates/gpu/cl;${flatbuffers-core}/bin/flatc -c compiled_program_cache.fbs)
+        (cd tensorflow/lite/delegates/gpu/cl;${flatbuffers-core}/bin/flatc -I $NIX_BUILD_TOP/source -c serialization.fbs)
+        (cd tensorflow/lite/delegates/gpu/common;${flatbuffers-core}/bin/flatc -I $NIX_BUILD_TOP/source -c gpu_model.fbs)
+        (cd tensorflow/lite/delegates/gpu/common/task;${flatbuffers-core}/bin/flatc -c serialization_base.fbs)
        patchShebangs .
      '';

@ -448,10 +490,7 @@ let
      license = licenses.asl20;
      maintainers = with maintainers; [ abbradar ];
      platforms = with platforms; linux ++ darwin;
-      # More vulnerabilities in 2.11.1 really; https://github.com/tensorflow/tensorflow/releases
-      knownVulnerabilities = [ "CVE-2023-33976" ];
-      broken = true || # most likely needs dealing with protobuf/abseil updates
-        !(xlaSupport -> cudaSupport) || python.pythonVersion == "3.11";
+      broken = stdenv.isDarwin || !(xlaSupport -> cudaSupport);
    } // lib.optionalAttrs stdenv.isDarwin {
      timeout = 86400; # 24 hours
      maxSilent = 14400; # 4h, double the default of 7200s
@ -460,7 +499,7 @@ let

 in buildPythonPackage {
  inherit version pname;
-  disabled = !isPy3k;
+  disabled = pythonOlder "3.8";

  src = bazel-build.python;

@ -493,6 +532,7 @@ in buildPythonPackage {
  # tensorflow/tools/pip_package/setup.py
  propagatedBuildInputs = [
    absl-py
+    abseil-cpp
    astunparse
    flatbuffers-python
    gast
@ -529,7 +569,6 @@ in buildPythonPackage {
  # TEST_PACKAGES in tensorflow/tools/pip_package/setup.py
  nativeCheckInputs = [
    dill
-    keras
    portpicker
    tblib
  ];
@ -540,18 +579,15 @@ in buildPythonPackage {
    hello = tf.constant("Hello, world!")
    tf.print(hello)

-    # Fit a simple model to random data
-    import numpy as np
-    np.random.seed(0)
    tf.random.set_seed(0)
-    model = tf.keras.models.Sequential([
-        tf.keras.layers.Dense(1, activation="linear")
-    ])
-    model.compile(optimizer="sgd", loss="mse")
-
-    x = np.random.uniform(size=(1,1))
-    y = np.random.uniform(size=(1,))
-    model.fit(x, y, epochs=1)
+    width = 512
+    choice = 48
+    t_in = tf.Variable(tf.random.uniform(shape=[width]))
+    with tf.GradientTape() as tape:
+        t_out = tf.slice(tf.nn.softmax(t_in), [choice], [1])
+    diff = tape.gradient(t_out, t_in)
+    assert(0 < tf.reduce_min(tf.slice(diff, [choice], [1])))
+    assert(0 > tf.reduce_max(tf.slice(diff, [1], [choice - 1])))
    EOF
  '';
  # Regression test for #77626 removed because not more `tensorflow.contrib`.
--- a/pkgs/development/python-modules/tensorflow/protobuf_python.patch
+++ b/pkgs/development/python-modules/tensorflow/protobuf_python.patch
@ -0,0 +1,19 @@
+diff -Naurd x/tensorflow/workspace2.bzl y/tensorflow/workspace2.bzl
+--- x/tensorflow/workspace2.bzl	2023-09-16 20:44:02.266422132 +0000
+++ y/tensorflow/workspace2.bzl	2023-09-16 20:50:40.370998305 +0000
+@@ -469,6 +469,7 @@
+         system_link_files = {
+             "//third_party/systemlibs:protobuf.bzl": "protobuf.bzl",
+             "//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl",
+            "//third_party/systemlibs:protobuf.python.BUILD": "python/BUILD",
+         },
+         urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"),
+     )
+diff -Naurd x/third_party/systemlibs/protobuf.python.BUILD y/third_party/systemlibs/protobuf.python.BUILD
+--- x/third_party/systemlibs/protobuf.python.BUILD	1970-01-01 00:00:00.000000000 +0000
+++ y/third_party/systemlibs/protobuf.python.BUILD	2023-09-16 20:49:12.514890584 +0000
+@@ -0,0 +1,4 @@
+cc_library(
+    name = "proto_api",
+    visibility = ["//visibility:public"]
+)
--- a/pkgs/development/python-modules/tensorflow/pybind11_protobuf_newer_version.patch
+++ b/pkgs/development/python-modules/tensorflow/pybind11_protobuf_newer_version.patch
@ -0,0 +1,16 @@
+diff -Naurd x/tensorflow/workspace2.bzl y/tensorflow/workspace2.bzl
+--- x/tensorflow/workspace2.bzl	2023-09-21 12:55:06.104407343 +0000
+++ y/tensorflow/workspace2.bzl	2023-09-21 12:55:39.732453203 +0000
+@@ -872,9 +872,9 @@
+ 
+     tf_http_archive(
+         name = "pybind11_protobuf",
+-        urls = tf_mirror_urls("https://github.com/pybind/pybind11_protobuf/archive/80f3440cd8fee124e077e2e47a8a17b78b451363.zip"),
+-        sha256 = "c7ab64b1ccf9a678694a89035a8c865a693e4e872803778f91f0965c2f281d78",
+-        strip_prefix = "pybind11_protobuf-80f3440cd8fee124e077e2e47a8a17b78b451363",
+        urls = tf_mirror_urls("https://github.com/pybind/pybind11_protobuf/archive/c8cc30e2495309e3499b7d76033446236d21c837.zip"),
+        sha256 = "0663f73b34c0b4af55003edbb066e62aceff99bfcf12e47ea9a15d6013d81413",
+        strip_prefix = "pybind11_protobuf-c8cc30e2495309e3499b7d76033446236d21c837",
+         patch_file = [
+             "//third_party/pybind11_protobuf:remove_license.patch",
+             "//third_party/pybind11_protobuf:python_runtime_dep.patch",
--- a/pkgs/development/python-modules/tensorflow/pybind11_protobuf_python_runtime_dep.patch
+++ b/pkgs/development/python-modules/tensorflow/pybind11_protobuf_python_runtime_dep.patch
@ -0,0 +1,30 @@
+diff -Naurd x/tensorflow/workspace2.bzl y/tensorflow/workspace2.bzl
+--- x/tensorflow/workspace2.bzl	2023-09-17 22:22:22.029075202 +0000
+++ y/tensorflow/workspace2.bzl	2023-09-17 22:25:27.501289587 +0000
+@@ -876,7 +876,10 @@
+         urls = tf_mirror_urls("https://github.com/pybind/pybind11_protobuf/archive/80f3440cd8fee124e077e2e47a8a17b78b451363.zip"),
+         sha256 = "c7ab64b1ccf9a678694a89035a8c865a693e4e872803778f91f0965c2f281d78",
+         strip_prefix = "pybind11_protobuf-80f3440cd8fee124e077e2e47a8a17b78b451363",
+-        patch_file = ["//third_party/pybind11_protobuf:remove_license.patch"],
+        patch_file = [
+            "//third_party/pybind11_protobuf:remove_license.patch",
+            "//third_party/pybind11_protobuf:python_runtime_dep.patch",
+	],
+     )
+ 
+     tf_http_archive(
+diff -Naurd x/third_party/pybind11_protobuf/python_runtime_dep.patch y/third_party/pybind11_protobuf/python_runtime_dep.patch
+--- x/third_party/pybind11_protobuf/python_runtime_dep.patch	1970-01-01 00:00:00.000000000 +0000
+++ y/third_party/pybind11_protobuf/python_runtime_dep.patch	2023-09-17 22:23:07.849128180 +0000
+@@ -0,0 +1,11 @@
+diff -Naurd x/pybind11_protobuf/BUILD y/pybind11_protobuf/BUILD
+--- x/pybind11_protobuf/BUILD	2023-09-17 22:17:19.932725814 +0000
++++ y/pybind11_protobuf/BUILD	2023-09-17 22:20:44.056961932 +0000
+@@ -86,6 +86,7 @@
+         "//visibility:private",
+     ],
+     deps = [
++        "@org_tensorflow//third_party/python_runtime:headers",
+         "@com_google_absl//absl/container:flat_hash_map",
+         "@com_google_absl//absl/container:flat_hash_set",
+         "@com_google_absl//absl/meta:type_traits",
--- a/pkgs/top-level/python-packages.nix
+++ b/pkgs/top-level/python-packages.nix
@ -13689,16 +13689,62 @@ self: super: with self; {
    inherit (pkgs.config) cudaSupport;
  };

-  tensorflow-build = callPackage ../development/python-modules/tensorflow {
+  tensorflow-build = let
+    compat = rec {
+      protobufTF = pkgs.protobuf3_21.override {
+        abseil-cpp = pkgs.abseil-cpp;
+      };
+      grpcTF = (pkgs.grpc.overrideAttrs (
+        oldAttrs: rec {
+          # nvcc fails on recent grpc versions, so we use the latest patch level
+          #  of the grpc version bundled by upstream tensorflow to allow CUDA
+          #  support
+          version = "1.27.3";
+          src = pkgs.fetchFromGitHub {
+            owner = "grpc";
+            repo = "grpc";
+            rev = "v${version}";
+            hash = "sha256-PpiOT4ZJe1uMp5j+ReQulC9jpT0xoR2sAl6vRYKA0AA=";
+            fetchSubmodules = true;
+          };
+          patches = [ ];
+          postPatch = ''
+            sed -i "s/-std=c++11/-std=c++17/" CMakeLists.txt
+            echo "set(CMAKE_CXX_STANDARD 17)" >> CMakeLists.txt
+          '';
+        })
+      ).override {
+        protobuf = protobufTF;
+      };
+      protobuf-pythonTF = self.protobuf.override {
+        protobuf = protobufTF;
+      };
+      grpcioTF = self.grpcio.override {
+        protobuf = protobufTF;
+        grpc = grpcTF;
+      };
+      tensorboard-plugin-profileTF = self.tensorboard-plugin-profile.override {
+        protobuf = protobuf-pythonTF;
+      };
+      tensorboardTF = self.tensorboard.override {
+        grpcio = grpcioTF;
+        protobuf = protobuf-pythonTF;
+        tensorboard-plugin-profile = tensorboard-plugin-profileTF;
+      };
+    };
+  in
+  callPackage ../development/python-modules/tensorflow {
    inherit (pkgs.darwin) cctools;
    inherit (pkgs.config) cudaSupport;
    inherit (self.tensorflow-bin) cudaPackages;
    inherit (pkgs.darwin.apple_sdk.frameworks) Foundation Security;
    flatbuffers-core = pkgs.flatbuffers;
    flatbuffers-python = self.flatbuffers;
-    protobuf-core = pkgs.protobuf;
-    protobuf-python = self.protobuf;
-    lmdb-core = pkgs.lmdb;
+    protobuf-core = compat.protobufTF;
+    protobuf-python = compat.protobuf-pythonTF;
+    grpc = compat.grpcTF;
+    grpcio = compat.grpcioTF;
+    tensorboard = compat.tensorboardTF;
  };

  tensorflow-datasets = callPackage ../development/python-modules/tensorflow-datasets { };