Upgrade to 2.18 (still failing)

tensorflow · karllessard · May 23, 2025 · Oct 18, 2024 · Mar 21, 2025 · Apr 11, 2025
commit 118a96cc9a6bc8ab9a836eb0279e442ceff1838f
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -82,7 +82,7 @@ To upgrade the version of TensorFlow that is embedded within TensorFlow Java, pl
 3. Update `urls`, `sha256` and `strip_prefix` fields of the `org_tensorflow` archive rule in Bazel [workspace](https://github.com/tensorflow/java/blob/master/tensorflow-core/tensorflow-core-native/WORKSPACE#L19)
 4. Extract the archive in a temporary folder
 5. Copy the content of `tensorflow-x.x.x/.bazelrc` file to `tensorflow-core/tensorflow-core-native/tensorflow.bazelrc` under TensorFlow Java source tree
-6. Copy the content of `tensorflow-x.x.x/WORKSPACE` after the "###### Copy content of..." notice if `tensorflow-core/tensorflow-core-native/WORKSPACE`, read notice for more details
+6. Copy the content of `tensorflow-x.x.x/WORKSPACE` after the "###### Copy content of..." notice to `tensorflow-core/tensorflow-core-native/WORKSPACE`, read notice for more details
 7. Copy the content of `tensorflow-x.x.x/.bazelversion` file to `tensorflow-core/tensorflow-core-native/.bazelversion`
 8. Validate that options in `tensorflow-core/tensorflow-core-native/.bazelrc` are still accurate or update them accordingly
 9. Update URLs of existing TensorFlow binaries in the `tensorflow-core/tensorflow-core-native/scripts/dist_download` script

diff --git a/tensorflow-core/tensorflow-core-api/scripts/test_download.sh b/tensorflow-core/tensorflow-core-api/scripts/test_download.sh
@@ -5,13 +5,13 @@ DOWNLOAD_FOLDER="$1"
 
 case ${PLATFORM:-} in
   'linux-x86_64')
-    TEXT_WHEEL_URL='https://files.pythonhosted.org/packages/43/dd/8f03331107b76e63313d2089ddfbd13f15e51fb8ed73517cdd0ab3341928/tensorflow-2.16.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl'
+    TEXT_WHEEL_URL='https://files.pythonhosted.org/packages/f3/73/3a906feb0d71d9353c6fb2363d4052856cc6eff5a78a097b1a6002d4e908/tensorflow_text-2.18.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl'
     ;;
-  'macosx-x86_64')
-    TEXT_WHEEL_URL='https://files.pythonhosted.org/packages/6d/69/9999c2d9e8a3b08dfcfc7e9259a05fb1da5f700936091d2eb4a7985c2776/tensorflow-2.16.2-cp311-cp311-macosx_10_15_x86_64.whl'
+  'linux-arm64')
+    TEXT_WHEEL_URL='https://files.pythonhosted.org/packages/8a/9a/ebba9f6274f8b51e5fe1ac2411b8b6bf680a32d10bd6e9c54be1faeec062/tensorflow_text-2.18.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl'
     ;;
   'macosx-arm64')
-    TEXT_WHEEL_URL='https://files.pythonhosted.org/packages/e7/0d/20b259aadf5f98bad45d55dcd3a7e2690058bb4bc1188dd9e36ab9bdd2ec/tensorflow_text-2.18.0rc0-cp310-cp310-macosx_11_0_arm64.whl'
+    TEXT_WHEEL_URL='https://files.pythonhosted.org/packages/18/b6/8ad233edb0732847db1da538cea941dcccc42f59304ff6fb449676e6dd5a/tensorflow_text-2.18.1-cp311-cp311-macosx_11_0_arm64.whl'
     ;;
   *)
     echo "TensorFlow Text distribution for ${PLATFORM} is not supported for download"

diff --git a/tensorflow-core/tensorflow-core-native/WORKSPACE b/tensorflow-core/tensorflow-core-native/WORKSPACE
@@ -18,14 +18,16 @@ http_archive(
         "find tensorflow third_party/xla/third_party/tsl -name \\*.proto | xargs sed -i.bak 's/^package tensorflow\\([^;]*\\).*$/package tensorflow\\1;\\noption java_package = \"org.tensorflow.proto\\1\";/'",
     ],
     urls = [
-       "https://github.com/tensorflow/tensorflow/archive/refs/tags/v2.18.0-rc2.tar.gz",
+       "https://github.com/tensorflow/tensorflow/archive/refs/tags/v2.18.0.tar.gz",
     ],
-    sha256 = "ed371d42f69b9029175b0a70667c4c65d02a269887353520d5cfca5ce8eb7855",
-    strip_prefix = "tensorflow-2.18.0-rc2"
+    sha256 = "d7876f4bb0235cac60eb6316392a7c48676729860da1ab659fb440379ad5186d",
+    strip_prefix = "tensorflow-2.18.0"
 )
 
 ##### Copy content of tensorflow/WORKSPACE here (make sure to change references of default package "//" to "@org_tensorflow//")
 
+# buildifier: disable=load-on-top
+
 # We must initialize hermetic python first.
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
@@ -138,4 +140,4 @@ load(
     "nccl_configure",
 )
 
-nccl_configure(name = "local_config_nccl")
+nccl_configure(name = "local_config_nccl")
diff --git a/tensorflow-core/tensorflow-core-native/scripts/dist_download.sh b/tensorflow-core/tensorflow-core-native/scripts/dist_download.sh
@@ -5,23 +5,20 @@ DOWNLOAD_FOLDER="$1"
 
 case ${PLATFORM:-} in
   'linux-x86_64')
-    WHEEL_URL='https://files.pythonhosted.org/packages/c6/d9/f2ff325194b8e8acb6b69f303c838b0486f41b8028ec42261f2eb037a031/tensorflow_cpu-2.16.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl'
+    WHEEL_URL='https://files.pythonhosted.org/packages/aa/1d/032a9d40762895e51cad06f382135c14d16487a0ad9dcc65aae5bd89c968/tensorflow_cpu-2.18.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl'
     ;;
   'linux-x86_64-gpu')
-    WHEEL_URL='https://files.pythonhosted.org/packages/43/dd/8f03331107b76e63313d2089ddfbd13f15e51fb8ed73517cdd0ab3341928/tensorflow-2.16.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl'
+    WHEEL_URL='https://files.pythonhosted.org/packages/84/76/c55967ac9968ddaede25a4dce37aba37e9030656f02c12676151ce1b6f22/tensorflow-2.18.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl'
     ;;
-  'macosx-x86_64')
-    WHEEL_URL='https://files.pythonhosted.org/packages/6d/69/9999c2d9e8a3b08dfcfc7e9259a05fb1da5f700936091d2eb4a7985c2776/tensorflow-2.16.2-cp311-cp311-macosx_10_15_x86_64.whl'
+  'linux-arm64')
+    WHEEL_URL='https://files.pythonhosted.org/packages/56/e4/55aaac2b15af4dad079e5af329a79d961e5206589d0e02b1e8da221472ed/tensorflow-2.18.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl'
     ;;
   'macosx-arm64')
-    WHEEL_URL='https://files.pythonhosted.org/packages/20/91/4358b8f5c83ff0dff679fdcdde0f7d6020dd47ef0c00d3e815ec3ceae426/tensorflow-2.18.0rc2-cp312-cp312-macosx_12_0_arm64.whl'
+    WHEEL_URL='https://files.pythonhosted.org/packages/26/08/556c4159675c1a30e077ec2a942eeeb81b457cc35c247a5b4a59a1274f05/tensorflow-2.18.0-cp311-cp311-macosx_12_0_arm64.whl'
     ;;
   'windows-x86_64')
-    WHEEL_URL='https://files.pythonhosted.org/packages/46/87/c3e4e9fe7c630f38a6984afdd1d4ed531ef9c74dc66b86f46f6bdd89d608/tensorflow_intel-2.16.2-cp311-cp311-win_amd64.whl'
-    CLIB_URL='https://storage.googleapis.com/tensorflow/versions/2.16.2/libtensorflow-cpu-windows-x86_64.zip'
-    ;;
-  'linux-arm64')
-    WHEEL_URL='https://files.pythonhosted.org/packages/b5/01/c03e98c8e97d151d9ce075fae210f838832eb53d8aa55669d384cb72925b/tensorflow-2.16.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl'
+    WHEEL_URL='https://files.pythonhosted.org/packages/76/ad/fa6c508a15ff79cb5409294c293388e0999b7d480f84b65e4287277434fe/tensorflow_intel-2.18.0-cp311-cp311-win_amd64.whl'
+    CLIB_URL='https://storage.googleapis.com/tensorflow/versions/2.18.0/libtensorflow-cpu-windows-x86_64.zip'
     ;;
   *)
     echo "TensorFlow distribution for ${PLATFORM} is not supported for download"

diff --git a/.../tensorflow-core-native/src/gen/java/org/tensorflow/internal/c_api/global/tensorflow.java b/.../tensorflow-core-native/src/gen/java/org/tensorflow/internal/c_api/global/tensorflow.java
@@ -274,9 +274,9 @@ public static native void TF_TString_Copy(TF_TString dst, String src,
 // #endif  // TENSORFLOW_TSL_PLATFORM_CTSTRING_H_
 
 
-// Parsed from tsl/c/tsl_status.h
+// Parsed from tsl/platform/status.h
 
-/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -291,67 +291,130 @@ public static native void TF_TString_Copy(TF_TString dst, String src,
 limitations under the License.
 ==============================================================================*/
 
-// #ifndef TENSORFLOW_TSL_C_TSL_STATUS_H_
-// #define TENSORFLOW_TSL_C_TSL_STATUS_H_
+// #ifndef TENSORFLOW_TSL_PLATFORM_STATUS_H_
+// #define TENSORFLOW_TSL_PLATFORM_STATUS_H_
 
-// #ifdef __cplusplus
+// #include <functional>
+// #include <iosfwd>
+// #include <memory>
+// #include <set>
+// #include <string>
+// #include <unordered_map>
+// #include <utility>
+// #include <vector>
+
+// #include "absl/base/attributes.h"
+// #include "absl/base/macros.h"
+// #include "absl/functional/function_ref.h"
+// #include "absl/status/status.h"
+// #include "absl/strings/cord.h"
+// #include "absl/strings/string_view.h"
+// #include "absl/types/optional.h"
+// #include "tsl/platform/logging.h"
+// #include "tsl/platform/macros.h"
+// #include "tsl/platform/platform.h"
+// #include "tsl/platform/stack_frame.h"
+// #include "tsl/platform/types.h"
+// #include "tsl/protobuf/error_codes.pb.h"
+
+// Include appropriate platform-dependent parts of status.
+// #if defined(PLATFORM_GOOGLE)
+// #include "tsl/platform/google/status.h"  // IWYU pragma: export
+// #else
+// #include "tsl/platform/default/status.h"  // IWYU pragma: export
 // #endif
 
-// --------------------------------------------------------------------------
-// TSL_Code holds an error code.  The enum values here are identical to
-// corresponding values in error_codes.proto.
-/** enum TSL_Code */
-public static final int
-  TSL_OK = 0,
-  TSL_CANCELLED = 1,
-  TSL_UNKNOWN = 2,
-  TSL_INVALID_ARGUMENT = 3,
-  TSL_DEADLINE_EXCEEDED = 4,
-  TSL_NOT_FOUND = 5,
-  TSL_ALREADY_EXISTS = 6,
-  TSL_PERMISSION_DENIED = 7,
-  TSL_UNAUTHENTICATED = 16,
-  TSL_RESOURCE_EXHAUSTED = 8,
-  TSL_FAILED_PRECONDITION = 9,
-  TSL_ABORTED = 10,
-  TSL_OUT_OF_RANGE = 11,
-  TSL_UNIMPLEMENTED = 12,
-  TSL_INTERNAL = 13,
-  TSL_UNAVAILABLE = 14,
-  TSL_DATA_LOSS = 15;
+// TODO: b/323943471 - This macro should eventually be provided by Abseil.
+// #ifndef ABSL_DEPRECATE_AND_INLINE
+// #define ABSL_DEPRECATE_AND_INLINE()
+// #endif
 
-// --------------------------------------------------------------------------
+// Since April 2023, tensorflow::Status is an alias to absl::Status. The first
+// TF release including this change will be TF 2.14 (the latest release in
+// April 2023 is 2.13).
+// At the same time `tsl::errors::Code` aliases `absl::StatusCode`.
+//
+// Here is a set of correspondences:
+// - Use `absl::OkStatus()` instead of `tsl::OkStatus()`.
+  // namespace errors
+  // namespace error
+  // namespace tsl
+
+// Transparent comparison between tensorflow::error::Code protobuf enum and
+// absl::Status.
+//
+// The longer term objective is to delete these when we have done the transition
+// to absl::Status.
+@Namespace("tensorflow::error") public static native @Cast("bool") @Name("operator ==") boolean equals(@Const @ByRef Code c1,
+                       @Const @ByRef StatusCode c2);
+
+@Namespace("tensorflow::error") public static native @Cast("bool") @Name("operator !=") boolean notEquals(@Const @ByRef Code c1,
+                       @Const @ByRef StatusCode c2);
+  // namespace tensorflow::error
+@Namespace("absl") public static native @Cast("bool") @Name("operator ==") boolean equals(@Const @ByRef StatusCode c1,
+                       @Const @ByRef Code c2);
+
+@Namespace("absl") public static native @Cast("bool") @Name("operator !=") boolean notEquals(@Const @ByRef StatusCode c1,
+                       @Const @ByRef Code c2);
+  // namespace absl
+
+// OkStatus()
+//
+// Returns an OK status, equivalent to a default constructed instance. Prefer
+// usage of `OkStatus()` when constructing such an OK status.
+@Namespace("tsl") public static native @ByVal Status OkStatus();
+
+@Namespace("tsl") public static native @ByVal Status FromAbslStatus(@Const @ByRef Status s);
+@Namespace("tsl") public static native @ByVal Status ToAbslStatus(@Const @ByRef Status s);
+
+// Given `Status.message()` does not guarantee to be always backed by a
+// null-terminated string, we have this utility function when it's needed for
+// the Tensorflow C-API.
+// A more robust API would be to get both a `char*` of the beginning of the
+// string, plus the size (see e.g. `XlaCustomCallStatusSetFailure`).
+// NB: This Windows-only implementation is exists only to avoid a linker error.
+// Remove if this is resolved.
+// #ifdef _WIN32
+@Namespace("tsl") public static native @Cast("const char*") BytePointer NullTerminatedMessage(@Const @ByRef Status status);
+// #else
+// #endif
 
-// Return a new status object.
+// TODO(b/197552541) Move this namespace to errors.h.
 
-// Delete a previously created status object.
+@Namespace("tsl::errors") public static native void SetStackTrace(@ByRef Status status, @StdVector StackFrame stack_trace);
 
-// Record <code, msg> in *s.  Any previous information is lost.
-// A common use is to clear a status: TSL_SetStatus(s, TSL_OK, "");
+@Namespace("tsl::errors") public static native @StdVector StackFrame GetStackTrace(@Const @ByRef Status status);
+  // namespace errors
 
-// Record <key, value> as a payload in *s. The previous payload having the
-// same key (if any) is overwritten. Payload will not be added if the Status
-// is OK.
+// Helper class to manage multiple child status values.
 
-// Iterates over the stored payloads and calls the `visitor(key, value)`
-// callable for each one. `key` and `value` is only usable during the callback.
-// `capture` will be passed to the callback without modification.
+@Namespace("tsl") public static native string TfCheckOpHelperOutOfLine(@Const @ByRef Status v,
+                                               @Cast("const char*") BytePointer msg);
+@Namespace("tsl") public static native string TfCheckOpHelperOutOfLine(@Const @ByRef Status v,
+                                               String msg);
 
-// Convert from an I/O error code (e.g., errno) to a TSL_Status value.
-// Any previous information is lost. Prefer to use this instead of TSL_SetStatus
-// when the error comes from I/O operations.
+@Namespace("tsl") public static native string TfCheckOpHelper(@ByVal Status v, @Cast("const char*") BytePointer msg);
+@Namespace("tsl") public static native string TfCheckOpHelper(@ByVal Status v, String msg);
 
-// Return the code record in *s.
+// #define TF_DO_CHECK_OK(val, level)
+//   while (auto* _result = ::tsl::TfCheckOpHelper(val, #val))
+//   LOG(level) << *(_result)
 
-// Return a pointer to the (null-terminated) error message in *s.  The
-// return value points to memory that is only usable until the next
-// mutation to *s.  Always returns an empty string if TSL_GetCode(s) is
-// TSL_OK.
+// #define TF_CHECK_OK(val) TF_DO_CHECK_OK(val, FATAL)
+// #define TF_QCHECK_OK(val) TF_DO_CHECK_OK(val, QFATAL)
 
-// #ifdef __cplusplus /* end extern "C" */
+// DEBUG only version of TF_CHECK_OK.  Compiler still parses 'val' even in opt
+// mode.
+// #ifndef NDEBUG
+// #define TF_DCHECK_OK(val) TF_CHECK_OK(val)
+// #else
+// #define TF_DCHECK_OK(val)
+//   while (false && (::tsl::OkStatus() == (val))) LOG(FATAL)
 // #endif
 
-// #endif  // TENSORFLOW_TSL_C_TSL_STATUS_H_
+  // namespace tsl
+
+// #endif  // TENSORFLOW_TSL_PLATFORM_STATUS_H_
 
 
 // Parsed from tensorflow/c/c_api_macros.h
@@ -507,7 +570,7 @@ public static native void TF_TString_Copy(TF_TString dst, String src,
 // #define TENSORFLOW_C_TF_STATUS_H_
 
 // #include "tensorflow/c/c_api_macros.h"
-// #include "tsl/c/tsl_status.h"
+// #include "xla/tsl/c/tsl_status.h"
 
 // #ifdef __cplusplus
 // Targeting ../TF_Status.java
@@ -547,9 +610,9 @@ public static native void TF_TString_Copy(TF_TString dst, String src,
 
 // Record <code, msg> in *s.  Any previous information is lost.
 // A common use is to clear a status: TF_SetStatus(s, TF_OK, "");
-public static native void TF_SetStatus(TF_Status s, @Cast("TF_Code") int code,
+public static native void TF_SetStatus(TF_Status s, @ByVal @Cast("TF_Code*") TSL_Code code,
                                         @Cast("const char*") BytePointer msg);
-public static native void TF_SetStatus(TF_Status s, @Cast("TF_Code") int code,
+public static native void TF_SetStatus(TF_Status s, @ByVal @Cast("TF_Code*") TSL_Code code,
                                         String msg);
 
 // Record <key, value> as a payload in *s. The previous payload having the
@@ -575,7 +638,7 @@ public static native void TF_SetStatusFromIOError(TF_Status s, int error_code,
                                                    String context);
 
 // Return the code record in *s.
-public static native @Cast("TF_Code") int TF_GetCode(@Const TF_Status s);
+public static native @ByVal @Cast("TF_Code*") TSL_Code TF_GetCode(@Const TF_Status s);
 
 // Return a pointer to the (null-terminated) error message in *s.  The
 // return value points to memory that is only usable until the next
@@ -689,6 +752,13 @@ public static native TF_Tensor TF_NewTensor(
     Deallocator_Pointer_long_Pointer deallocator,
     Pointer deallocator_arg);
 
+// Returns the alignment, in bytes, required for allocating aligned tensors.
+//
+// This can be used in combination with TF_NewTensor to manually manage
+// memory while ensuring the resulting tensors satisfy TensorFlow's
+// memory alignment preferences.
+public static native @Cast("size_t") long TF_TensorDefaultAlignment();
+
 // Allocate and return a new Tensor.
 //
 // This function is an alternative to TF_NewTensor and should be used when
@@ -4367,15 +4437,17 @@ public static native void TFE_ContextUpdateServerDefWithTimeout(
 // This API is for experimental usage and may be subject to change.
 public static native void TFE_ContextSetServerDefWithTimeout(
     TFE_Context ctx, int keep_alive_secs, @Const Pointer proto, @Cast("size_t") long proto_len,
-    @Cast("int64_t") long init_timeout_in_ms, TF_Status status);
+    @Cast("int64_t") long init_timeout_in_ms, TF_Status status,
+    @Cast("bool") boolean clear_existing_contexts);
 
 // Set server def with retries and timeout. This is helpful for fault-tolerant
 // initial connection in high-preemption environments, such as
 // ParameterServerStrategy training.
 // This API is for experimental usage and may be subject to change.
 public static native void TFE_ContextSetServerDefWithTimeoutAndRetries(
     TFE_Context ctx, int keep_alive_secs, @Const Pointer proto, @Cast("size_t") long proto_len,
-    @Cast("int64_t") long init_timeout_in_ms, int retries, TF_Status status);
+    @Cast("int64_t") long init_timeout_in_ms, int retries, TF_Status status,
+    @Cast("bool") boolean clear_existing_contexts);
 
 // Checks whether a remote worker is alive or not. This will return true even if
 // the context doesn't exist on the remote worker.