Skip to content

Commit 72c33b8

Browse files
karllessardkarllessard
committed
Fix proto packaging
1 parent ec2aa99 commit 72c33b8

400 files changed

Lines changed: 163 additions & 367964 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎CONTRIBUTING.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@ To upgrade the version of TensorFlow that is embedded within TensorFlow Java, pl
119119
3. Update `urls`, `sha256` and `strip_prefix` fields of the `org_tensorflow` archive rule in Bazel [workspace](https://github.com/tensorflow/java/blob/master/tensorflow-core/tensorflow-core-api/WORKSPACE#L19)
120120
4. Extract the archive in a temporary folder
121121
5. Copy the content of `tensorflow-x.x.x/.bazelrc` file to `tensorflow-core/tensorflow-core-api/tensorflow.bazelrc` under TensorFlow Java source tree
122+
6. Copy the content of `tensorflow-x.x.x/WORKSPACE` after the "###### Copy content of..." notice if `tensorflow-core/tensorflow-core-api/WORKSPACE`, read notice for more details
122123

123124
If the version of `tensorflow-x.x.x/.bazelversion` is different than the one found in `tensorflow-core/tensorflow-core-api/.bazelversion`
124125

‎tensorflow-core/tensorflow-core-api/.bazelversion‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
5.3.0
1+
6.1.0

‎tensorflow-core/tensorflow-core-api/BUILD‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,6 @@ tfjava_cc_binary(
5454
java_proto_library(
5555
name = "java_proto_gen_sources",
5656
deps = [
57-
clean_dep("//tensorflow/tsl/protobuf:protos_all"),
58-
clean_dep("//tensorflow/tsl/profiler/protobuf:protos_all"),
5957
clean_dep("//tensorflow/core:protos_all")
6058
]
6159
)

‎tensorflow-core/tensorflow-core-api/WORKSPACE‎

Lines changed: 76 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,8 @@ http_archive(
1717
patch_tool = "patch",
1818
patch_args = ["-p1"],
1919
patch_cmds = [
20-
# "grep -rl 'syntax = "proto3";' tensorflow | xargs sed -i.bak 's/^\\(.* java_package = \"org\\.tensorflow\\.\\)\\(.*\"\\)/\\1proto.\\2'/",
21-
"find tensorflow -name \\*.proto | xargs sed -i.bak 's/^\\(.* java_package = \"org\\.tensorflow\\.\\)\\(.*\"\\)/\\1proto.\\2/'",
22-
"find tensorflow -name \\*.proto | xargs grep -L 'option java_package' | xargs sed -i.bak 's/^\\(syntax = \"proto3\";\\)/\\1\\roption java_package = \"org.tensorflow.proto\";/'",
20+
"find tensorflow -name \\*.proto | xargs sed -i.bak '/^option java_package/d'",
21+
"find tensorflow -name \\*.proto | xargs sed -i.bak 's/^package tensorflow\\([^;\\s]*\\).*$/package tensorflow\\1;\\roption java_package = \"org.tensorflow.proto\\1\";/'",
2322
],
2423
urls = [
2524
"https://github.com/tensorflow/tensorflow/archive/refs/tags/v2.14.0.tar.gz",
@@ -28,9 +27,79 @@ http_archive(
2827
strip_prefix = "tensorflow-2.14.0"
2928
)
3029

31-
# START: Upstream TensorFlow dependencies
32-
# TensorFlow build depends on these dependencies.
33-
# Needs to be in-sync with TensorFlow sources.
30+
##### Copy content of tensorflow/WORKSPACE here (make sure to change references of default package "//" to "@org_tensorflow//")
31+
32+
# We must initialize hermetic python first.
33+
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
34+
35+
http_archive(
36+
name = "bazel_skylib",
37+
sha256 = "74d544d96f4a5bb630d465ca8bbcfe231e3594e5aae57e1edbf17a6eb3ca2506",
38+
urls = [
39+
"https://storage.googleapis.com/mirror.tensorflow.org/github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz",
40+
"https://github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz",
41+
],
42+
)
43+
44+
http_archive(
45+
name = "rules_python",
46+
sha256 = "84aec9e21cc56fbc7f1335035a71c850d1b9b5cc6ff497306f84cced9a769841",
47+
strip_prefix = "rules_python-0.23.1",
48+
url = "https://github.com/bazelbuild/rules_python/releases/download/0.23.1/rules_python-0.23.1.tar.gz",
49+
)
50+
51+
load("@rules_python//python:repositories.bzl", "python_register_toolchains")
52+
load(
53+
"@org_tensorflow//tensorflow/tools/toolchains/python:python_repo.bzl",
54+
"python_repository",
55+
)
56+
57+
python_repository(name = "python_version_repo")
58+
59+
load("@python_version_repo//:py_version.bzl", "HERMETIC_PYTHON_VERSION")
60+
61+
python_register_toolchains(
62+
name = "python",
63+
ignore_root_user_error = True,
64+
python_version = HERMETIC_PYTHON_VERSION,
65+
)
66+
67+
load("@python//:defs.bzl", "interpreter")
68+
load("@rules_python//python:pip.bzl", "package_annotation", "pip_parse")
69+
70+
NUMPY_ANNOTATIONS = {
71+
"numpy": package_annotation(
72+
additive_build_content = """\
73+
filegroup(
74+
name = "includes",
75+
srcs = glob(["site-packages/numpy/core/include/**/*.h"]),
76+
)
77+
cc_library(
78+
name = "numpy_headers",
79+
hdrs = [":includes"],
80+
strip_include_prefix="site-packages/numpy/core/include/",
81+
)
82+
""",
83+
),
84+
}
85+
86+
pip_parse(
87+
name = "pypi",
88+
annotations = NUMPY_ANNOTATIONS,
89+
python_interpreter_target = interpreter,
90+
requirements = "@org_tensorflow//:requirements_lock_" + HERMETIC_PYTHON_VERSION.replace(".", "_") + ".txt",
91+
)
92+
93+
load("@pypi//:requirements.bzl", "install_deps")
94+
95+
install_deps()
96+
97+
# Initialize the TensorFlow repository and all dependencies.
98+
#
99+
# The cascade of load() statements and tf_workspace?() calls works around the
100+
# restriction that load() statements need to be at the top of .bzl files.
101+
# E.g. we can not retrieve a new repository with http_archive and then load()
102+
# a macro from that repository in the same file.
34103
load("@org_tensorflow//tensorflow:workspace3.bzl", "tf_workspace3")
35104

36105
tf_workspace3()
@@ -45,11 +114,4 @@ tf_workspace1()
45114

46115
load("@org_tensorflow//tensorflow:workspace0.bzl", "tf_workspace0")
47116

48-
tf_workspace0()
49-
# END: Upstream TensorFlow dependencies
50-
51-
load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps")
52-
grpc_deps()
53-
54-
load("@upb//bazel:repository_defs.bzl", "bazel_version_repository")
55-
bazel_version_repository(name = "bazel_version")
117+
tf_workspace0()

‎tensorflow-core/tensorflow-core-api/scripts/native_build.sh‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -90,16 +90,16 @@ GEN_RESOURCE_DIR=src/gen/resources
9090
rm -rf $GEN_RESOURCE_DIR
9191
mkdir -p $GEN_RESOURCE_DIR
9292

93+
# Copy generated Java protos from source jars
94+
echo "Copying generated protos"
95+
cd $GEN_SRCS_DIR
96+
find $TENSORFLOW_BIN -name \*-speed-src.jar -exec jar xf {} \;
97+
rm -rf META-INF
98+
9399
# Export op defs
94-
echo "Exporting Ops..."
100+
echo "Exporting Ops"
95101
$BAZEL_BIN/java_op_exporter \
96102
$GEN_RESOURCE_DIR/ops.pb \
97103
$GEN_RESOURCE_DIR/ops.pbtxt \
98104
$BAZEL_SRCS/external/org_tensorflow/tensorflow/core/api_def/base_api \
99105
src/bazel/api_def
100-
101-
# Copy generated Java protos from source jars
102-
echo "Copying generated protos"
103-
cd $GEN_SRCS_DIR
104-
find $TENSORFLOW_BIN -name \*-speed-src.jar -exec jar xf {} \;
105-
rm -rf META-INF

‎tensorflow-core/tensorflow-core-api/src/gen/java/ErrorCodes.java‎

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)