Skip to content

Commit 9e3f293

Browse files
karllessardkarllessard
committed
Move URLs to download script + new CI
1 parent d3d3763 commit 9e3f293

5 files changed

Lines changed: 226 additions & 64 deletions

File tree

‎.github/workflows/build.yml‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
name: CI jobs
2+
on:
3+
push:
4+
branches:
5+
- master
6+
- staging
7+
- r[0-9]+.*
8+
- bazelcism-superclean2000-214 # temporary
9+
pull_request:
10+
branches:
11+
- master
12+
- r[0-9]+.*
13+
types: [opened, reopened, synchronize, labeled, unlabeled]
14+
env:
15+
STAGING_PROFILE_ID: 46f80d0729c92d
16+
GCP_CREDS: ${{ secrets.GCP_CREDS }}
17+
jobs:
18+
check-format:
19+
if: github.event_name == 'pull_request'
20+
runs-on: ubuntu-18.04
21+
steps:
22+
- name: Configure Java
23+
uses: actions/setup-java@v2
24+
with:
25+
distribution: 'adopt'
26+
java-version: '17'
27+
- name: Checkout repository
28+
uses: actions/checkout@v1
29+
- name: Build project
30+
run: |
31+
gcc --version
32+
mvn -version
33+
mvn clean install -Pjdk17 -B -U -e -Dlint.skip=true -Dmaven.test.skip=true
34+
- name: Run format checks
35+
run: |
36+
mvn spotless:check -Pjdk17 -B -U -e
37+
prepare:
38+
runs-on: ubuntu-18.04
39+
outputs:
40+
stagingRepositoryId: ${{ steps.staging.outputs.stagingRepositoryId }}
41+
steps:
42+
- name: Create staging repository
43+
if: github.event_name == 'push' && startsWith(github.ref, 'refs/heads/r')
44+
id: staging
45+
run: |
46+
echo "Creating staging repository with profile $STAGING_PROFILE_ID"
47+
echo "<promoteRequest><data><description>Releasing TF Java - created by CI build</description></data></promoteRequest>" > request.xml
48+
curl -X POST -d @request.xml -s -o response.xml -u ${{ secrets.CI_DEPLOY_USERNAME }}:${{ secrets.CI_DEPLOY_PASSWORD }} -H "Content-Type:application/xml" \
49+
https://oss.sonatype.org/service/local/staging/profiles/$STAGING_PROFILE_ID/start
50+
STAGING_REPOSITORY_ID=`awk -F'[<>]' '/stagedRepositoryId/{print $3}' response.xml`
51+
echo "Staging repository created: $STAGING_REPOSITORY_ID"
52+
echo "::set-output name=stagingRepositoryId::$STAGING_REPOSITORY_ID"
53+
linux-x86_64:
54+
if: github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'CI build')
55+
runs-on: ubuntu-18.04
56+
needs: prepare
57+
strategy:
58+
matrix:
59+
ext: ["", -gpu] #, -mkl, -mkl-gpu]
60+
steps:
61+
- name: Configure Java
62+
uses: actions/setup-java@v2
63+
with:
64+
distribution: 'adopt'
65+
java-version: '11'
66+
- name: Checkout repository
67+
uses: actions/checkout@v1
68+
- name: Build project
69+
run: |
70+
gcc --version
71+
mvn -version
72+
[[ "${{ github.event_name }}" == "push" ]] && MAVEN_PHASE=deploy || MAVEN_PHASE=install
73+
echo "<settings><servers><server><id>ossrh</id><username>${{ secrets.CI_DEPLOY_USERNAME }}</username><password>${{ secrets.CI_DEPLOY_PASSWORD }}</password></server></servers></settings>" > $HOME/.m2/settings.xml
74+
echo Executing Maven $MAVEN_PHASE
75+
mvn clean $MAVEN_PHASE -B -U -e -Djavacpp.platform=linux-x86_64 -Djavacpp.platform.extension=${{ matrix.ext }} -am -DstagingRepositoryId=${{ needs.prepare.outputs.stagingRepositoryId }} "-Dnative.build.flags=$BAZEL_CACHE"
76+
macosx-x86_64:
77+
if: github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'CI build')
78+
runs-on: macos-10.15
79+
needs: prepare
80+
strategy:
81+
matrix:
82+
ext: [""] # , -mkl]
83+
steps:
84+
- name: Configure Java
85+
uses: actions/setup-java@v2
86+
with:
87+
distribution: 'adopt'
88+
java-version: '11'
89+
- name: Checkout repository
90+
uses: actions/checkout@v1
91+
- name: Build project
92+
run: |
93+
clang --version
94+
mvn -version
95+
[[ "${{ github.event_name }}" == "push" ]] && MAVEN_PHASE=deploy || MAVEN_PHASE=install
96+
echo "<settings><servers><server><id>ossrh</id><username>${{ secrets.CI_DEPLOY_USERNAME }}</username><password>${{ secrets.CI_DEPLOY_PASSWORD }}</password></server></servers></settings>" > $HOME/.m2/settings.xml
97+
echo Executing Maven $MAVEN_PHASE
98+
mvn clean $MAVEN_PHASE -B -U -e -Djavacpp.platform=macosx-x86_64 -Djavacpp.platform.extension=${{ matrix.ext }} -am -DstagingRepositoryId=${{ needs.prepare.outputs.stagingRepositoryId }} "-Dnative.build.flags=$BAZEL_CACHE"
99+
windows-x86_64:
100+
if: github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'CI build')
101+
runs-on: windows-2019
102+
needs: prepare
103+
strategy:
104+
matrix:
105+
ext: ["", -gpu] #, -mkl, -mkl-gpu]
106+
steps:
107+
- name: Configure page file
108+
uses: al-cheb/configure-pagefile-action@v1.2
109+
with:
110+
minimum-size: 8GB
111+
maximum-size: 16GB
112+
disk-root: "C:"
113+
- name: Install environment
114+
shell: cmd
115+
run: |
116+
set "PATH=C:\msys64\usr\bin;%PATH%"
117+
echo Removing broken stuff from WSL and MSYS2
118+
rm "C:/WINDOWS/system32/bash.EXE" "C:/msys64/usr/bin/python.exe" "C:/msys64/usr/bin/python3.exe"
119+
python -m pip install numpy six
120+
echo Removing old versions of MSVC that interfere with Bazel
121+
bash.exe -lc "find 'C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/' -iname '14.1*' -exec rm -Rf {} \;"
122+
set "EXT=${{ matrix.ext }}"
123+
echo %JAVA_HOME%
124+
- name: Configure Java
125+
uses: actions/setup-java@v2
126+
with:
127+
distribution: 'adopt'
128+
java-version: '11'
129+
- name: Checkout repository
130+
uses: actions/checkout@v1
131+
- name: Build project
132+
shell: cmd
133+
run: |
134+
call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" amd64
135+
set "PATH=C:\msys64\usr\bin;C:\bazel;C:\Program Files\Git\bin;%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v11.2\bin;%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v11.2\libnvvp;%PATH%"
136+
echo Shorten work paths to prevent Bazel from reaching MAX_PATH limit
137+
set "TEST_TMPDIR=C:\tmp"
138+
set "TMPDIR=C:\tmp"
139+
set "TEMP=C:\tmp"
140+
set "TMP=C:\tmp"
141+
mkdir C:\tmp
142+
bash --version
143+
git --version
144+
cl
145+
call mvn -version
146+
if "${{ github.event_name }}" == "push" (set MAVEN_PHASE=deploy) else (set MAVEN_PHASE=install)
147+
echo ^<settings^>^<servers^>^<server^>^<id^>ossrh^</id^>^<username^>${{ secrets.CI_DEPLOY_USERNAME }}^</username^>^<password^>${{ secrets.CI_DEPLOY_PASSWORD }}^</password^>^</server^>^</servers^>^</settings^> > %USERPROFILE%\.m2\settings.xml
148+
set "SKIP_EXPORT=true"
149+
echo Executing Maven %MAVEN_PHASE%
150+
call mvn clean %MAVEN_PHASE% -B -U -e -Djavacpp.platform=windows-x86_64 -Djavacpp.platform.extension=${{ matrix.ext }} -am -DstagingRepositoryId=${{ needs.prepare.outputs.stagingRepositoryId }} "-Dnative.build.flags=%BAZEL_CACHE%"
151+
if ERRORLEVEL 1 exit /b
152+
deploy:
153+
if: github.event_name == 'push' && contains(github.ref, 'master')
154+
needs: [linux-x86_64, macosx-x86_64, windows-x86_64]
155+
runs-on: ubuntu-18.04
156+
steps:
157+
- name: Configure Java
158+
uses: actions/setup-java@v2
159+
with:
160+
distribution: 'adopt'
161+
java-version: '11'
162+
- name: Checkout repository
163+
uses: actions/checkout@v1
164+
- name: Deploy snapshot artifacts
165+
run: |
166+
echo "<settings><servers><server><id>ossrh</id><username>${{ secrets.CI_DEPLOY_USERNAME }}</username><password>${{ secrets.CI_DEPLOY_PASSWORD }}</password></server></servers></settings>" > settings.xml
167+
bash deploy.sh

‎pom.xml‎

Lines changed: 0 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -48,13 +48,6 @@
4848
<maven.source.skip>true</maven.source.skip>
4949
<gpg.skip>true</gpg.skip>
5050
<spotless.version>2.38.0</spotless.version>
51-
52-
<wheel.download.url/>
53-
<wheel.download.url.linux-x86_64.gpu>https://files.pythonhosted.org/packages/09/63/25e76075081ea98ec48f23929cefee58be0b42212e38074a9ec5c19e838c/tensorflow-2.14.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl</wheel.download.url.linux-x86_64.gpu>
54-
<wheel.download.url.linux-x86_64>https://files.pythonhosted.org/packages/57/bf/03debeeaeca996543478db89c8729a40dcb97a6be811804f4a08c8e1c75a/tensorflow_cpu-2.14.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl</wheel.download.url.linux-x86_64>
55-
<wheel.download.url.macosx-x86_64>https://files.pythonhosted.org/packages/ad/e2/ff29a5e514ec25470b59e5d0134f08f75bd28d6ec6941a22bea6b78cb2fb/tensorflow_cpu-2.14.0-cp311-cp311-macosx_10_15_x86_64.whl</wheel.download.url.macosx-x86_64>
56-
<wheel.download.url.macosx-arm64>https://files.pythonhosted.org/packages/d3/4b/ae9037ea22ba94eb2cf267e991384c3444f3e6142fa49923352b4ab73e14/tensorflow_macos-2.14.0-cp311-cp311-macosx_12_0_arm64.whl</wheel.download.url.macosx-arm64>
57-
<wheel.download.url.windows-x86_64>https://files.pythonhosted.org/packages/ad/6e/1bfe367855dd87467564f7bf9fa14f3b17889988e79598bc37bf18f5ffb6/tensorflow_intel-2.14.0-cp311-cp311-win_amd64.whl</wheel.download.url.windows-x86_64>
5851
</properties>
5952

6053
<repositories>
@@ -318,9 +311,6 @@
318311
<name>!javacpp.platform.extension</name>
319312
</property>
320313
</activation>
321-
<properties>
322-
<wheel.download.url>${wheel.download.url.linux-x86_64}</wheel.download.url>
323-
</properties>
324314
</profile>
325315
<profile>
326316
<id>linux-x86_64-gpu</id>
@@ -334,9 +324,6 @@
334324
<value>-gpu</value>
335325
</property>
336326
</activation>
337-
<properties>
338-
<wheel.download.url>${wheel.download.url.linux-x86_64.gpu}</wheel.download.url>
339-
</properties>
340327
</profile>
341328
<profile>
342329
<id>macosx</id>
@@ -356,9 +343,6 @@
356343
<arch>x86_64</arch>
357344
</os>
358345
</activation>
359-
<properties>
360-
<wheel.download.url>${wheel.download.url.macosx-x86_64}</wheel.download.url>
361-
</properties>
362346
</profile>
363347
<profile>
364348
<id>macosx-arm64</id>
@@ -368,9 +352,6 @@
368352
<arch>aarch64</arch>
369353
</os>
370354
</activation>
371-
<properties>
372-
<wheel.download.url>${wheel.download.url.macosx-arm64}</wheel.download.url>
373-
</properties>
374355
</profile>
375356
<profile>
376357
<id>windowsos</id>
@@ -390,9 +371,6 @@
390371
<arch>x86_64</arch>
391372
</os>
392373
</activation>
393-
<properties>
394-
<wheel.download.url>${wheel.download.url.windows-x86_64}</wheel.download.url>
395-
</properties>
396374
</profile>
397375
<profile>
398376
<id>arm</id>

‎tensorflow-core/tensorflow-core-native/pom.xml‎

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,9 @@
2121
<javacpp.build.skip>false</javacpp.build.skip>
2222
<javacpp.generate.skip>true</javacpp.generate.skip>
2323
<javacpp.clean.skip>true</javacpp.clean.skip>
24-
<wheel.download.skip>false</wheel.download.skip>
25-
<wheel.download.folder>${project.build.directory}/wheel/</wheel.download.folder>
24+
<dist.download.skip>false</dist.download.skip>
25+
<dist.download.url/>
26+
<dist.download.folder>${project.build.directory}/dist/</dist.download.folder>
2627
<target.folder>${project.build.directory}/native/org/tensorflow/internal/c_api/${native.classifier}/</target.folder>
2728
<java.module.name>org.tensorflow.core.native</java.module.name>
2829
</properties>
@@ -69,7 +70,7 @@
6970
<bazel.build.skip>false</bazel.build.skip>
7071
<bazel.clean.skip>false</bazel.clean.skip>
7172
<javacpp.clean.skip>false</javacpp.clean.skip>
72-
<wheel.download.skip>true</wheel.download.skip>
73+
<dist.download.skip>true</dist.download.skip>
7374
</properties>
7475
</profile>
7576

@@ -87,6 +88,7 @@
8788
<javacpp.clean.skip>false</javacpp.clean.skip>
8889
</properties>
8990
</profile>
91+
9092
</profiles>
9193

9294
<build>
@@ -100,7 +102,7 @@
100102
<!--
101103
Run the full native build with Bazel
102104
This will compile TensorFlow from sources to generate the runtime libraries, instead of using those
103-
retrieved from the official Python wheels. It can be useful when someone wants to build a version
105+
retrieved from the official Python distribution. It can be useful when someone wants to build a version
104106
of TensorFlow Java for an architecture not officially supported.
105107
-->
106108
<id>bazel-build</id>
@@ -165,24 +167,25 @@
165167
<execution>
166168
<!--
167169
Download TensorFlow native libraries
168-
This will download the official Python wheel distribution for the active platform, and extract the `tensorflow_cc` library
170+
This will download the official Python distribution for the active platform, and extract the `tensorflow_cc` library
169171
from it so that we can generate the JavaCPP API bindings and distribute it as a JAR. This will be executed only
170172
when not building a full native build.
171173
-->
172-
<id>wheel-download</id>
174+
<id>dist-download</id>
173175
<phase>initialize</phase>
174176
<goals>
175177
<goal>exec</goal>
176178
</goals>
177179
<configuration>
178-
<skip>${wheel.download.skip}</skip> <!-- skipped when full native build is enabled -->
180+
<skip>${dist.download.skip}</skip> <!-- skipped when full native build is enabled -->
179181
<executable>/bin/bash</executable>
180182
<arguments>
181-
<argument>scripts/wheel_download.sh</argument>
182-
<argument>${wheel.download.url}</argument>
183-
<argument>${wheel.download.folder}</argument>
184-
<argument>${os.name}</argument>
183+
<argument>scripts/dist_download.sh</argument>
184+
<argument>${dist.download.folder}</argument>
185185
</arguments>
186+
<environmentVariables>
187+
<PLATFORM>${native.classifier}</PLATFORM>
188+
</environmentVariables>
186189
<workingDirectory>${project.basedir}</workingDirectory>
187190
</configuration>
188191
</execution>
@@ -212,7 +215,7 @@
212215
<classPath>${project.build.outputDirectory}</classPath>
213216
<includePaths>
214217
<includePath>${project.basedir}/</includePath>
215-
<includePath>${wheel.download.folder}/tensorflow/include/</includePath>
218+
<includePath>${dist.download.folder}/tensorflow/include/</includePath>
216219
<includePath>${project.basedir}/target/classes/org/tensorflow/internal/c_api/include/</includePath>
217220
<!-- additional include paths in case of a full native build -->
218221
<includePath>${project.basedir}/bazel-${project.artifactId}/external/org_tensorflow/</includePath>
@@ -222,7 +225,7 @@
222225
<includePath>${project.basedir}/bazel-${project.artifactId}/external/com_google_protobuf/src/</includePath>
223226
</includePaths>
224227
<linkPaths>
225-
<linkPath>${wheel.download.folder}/tensorflow</linkPath>
228+
<linkPath>${dist.download.folder}/tensorflow</linkPath>
226229
<!-- additional link paths in case of a full native build -->
227230
<linkPath>${project.basedir}/bazel-bin/external/llvm_openmp/</linkPath>
228231
<linkPath>${project.basedir}/bazel-bin/external/org_tensorflow/tensorflow/</linkPath>

‎tensorflow-core/tensorflow-core-native/scripts/dist_download.sh‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
#!/bin/bash
2+
set -eu
3+
4+
DOWNLOAD_FOLDER="$1"
5+
DOWNLOADED_FILE="tensorflow.zip" # TODO extract name from wheel url, change .whl to .zip
6+
7+
case ${PLATFORM:-} in
8+
'linux-x86_64')
9+
DOWNLOAD_URL='https://files.pythonhosted.org/packages/57/bf/03debeeaeca996543478db89c8729a40dcb97a6be811804f4a08c8e1c75a/tensorflow_cpu-2.14.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl'
10+
;;
11+
'linux-x86_64-gpu')
12+
DOWNLOAD_URL='https://files.pythonhosted.org/packages/09/63/25e76075081ea98ec48f23929cefee58be0b42212e38074a9ec5c19e838c/tensorflow-2.14.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl'
13+
;;
14+
'macosx-x86_64')
15+
DOWNLOAD_URL='https://files.pythonhosted.org/packages/ad/e2/ff29a5e514ec25470b59e5d0134f08f75bd28d6ec6941a22bea6b78cb2fb/tensorflow_cpu-2.14.0-cp311-cp311-macosx_10_15_x86_64.whl'
16+
;;
17+
'macosx-arm64')
18+
DOWNLOAD_URL='https://files.pythonhosted.org/packages/d3/4b/ae9037ea22ba94eb2cf267e991384c3444f3e6142fa49923352b4ab73e14/tensorflow_macos-2.14.0-cp311-cp311-macosx_12_0_arm64.whl'
19+
;;
20+
'windows')
21+
DOWNLOAD_URL='https://files.pythonhosted.org/packages/ad/6e/1bfe367855dd87467564f7bf9fa14f3b17889988e79598bc37bf18f5ffb6/tensorflow_intel-2.14.0-cp311-cp311-win_amd64.whl'
22+
;;
23+
*)
24+
echo "TensorFlow distribution for ${PLATFORM} is not supported for download"
25+
return 1;
26+
esac
27+
28+
mkdir -p $DOWNLOAD_FOLDER
29+
30+
if [ ! -f $DOWNLOAD_FOLDER/$DOWNLOADED_FILE ]; then
31+
curl -L $DOWNLOAD_URL --output-dir $DOWNLOAD_FOLDER --output $DOWNLOADED_FILE
32+
fi
33+
34+
unzip -q -u -d $DOWNLOAD_FOLDER $DOWNLOAD_FOLDER/$DOWNLOADED_FILE
35+
36+
if [[ "$PLATFORM" =~ "linux" ]]; then
37+
ln -fs $DOWNLOAD_FOLDER/tensorflow/libtensorflow_cc.so.2 $DOWNLOAD_FOLDER/tensorflow/libtensorflow_cc.so
38+
ln -fs $DOWNLOAD_FOLDER/tensorflow/libtensorflow_framework.so.2 $DOWNLOAD_FOLDER/tensorflow/libtensorflow_framework.so
39+
40+
elif [[ "$PLATFORM" =~ "macosx" ]]; then
41+
ln -fs $DOWNLOAD_FOLDER/tensorflow/libtensorflow_cc.2.dylib $DOWNLOAD_FOLDER/tensorflow/libtensorflow_cc.dylib
42+
ln -fs $DOWNLOAD_FOLDER/tensorflow/libtensorflow_framework.2.dylib $DOWNLOAD_FOLDER/tensorflow/libtensorflow_framework.dylib
43+
fi

‎tensorflow-core/tensorflow-core-native/scripts/wheel_download.sh‎

Lines changed: 0 additions & 29 deletions
This file was deleted.

0 commit comments

Comments
 (0)