Skip to content

Commit d51a4c8

Browse files
coryancoryan
authored
cleanup: restructure all windows builds (#3465)
Use the same structure as the builds in `-common` and `-pubsub`, these scripts can be executed manually, which makes them much better for development. This PR does not remove the old scripts, first we need to switch to the new ones.
1 parent a2380a7 commit d51a4c8

17 files changed

Lines changed: 469 additions & 8 deletions

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ cmake-out/
66

77
# Common bazel output directories
88
bazel-*
9+
!bazel-*.cfg
910

1011
# Backup files for Emacs
1112
*~

‎ci/kokoro/windows/bazel-presubmit.cfg‎

Whitespace-only changes.

‎ci/kokoro/windows/bazel.cfg‎

Whitespace-only changes.

‎ci/kokoro/windows/bazel/build.bat‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ call refreshenv.cmd
2424
@echo %date% %time%
2525
@if not exist "C:\b\" mkdir C:\b
2626

27-
call "c:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat"
27+
call "c:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars64.bat"
2828

2929
@echo %date% %time%
3030
C:\ProgramData\chocolatey\bin\bazel version

‎ci/kokoro/windows/build-bazel-dependencies.ps1‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# !/usr/bin/env powershell
2+
# Copyright 2020 Google LLC
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
Write-Host -ForegroundColor Yellow "`n$(Get-Date -Format o) No dependencies required for Bazel builds."

‎ci/kokoro/windows/build-bazel.ps1‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
# !/usr/bin/env powershell
2+
# Copyright 2020 Google LLC
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
# Stop on errors. This is similar to `set -e` on Unix shells.
17+
$ErrorActionPreference = "Stop"
18+
19+
Write-Host -ForegroundColor Yellow "`n$(Get-Date -Format o) Capture Bazel information for troubleshooting"
20+
bazel version
21+
22+
$common_flags = @()
23+
# Create output directory for Bazel. Bazel creates really long paths,
24+
# sometimes exceeding the Windows limits. Using a short name for the
25+
# root of the Bazel output directory works around this problem.
26+
$bazel_root="C:\b"
27+
if (-not (Test-Path $bazel_root)) {
28+
Write-Host -ForegroundColor Yellow "`n$(Get-Date -Format o) Create bazel user root (${bazel_root})"
29+
New-Item -ItemType Directory -Path $bazel_root | Out-Null
30+
}
31+
$common_flags += ("--output_user_root=${bazel_root}")
32+
33+
$test_flags = @("--test_output=errors",
34+
"--verbose_failures=true",
35+
"--keep_going")
36+
$build_flags = @("--keep_going")
37+
38+
$env:BAZEL_VC="C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC"
39+
40+
1..3 | % {
41+
Write-Host -ForegroundColor Yellow "`n$(Get-Date -Format o) Fetch dependencies [$_]"
42+
bazel $common_flags fetch -- //google/cloud/...:all
43+
if ($LastExitCode -eq 0) {
44+
return
45+
}
46+
}
47+
48+
Write-Host -ForegroundColor Yellow "`n$(Get-Date -Format o) Compiling and running unit tests"
49+
bazel $common_flags test $test_flags `
50+
--test_tag_filters=-integration-tests `
51+
-- //google/cloud/...:all
52+
if ($LastExitCode) {
53+
throw "bazel test failed with exit code $LastExitCode"
54+
}
55+
56+
Write-Host -ForegroundColor Yellow "`n$(Get-Date -Format o) Compiling extra programs"
57+
bazel $common_flags build $build_flags -- //google/cloud/...:all
58+
if ($LastExitCode) {
59+
throw "bazel test failed with exit code $LastExitCode"
60+
}
61+
62+
if ((Test-Path env:RUN_INTEGRATION_TESTS) -and ($env:RUN_INTEGRATION_TESTS -eq "true")) {
63+
Write-Host -ForegroundColor Yellow "`n$(Get-Date -Format o) Running integration tests $env:CONFIG"
64+
$integration_flags=@(
65+
"--test_env", "GOOGLE_CLOUD_PROJECT=${env:GOOGLE_CLOUD_PROJECT}",
66+
"--test_env", "GOOGLE_APPLICATION_CREDENTIALS=${env:GOOGLE_APPLICATION_CREDENTIALS}",
67+
"--test_env", "GOOGLE_CLOUD_CPP_AUTO_RUN_EXAMPLES=yes",
68+
"--test_env", "RUN_SLOW_INTEGRATION_TESTS=${env:RUN_SLOW_INTEGRATION_TESTS}",
69+
"--test_env", "GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=${env:GRPC_DEFAULT_SSL_ROOTS_FILE_PATH}"
70+
)
71+
bazel $common_flags test $test_flags $integration_flags `
72+
--test_tag_filters=integration-tests `
73+
-- //google/cloud/...:all
74+
if ($LastExitCode) {
75+
throw "Integration tests failed with exit code $LastExitCode"
76+
}
77+
}
78+
79+
Write-Host -ForegroundColor Yellow "`n$(Get-Date -Format o) DONE"

‎ci/kokoro/windows/build-cmake-dependencies.ps1‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
# !/usr/bin/env powershell
2+
# Copyright 2020 Google LLC
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
# Stop on errors. This is similar to `set -e` on Unix shells.
17+
$ErrorActionPreference = "Stop"
18+
19+
# First check the required environment variables.
20+
if (-not (Test-Path env:CONFIG)) {
21+
throw "Aborting build because the CONFIG environment variable is not set."
22+
}
23+
$CONFIG = $env:CONFIG
24+
25+
$project_root = (Get-Item -Path ".\" -Verbose).FullName
26+
27+
# Update or clone the 'vcpkg' package manager, this is a bit overly complicated,
28+
# but it works well on your workstation where you may want to run this script
29+
# multiple times while debugging vcpkg installs. It also works on AppVeyor
30+
# where we cache the vcpkg installation, but it might be empty on the first
31+
# build.
32+
Set-Location ..
33+
if (Test-Path env:RUNNING_CI) {
34+
Write-Host -ForegroundColor Yellow "`n$(Get-Date -Format o) Cloning vcpkg repository."
35+
if (Test-Path "vcpkg") {
36+
Remove-Item -LiteralPath "vcpkg" -Force -Recurse
37+
}
38+
git clone --depth 10 https://github.com/Microsoft/vcpkg.git
39+
if ($LastExitCode) {
40+
throw "vcpkg git setup failed with exit code $LastExitCode"
41+
}
42+
}
43+
if (-not (Test-Path "vcpkg")) {
44+
throw "Missing vcpkg directory."
45+
}
46+
Set-Location vcpkg
47+
48+
# If BUILD_CACHE is set (which typically is on Kokoro builds), try
49+
# to download and extract the build cache.
50+
if (Test-Path env:BUILD_CACHE) {
51+
gcloud auth activate-service-account --key-file "${env:KOKORO_GFILE_DIR}/build-results-service-account.json"
52+
Write-Host -ForegroundColor Yellow "`n$(Get-Date -Format o) Downloading build cache."
53+
gsutil cp $env:BUILD_CACHE vcpkg-installed.zip
54+
if ($LastExitCode) {
55+
# Ignore errors, caching failures should not break the build.
56+
Write-Host "gsutil download failed with exit code $LastExitCode"
57+
}
58+
Write-Host -ForegroundColor Yellow "`n$(Get-Date -Format o) Extracting build cache."
59+
7z x vcpkg-installed.zip -aoa
60+
if ($LastExitCode) {
61+
# Ignore errors, caching failures should not break the build.
62+
Write-Host "extracting build cache failed with exit code $LastExitCode"
63+
}
64+
}
65+
66+
Write-Host -ForegroundColor Yellow "`n$(Get-Date -Format o) Bootstrap vcpkg."
67+
powershell -exec bypass scripts\bootstrap.ps1
68+
if ($LastExitCode) {
69+
throw "Error bootstrapping vcpkg: $LastExitCode"
70+
Write-Host -ForegroundColor Red "bootstrap[1] failed"
71+
}
72+
73+
# Only compile the release version of the packages we need, for Debug builds
74+
# this trick does not work, so we need to compile all versions (yuck).
75+
if ($CONFIG -eq "Release") {
76+
# Add-Content triplets\x64-windows-static.cmake "set(VCPKG_BUILD_TYPE release)"
77+
}
78+
79+
# Integrate installed packages into the build environment.
80+
.\vcpkg.exe integrate install
81+
if ($LastExitCode) {
82+
throw "vcpkg integrate failed with exit code $LastExitCode"
83+
}
84+
85+
$vcpkg_flags=@(
86+
"--triplet", "${env:VCPKG_TRIPLET}",
87+
"--overlay-ports=${project_root}/ci/kokoro/windows/vcpkg-ports")
88+
89+
# Remove old versions of the packages.
90+
Write-Host -ForegroundColor Yellow "`n$(Get-Date -Format o) Cleanup old vcpkg package versions."
91+
.\vcpkg.exe remove ${vcpkg_flags} --outdated --recurse
92+
if ($LastExitCode) {
93+
throw "vcpkg remove --outdated failed with exit code $LastExitCode"
94+
}
95+
96+
Write-Host -ForegroundColor Yellow "`n$(Get-Date -Format o) Building vcpkg package versions."
97+
$packages = @("zlib", "openssl",
98+
"protobuf", "c-ares",
99+
"grpc", "gtest", "crc32c", "curl",
100+
"googleapis", "google-cloud-cpp-common[test]")
101+
foreach ($pkg in $packages) {
102+
.\vcpkg.exe install ${vcpkg_flags} "${pkg}"
103+
if ($LastExitCode) {
104+
throw "vcpkg install $pkg failed with exit code $LastExitCode"
105+
}
106+
}
107+
108+
Write-Host -ForegroundColor Yellow "`n$(Get-Date -Format o) vcpkg list"
109+
.\vcpkg.exe list
110+
111+
if (Test-Path env:BUILD_CACHE) {
112+
Write-Host -ForegroundColor Yellow "`n$(Get-Date -Format o) Create cache zip file."
113+
7z a vcpkg-installed.zip installed\
114+
if ($LastExitCode) {
115+
# Ignore errors, caching failures should not break the build.
116+
Write-Host "zip build cache failed with exit code $LastExitCode"
117+
}
118+
119+
Write-Host -ForegroundColor Yellow "`n$(Get-Date -Format o) Upload cache zip file."
120+
gsutil cp vcpkg-installed.zip $env:BUILD_CACHE
121+
if ($LastExitCode) {
122+
# Ignore errors, caching failures should not break the build.
123+
Write-Host "gsutil upload failed with exit code $LastExitCode"
124+
}
125+
}

‎ci/kokoro/windows/build-cmake.ps1‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
# !/usr/bin/env powershell
2+
# Copyright 2020 Google LLC
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
# Stop on errors. This is similar to `set -e` on Unix shells.
17+
$ErrorActionPreference = "Stop"
18+
19+
# First check the required environment variables.
20+
if (-not (Test-Path env:CONFIG)) {
21+
throw "Aborting build because the CONFIG environment variable is not set."
22+
}
23+
24+
# By default assume "module", use the configuration parameters and build in the `cmake-out` directory.
25+
$cmake_flags=@("-G$env:GENERATOR", "-DCMAKE_BUILD_TYPE=$env:CONFIG", "-H.", "-Bcmake-out")
26+
27+
# This script expects vcpkg to be installed in ..\vcpkg, discover the full
28+
# path to that directory:
29+
$dir = Split-Path (Get-Item -Path ".\" -Verbose).FullName
30+
31+
# Setup the environment for vcpkg:
32+
$cmake_flags += "-DCMAKE_TOOLCHAIN_FILE=`"$dir\vcpkg\scripts\buildsystems\vcpkg.cmake`""
33+
$cmake_flags += "-DVCPKG_TARGET_TRIPLET=$env:VCPKG_TRIPLET"
34+
$cmake_flags += "-DCMAKE_C_COMPILER=cl.exe"
35+
$cmake_flags += "-DCMAKE_CXX_COMPILER=cl.exe"
36+
37+
# Configure CMake and create the build directory.
38+
Write-Host -ForegroundColor Yellow "`n$(Get-Date -Format o) Configuring CMake with $cmake_flags"
39+
cmake $cmake_flags
40+
if ($LastExitCode) {
41+
throw "cmake config failed with exit code $LastExitCode"
42+
}
43+
44+
Write-Host -ForegroundColor Yellow "`n$(Get-Date -Format o) Compiling with CMake $env:CONFIG"
45+
cmake --build cmake-out --config $env:CONFIG
46+
if ($LastExitCode) {
47+
throw "cmake for 'all' target failed with exit code $LastExitCode"
48+
}
49+
50+
Write-Host -ForegroundColor Yellow "`n$(Get-Date -Format o) Running unit tests $env:CONFIG"
51+
Set-Location cmake-out
52+
if (Test-Path env:RUNNING_CI) {
53+
# On Kokoro we need to define %TEMP% or the tests do not have a valid directory for
54+
# temporary files.
55+
$env:TEMP="T:\tmp"
56+
}
57+
ctest --output-on-failure -LE integration-tests -C $env:CONFIG
58+
if ($LastExitCode) {
59+
throw "ctest failed with exit code $LastExitCode"
60+
}
61+
62+
if ((Test-Path env:RUN_INTEGRATION_TESTS) -and ($env:RUN_INTEGRATION_TESTS -eq "true")) {
63+
Write-Host -ForegroundColor Yellow "`n$(Get-Date -Format o) Running integration tests $env:CONFIG"
64+
ctest --output-on-failure -L integration-tests
65+
if ($LastExitCode) {
66+
throw "Integration tests failed with exit code $LastExitCode"
67+
}
68+
}
69+
70+
Write-Host -ForegroundColor Yellow "`n$(Get-Date -Format o) DONE"

‎ci/kokoro/windows/build-dependencies.ps1‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ $packages = @("zlib", "openssl",
110110
"protobuf", "c-ares",
111111
"grpc", "curl",
112112
"gtest", "crc32c"
113-
"googleapis", "google-cloud-cpp-common")
113+
"googleapis", "google-cloud-cpp-common[test]")
114114
foreach ($pkg in $packages) {
115115
.\vcpkg.exe install ${vcpkg_flags} ${pkg}
116116
if ($LastExitCode) {

‎ci/kokoro/windows/build-new.bat‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
@REM Copyright 2020 Google LLC
2+
@REM
3+
@REM Licensed under the Apache License, Version 2.0 (the "License");
4+
@REM you may not use this file except in compliance with the License.
5+
@REM You may obtain a copy of the License at
6+
@REM
7+
@REM http://www.apache.org/licenses/LICENSE-2.0
8+
@REM
9+
@REM Unless required by applicable law or agreed to in writing, software
10+
@REM distributed under the License is distributed on an "AS IS" BASIS,
11+
@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
@REM See the License for the specific language governing permissions and
13+
@REM limitations under the License.
14+
15+
REM Install Bazel using Chocolatey.
16+
choco install -y bazel --version 2.0.0
17+
18+
REM Configure the environment to use MSVC 2019 and then switch to PowerShell.
19+
call "c:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars64.bat"
20+
21+
REM The remaining of the build script is implemented in PowerShell.
22+
echo %date% %time%
23+
cd github\google-cloud-cpp
24+
powershell -exec bypass ci\kokoro\windows\build.ps1
25+
if %errorlevel% neq 0 exit /b %errorlevel%
26+
27+
@REM Kokoro rsyncs all the files in the %KOKORO_ARTIFACTS_DIR%, which takes a
28+
@REM long time. The recommended workaround is to remove all the files that are
29+
@REM not interesting artifacts.
30+
if defined KOKORO_ARTIFACTS_DIR (
31+
@echo %date% %time% "Cleanup Kokoro artifacts directory"
32+
cd "%KOKORO_ARTIFACTS_DIR%"
33+
powershell -Command "& {Get-ChildItem -Recurse -File -Exclude test.xml,sponge_log.xml,build.bat | Remove-Item -Recurse -Force}"
34+
if %errorlevel% neq 0 exit /b %errorlevel%
35+
)
36+
37+
@echo DONE "============================================="
38+
@echo %date% %time%

0 commit comments

Comments
 (0)