Skip to content

Commit 3851704

Browse files
author
David Cavazos
authored
ppai/weather: add timeout to vertex training (GoogleCloudPlatform#9273)
1 parent cfcf91b commit 3851704

4 files changed

Lines changed: 5 additions & 2 deletions

File tree

‎people-and-planet-ai/weather-forecasting/notebooks/3-training.ipynb‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1364,6 +1364,7 @@
13641364
"from google.cloud import aiplatform\n",
13651365
"\n",
13661366
"epochs = 100\n",
1367+
"timeout_min = 60 # 1 hour\n",
13671368
"\n",
13681369
"# Cloud Storage paths.\n",
13691370
"data_path = f\"/gcs/{bucket}/weather/data\"\n",
@@ -1376,7 +1377,7 @@
13761377
" display_name=\"weather-forecasting\",\n",
13771378
" python_package_gcs_uri=f\"gs://{bucket}/weather/weather-model-1.0.0.tar.gz\",\n",
13781379
" python_module_name=\"weather.trainer\",\n",
1379-
" container_uri=\"us-docker.pkg.dev/vertex-ai/training/pytorch-gpu.1-11:latest\",\n",
1380+
" container_uri=\"us-docker.pkg.dev/vertex-ai/training/pytorch-gpu.1-13:latest\",\n",
13801381
")\n",
13811382
"job.run(\n",
13821383
" machine_type=\"n1-highmem-8\",\n",
@@ -1387,6 +1388,7 @@
13871388
" f\"--model-path={model_path}\",\n",
13881389
" f\"--epochs={epochs}\",\n",
13891390
" ],\n",
1391+
" timeout=timeout_min * 60, # in seconds\n",
13901392
")"
13911393
],
13921394
"id": "Ny4x99GiS2Lm"

‎people-and-planet-ai/weather-forecasting/serving/weather-model/pyproject.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ name = "weather-model"
1818
version = "1.0.0"
1919
dependencies = [
2020
"datasets==2.8.0",
21+
"torch==1.13.1", # make sure this matches the `container_uri` in `notebooks/3-training.ipynb`
2122
"transformers==4.25.1",
2223
]
2324

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
11
../../serving/weather-model
22
build==0.10.0
33
google-cloud-aiplatform==1.21.0
4-
torch==1.13.1

‎people-and-planet-ai/weather-forecasting/tests/training_tests/test_training.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ def test_train_model(
8282
"data_path": data_path_gcs.replace("gs://", "/gcs/"),
8383
"model_path": f"/gcs/{bucket_name}/test/weather/model-vertex",
8484
"epochs": 2,
85+
"timeout_min": 5, # should take a couple seconds, so timeout in 5 minutes
8586
}
8687
},
8788
},

0 commit comments

Comments
 (0)