Skip to content

Commit e782373

Browse files
bradmirobradmiro
authored
fix(data-science-onramp): Change file names, remove extra dependencies for model training (GoogleCloudPlatform#5770)
* name changes, changes to model code * changing file names * updating to vertex * fixes * fixes and removing noxfile
1 parent d376b3c commit e782373

16 files changed

Lines changed: 67 additions & 75 deletions

File tree

‎data-science-onramp/ai-platform/modules/setup.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
include_package_data=True,
2424
description="Tutorial Package",
2525
install_requires=[
26-
"gcsfs==0.8.0"
26+
"gcsfs"
2727
]
2828
)
2929
# [END ai_platform_setup]

‎data-science-onramp/ai-platform/modules/trainer/sklearn_model/task.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,10 +47,10 @@ def get_args() -> argparse.Namespace:
4747
help="Regularization strength, default=0 (Standard Regression)",
4848
)
4949
parser.add_argument(
50-
"--model_dir",
50+
"--model-dir",
5151
type=str,
5252
help="Output directory for the model.",
53-
default=os.environ["AIP_MODEL_DIR"],
53+
default=os.getenv("AIP_MODEL_DIR"),
5454
)
5555
return parser.parse_args()
5656
# [END ai_platform_sklearn_task_args]

‎data-science-onramp/ai-platform/modules/trainer/tfkeras_model/task.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,10 +57,10 @@ def get_args() -> argparse.Namespace:
5757
default="INFO",
5858
)
5959
parser.add_argument(
60-
"--model_dir",
60+
"--model-dir",
6161
type=str,
6262
help="Output directory for the model.",
63-
default=os.environ["AIP_MODEL_DIR"],
63+
default=os.getenv("AIP_MODEL_DIR"),
6464
)
6565
return parser.parse_args()
6666
# [END ai_platform_tfkeras_task_args]

‎data-science-onramp/ai-platform/noxfile_config.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222

2323
TEST_CONFIG_OVERRIDE = {
2424
# You can opt out from the test for specific Python versions.
25-
'ignored_versions': ["2.7"],
25+
"ignored_versions": ["2.7"],
2626
# Old samples are opted out of enforcing Python type hints
2727
# All new samples should feature them
2828
"enforce_type_hints": True,

‎data-science-onramp/ai-platform/sklearn_test.py‎

Lines changed: 24 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -35,13 +35,13 @@
3535
REGION = "us-central1"
3636
MODEL_NAME = f"sklearn-test-{uuid.uuid4()}"
3737
JOB_ID = f"sklearn_{str(uuid.uuid4())[:7]}"
38-
DEPLOY_IMAGE = "gcr.io/cloud-aiplatform/training/scikit-learn-cpu.0-23:latest"
38+
DEPLOY_IMAGE = "us-docker.pkg.dev/vertex-ai/training/scikit-learn-cpu.0-23:latest"
3939

4040
TERMINAL_STATES = [
4141
aip.JobState.JOB_STATE_SUCCEEDED,
4242
aip.JobState.JOB_STATE_FAILED,
4343
aip.JobState.JOB_STATE_CANCELLING,
44-
aip.JobState.JOB_STATE_CANCELLED
44+
aip.JobState.JOB_STATE_CANCELLED,
4545
]
4646

4747

@@ -53,21 +53,21 @@ def shared_state() -> dict:
5353

5454
@pytest.fixture(autouse=True)
5555
def setup_teardown(
56-
shared_state: dict
56+
shared_state: dict,
5757
) -> Tuple[storage.bucket.Bucket, aip.JobServiceClient]:
5858
storage_client = storage.Client()
5959
bucket = storage_client.create_bucket(STAGING_BUCKET, location=REGION)
60-
bucket.blob(f"{INPUT_DIR}/{TRAIN_DATA}").upload_from_filename(TRAIN_DATA, timeout=600)
60+
bucket.blob(f"{INPUT_DIR}/{TRAIN_DATA}").upload_from_filename(
61+
TRAIN_DATA, timeout=600
62+
)
6163

6264
with tarfile.open(TRAINER_TAR, mode="x:gz") as tar:
6365
tar.add(f"{TRAINER_DIR}/")
6466

6567
bucket.blob(TRAINER_TAR).upload_from_filename(TRAINER_TAR)
6668

6769
aip_job_client = aip.JobServiceClient(
68-
client_options={
69-
"api_endpoint": f"{REGION}-aiplatform.googleapis.com"
70-
}
70+
client_options={"api_endpoint": f"{REGION}-aiplatform.googleapis.com"}
7171
)
7272

7373
yield bucket, aip_job_client
@@ -85,40 +85,38 @@ def setup_teardown(
8585
@pytest.mark.timeout(1800)
8686
def test_sklearn(
8787
setup_teardown: Tuple[storage.bucket.Bucket, aip.JobServiceClient],
88-
shared_state: dict
88+
shared_state: dict,
8989
) -> None:
9090
bucket, aip_job_client = setup_teardown
9191

9292
custom_job = {
9393
"display_name": JOB_ID,
9494
"job_spec": {
9595
"base_output_directory": {"output_uri_prefix": f"gs://{STAGING_BUCKET}"},
96-
"worker_pool_specs": [{
97-
"replica_count": 1,
98-
"machine_spec": {
99-
"machine_type": "n1-standard-4",
100-
},
101-
"python_package_spec": {
102-
"executor_image_uri": DEPLOY_IMAGE,
103-
"package_uris": [f"gs://{STAGING_BUCKET}/{TRAINER_TAR}"],
104-
"python_module": "trainer.sklearn_model.task",
105-
"args": [
106-
f"--input-path={TRAIN_DATA_PATH}"
107-
]
96+
"worker_pool_specs": [
97+
{
98+
"replica_count": 1,
99+
"machine_spec": {
100+
"machine_type": "n1-standard-4",
101+
},
102+
"python_package_spec": {
103+
"executor_image_uri": DEPLOY_IMAGE,
104+
"package_uris": [f"gs://{STAGING_BUCKET}/{TRAINER_TAR}"],
105+
"python_module": "trainer.sklearn_model.task",
106+
"args": [f"--input-path={TRAIN_DATA_PATH}"],
107+
},
108108
}
109-
}]
110-
}
109+
],
110+
},
111111
}
112112

113113
parent = f"projects/{PROJECT_ID}/locations/{REGION}"
114-
response = aip_job_client.create_custom_job(
115-
parent=parent, custom_job=custom_job
116-
)
114+
response = aip_job_client.create_custom_job(parent=parent, custom_job=custom_job)
117115
resource_name = response.name
118116
shared_state["model_name"] = resource_name
119117

120118
# Subject to change with LRO availability
121-
while (response.state not in TERMINAL_STATES):
119+
while response.state not in TERMINAL_STATES:
122120
time.sleep(60)
123121
response = aip_job_client.get_custom_job(name=resource_name)
124122

‎data-science-onramp/ai-platform/tfkeras_test.py‎

Lines changed: 24 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -35,13 +35,13 @@
3535
REGION = "us-central1"
3636
MODEL_NAME = f"tfkeras-test-{uuid.uuid4()}"
3737
JOB_ID = f"tfkeras_{str(uuid.uuid4())[:7]}"
38-
DEPLOY_IMAGE = "gcr.io/cloud-aiplatform/training/tf-cpu.2-3:latest"
38+
DEPLOY_IMAGE = "us-docker.pkg.dev/vertex-ai/training/tf-cpu.2-3:latest"
3939

4040
TERMINAL_STATES = [
4141
aip.JobState.JOB_STATE_SUCCEEDED,
4242
aip.JobState.JOB_STATE_FAILED,
4343
aip.JobState.JOB_STATE_CANCELLING,
44-
aip.JobState.JOB_STATE_CANCELLED
44+
aip.JobState.JOB_STATE_CANCELLED,
4545
]
4646

4747

@@ -53,22 +53,22 @@ def shared_state() -> dict:
5353

5454
@pytest.fixture(autouse=True)
5555
def setup_teardown(
56-
shared_state: dict
56+
shared_state: dict,
5757
) -> Tuple[storage.bucket.Bucket, aip.JobServiceClient]:
5858

5959
storage_client = storage.Client()
6060
bucket = storage_client.create_bucket(STAGING_BUCKET, location=REGION)
61-
bucket.blob(f"{INPUT_DIR}/{TRAIN_DATA}").upload_from_filename(TRAIN_DATA, timeout=600)
61+
bucket.blob(f"{INPUT_DIR}/{TRAIN_DATA}").upload_from_filename(
62+
TRAIN_DATA, timeout=600
63+
)
6264

6365
with tarfile.open(TRAINER_TAR, mode="x:gz") as tar:
6466
tar.add(f"{TRAINER_DIR}/")
6567

6668
bucket.blob(TRAINER_TAR).upload_from_filename(TRAINER_TAR)
6769

6870
aip_job_client = aip.JobServiceClient(
69-
client_options={
70-
"api_endpoint": f"{REGION}-aiplatform.googleapis.com"
71-
}
71+
client_options={"api_endpoint": f"{REGION}-aiplatform.googleapis.com"}
7272
)
7373

7474
yield bucket, aip_job_client
@@ -86,40 +86,38 @@ def setup_teardown(
8686
@pytest.mark.timeout(1800)
8787
def test_tfkeras(
8888
setup_teardown: Tuple[storage.bucket.Bucket, aip.JobServiceClient],
89-
shared_state: dict
89+
shared_state: dict,
9090
) -> None:
9191
bucket, aip_job_client = setup_teardown
9292

9393
custom_job = {
9494
"display_name": JOB_ID,
9595
"job_spec": {
9696
"base_output_directory": {"output_uri_prefix": f"gs://{STAGING_BUCKET}"},
97-
"worker_pool_specs": [{
98-
"replica_count": 1,
99-
"machine_spec": {
100-
"machine_type": "n1-standard-4",
101-
},
102-
"python_package_spec": {
103-
"executor_image_uri": DEPLOY_IMAGE,
104-
"package_uris": [f"gs://{STAGING_BUCKET}/{TRAINER_TAR}"],
105-
"python_module": "trainer.tfkeras_model.task",
106-
"args": [
107-
f"--input-path={TRAIN_DATA_PATH}"
108-
]
97+
"worker_pool_specs": [
98+
{
99+
"replica_count": 1,
100+
"machine_spec": {
101+
"machine_type": "n1-standard-4",
102+
},
103+
"python_package_spec": {
104+
"executor_image_uri": DEPLOY_IMAGE,
105+
"package_uris": [f"gs://{STAGING_BUCKET}/{TRAINER_TAR}"],
106+
"python_module": "trainer.tfkeras_model.task",
107+
"args": [f"--input-path={TRAIN_DATA_PATH}"],
108+
},
109109
}
110-
}]
111-
}
110+
],
111+
},
112112
}
113113

114114
parent = f"projects/{PROJECT_ID}/locations/{REGION}"
115-
response = aip_job_client.create_custom_job(
116-
parent=parent, custom_job=custom_job
117-
)
115+
response = aip_job_client.create_custom_job(parent=parent, custom_job=custom_job)
118116
resource_name = response.name
119117
shared_state["model_name"] = resource_name
120118

121119
# Subject to change with LRO availability
122-
while (response.state not in TERMINAL_STATES):
120+
while response.state not in TERMINAL_STATES:
123121
time.sleep(60)
124122
response = aip_job_client.get_custom_job(name=resource_name)
125123

data-science-onramp/data-ingestion/setup.py renamed to data-science-onramp/data-ingestion/ingestion.py

File renamed without changes.

data-science-onramp/data-ingestion/setup.sh renamed to data-science-onramp/data-ingestion/ingestion.sh

File renamed without changes.

data-science-onramp/data-ingestion/setup_test.py renamed to data-science-onramp/data-ingestion/ingestion_test.py

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@
3131
TEST_ID = uuid.uuid4()
3232

3333
# Google Cloud Storage constants
34-
BUCKET_NAME = f"setup-test-{TEST_ID}"
35-
BUCKET_BLOB = "setup.py"
34+
BUCKET_NAME = f"ingestion-test-{TEST_ID}"
35+
BUCKET_BLOB = "ingestion.py"
3636

3737
BQ_DATASET = f"setup-test-{TEST_ID}".replace("-", "_")
3838
BQ_CITIBIKE_TABLE = "RAW_DATA"
@@ -42,9 +42,9 @@
4242
]
4343

4444
# Dataproc constants
45-
DATAPROC_CLUSTER = f"setup-test-{TEST_ID}"
45+
DATAPROC_CLUSTER = f"ingestion-test-{TEST_ID}"
4646
CLUSTER_REGION = "us-central1"
47-
CLUSTER_IMAGE = "1.5.4-debian10"
47+
CLUSTER_IMAGE = "2.0-debian10"
4848
CLUSTER_CONFIG = { # Dataproc cluster configuration
4949
"project_id": PROJECT_ID,
5050
"cluster_name": DATAPROC_CLUSTER,
@@ -58,7 +58,6 @@
5858
"worker_config": {"num_instances": 2, "machine_type_uri": "n1-standard-4"},
5959
"software_config": {
6060
"image_version": CLUSTER_IMAGE,
61-
"optional_components": [5],
6261
},
6362
},
6463
}
@@ -103,7 +102,7 @@ def setup_and_teardown_bucket():
103102

104103
# Upload file
105104
blob = bucket.blob(BUCKET_BLOB)
106-
blob.upload_from_filename("setup.py")
105+
blob.upload_from_filename("ingestion.py")
107106

108107
yield
109108

data-science-onramp/data-cleaning/README.md renamed to data-science-onramp/data-processing/README.md

File renamed without changes.

0 commit comments

Comments
 (0)