Skip to content

Commit 06508d8

Browse files
HowalHowal
committed
update dnl scripts
1 parent 75c67f9 commit 06508d8

6 files changed

Lines changed: 79 additions & 24 deletions

‎PyTorch/LanguageModeling/BERT/run_pretraining.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@
4343
from schedulers import PolyWarmUpScheduler
4444

4545
from file_utils import PYTORCH_PRETRAINED_BERT_CACHE
46-
from utils import is_main_process, format_step, get_world_size, get_rank, get_local_rank
46+
from utils import is_main_process, format_step, get_world_size, get_local_size, get_rank, get_local_rank, set_environment_variables_for_nccl_backend
4747
from apex.parallel import DistributedDataParallel as DDP
4848
from schedulers import LinearWarmUpScheduler
4949
from apex.parallel.distributed import flat_dist_call
@@ -280,6 +280,7 @@ def parse_arguments():
280280
def setup_training(args):
281281

282282
assert (torch.cuda.is_available())
283+
set_environment_variables_for_nccl_backend(get_local_size() == get_world_size())
283284

284285
if args.local_rank == -1:
285286
device = torch.device("cuda")
@@ -625,7 +626,7 @@ def main():
625626
dllogger.log(step=(epoch, global_step, ), data={"average_loss": average_loss / (args.log_freq * divisor),
626627
"step_loss": loss.item() * args.gradient_accumulation_steps / divisor,
627628
"learning_rate": optimizer.param_groups[0]['lr']})
628-
run.log_row("train loss over steps", global_step = global_step, loss = np.float(loss.item()))
629+
run.log_row("train loss over steps", global_step = global_step + args.phase1_end_step if args.phase2 else global_step, loss = np.float(average_loss / (args.log_freq * divisor)))
629630
average_loss = 0
630631

631632
if global_step >= args.max_steps or training_steps % (

‎PyTorch/LanguageModeling/BERT/scripts/aml_run_pretrain.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,12 @@
66
default=None,
77
type=str,
88
required=True)
9+
parser.add_argument("--data_dir",
10+
default=None,
11+
type=str,
12+
required=True)
913

1014
args = parser.parse_args()
1115

1216
print(os.environ)
13-
os.system('bash ' + args.cfg)
17+
os.system('bash ' + args.cfg + ' ' + args.data_dir)

‎PyTorch/LanguageModeling/BERT/scripts/aml_run_pretraining_bert_base.sh‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
# limitations under the License.
1515

1616
echo "Container nvidia build = " $NVIDIA_BUILD_ID
17-
train_batch_size=${1:-2048}
17+
train_batch_size=2048
1818
learning_rate=${2:-"6e-3"}
1919
precision=${3:-"fp16"}
2020
num_gpus=${4:-8}
@@ -34,12 +34,12 @@ learning_rate_phase2=${18:-"4e-3"}
3434
warmup_proportion_phase2=${19:-"0.128"}
3535
train_steps_phase2=${20:-1563}
3636
gradient_accumulation_steps_phase2=${21:-256}
37-
DATASET=./nvidia_bert_data/hdf5_lower_case_1_seq_len_128_max_pred_20_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5/books_wiki_en_corpus # change this for other datasets
37+
DATASET=${1}hdf5_lower_case_1_seq_len_128_max_pred_20_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5/books_wiki_en_corpus # change this for other datasets
3838
DATA_DIR_PHASE1=${22:-${DATASET}/}
3939
BERT_CONFIG=bert_base_config.json
40-
CODEDIR=${24:-"./nvidia_bert_data"}
40+
CODEDIR=${24:-"."}
4141
init_checkpoint=${25:-"None"}
42-
RESULTS_DIR=$CODEDIR/results/bert_base
42+
RESULTS_DIR=${1}results/bert_base
4343
CHECKPOINTS_DIR=$RESULTS_DIR/checkpoints
4444

4545
mkdir -p $CHECKPOINTS_DIR
@@ -119,6 +119,7 @@ CMD+=" $ALL_REDUCE_POST_ACCUMULATION"
119119
CMD+=" $ALL_REDUCE_POST_ACCUMULATION_FP16"
120120
CMD+=" $INIT_CHECKPOINT"
121121
CMD+=" --do_train"
122+
CMD+=" --use_env"
122123
CMD+=" --json-summary ${RESULTS_DIR}/dllogger.json "
123124

124125
#CMD="python3 -m torch.distributed.launch --nproc_per_node=$num_gpus $CMD"
@@ -145,9 +146,12 @@ set +x
145146

146147
echo "finished pretraining"
147148

148-
#Start Phase2
149+
Start Phase2
150+
CMD="sleep 300s"
151+
CMD="pkill -9 python"
152+
CMD="sleep 300s"
149153

150-
DATASET=./nvidia_bert_data/hdf5_lower_case_1_seq_len_512_max_pred_80_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5/books_wiki_en_corpus # change this for other datasets
154+
DATASET=${1}hdf5_lower_case_1_seq_len_512_max_pred_80_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5/books_wiki_en_corpus # change this for other datasets
151155
DATA_DIR_PHASE2=${23:-${DATASET}/}
152156

153157
PREC=""
@@ -196,6 +200,7 @@ CMD+=" $CHECKPOINT"
196200
CMD+=" $ALL_REDUCE_POST_ACCUMULATION"
197201
CMD+=" $ALL_REDUCE_POST_ACCUMULATION_FP16"
198202
CMD+=" --do_train --phase2 --resume_from_checkpoint --phase1_end_step=$train_steps"
203+
CMD+=" --use_env"
199204
CMD+=" --json-summary ${RESULTS_DIR}/dllogger.json "
200205

201206
#CMD="python3 -m torch.distributed.launch --nproc_per_node=$num_gpus $CMD"

‎PyTorch/LanguageModeling/BERT/scripts/aml_run_pretraining_bert_base_dnl_wd0_scale1.sh‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
# limitations under the License.
1515

1616
echo "Container nvidia build = " $NVIDIA_BUILD_ID
17-
train_batch_size=${1:-2048}
17+
train_batch_size=2048
1818
learning_rate=${2:-"6e-3"}
1919
precision=${3:-"fp16"}
2020
num_gpus=${4:-8}
@@ -34,12 +34,12 @@ learning_rate_phase2=${18:-"4e-3"}
3434
warmup_proportion_phase2=${19:-"0.128"}
3535
train_steps_phase2=${20:-1563}
3636
gradient_accumulation_steps_phase2=${21:-256}
37-
DATASET=./nvidia_bert_data/hdf5_lower_case_1_seq_len_128_max_pred_20_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5/books_wiki_en_corpus # change this for other datasets
37+
DATASET=${1}hdf5_lower_case_1_seq_len_128_max_pred_20_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5/books_wiki_en_corpus # change this for other datasets
3838
DATA_DIR_PHASE1=${22:-${DATASET}/}
3939
BERT_CONFIG=bert_base_dnl_wd0_scale1_config.json
40-
CODEDIR=${24:-"./nvidia_bert_data"}
40+
CODEDIR=${24:-"."}
4141
init_checkpoint=${25:-"None"}
42-
RESULTS_DIR=$CODEDIR/results/bert_base_dnl_wd0_scale1
42+
RESULTS_DIR=${1}results/bert_base_dnl_wd0_scale1
4343
CHECKPOINTS_DIR=$RESULTS_DIR/checkpoints
4444

4545
mkdir -p $CHECKPOINTS_DIR
@@ -119,6 +119,7 @@ CMD+=" $ALL_REDUCE_POST_ACCUMULATION"
119119
CMD+=" $ALL_REDUCE_POST_ACCUMULATION_FP16"
120120
CMD+=" $INIT_CHECKPOINT"
121121
CMD+=" --do_train"
122+
CMD+=" --use_env"
122123
CMD+=" --json-summary ${RESULTS_DIR}/dllogger.json "
123124

124125
#CMD="python3 -m torch.distributed.launch --nproc_per_node=$num_gpus $CMD"
@@ -145,9 +146,13 @@ set +x
145146

146147
echo "finished pretraining"
147148

148-
#Start Phase2
149+
Start Phase2
149150

150-
DATASET=./nvidia_bert_data/hdf5_lower_case_1_seq_len_512_max_pred_80_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5/books_wiki_en_corpus # change this for other datasets
151+
CMD="sleep 300s"
152+
CMD="pkill -9 python"
153+
CMD="sleep 300s"
154+
155+
DATASET=${1}hdf5_lower_case_1_seq_len_512_max_pred_80_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5/books_wiki_en_corpus # change this for other datasets
151156
DATA_DIR_PHASE2=${23:-${DATASET}/}
152157

153158
PREC=""
@@ -196,6 +201,7 @@ CMD+=" $CHECKPOINT"
196201
CMD+=" $ALL_REDUCE_POST_ACCUMULATION"
197202
CMD+=" $ALL_REDUCE_POST_ACCUMULATION_FP16"
198203
CMD+=" --do_train --phase2 --resume_from_checkpoint --phase1_end_step=$train_steps"
204+
CMD+=" --use_env"
199205
CMD+=" --json-summary ${RESULTS_DIR}/dllogger.json "
200206

201207
#CMD="python3 -m torch.distributed.launch --nproc_per_node=$num_gpus $CMD"

‎PyTorch/LanguageModeling/BERT/scripts/aml_run_pretraining_bert_base_dnl_wd0_scale1_keep-mean.sh‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
# limitations under the License.
1515

1616
echo "Container nvidia build = " $NVIDIA_BUILD_ID
17-
train_batch_size=${1:-2048}
17+
train_batch_size=2048
1818
learning_rate=${2:-"6e-3"}
1919
precision=${3:-"fp16"}
2020
num_gpus=${4:-8}
@@ -34,12 +34,12 @@ learning_rate_phase2=${18:-"4e-3"}
3434
warmup_proportion_phase2=${19:-"0.128"}
3535
train_steps_phase2=${20:-1563}
3636
gradient_accumulation_steps_phase2=${21:-256}
37-
DATASET=./nvidia_bert_data/hdf5_lower_case_1_seq_len_128_max_pred_20_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5/books_wiki_en_corpus # change this for other datasets
37+
DATASET=${1}hdf5_lower_case_1_seq_len_128_max_pred_20_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5/books_wiki_en_corpus # change this for other datasets
3838
DATA_DIR_PHASE1=${22:-${DATASET}/}
3939
BERT_CONFIG=bert_base_dnl_wd0_scale1_keep-mean_config.json
40-
CODEDIR=${24:-"./nvidia_bert_data"}
40+
CODEDIR=${24:-"."}
4141
init_checkpoint=${25:-"None"}
42-
RESULTS_DIR=$CODEDIR/results/bert_base_dnl_wd0_scale1_keep-mean
42+
RESULTS_DIR=${1}results/bert_base_dnl_wd0_scale1_keep-mean
4343
CHECKPOINTS_DIR=$RESULTS_DIR/checkpoints
4444

4545
mkdir -p $CHECKPOINTS_DIR
@@ -119,6 +119,7 @@ CMD+=" $ALL_REDUCE_POST_ACCUMULATION"
119119
CMD+=" $ALL_REDUCE_POST_ACCUMULATION_FP16"
120120
CMD+=" $INIT_CHECKPOINT"
121121
CMD+=" --do_train"
122+
CMD+=" --use_env"
122123
CMD+=" --json-summary ${RESULTS_DIR}/dllogger.json "
123124

124125
#CMD="python3 -m torch.distributed.launch --nproc_per_node=$num_gpus $CMD"
@@ -145,9 +146,13 @@ set +x
145146

146147
echo "finished pretraining"
147148

148-
#Start Phase2
149+
Start Phase2
149150

150-
DATASET=./nvidia_bert_data/hdf5_lower_case_1_seq_len_512_max_pred_80_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5/books_wiki_en_corpus # change this for other datasets
151+
CMD="sleep 300s"
152+
CMD="pkill -9 python"
153+
CMD="sleep 300s"
154+
155+
DATASET=${1}hdf5_lower_case_1_seq_len_512_max_pred_80_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5/books_wiki_en_corpus # change this for other datasets
151156
DATA_DIR_PHASE2=${23:-${DATASET}/}
152157

153158
PREC=""
@@ -196,6 +201,7 @@ CMD+=" $CHECKPOINT"
196201
CMD+=" $ALL_REDUCE_POST_ACCUMULATION"
197202
CMD+=" $ALL_REDUCE_POST_ACCUMULATION_FP16"
198203
CMD+=" --do_train --phase2 --resume_from_checkpoint --phase1_end_step=$train_steps"
204+
CMD+=" --use_env"
199205
CMD+=" --json-summary ${RESULTS_DIR}/dllogger.json "
200206

201207
#CMD="python3 -m torch.distributed.launch --nproc_per_node=$num_gpus $CMD"

‎PyTorch/LanguageModeling/BERT/utils.py‎

Lines changed: 36 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,24 +11,57 @@
1111
# See the License for the specific language governing permissions and
1212
# limitations under the License.
1313

14+
import os
1415
import torch
1516
import torch.distributed as dist
1617

18+
def set_environment_variables_for_nccl_backend(single_node=False):
19+
os.environ['RANK'] = os.environ['OMPI_COMM_WORLD_RANK']
20+
os.environ['WORLD_SIZE'] = os.environ['OMPI_COMM_WORLD_SIZE']
21+
22+
if 'AZ_BATCH_MASTER_NODE' in os.environ.keys():
23+
if not single_node:
24+
master_node_params = os.environ['AZ_BATCH_MASTER_NODE'].split(':')
25+
os.environ['MASTER_ADDR'] = master_node_params[0]
26+
os.environ['MASTER_PORT'] = master_node_params[1]
27+
else:
28+
os.environ['MASTER_ADDR'] = os.environ['AZ_BATCHAI_MPI_MASTER_NODE']
29+
os.environ['MASTER_PORT'] = '54965'
30+
print('NCCL_SOCKET_IFNAME original value = {}'.format(os.environ['NCCL_SOCKET_IFNAME']))
31+
elif 'OMPI_MCA_orte_hnp_uri' in os.environ.keys():
32+
# ITP
33+
master_node_params = os.environ['OMPI_MCA_orte_hnp_uri'].split('tcp://')[1].split(':')
34+
os.environ['MASTER_ADDR'] = master_node_params[0]
35+
os.environ['MASTER_PORT'] = '54965'
36+
else:
37+
print('no master node info in env')
38+
exit(1)
39+
# TODO make this parameterizable
40+
os.environ['NCCL_SOCKET_IFNAME'] = '^docker0,lo'
41+
#os.environ['NCCL_IB_DISABLE'] = '0'
42+
43+
print('RANK = {}'.format(os.environ['RANK']))
44+
print('WORLD_SIZE = {}'.format(os.environ['WORLD_SIZE']))
45+
print('MASTER_ADDR = {}'.format(os.environ['MASTER_ADDR']))
46+
print('MASTER_PORT = {}'.format(os.environ['MASTER_PORT']))
47+
# print('MASTER_NODE = {}'.format(os.environ['MASTER_NODE']))
48+
print('NCCL_SOCKET_IFNAME new value = {}'.format(os.environ['NCCL_SOCKET_IFNAME']))
49+
1750
def get_local_rank():
1851
return int(os.environ['OMPI_COMM_WORLD_LOCAL_RANK'])
1952

2053
def get_rank(): # world_rank
2154
return int(os.environ['OMPI_COMM_WORLD_RANK'])
2255

56+
def get_world_size():
57+
return int(os.environ['OMPI_COMM_WORLD_SIZE'])
58+
2359
def get_global_size():
2460
return int(os.environ['OMPI_COMM_WORLD_SIZE'])
2561

2662
def get_local_size():
2763
return int(os.environ['OMPI_COMM_WORLD_LOCAL_SIZE'])
2864

29-
def get_world_size():
30-
return int(os.environ['WORLD_SIZE'])
31-
3265
def is_main_process():
3366
return get_rank() == 0
3467

0 commit comments

Comments
 (0)