Skip to content

Commit 9526ddc

Browse files
kerryjiangOceania2018
kerryjiang
authored and
Oceania2018
committed
fixed issues about displaying progress in console when download/unzip resources
1 parent 1bf44ec commit 9526ddc

8 files changed

Lines changed: 57 additions & 34 deletions

File tree

‎src/TensorFlowHub/MnistModelLoader.cs‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,15 @@ public class MnistModelLoader : IModelLoader<MnistDataSet>
1515
private const string TEST_IMAGES = "t10k-images-idx3-ubyte.gz";
1616
private const string TEST_LABELS = "t10k-labels-idx1-ubyte.gz";
1717

18-
public static async Task<Datasets<MnistDataSet>> LoadAsync(string trainDir, bool oneHot = false, int? trainSize = null, int? validationSize = null, int? testSize = null)
18+
public static async Task<Datasets<MnistDataSet>> LoadAsync(string trainDir, bool oneHot = false, int? trainSize = null, int? validationSize = null, int? testSize = null, bool showProgressInConsole = false)
1919
{
2020
var loader = new MnistModelLoader();
2121

2222
var setting = new ModelLoadSetting
2323
{
2424
TrainDir = trainDir,
25-
OneHot = oneHot
25+
OneHot = oneHot,
26+
ShowProgressInConsole = showProgressInConsole
2627
};
2728

2829
if (trainSize.HasValue)
@@ -48,37 +49,37 @@ public async Task<Datasets<MnistDataSet>> LoadAsync(ModelLoadSetting setting)
4849
sourceUrl = DEFAULT_SOURCE_URL;
4950

5051
// load train images
51-
await this.DownloadAsync(sourceUrl + TRAIN_IMAGES, setting.TrainDir, TRAIN_IMAGES)
52+
await this.DownloadAsync(sourceUrl + TRAIN_IMAGES, setting.TrainDir, TRAIN_IMAGES, showProgressInConsole: setting.ShowProgressInConsole)
5253
.ShowProgressInConsole(setting.ShowProgressInConsole);
5354

54-
await this.UnzipAsync(Path.Combine(setting.TrainDir, TRAIN_IMAGES), setting.TrainDir)
55+
await this.UnzipAsync(Path.Combine(setting.TrainDir, TRAIN_IMAGES), setting.TrainDir, showProgressInConsole: setting.ShowProgressInConsole)
5556
.ShowProgressInConsole(setting.ShowProgressInConsole);
5657

5758
var trainImages = ExtractImages(Path.Combine(setting.TrainDir, Path.GetFileNameWithoutExtension(TRAIN_IMAGES)), limit: setting.TrainSize);
5859

5960
// load train labels
60-
await this.DownloadAsync(sourceUrl + TRAIN_LABELS, setting.TrainDir, TRAIN_LABELS)
61+
await this.DownloadAsync(sourceUrl + TRAIN_LABELS, setting.TrainDir, TRAIN_LABELS, showProgressInConsole: setting.ShowProgressInConsole)
6162
.ShowProgressInConsole(setting.ShowProgressInConsole);
6263

63-
await this.UnzipAsync(Path.Combine(setting.TrainDir, TRAIN_LABELS), setting.TrainDir)
64+
await this.UnzipAsync(Path.Combine(setting.TrainDir, TRAIN_LABELS), setting.TrainDir, showProgressInConsole: setting.ShowProgressInConsole)
6465
.ShowProgressInConsole(setting.ShowProgressInConsole);
6566

6667
var trainLabels = ExtractLabels(Path.Combine(setting.TrainDir, Path.GetFileNameWithoutExtension(TRAIN_LABELS)), one_hot: setting.OneHot, limit: setting.TrainSize);
6768

6869
// load test images
69-
await this.DownloadAsync(sourceUrl + TEST_IMAGES, setting.TrainDir, TEST_IMAGES)
70+
await this.DownloadAsync(sourceUrl + TEST_IMAGES, setting.TrainDir, TEST_IMAGES, showProgressInConsole: setting.ShowProgressInConsole)
7071
.ShowProgressInConsole(setting.ShowProgressInConsole);
7172

72-
await this.UnzipAsync(Path.Combine(setting.TrainDir, TEST_IMAGES), setting.TrainDir)
73+
await this.UnzipAsync(Path.Combine(setting.TrainDir, TEST_IMAGES), setting.TrainDir, showProgressInConsole: setting.ShowProgressInConsole)
7374
.ShowProgressInConsole(setting.ShowProgressInConsole);
7475

7576
var testImages = ExtractImages(Path.Combine(setting.TrainDir, Path.GetFileNameWithoutExtension(TEST_IMAGES)), limit: setting.TestSize);
7677

7778
// load test labels
78-
await this.DownloadAsync(sourceUrl + TEST_LABELS, setting.TrainDir, TEST_LABELS)
79+
await this.DownloadAsync(sourceUrl + TEST_LABELS, setting.TrainDir, TEST_LABELS, showProgressInConsole: setting.ShowProgressInConsole)
7980
.ShowProgressInConsole(setting.ShowProgressInConsole);
8081

81-
await this.UnzipAsync(Path.Combine(setting.TrainDir, TEST_LABELS), setting.TrainDir)
82+
await this.UnzipAsync(Path.Combine(setting.TrainDir, TEST_LABELS), setting.TrainDir, showProgressInConsole: setting.ShowProgressInConsole)
8283
.ShowProgressInConsole(setting.ShowProgressInConsole);
8384

8485
var testLabels = ExtractLabels(Path.Combine(setting.TrainDir, Path.GetFileNameWithoutExtension(TEST_LABELS)), one_hot: setting.OneHot, limit: setting.TestSize);

‎src/TensorFlowHub/Utils.cs‎

Lines changed: 39 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -19,27 +19,39 @@ public static async Task DownloadAsync<TDataSet>(this IModelLoader<TDataSet> mod
1919
await modelLoader.DownloadAsync(url, dir, fileName);
2020
}
2121

22-
public static async Task DownloadAsync<TDataSet>(this IModelLoader<TDataSet> modelLoader, string url, string dirSaveTo, string fileName)
22+
public static async Task DownloadAsync<TDataSet>(this IModelLoader<TDataSet> modelLoader, string url, string dirSaveTo, string fileName, bool showProgressInConsole = false)
2323
where TDataSet : IDataSet
2424
{
2525
if (!Path.IsPathRooted(dirSaveTo))
2626
dirSaveTo = Path.Combine(AppContext.BaseDirectory, dirSaveTo);
2727

2828
var fileSaveTo = Path.Combine(dirSaveTo, fileName);
2929

30+
if (showProgressInConsole)
31+
{
32+
Console.WriteLine($"Downloading {fileName}");
33+
}
34+
3035
if (File.Exists(fileSaveTo))
36+
{
37+
if (showProgressInConsole)
38+
{
39+
Console.WriteLine($"The file {fileName} already exists");
40+
}
41+
3142
return;
43+
}
3244

3345
Directory.CreateDirectory(dirSaveTo);
3446

3547
using (var wc = new WebClient())
3648
{
37-
await wc.DownloadFileTaskAsync(url, fileSaveTo);
49+
await wc.DownloadFileTaskAsync(url, fileSaveTo).ConfigureAwait(false);
3850
}
3951

4052
}
4153

42-
public static async Task UnzipAsync<TDataSet>(this IModelLoader<TDataSet> modelLoader, string zipFile, string saveTo)
54+
public static async Task UnzipAsync<TDataSet>(this IModelLoader<TDataSet> modelLoader, string zipFile, string saveTo, bool showProgressInConsole = false)
4355
where TDataSet : IDataSet
4456
{
4557
if (!Path.IsPathRooted(saveTo))
@@ -50,67 +62,76 @@ public static async Task UnzipAsync<TDataSet>(this IModelLoader<TDataSet> modelL
5062
if (!Path.IsPathRooted(zipFile))
5163
zipFile = Path.Combine(AppContext.BaseDirectory, zipFile);
5264

53-
var destFilePath = Path.Combine(saveTo, Path.GetFileNameWithoutExtension(zipFile));
65+
var destFileName = Path.GetFileNameWithoutExtension(zipFile);
66+
var destFilePath = Path.Combine(saveTo, destFileName);
67+
68+
if (showProgressInConsole)
69+
Console.WriteLine($"Unzippinng {Path.GetFileName(zipFile)}");
5470

5571
if (File.Exists(destFilePath))
56-
File.Delete(destFilePath);
72+
{
73+
if (showProgressInConsole)
74+
Console.WriteLine($"The file {destFileName} already exists");
75+
}
5776

5877
using (GZipStream unzipStream = new GZipStream(File.OpenRead(zipFile), CompressionMode.Decompress))
5978
{
6079
using (var destStream = File.Create(destFilePath))
6180
{
62-
await unzipStream.CopyToAsync(destStream);
63-
await destStream.FlushAsync();
81+
await unzipStream.CopyToAsync(destStream).ConfigureAwait(false);
82+
await destStream.FlushAsync().ConfigureAwait(false);
6483
destStream.Close();
6584
}
6685

6786
unzipStream.Close();
6887
}
69-
}
70-
71-
public static async Task ShowProgressInConsole(this Task task)
72-
{
73-
await ShowProgressInConsole(task, true);
74-
}
88+
}
7589

7690
public static async Task ShowProgressInConsole(this Task task, bool enable)
7791
{
7892
if (!enable)
7993
{
8094
await task;
95+
return;
8196
}
8297

8398
var cts = new CancellationTokenSource();
99+
84100
var showProgressTask = ShowProgressInConsole(cts);
85101

86102
try
87-
{
103+
{
88104
await task;
89105
}
90106
finally
91107
{
92-
cts.Cancel();
108+
cts.Cancel();
93109
}
110+
111+
await showProgressTask;
112+
Console.WriteLine("Done.");
94113
}
95114

96115
private static async Task ShowProgressInConsole(CancellationTokenSource cts)
97116
{
98117
var cols = 0;
99118

119+
await Task.Delay(1000);
120+
100121
while (!cts.IsCancellationRequested)
101122
{
102123
await Task.Delay(1000);
103124
Console.Write(".");
104125
cols++;
105126

106-
if (cols >= 50)
127+
if (cols % 50 == 0)
107128
{
108-
cols = 0;
109129
Console.WriteLine();
110130
}
111131
}
112132

113-
Console.WriteLine();
133+
if (cols > 0)
134+
Console.WriteLine();
114135
}
115136
}
116137
}

‎test/TensorFlowNET.Examples/BasicModels/KMeansClustering.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,8 @@ public void PrepareData()
7070
OneHot = true,
7171
TrainSize = train_size,
7272
ValidationSize = validation_size,
73-
TestSize = test_size
73+
TestSize = test_size,
74+
ShowProgressInConsole = true
7475
};
7576

7677
mnist = loader.LoadAsync(setting).Result;

‎test/TensorFlowNET.Examples/BasicModels/LogisticRegression.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ public bool Run()
124124

125125
public void PrepareData()
126126
{
127-
mnist = MnistModelLoader.LoadAsync(".resources/mnist", oneHot: true, trainSize: train_size, validationSize: validation_size, testSize: test_size).Result;
127+
mnist = MnistModelLoader.LoadAsync(".resources/mnist", oneHot: true, trainSize: train_size, validationSize: validation_size, testSize: test_size, showProgressInConsole: true).Result;
128128
}
129129

130130
public void SaveModel(Session sess)

‎test/TensorFlowNET.Examples/BasicModels/NearestNeighbor.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ public bool Run()
8484

8585
public void PrepareData()
8686
{
87-
mnist = MnistModelLoader.LoadAsync(".resources/mnist", oneHot: true, trainSize: TrainSize, validationSize: ValidationSize, testSize: TestSize).Result;
87+
mnist = MnistModelLoader.LoadAsync(".resources/mnist", oneHot: true, trainSize: TrainSize, validationSize: ValidationSize, testSize: TestSize, showProgressInConsole: true).Result;
8888
// In this example, we limit mnist data
8989
(Xtr, Ytr) = mnist.Train.GetNextBatch(TrainSize == null ? 5000 : TrainSize.Value / 100); // 5000 for training (nn candidates)
9090
(Xte, Yte) = mnist.Test.GetNextBatch(TestSize == null ? 200 : TestSize.Value / 100); // 200 for testing

‎test/TensorFlowNET.Examples/ImageProcessing/DigitRecognitionCNN.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -310,7 +310,7 @@ private Tensor fc_layer(Tensor x, int num_units, string name, bool use_relu = tr
310310

311311
public void PrepareData()
312312
{
313-
mnist = MnistModelLoader.LoadAsync(".resources/mnist", oneHot: true).Result;
313+
mnist = MnistModelLoader.LoadAsync(".resources/mnist", oneHot: true, showProgressInConsole: true).Result;
314314
(x_train, y_train) = Reformat(mnist.Train.Data, mnist.Train.Labels);
315315
(x_valid, y_valid) = Reformat(mnist.Validation.Data, mnist.Validation.Labels);
316316
(x_test, y_test) = Reformat(mnist.Test.Data, mnist.Test.Labels);

‎test/TensorFlowNET.Examples/ImageProcessing/DigitRecognitionNN.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ private Tensor fc_layer(Tensor x, int num_units, string name, bool use_relu = tr
121121

122122
public void PrepareData()
123123
{
124-
mnist = MnistModelLoader.LoadAsync(".resources/mnist", oneHot: true).Result;
124+
mnist = MnistModelLoader.LoadAsync(".resources/mnist", oneHot: true, showProgressInConsole: true).Result;
125125
}
126126

127127
public void Train(Session sess)

‎test/TensorFlowNET.Examples/ImageProcessing/DigitRecognitionRNN.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ public void Test(Session sess)
143143

144144
public void PrepareData()
145145
{
146-
mnist = MnistModelLoader.LoadAsync(".resources/mnist", oneHot: true).Result;
146+
mnist = MnistModelLoader.LoadAsync(".resources/mnist", oneHot: true, showProgressInConsole: true).Result;
147147
(x_train, y_train) = (mnist.Train.Data, mnist.Train.Labels);
148148
(x_valid, y_valid) = (mnist.Validation.Data, mnist.Validation.Labels);
149149
(x_test, y_test) = (mnist.Test.Data, mnist.Test.Labels);

0 commit comments

Comments
 (0)