Skip to content

Commit 10902ae

Browse files
dsmilkovdsmilkov
authored
[wasm] Fix bug in CropAndResize, and add AutoML models to bench… (tensorflow#2510)
BUG DEV Fix bug in CropAndResize, and add coverage in core. Also add AutoML models to benchmark.
1 parent ab0013d commit 10902ae

5 files changed

Lines changed: 77 additions & 12 deletions

File tree

‎tfjs-backend-wasm/src/cc/kernels/CropAndResize.cc‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ namespace {
3434
void interpolate_nearest(float* out_buf_ptr, const float* images_buf,
3535
std::vector<int> images_strides, int crop_width,
3636
int image_width, int image_width_m1, int num_channels,
37-
float extrapolation_value, int box_ind, float y_ind,
37+
float extrapolation_value, int box_offset, float y_ind,
3838
float width_scale, float x1, float x2) {
3939
for (int x = 0; x < crop_width; ++x) {
4040
const float x_ind = (crop_width > 1) ? x1 * image_width_m1 + x * width_scale
@@ -52,7 +52,7 @@ void interpolate_nearest(float* out_buf_ptr, const float* images_buf,
5252
float closest_y = round(y_ind);
5353
for (int c = 0; c < num_channels; ++c) {
5454
const int in_ind = c + closest_x * images_strides[2] +
55-
closest_y * images_strides[1] + box_ind;
55+
closest_y * images_strides[1] + box_offset;
5656
*out_buf_ptr = images_buf[in_ind];
5757
out_buf_ptr++;
5858
}
@@ -123,7 +123,7 @@ void CropAndResize(int images_id, int boxes_id, int box_ind_id, int num_boxes,
123123
continue;
124124
}
125125

126-
const int box_ind = *box_ind_buf * images_strides[0];
126+
const int box_offset = *box_ind_buf * images_strides[0];
127127

128128
const float height_scale =
129129
(crop_height > 1) ? (y2 - y1) * image_height_m1 / (crop_height - 1) : 0;
@@ -175,22 +175,22 @@ void CropAndResize(int images_id, int boxes_id, int box_ind_id, int num_boxes,
175175

176176
if (should_memcpy) {
177177
int y_ind_int = y_ind;
178-
images_buf += (y_ind_int * images_strides[1] + box_ind);
179-
180-
memcpy(out_buf_ptr, images_buf, sizeof(float) * crop_width);
178+
int offset = box_offset + y_ind_int * images_strides[1];
179+
memcpy(out_buf_ptr, images_buf + offset,
180+
sizeof(float) * crop_width * num_channels);
181181
continue;
182182
}
183183

184184
if (method == InterpolationMethod::BILINEAR) {
185185
tfjs::wasm::interpolate_bilinear(
186186
out_buf_ptr, images_buf, images_strides, crop_width, image_width,
187187
image_width_m1, image_height_m1, num_channels, should_extrapolate,
188-
extrapolation_value, box_ind, y_ind, width_scale, x1, x2);
188+
extrapolation_value, box_offset, y_ind, width_scale, x1, x2);
189189

190190
} else {
191191
interpolate_nearest(out_buf_ptr, images_buf, images_strides, crop_width,
192192
image_width, image_width_m1, num_channels,
193-
extrapolation_value, box_ind, y_ind, width_scale,
193+
extrapolation_value, box_offset, y_ind, width_scale,
194194
x1, x2);
195195
}
196196
}

‎tfjs-core/benchmarks/index.html‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,9 +88,11 @@ <h2>TensorFlow.js Model Benchmark</h2>
8888
<script src="https://unpkg.com/@tensorflow/tfjs-layers@latest/dist/tf-layers.js"></script>
8989
<script src="https://unpkg.com/@tensorflow/tfjs-converter@latest/dist/tf-converter.js"></script>
9090
<script src="https://unpkg.com/@tensorflow/tfjs-backend-wasm@latest/dist/tf-backend-wasm.js"></script>
91+
<script src="https://unpkg.com/@tensorflow/tfjs-automl@latest/dist/tf-automl.js"></script>
9192
<script src="https://cdn.jsdelivr.net/npm/@tensorflow-models/universal-sentence-encoder"></script>
9293
<script src="https://cdn.jsdelivr.net/npm/@tensorflow-models/posenet@2"></script>
9394
<script src="https://cdn.jsdelivr.net/npm/@tensorflow-models/body-pix@2"></script>
95+
9496
<script src="./modelConfig.js"></script>
9597
<script src="./util.js"></script>
9698
<script>
@@ -214,7 +216,7 @@ <h2>TensorFlow.js Model Benchmark</h2>
214216
const start = performance.now();
215217
let res = predict(model);
216218
if (res instanceof Promise) {
217-
await res;
219+
res = await res;
218220
}
219221

220222
if (res instanceof tf.Tensor) {

‎tfjs-core/benchmarks/modelConfig.js‎

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,8 @@ const sentences = [
7474
const benchmarks = {
7575
'mobilenet_v2': {
7676
load: async () => {
77-
const url = 'https://storage.googleapis.com/learnjs-data/mobilenet_v2_100_fused/model.json';
77+
const url =
78+
'https://storage.googleapis.com/learnjs-data/mobilenet_v2_100_fused/model.json';
7879
return tf.loadGraphModel(url);
7980
},
8081
predictFunc: () => {
@@ -97,7 +98,8 @@ const benchmarks = {
9798
},
9899
'face_detector': {
99100
load: async () => {
100-
const url = 'https://storage.googleapis.com/learnjs-data/face_detector_front/model.json';
101+
const url =
102+
'https://storage.googleapis.com/learnjs-data/face_detector_front/model.json';
101103
return tf.loadGraphModel(url);
102104
},
103105
predictFunc: () => {
@@ -107,6 +109,28 @@ const benchmarks = {
107109
};
108110
},
109111
},
112+
'AutoML Image': {
113+
load: async () => {
114+
const url =
115+
'https://storage.googleapis.com/tfjs-testing/tfjs-automl/img_classification/model.json';
116+
return tf.automl.loadImageClassification(url);
117+
},
118+
predictFunc: () => {
119+
const zeros = tf.zeros([224, 224, 3]);
120+
return model => model.classify(zeros);
121+
}
122+
},
123+
'AutoML Object': {
124+
load: async () => {
125+
const url =
126+
'https://storage.googleapis.com/tfjs-testing/tfjs-automl/object_detection/model.json';
127+
return tf.automl.loadObjectDetection(url);
128+
},
129+
predictFunc: () => {
130+
const zeros = tf.zeros([224, 224, 3]);
131+
return model => model.detect(zeros);
132+
}
133+
},
110134
'USE - batchsize 30': {
111135
load: async () => {
112136
return use.load();
@@ -160,7 +184,8 @@ const benchmarks = {
160184
},
161185
};
162186

163-
const imageBucket = 'https://storage.googleapis.com/tfjs-models/assets/posenet/';
187+
const imageBucket =
188+
'https://storage.googleapis.com/tfjs-models/assets/posenet/';
164189
async function loadImage(imagePath) {
165190
const image = new Image();
166191
const promise = new Promise((resolve, reject) => {

‎tfjs-core/src/ops/image_ops_test.ts‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -229,6 +229,33 @@ describeWithFlags('cropAndResize', ALL_ENVS, () => {
229229
expect(output.dtype).toBe('float32');
230230
expectArraysClose(await output.data(), [2.5]);
231231
});
232+
233+
it('5x5-bilinear, no change in shape', async () => {
234+
const image: tf.Tensor4D = tf.ones([1, 5, 5, 3]);
235+
const boxes: tf.Tensor2D = tf.tensor2d([0, 0, 1, 1], [1, 4]);
236+
const boxInd: tf.Tensor1D = tf.tensor1d([0], 'int32');
237+
238+
const output =
239+
tf.image.cropAndResize(image, boxes, boxInd, [5, 5], 'bilinear', 0);
240+
241+
expect(output.shape).toEqual([1, 5, 5, 3]);
242+
expect(output.dtype).toBe('float32');
243+
expectArraysClose(await output.data(), await image.data());
244+
});
245+
246+
it('5x5-bilinear, just a crop, no resize', async () => {
247+
const image: tf.Tensor4D = tf.ones([1, 6, 6, 3]);
248+
const boxes: tf.Tensor2D = tf.tensor2d([0.5, 0.5, 1, 1], [1, 4]);
249+
const boxInd: tf.Tensor1D = tf.tensor1d([0], 'int32');
250+
251+
const output =
252+
tf.image.cropAndResize(image, boxes, boxInd, [3, 3], 'bilinear', 0);
253+
254+
expect(output.shape).toEqual([1, 3, 3, 3]);
255+
expect(output.dtype).toBe('float32');
256+
expectArraysClose(await output.data(), await tf.ones([1, 3, 3, 3]).data());
257+
});
258+
232259
it('1x1-nearest', async () => {
233260
const image: tf.Tensor4D = tf.tensor4d([1, 2, 3, 4], [1, 2, 2, 1]);
234261
const boxes: tf.Tensor2D = tf.tensor2d([0, 0, 1, 1], [1, 4]);

‎tfjs-core/src/ops/resize_bilinear_test.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,17 @@ describeWithFlags('resizeBilinear', ALL_ENVS, () => {
2828
await output.data(), [2, 2, 2, 10 / 3, 10 / 3, 10 / 3, 4, 4, 4]);
2929
});
3030

31+
it('5x5-bilinear, no change in shape', async () => {
32+
const image: tf.Tensor4D = tf.ones([1, 5, 5, 3]);
33+
34+
const alignCorners = false;
35+
const output = tf.image.resizeBilinear(image, [5, 5], alignCorners);
36+
37+
expect(output.shape).toEqual([1, 5, 5, 3]);
38+
expect(output.dtype).toBe('float32');
39+
expectArraysClose(await output.data(), await image.data());
40+
});
41+
3142
it('simple alignCorners=true', async () => {
3243
const input = tf.tensor3d([2, 2, 4, 4], [2, 2, 1]);
3344
const output = input.resizeBilinear([3, 3], true);

0 commit comments

Comments
 (0)