Skip to content

Commit 8ee2191

Browse files
Lewington-pitsospatrickvonplaten
Lewington-pitsos
and
patrickvonplaten
authored
Integration tests precision improvement for inpainting (huggingface#1052)
* improve test precision get tests passing with greater precision using lewington images * make old numpy load function a wrapper around a more flexible numpy loading function * adhere to black formatting * add more black formatting * adhere to isort * loosen precision and replace path Co-authored-by: Patrick von Platen <patrick.v.platen@gmail.com>
1 parent 8608795 commit 8ee2191

6 files changed

Lines changed: 60 additions & 37 deletions

File tree

‎src/diffusers/utils/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
if is_torch_available():
4343
from .testing_utils import (
4444
floats_tensor,
45+
load_hf_numpy,
4546
load_image,
4647
load_numpy,
4748
parse_flag_from_env,

‎src/diffusers/utils/testing_utils.py‎

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,29 @@ def require_onnxruntime(test_case):
139139
return unittest.skipUnless(is_onnx_available(), "test requires onnxruntime")(test_case)
140140

141141

142+
def load_numpy(arry: Union[str, np.ndarray]) -> np.ndarray:
143+
if isinstance(arry, str):
144+
if arry.startswith("http://") or arry.startswith("https://"):
145+
response = requests.get(arry)
146+
response.raise_for_status()
147+
arry = np.load(BytesIO(response.content))
148+
elif os.path.isfile(arry):
149+
arry = np.load(arry)
150+
else:
151+
raise ValueError(
152+
f"Incorrect path or url, URLs must start with `http://` or `https://`, and {arry} is not a valid path"
153+
)
154+
elif isinstance(arry, np.ndarray):
155+
pass
156+
else:
157+
raise ValueError(
158+
"Incorrect format used for numpy ndarray. Should be an url linking to an image, a local path, or a"
159+
" ndarray."
160+
)
161+
162+
return arry
163+
164+
142165
def load_image(image: Union[str, PIL.Image.Image]) -> PIL.Image.Image:
143166
"""
144167
Args:
@@ -168,17 +191,13 @@ def load_image(image: Union[str, PIL.Image.Image]) -> PIL.Image.Image:
168191
return image
169192

170193

171-
def load_numpy(path) -> np.ndarray:
194+
def load_hf_numpy(path) -> np.ndarray:
172195
if not path.startswith("http://") or path.startswith("https://"):
173196
path = os.path.join(
174197
"https://huggingface.co/datasets/fusing/diffusers-testing/resolve/main", urllib.parse.quote(path)
175198
)
176199

177-
response = requests.get(path)
178-
response.raise_for_status()
179-
array = np.load(BytesIO(response.content))
180-
181-
return array
200+
return load_numpy(path)
182201

183202

184203
# --- pytest conf functions --- #

‎tests/models/test_models_unet_2d.py‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,15 @@
2121
import torch
2222

2323
from diffusers import UNet2DConditionModel, UNet2DModel
24-
from diffusers.utils import floats_tensor, load_numpy, logging, require_torch_gpu, slow, torch_all_close, torch_device
24+
from diffusers.utils import (
25+
floats_tensor,
26+
load_hf_numpy,
27+
logging,
28+
require_torch_gpu,
29+
slow,
30+
torch_all_close,
31+
torch_device,
32+
)
2533
from parameterized import parameterized
2634

2735
from ..test_modeling_common import ModelTesterMixin
@@ -423,7 +431,7 @@ def tearDown(self):
423431

424432
def get_latents(self, seed=0, shape=(4, 4, 64, 64), fp16=False):
425433
dtype = torch.float16 if fp16 else torch.float32
426-
image = torch.from_numpy(load_numpy(self.get_file_format(seed, shape))).to(torch_device).to(dtype)
434+
image = torch.from_numpy(load_hf_numpy(self.get_file_format(seed, shape))).to(torch_device).to(dtype)
427435
return image
428436

429437
def get_unet_model(self, fp16=False, model_id="CompVis/stable-diffusion-v1-4"):
@@ -439,7 +447,7 @@ def get_unet_model(self, fp16=False, model_id="CompVis/stable-diffusion-v1-4"):
439447

440448
def get_encoder_hidden_states(self, seed=0, shape=(4, 77, 768), fp16=False):
441449
dtype = torch.float16 if fp16 else torch.float32
442-
hidden_states = torch.from_numpy(load_numpy(self.get_file_format(seed, shape))).to(torch_device).to(dtype)
450+
hidden_states = torch.from_numpy(load_hf_numpy(self.get_file_format(seed, shape))).to(torch_device).to(dtype)
443451
return hidden_states
444452

445453
@parameterized.expand(

‎tests/models/test_models_vae.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020

2121
from diffusers import AutoencoderKL
2222
from diffusers.modeling_utils import ModelMixin
23-
from diffusers.utils import floats_tensor, load_numpy, require_torch_gpu, slow, torch_all_close, torch_device
23+
from diffusers.utils import floats_tensor, load_hf_numpy, require_torch_gpu, slow, torch_all_close, torch_device
2424
from parameterized import parameterized
2525

2626
from ..test_modeling_common import ModelTesterMixin
@@ -147,7 +147,7 @@ def tearDown(self):
147147

148148
def get_sd_image(self, seed=0, shape=(4, 3, 512, 512), fp16=False):
149149
dtype = torch.float16 if fp16 else torch.float32
150-
image = torch.from_numpy(load_numpy(self.get_file_format(seed, shape))).to(torch_device).to(dtype)
150+
image = torch.from_numpy(load_hf_numpy(self.get_file_format(seed, shape))).to(torch_device).to(dtype)
151151
return image
152152

153153
def get_sd_vae_model(self, model_id="CompVis/stable-diffusion-v1-4", fp16=False):

‎tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py‎

Lines changed: 12 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
UNet2DModel,
2929
VQModel,
3030
)
31-
from diffusers.utils import floats_tensor, load_image, slow, torch_device
31+
from diffusers.utils import floats_tensor, load_image, load_numpy, slow, torch_device
3232
from diffusers.utils.testing_utils import require_torch_gpu
3333
from PIL import Image
3434
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer
@@ -278,11 +278,10 @@ def test_stable_diffusion_inpaint_pipeline(self):
278278
"https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"
279279
"/in_paint/overture-creations-5sI6fQgYIuo_mask.png"
280280
)
281-
expected_image = load_image(
282-
"https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"
283-
"/in_paint/yellow_cat_sitting_on_a_park_bench.png"
281+
expected_image = load_numpy(
282+
"https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/in_paint"
283+
"/yellow_cat_sitting_on_a_park_bench.npy"
284284
)
285-
expected_image = np.array(expected_image, dtype=np.float32) / 255.0
286285

287286
model_id = "runwayml/stable-diffusion-inpainting"
288287
pipe = StableDiffusionInpaintPipeline.from_pretrained(
@@ -307,7 +306,7 @@ def test_stable_diffusion_inpaint_pipeline(self):
307306
image = output.images[0]
308307

309308
assert image.shape == (512, 512, 3)
310-
assert np.abs(expected_image - image).max() < 1e-2
309+
assert np.abs(expected_image - image).max() < 1e-3
311310

312311
def test_stable_diffusion_inpaint_pipeline_fp16(self):
313312
init_image = load_image(
@@ -318,11 +317,10 @@ def test_stable_diffusion_inpaint_pipeline_fp16(self):
318317
"https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"
319318
"/in_paint/overture-creations-5sI6fQgYIuo_mask.png"
320319
)
321-
expected_image = load_image(
322-
"https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"
323-
"/in_paint/yellow_cat_sitting_on_a_park_bench_fp16.png"
320+
expected_image = load_numpy(
321+
"https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/in_paint"
322+
"/yellow_cat_sitting_on_a_park_bench_fp16.npy"
324323
)
325-
expected_image = np.array(expected_image, dtype=np.float32) / 255.0
326324

327325
model_id = "runwayml/stable-diffusion-inpainting"
328326
pipe = StableDiffusionInpaintPipeline.from_pretrained(
@@ -360,11 +358,10 @@ def test_stable_diffusion_inpaint_pipeline_pndm(self):
360358
"https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"
361359
"/in_paint/overture-creations-5sI6fQgYIuo_mask.png"
362360
)
363-
expected_image = load_image(
364-
"https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"
365-
"/in_paint/yellow_cat_sitting_on_a_park_bench_pndm.png"
361+
expected_image = load_numpy(
362+
"https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/in_paint"
363+
"/yellow_cat_sitting_on_a_park_bench_pndm.npy"
366364
)
367-
expected_image = np.array(expected_image, dtype=np.float32) / 255.0
368365

369366
model_id = "runwayml/stable-diffusion-inpainting"
370367
pndm = PNDMScheduler.from_config(model_id, subfolder="scheduler")
@@ -388,4 +385,4 @@ def test_stable_diffusion_inpaint_pipeline_pndm(self):
388385
image = output.images[0]
389386

390387
assert image.shape == (512, 512, 3)
391-
assert np.abs(expected_image - image).max() < 1e-2
388+
assert np.abs(expected_image - image).max() < 1e-3

‎tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint_legacy.py‎

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
VQModel,
3232
)
3333
from diffusers.utils import floats_tensor, load_image, slow, torch_device
34-
from diffusers.utils.testing_utils import require_torch_gpu
34+
from diffusers.utils.testing_utils import load_numpy, require_torch_gpu
3535
from PIL import Image
3636
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer
3737

@@ -358,11 +358,10 @@ def test_stable_diffusion_inpaint_legacy_pipeline(self):
358358
"https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"
359359
"/in_paint/overture-creations-5sI6fQgYIuo_mask.png"
360360
)
361-
expected_image = load_image(
362-
"https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"
363-
"/in_paint/red_cat_sitting_on_a_park_bench.png"
361+
expected_image = load_numpy(
362+
"https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/in_paint"
363+
"/red_cat_sitting_on_a_park_bench.npy"
364364
)
365-
expected_image = np.array(expected_image, dtype=np.float32) / 255.0
366365

367366
model_id = "CompVis/stable-diffusion-v1-4"
368367
pipe = StableDiffusionInpaintPipeline.from_pretrained(
@@ -389,7 +388,7 @@ def test_stable_diffusion_inpaint_legacy_pipeline(self):
389388
image = output.images[0]
390389

391390
assert image.shape == (512, 512, 3)
392-
assert np.abs(expected_image - image).max() < 1e-2
391+
assert np.abs(expected_image - image).max() < 1e-3
393392

394393
def test_stable_diffusion_inpaint_legacy_pipeline_k_lms(self):
395394
# TODO(Anton, Patrick) - I think we can remove this test soon
@@ -401,11 +400,10 @@ def test_stable_diffusion_inpaint_legacy_pipeline_k_lms(self):
401400
"https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"
402401
"/in_paint/overture-creations-5sI6fQgYIuo_mask.png"
403402
)
404-
expected_image = load_image(
405-
"https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"
406-
"/in_paint/red_cat_sitting_on_a_park_bench_k_lms.png"
403+
expected_image = load_numpy(
404+
"https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/in_paint"
405+
"/red_cat_sitting_on_a_park_bench_k_lms.npy"
407406
)
408-
expected_image = np.array(expected_image, dtype=np.float32) / 255.0
409407

410408
model_id = "CompVis/stable-diffusion-v1-4"
411409
lms = LMSDiscreteScheduler.from_config(model_id, subfolder="scheduler")
@@ -434,7 +432,7 @@ def test_stable_diffusion_inpaint_legacy_pipeline_k_lms(self):
434432
image = output.images[0]
435433

436434
assert image.shape == (512, 512, 3)
437-
assert np.abs(expected_image - image).max() < 1e-2
435+
assert np.abs(expected_image - image).max() < 1e-3
438436

439437
def test_stable_diffusion_inpaint_legacy_intermediate_state(self):
440438
number_of_steps = 0

0 commit comments

Comments
 (0)