Fix error block output color handling

solidpixel · solidpixel · commit ffbccca244a0 · 2022-02-23T15:10:59.000Z
diff --git a/Source/astcenc_entry.cpp b/Source/astcenc_entry.cpp
@@ -1119,7 +1119,7 @@ astcenc_error astcenc_decompress_image(
 			                          scb, blk);
 
 			write_image_block(image_out, blk, *ctx->bsd,
-			                 x * block_x, y * block_y, z * block_z, *swizzle);
+			                  x * block_x, y * block_y, z * block_z, *swizzle);
 		}
 
 		ctx->manage_decompress.complete_task_assignment(count);
diff --git a/Source/astcenc_image.cpp b/Source/astcenc_image.cpp
@@ -318,9 +318,9 @@ void write_image_block(
 				{
 					vint4 colori = vint4::zero();
 
-					if (blk.data_r[idx] == std::numeric_limits<float>::quiet_NaN())
+					// Errors are NaN encoded - convert to magenta error color
+					if (blk.data_r[idx] != blk.data_r[idx])
 					{
-						// Can't display NaN - show magenta error color
 						colori = vint4(0xFF, 0x00, 0xFF, 0xFF);
 					}
 					else if (needs_swz)
@@ -374,7 +374,8 @@ void write_image_block(
 				{
 					vint4 color;
 
-					if (blk.data_r[idx] == std::numeric_limits<float>::quiet_NaN())
+					// Errors are NaN encoded - convert to FP16 NaN
+					if (blk.data_r[idx] != blk.data_r[idx])
 					{
 						color = vint4(0xFFFF);
 					}
@@ -433,7 +434,8 @@ void write_image_block(
 				{
 					vfloat4 color = blk.texel(idx);
 
-					if (color.lane<0>() == std::numeric_limits<float>::quiet_NaN())
+					// Errors are NaN encoded - convert to FP32 NaN
+					if (color.lane<0>() != color.lane<0>())
 					{
 						color = vfloat4(std::numeric_limits<float>::quiet_NaN());
 					}

Original file line number	Diff line number	Diff line change
`@@ -1119,7 +1119,7 @@ astcenc_error astcenc_decompress_image(`
`1119`	`1119`	`scb, blk);`
`1120`	`1120`
`1121`	`1121`	`write_image_block(image_out, blk, *ctx->bsd,`
`1122`		`- x * block_x, y * block_y, z * block_z, *swizzle);`
	`1122`	`+ x * block_x, y * block_y, z * block_z, *swizzle);`
`1123`	`1123`	`}`
`1124`	`1124`
`1125`	`1125`	`ctx->manage_decompress.complete_task_assignment(count);`
Original file line number	Diff line number	Diff line change
`@@ -318,9 +318,9 @@ void write_image_block(`
`318`	`318`	`{`
`319`	`319`	`vint4 colori = vint4::zero();`
`320`	`320`
`321`		`- if (blk.data_r[idx] == std::numeric_limits<float>::quiet_NaN())`
	`321`	`+ // Errors are NaN encoded - convert to magenta error color`
	`322`	`+ if (blk.data_r[idx] != blk.data_r[idx])`
`322`	`323`	`{`
`323`		`- // Can't display NaN - show magenta error color`
`324`	`324`	`colori = vint4(0xFF, 0x00, 0xFF, 0xFF);`
`325`	`325`	`}`
`326`	`326`	`else if (needs_swz)`
`@@ -374,7 +374,8 @@ void write_image_block(`
`374`	`374`	`{`
`375`	`375`	`vint4 color;`
`376`	`376`
`377`		`- if (blk.data_r[idx] == std::numeric_limits<float>::quiet_NaN())`
	`377`	`+ // Errors are NaN encoded - convert to FP16 NaN`
	`378`	`+ if (blk.data_r[idx] != blk.data_r[idx])`
`378`	`379`	`{`
`379`	`380`	`color = vint4(0xFFFF);`
`380`	`381`	`}`
`@@ -433,7 +434,8 @@ void write_image_block(`
`433`	`434`	`{`
`434`	`435`	`vfloat4 color = blk.texel(idx);`
`435`	`436`
`436`		`- if (color.lane<0>() == std::numeric_limits<float>::quiet_NaN())`
	`437`	`+ // Errors are NaN encoded - convert to FP32 NaN`
	`438`	`+ if (color.lane<0>() != color.lane<0>())`
`437`	`439`	`{`
`438`	`440`	`color = vfloat4(std::numeric_limits<float>::quiet_NaN());`
`439`	`441`	`}`