Skip to content

Commit d596f6d

Browse files
solidpixelsolidpixel
authored
Add optional compression progress callback (#451)
This PR adds support for an optional progress reporting callback function, which is used by compression to report incremental progress as an image is processed. The callback function is called by one of the compression threads, so doing any significant work in the callback function will slow down compression performance. E.g. emitting a progress message to stdout on a test machine slows down -fast compression by ~3%. The command line compressor will emit a progress bar when called from a terminal, unless invoked with -silent.
1 parent e5e4ca4 commit d596f6d

6 files changed

Lines changed: 151 additions & 12 deletions

File tree

‎Docs/ChangeLog-4x.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,11 @@ The 4.7.0 release is a maintenance release.
3232
is stored to an 8-bit per component file format. This option must be set
3333
maually for compression (`-c*`) tool operation, as the desired decode mode
3434
cannot be reliably determined.
35+
* **Feature:** Library configuration supports a new optional progress
36+
reporting callback to be specified. This is called during compression to
37+
to allow interactive tooling use cases to display incremental progress. The
38+
command line tool uses this feature to show compression progress unless
39+
`-silent` is used.
3540

3641
<!-- ---------------------------------------------------------------------- -->
3742
## 4.6.1

‎Source/astcenc.h‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
// SPDX-License-Identifier: Apache-2.0
22
// ----------------------------------------------------------------------------
3-
// Copyright 2020-2023 Arm Limited
3+
// Copyright 2020-2024 Arm Limited
44
//
55
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
66
// use this file except in compliance with the License. You may obtain a copy
@@ -304,6 +304,11 @@ enum astcenc_type
304304
ASTCENC_TYPE_F32 = 2
305305
};
306306

307+
/**
308+
* @brief Function pointer type for compression progress reporting callback.
309+
*/
310+
extern "C" typedef void (*astcenc_progress_callback)(float);
311+
307312
/**
308313
* @brief Enable normal map compression.
309314
*
@@ -566,6 +571,16 @@ struct astcenc_config
566571
*/
567572
float tune_search_mode0_enable;
568573

574+
/**
575+
* @brief The progress callback, can be @c nullptr.
576+
*
577+
* If this is specified the codec will peridocially report progress for
578+
* compression as a percentage between 0 and 100. The callback is called from one
579+
* of the compressor threads, so doing significant work in the callback will
580+
* reduce compression performance.
581+
*/
582+
astcenc_progress_callback progress_callback;
583+
569584
#if defined(ASTCENC_DIAGNOSTICS)
570585
/**
571586
* @brief The path to save the diagnostic trace data to.

‎Source/astcenc_entry.cpp‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
// SPDX-License-Identifier: Apache-2.0
22
// ----------------------------------------------------------------------------
3-
// Copyright 2011-2023 Arm Limited
3+
// Copyright 2011-2024 Arm Limited
44
//
55
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
66
// use this file except in compliance with the License. You may obtain a copy
@@ -830,7 +830,7 @@ static void compress_image(
830830
auto& temp_buffers = ctx.working_buffers[thread_index];
831831

832832
// Only the first thread actually runs the initializer
833-
ctxo.manage_compress.init(block_count);
833+
ctxo.manage_compress.init(block_count, ctx.config.progress_callback);
834834

835835
// Determine if we can use an optimized load function
836836
bool needs_swz = (swizzle.r != ASTCENC_SWZ_R) || (swizzle.g != ASTCENC_SWZ_G) ||
@@ -1155,6 +1155,7 @@ astcenc_error astcenc_decompress_image(
11551155
unsigned int xblocks = (image_out.dim_x + block_x - 1) / block_x;
11561156
unsigned int yblocks = (image_out.dim_y + block_y - 1) / block_y;
11571157
unsigned int zblocks = (image_out.dim_z + block_z - 1) / block_z;
1158+
unsigned int block_count = zblocks * yblocks * xblocks;
11581159

11591160
int row_blocks = xblocks;
11601161
int plane_blocks = xblocks * yblocks;
@@ -1179,7 +1180,7 @@ astcenc_error astcenc_decompress_image(
11791180
}
11801181

11811182
// Only the first thread actually runs the initializer
1182-
ctxo->manage_decompress.init(zblocks * yblocks * xblocks);
1183+
ctxo->manage_decompress.init(block_count, nullptr);
11831184

11841185
// All threads run this processing loop until there is no work remaining
11851186
while (true)

‎Source/astcenc_internal_entry.h‎

Lines changed: 64 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
// SPDX-License-Identifier: Apache-2.0
22
// ----------------------------------------------------------------------------
3-
// Copyright 2011-2022 Arm Limited
3+
// Copyright 2011-2024 Arm Limited
44
//
55
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
66
// use this file except in compliance with the License. You may obtain a copy
@@ -118,6 +118,18 @@ class ParallelManager
118118
/** @brief Number of tasks that need to be processed. */
119119
unsigned int m_task_count;
120120

121+
/** @brief Progress callback (optional). */
122+
astcenc_progress_callback m_callback;
123+
124+
/** @brief Lock used for callback synchronization. */
125+
std::mutex m_callback_lock;
126+
127+
/** @brief Minimum progress before making a callback. */
128+
float m_callback_min_diff;
129+
130+
/** @brief Last progress callback value. */
131+
float m_callback_last_value;
132+
121133
public:
122134
/** @brief Create a new ParallelManager. */
123135
ParallelManager()
@@ -138,6 +150,8 @@ class ParallelManager
138150
m_start_count = 0;
139151
m_done_count = 0;
140152
m_task_count = 0;
153+
m_callback_last_value = 0.0f;
154+
m_callback_min_diff = 1.0f;
141155
}
142156

143157
/**
@@ -166,14 +180,20 @@ class ParallelManager
166180
* initialization. Other threads will block and wait for it to complete.
167181
*
168182
* @param task_count Total number of tasks needing processing.
183+
* @param callback Function pointer for progress status callbacks.
169184
*/
170-
void init(unsigned int task_count)
185+
void init(unsigned int task_count, astcenc_progress_callback callback)
171186
{
172187
std::lock_guard<std::mutex> lck(m_lock);
173188
if (!m_init_done)
174189
{
190+
m_callback = callback;
175191
m_task_count = task_count;
176192
m_init_done = true;
193+
194+
// Report every 1% or 4096 blocks, whichever is larger, to avoid callback overhead
195+
float min_diff = (4096.0f / static_cast<float>(task_count)) * 100.0f;
196+
m_callback_min_diff = astc::max(min_diff, 1.0f);
177197
}
178198
}
179199

@@ -212,12 +232,49 @@ class ParallelManager
212232
{
213233
// Note: m_done_count cannot use an atomic without the mutex; this has a race between the
214234
// update here and the wait() for other threads
215-
std::unique_lock<std::mutex> lck(m_lock);
216-
this->m_done_count += count;
217-
if (m_done_count == m_task_count)
235+
unsigned int local_count;
236+
float local_last_value;
218237
{
219-
lck.unlock();
220-
m_complete.notify_all();
238+
std::unique_lock<std::mutex> lck(m_lock);
239+
m_done_count += count;
240+
local_count = m_done_count;
241+
local_last_value = m_callback_last_value;
242+
243+
if (m_done_count == m_task_count)
244+
{
245+
// Ensure the progress bar hits 100%
246+
if (m_callback)
247+
{
248+
std::unique_lock<std::mutex> cblck(m_callback_lock);
249+
m_callback(100.0f);
250+
m_callback_last_value = 100.0f;
251+
}
252+
253+
lck.unlock();
254+
m_complete.notify_all();
255+
}
256+
}
257+
258+
// Process progress callback if we have one
259+
if (m_callback)
260+
{
261+
// Initial lockless test - have we progressed enough to emit?
262+
float num = static_cast<float>(local_count);
263+
float den = static_cast<float>(m_task_count);
264+
float this_value = (num / den) * 100.0f;
265+
bool report_test = (this_value - local_last_value) > m_callback_min_diff;
266+
267+
// Recheck under lock, because another thread might report first
268+
if (report_test)
269+
{
270+
std::unique_lock<std::mutex> cblck(m_callback_lock);
271+
bool report_retest = (this_value - m_callback_last_value) > m_callback_min_diff;
272+
if (report_retest)
273+
{
274+
m_callback(this_value);
275+
m_callback_last_value = this_value;
276+
}
277+
}
221278
}
222279
}
223280

‎Source/astcenccli_toplevel.cpp‎

Lines changed: 60 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
// SPDX-License-Identifier: Apache-2.0
22
// ----------------------------------------------------------------------------
3-
// Copyright 2011-2023 Arm Limited
3+
// Copyright 2011-2024 Arm Limited
44
//
55
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
66
// use this file except in compliance with the License. You may obtain a copy
@@ -22,6 +22,12 @@
2222
#include "astcenc.h"
2323
#include "astcenccli_internal.h"
2424

25+
#if defined(_WIN32)
26+
#include <io.h>
27+
#define isatty _isatty
28+
#else
29+
#include <unistd.h>
30+
#endif
2531
#include <cassert>
2632
#include <cstring>
2733
#include <functional>
@@ -156,6 +162,35 @@ struct decompression_workload
156162
astcenc_error error;
157163
};
158164

165+
/**
166+
* @brief Callback emitting a progress bar
167+
*/
168+
extern "C" void progress_emitter(
169+
float value
170+
) {
171+
const unsigned int bar_size = 25;
172+
unsigned int parts = static_cast<int>(value / 4.0f);
173+
174+
char buffer[bar_size + 3];
175+
buffer[0] = '[';
176+
177+
for (unsigned int i = 0; i < parts; i++)
178+
{
179+
buffer[i + 1] = '=';
180+
}
181+
182+
for (unsigned int i = parts; i < bar_size; i++)
183+
{
184+
buffer[i + 1] = ' ';
185+
}
186+
187+
buffer[bar_size + 1] = ']';
188+
buffer[bar_size + 2] = '\0';
189+
190+
printf(" Progress: %s %03.1f%%\r", buffer, static_cast<double>(value));
191+
fflush(stdout);
192+
}
193+
159194
/**
160195
* @brief Test if a string argument is a well formed float.
161196
*/
@@ -1954,6 +1989,18 @@ int astcenc_main(
19541989
return 1;
19551990
}
19561991

1992+
// Enable progress callback if not in silent mode and using a terminal
1993+
#if defined(_WIN32)
1994+
int stdoutfno = _fileno(stdout);
1995+
#else
1996+
int stdoutfno = STDOUT_FILENO;
1997+
#endif
1998+
1999+
if ((!cli_config.silentmode) && isatty(stdoutfno))
2000+
{
2001+
config.progress_callback = progress_emitter;
2002+
}
2003+
19572004
astcenc_image* image_uncomp_in = nullptr ;
19582005
unsigned int image_uncomp_in_component_count = 0;
19592006
bool image_uncomp_in_is_hdr = false;
@@ -2122,6 +2169,13 @@ int astcenc_main(
21222169
double start_compression_time = get_time();
21232170
for (unsigned int i = 0; i < cli_config.repeat_count; i++)
21242171
{
2172+
if (config.progress_callback)
2173+
{
2174+
printf("Compression\n");
2175+
printf("===========\n");
2176+
printf("\n");
2177+
}
2178+
21252179
double start_iter_time = get_time();
21262180
if (cli_config.thread_count > 1)
21272181
{
@@ -2136,6 +2190,11 @@ int astcenc_main(
21362190

21372191
astcenc_compress_reset(codec_context);
21382192

2193+
if (config.progress_callback)
2194+
{
2195+
printf("\n\n");
2196+
}
2197+
21392198
double iter_time = get_time() - start_iter_time;
21402199
best_compression_time = astc::min(iter_time, best_compression_time);
21412200
}

‎jenkins/nightly.Jenkinsfile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,8 @@ pipeline {
204204
cd build_rel_arm64
205205
cmake -G "Visual Studio 17 2022" -A ARM64 -T ClangCL -DASTCENC_ISA_NEON=ON -DASTCENC_PACKAGE=arm64-clangcl ..
206206
msbuild astcencoder.sln -property:Configuration=Release
207+
msbuild PACKAGE.vcxproj -property:Configuration=Release
208+
msbuild INSTALL.vcxproj -property:Configuration=Release
207209
'''
208210
}
209211
}

0 commit comments

Comments
 (0)