From 68016ffb9b97a6da85ab0d98aa145cadda8ecbf5 Mon Sep 17 00:00:00 2001 From: Vinicius da Costa Date: Tue, 12 May 2026 19:06:33 +0000 Subject: [PATCH 1/3] Try to fix butler deploy by setting CLOUDSDK_PYTHON --- butler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/butler.py b/butler.py index 29b23c3740..7ae86d3f51 100644 --- a/butler.py +++ b/butler.py @@ -509,6 +509,7 @@ def _setup(args): """Set up configs and import paths.""" os.environ['ROOT_DIR'] = os.path.abspath('.') os.environ['PYTHONIOENCODING'] = 'UTF-8' + os.environ['CLOUDSDK_PYTHON'] = sys.executable sys.path.insert(0, os.path.abspath(os.path.join('src'))) from clusterfuzz._internal.base import modules From d830ce54759f98702f2b00589836d554fdc19199 Mon Sep 17 00:00:00 2001 From: Paulo Borges Date: Tue, 12 May 2026 19:10:18 +0000 Subject: [PATCH 2/3] retrieve job info from env instead of datastore in uworkers --- .../_internal/datastore/data_handler.py | 15 ++++++++++ .../tests/core/datastore/data_handler_test.py | 30 +++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/src/clusterfuzz/_internal/datastore/data_handler.py b/src/clusterfuzz/_internal/datastore/data_handler.py index 7b92f1ac52..8ddc70fb95 100644 --- a/src/clusterfuzz/_internal/datastore/data_handler.py +++ b/src/clusterfuzz/_internal/datastore/data_handler.py @@ -1214,6 +1214,15 @@ def bot_run_timed_out(): @memoize.wrap(memoize.Memcache(MEMCACHE_TTL_IN_SECONDS)) def get_component_name(job_type): """Gets component name for a job type.""" + if environment.is_uworker(): + # On uworkers, read from environment instead of Datastore. + for key, value in os.environ.items(): + if 'BUCKET_PATH' in key: + match = re.match(r'.*-([a-zA-Z0-9]+)-component', value) + if match: + return match.group(1) + return '' + job = data_types.Job.query(data_types.Job.name == job_type).get() if not job: return '' @@ -1230,6 +1239,9 @@ def get_component_name(job_type): @memoize.wrap(memoize.Memcache(MEMCACHE_TTL_IN_SECONDS)) def get_repository_for_component(component): """Get the repository based on component.""" + if environment.is_uworker(): + return '' + default_repository = '' repository = '' repository_mappings = db_config.get_value('component_repository_mappings') @@ -1262,6 +1274,9 @@ def get_value_from_job_definition(job_type, variable_pattern, default=None): if not job_type: return default + if environment.is_uworker(): + return environment.get_value(variable_pattern, default) + job = data_types.Job.query(data_types.Job.name == job_type).get() if not job: return default diff --git a/src/clusterfuzz/_internal/tests/core/datastore/data_handler_test.py b/src/clusterfuzz/_internal/tests/core/datastore/data_handler_test.py index ad6520284a..3d6a3f6ed7 100644 --- a/src/clusterfuzz/_internal/tests/core/datastore/data_handler_test.py +++ b/src/clusterfuzz/_internal/tests/core/datastore/data_handler_test.py @@ -1223,3 +1223,33 @@ def test_no_entities_found(self): self.GetEntitiesTestModel, equality_filters={'value': 5}) self.assertIsInstance(result, Generator) self.assertCountEqual(result, []) + + +class UworkerDataHandlerTest(unittest.TestCase): + """Tests for data_handler functions under uworker status.""" + + def setUp(self): + helpers.patch_environ(self) + helpers.patch(self, [ + 'clusterfuzz._internal.system.environment.is_uworker', + ]) + + def test_get_value_from_job_definition_uworker(self): + """Ensure that get_value_from_job_definition reads from environment on uworker.""" + self.mock.is_uworker.return_value = True + os.environ['CUSTOM_BINARY'] = 'True' + self.assertEqual( + True, data_handler.get_value_from_job_definition( + 'job', 'CUSTOM_BINARY')) + + def test_get_component_name_uworker(self): + """Ensure that get_component_name parses environment variable suffix on uworker.""" + self.mock.is_uworker.return_value = True + os.environ['RELEASE_BUILD_BUCKET_PATH'] = ( + 'gs://chromium-browser-component/releases') + self.assertEqual('browser', data_handler.get_component_name('job')) + + def test_get_repository_for_component_uworker(self): + """Ensure that get_repository_for_component returns empty string on uworker.""" + self.mock.is_uworker.return_value = True + self.assertEqual('', data_handler.get_repository_for_component('component')) From f1d0b125ed54ff022220dc829b1b1542c617b3c3 Mon Sep 17 00:00:00 2001 From: Dylan Jew Date: Fri, 15 May 2026 14:07:46 -0400 Subject: [PATCH 3/3] Replace INTERVAL duration fields with DOUBLE duration_seconds F1 doesn't support INTERVAL types natively (b/213368390). This PR replaces the INTERVAL duration fields with DOUBLE duration_seconds. ### Rollout I plan to just drop the existing cluster-fuzz.fuzzer_stats.daily_stats table after this is deployed and rerunning the job to backfill the data. Since there are no consumers of this data yet, this seems simpler than doing a dual write + alter table drop column, then changing the job load to stop including the old rows. The concern I had is that a BigQuery load without the old rows would fail, so dropping the table after this is deployed and then backfilling seems like a simpler approach. ### Testing I verified the SQL query by running it in BigQuery against the cluster-fuzz.fuzzer_stats.daily_stats table. --- .../_internal/cron/aggregate_fuzzer_stats.py | 56 ++++++++----------- .../cron/aggregate_fuzzer_stats_test.py | 28 +++++----- 2 files changed, 37 insertions(+), 47 deletions(-) diff --git a/src/clusterfuzz/_internal/cron/aggregate_fuzzer_stats.py b/src/clusterfuzz/_internal/cron/aggregate_fuzzer_stats.py index 2fd18c0429..534fdfbfd5 100644 --- a/src/clusterfuzz/_internal/cron/aggregate_fuzzer_stats.py +++ b/src/clusterfuzz/_internal/cron/aggregate_fuzzer_stats.py @@ -55,20 +55,20 @@ 'type': 'INTEGER', 'mode': 'NULLABLE' }, { - 'name': 'testcase_execution_duration', - 'type': 'INTERVAL', + 'name': 'testcase_execution_duration_seconds', + 'type': 'FLOAT', 'mode': 'NULLABLE' }, { 'name': 'testcases_generated', 'type': 'INTEGER', 'mode': 'NULLABLE' }, { - 'name': 'testcase_generation_duration', - 'type': 'INTERVAL', + 'name': 'testcase_generation_duration_seconds', + 'type': 'FLOAT', 'mode': 'NULLABLE' }, { - 'name': 'fuzzing_duration', - 'type': 'INTERVAL', + 'name': 'fuzzing_duration_seconds', + 'type': 'FLOAT', 'mode': 'NULLABLE' }] } @@ -151,33 +151,23 @@ def _query_fuzzer_stats(fuzzer_name, project_id, target_date_str): query = f""" SELECT - '{fuzzer_name}' as fuzzer_name, - CAST(DATE(TIMESTAMP_SECONDS(CAST(timestamp AS INT64))) AS STRING) as date, - SUM(testcases_executed) as testcases_executed, - CONCAT( - 'P', - CAST(EXTRACT(DAY FROM SUM(testcase_execution_duration)) AS STRING), 'DT', - CAST(EXTRACT(HOUR FROM SUM(testcase_execution_duration)) AS STRING), 'H', - CAST(EXTRACT(MINUTE FROM SUM(testcase_execution_duration)) AS STRING), 'M', - CAST(EXTRACT(SECOND FROM SUM(testcase_execution_duration)) AS STRING), 'S' - ) as testcase_execution_duration, - SUM(testcases_generated) as testcases_generated, - CONCAT( - 'P', - CAST(EXTRACT(DAY FROM SUM(testcase_generation_duration)) AS STRING), 'DT', - CAST(EXTRACT(HOUR FROM SUM(testcase_generation_duration)) AS STRING), 'H', - CAST(EXTRACT(MINUTE FROM SUM(testcase_generation_duration)) AS STRING), 'M', - CAST(EXTRACT(SECOND FROM SUM(testcase_generation_duration)) AS STRING), 'S' - ) as testcase_generation_duration, - CONCAT( - 'P', - CAST(EXTRACT(DAY FROM SUM(fuzzing_duration)) AS STRING), 'DT', - CAST(EXTRACT(HOUR FROM SUM(fuzzing_duration)) AS STRING), 'H', - CAST(EXTRACT(MINUTE FROM SUM(fuzzing_duration)) AS STRING), 'M', - CAST(EXTRACT(SECOND FROM SUM(fuzzing_duration)) AS STRING), 'S' - ) as fuzzing_duration - FROM - `{project_id}.{dataset_id}.{table_id}` + '{fuzzer_name}' as fuzzer_name, + CAST(DATE(TIMESTAMP_SECONDS(CAST(timestamp AS INT64))) AS STRING) as date, + SUM(testcases_executed) as testcases_executed, + (EXTRACT(DAY FROM SUM(testcase_execution_duration)) * 86400 + + EXTRACT(HOUR FROM SUM(testcase_execution_duration)) * 3600 + + EXTRACT(MINUTE FROM SUM(testcase_execution_duration)) * 60 + + EXTRACT(SECOND FROM SUM(testcase_execution_duration))) AS testcase_execution_duration_seconds, + SUM(testcases_generated) as testcases_generated, + (EXTRACT(DAY FROM SUM(testcase_generation_duration)) * 86400 + + EXTRACT(HOUR FROM SUM(testcase_generation_duration)) * 3600 + + EXTRACT(MINUTE FROM SUM(testcase_generation_duration)) * 60 + + EXTRACT(SECOND FROM SUM(testcase_generation_duration))) AS testcase_generation_duration_seconds, + (EXTRACT(DAY FROM SUM(fuzzing_duration)) * 86400 + + EXTRACT(HOUR FROM SUM(fuzzing_duration)) * 3600 + + EXTRACT(MINUTE FROM SUM(fuzzing_duration)) * 60 + + EXTRACT(SECOND FROM SUM(fuzzing_duration))) AS fuzzing_duration_seconds + FROM `{project_id}.{dataset_id}.{table_id}` WHERE DATE(TIMESTAMP_SECONDS(CAST(timestamp AS INT64))) = '{target_date_str}' GROUP BY diff --git a/src/clusterfuzz/_internal/tests/appengine/handlers/cron/aggregate_fuzzer_stats_test.py b/src/clusterfuzz/_internal/tests/appengine/handlers/cron/aggregate_fuzzer_stats_test.py index 611bccfaf4..d486af7e66 100644 --- a/src/clusterfuzz/_internal/tests/appengine/handlers/cron/aggregate_fuzzer_stats_test.py +++ b/src/clusterfuzz/_internal/tests/appengine/handlers/cron/aggregate_fuzzer_stats_test.py @@ -65,10 +65,10 @@ def test_aggregate_fuzzer_stats(self): 'fuzzer_name': 'ochang_js_fuzzer', 'date': '2026-04-30', 'testcases_executed': 10495, - 'testcase_execution_duration': 'P0DT11H12M11S', + 'testcase_execution_duration_seconds': 40331.0, 'testcases_generated': 10495, - 'testcase_generation_duration': 'P0DT1H15M33S', - 'fuzzing_duration': 'P0DT12H49M49S' + 'testcase_generation_duration_seconds': 4533.0, + 'fuzzing_duration_seconds': 46189.0 }], total_count=1) @@ -128,12 +128,12 @@ def test_aggregate_fuzzer_stats(self): self.assertEqual(uploaded_dict['fuzzer_name'], 'ochang_js_fuzzer') self.assertEqual(uploaded_dict['date'], '2026-04-30') self.assertEqual(uploaded_dict['testcases_executed'], 10495) - self.assertEqual(uploaded_dict['testcase_execution_duration'], - 'P0DT11H12M11S') + self.assertEqual(uploaded_dict['testcase_execution_duration_seconds'], + 40331.0) self.assertEqual(uploaded_dict['testcases_generated'], 10495) - self.assertEqual(uploaded_dict['testcase_generation_duration'], - 'P0DT1H15M33S') - self.assertEqual(uploaded_dict['fuzzing_duration'], 'P0DT12H49M49S') + self.assertEqual(uploaded_dict['testcase_generation_duration_seconds'], + 4533.0) + self.assertEqual(uploaded_dict['fuzzing_duration_seconds'], 46189.0) def test_aggregate_fuzzer_stats_ignoring_409(self): """Tests that execution successfully proceeds when the table already exists.""" @@ -146,10 +146,10 @@ def test_aggregate_fuzzer_stats_ignoring_409(self): 'fuzzer_name': 'ochang_js_fuzzer', 'date': '2026-04-30', 'testcases_executed': 10495, - 'testcase_execution_duration': 'P0DT11H12M11S', + 'testcase_execution_duration_seconds': 40331.0, 'testcases_generated': 10495, - 'testcase_generation_duration': 'P0DT1H15M33S', - 'fuzzing_duration': 'P0DT12H49M49S' + 'testcase_generation_duration_seconds': 4533.0, + 'fuzzing_duration_seconds': 46189.0 }], total_count=1) @@ -173,10 +173,10 @@ def test_aggregate_fuzzer_stats_with_date_flag(self): 'fuzzer_name': 'ochang_js_fuzzer', 'date': '2026-04-30', 'testcases_executed': 100, - 'testcase_execution_duration': 'P0DT1H0M0S', + 'testcase_execution_duration_seconds': 3600.0, 'testcases_generated': 100, - 'testcase_generation_duration': 'P0DT0H10M0S', - 'fuzzing_duration': 'P0DT1H10M0S' + 'testcase_generation_duration_seconds': 600.0, + 'fuzzing_duration_seconds': 4200.0 }], total_count=1)