diff --git a/butler.py b/butler.py index 29b23c3740..7ae86d3f51 100644 --- a/butler.py +++ b/butler.py @@ -509,6 +509,7 @@ def _setup(args): """Set up configs and import paths.""" os.environ['ROOT_DIR'] = os.path.abspath('.') os.environ['PYTHONIOENCODING'] = 'UTF-8' + os.environ['CLOUDSDK_PYTHON'] = sys.executable sys.path.insert(0, os.path.abspath(os.path.join('src'))) from clusterfuzz._internal.base import modules diff --git a/src/clusterfuzz/_internal/cron/aggregate_fuzzer_stats.py b/src/clusterfuzz/_internal/cron/aggregate_fuzzer_stats.py index 2fd18c0429..534fdfbfd5 100644 --- a/src/clusterfuzz/_internal/cron/aggregate_fuzzer_stats.py +++ b/src/clusterfuzz/_internal/cron/aggregate_fuzzer_stats.py @@ -55,20 +55,20 @@ 'type': 'INTEGER', 'mode': 'NULLABLE' }, { - 'name': 'testcase_execution_duration', - 'type': 'INTERVAL', + 'name': 'testcase_execution_duration_seconds', + 'type': 'FLOAT', 'mode': 'NULLABLE' }, { 'name': 'testcases_generated', 'type': 'INTEGER', 'mode': 'NULLABLE' }, { - 'name': 'testcase_generation_duration', - 'type': 'INTERVAL', + 'name': 'testcase_generation_duration_seconds', + 'type': 'FLOAT', 'mode': 'NULLABLE' }, { - 'name': 'fuzzing_duration', - 'type': 'INTERVAL', + 'name': 'fuzzing_duration_seconds', + 'type': 'FLOAT', 'mode': 'NULLABLE' }] } @@ -151,33 +151,23 @@ def _query_fuzzer_stats(fuzzer_name, project_id, target_date_str): query = f""" SELECT - '{fuzzer_name}' as fuzzer_name, - CAST(DATE(TIMESTAMP_SECONDS(CAST(timestamp AS INT64))) AS STRING) as date, - SUM(testcases_executed) as testcases_executed, - CONCAT( - 'P', - CAST(EXTRACT(DAY FROM SUM(testcase_execution_duration)) AS STRING), 'DT', - CAST(EXTRACT(HOUR FROM SUM(testcase_execution_duration)) AS STRING), 'H', - CAST(EXTRACT(MINUTE FROM SUM(testcase_execution_duration)) AS STRING), 'M', - CAST(EXTRACT(SECOND FROM SUM(testcase_execution_duration)) AS STRING), 'S' - ) as testcase_execution_duration, - SUM(testcases_generated) as testcases_generated, - CONCAT( - 'P', - CAST(EXTRACT(DAY FROM SUM(testcase_generation_duration)) AS STRING), 'DT', - CAST(EXTRACT(HOUR FROM SUM(testcase_generation_duration)) AS STRING), 'H', - CAST(EXTRACT(MINUTE FROM SUM(testcase_generation_duration)) AS STRING), 'M', - CAST(EXTRACT(SECOND FROM SUM(testcase_generation_duration)) AS STRING), 'S' - ) as testcase_generation_duration, - CONCAT( - 'P', - CAST(EXTRACT(DAY FROM SUM(fuzzing_duration)) AS STRING), 'DT', - CAST(EXTRACT(HOUR FROM SUM(fuzzing_duration)) AS STRING), 'H', - CAST(EXTRACT(MINUTE FROM SUM(fuzzing_duration)) AS STRING), 'M', - CAST(EXTRACT(SECOND FROM SUM(fuzzing_duration)) AS STRING), 'S' - ) as fuzzing_duration - FROM - `{project_id}.{dataset_id}.{table_id}` + '{fuzzer_name}' as fuzzer_name, + CAST(DATE(TIMESTAMP_SECONDS(CAST(timestamp AS INT64))) AS STRING) as date, + SUM(testcases_executed) as testcases_executed, + (EXTRACT(DAY FROM SUM(testcase_execution_duration)) * 86400 + + EXTRACT(HOUR FROM SUM(testcase_execution_duration)) * 3600 + + EXTRACT(MINUTE FROM SUM(testcase_execution_duration)) * 60 + + EXTRACT(SECOND FROM SUM(testcase_execution_duration))) AS testcase_execution_duration_seconds, + SUM(testcases_generated) as testcases_generated, + (EXTRACT(DAY FROM SUM(testcase_generation_duration)) * 86400 + + EXTRACT(HOUR FROM SUM(testcase_generation_duration)) * 3600 + + EXTRACT(MINUTE FROM SUM(testcase_generation_duration)) * 60 + + EXTRACT(SECOND FROM SUM(testcase_generation_duration))) AS testcase_generation_duration_seconds, + (EXTRACT(DAY FROM SUM(fuzzing_duration)) * 86400 + + EXTRACT(HOUR FROM SUM(fuzzing_duration)) * 3600 + + EXTRACT(MINUTE FROM SUM(fuzzing_duration)) * 60 + + EXTRACT(SECOND FROM SUM(fuzzing_duration))) AS fuzzing_duration_seconds + FROM `{project_id}.{dataset_id}.{table_id}` WHERE DATE(TIMESTAMP_SECONDS(CAST(timestamp AS INT64))) = '{target_date_str}' GROUP BY diff --git a/src/clusterfuzz/_internal/datastore/data_handler.py b/src/clusterfuzz/_internal/datastore/data_handler.py index 7b92f1ac52..8ddc70fb95 100644 --- a/src/clusterfuzz/_internal/datastore/data_handler.py +++ b/src/clusterfuzz/_internal/datastore/data_handler.py @@ -1214,6 +1214,15 @@ def bot_run_timed_out(): @memoize.wrap(memoize.Memcache(MEMCACHE_TTL_IN_SECONDS)) def get_component_name(job_type): """Gets component name for a job type.""" + if environment.is_uworker(): + # On uworkers, read from environment instead of Datastore. + for key, value in os.environ.items(): + if 'BUCKET_PATH' in key: + match = re.match(r'.*-([a-zA-Z0-9]+)-component', value) + if match: + return match.group(1) + return '' + job = data_types.Job.query(data_types.Job.name == job_type).get() if not job: return '' @@ -1230,6 +1239,9 @@ def get_component_name(job_type): @memoize.wrap(memoize.Memcache(MEMCACHE_TTL_IN_SECONDS)) def get_repository_for_component(component): """Get the repository based on component.""" + if environment.is_uworker(): + return '' + default_repository = '' repository = '' repository_mappings = db_config.get_value('component_repository_mappings') @@ -1262,6 +1274,9 @@ def get_value_from_job_definition(job_type, variable_pattern, default=None): if not job_type: return default + if environment.is_uworker(): + return environment.get_value(variable_pattern, default) + job = data_types.Job.query(data_types.Job.name == job_type).get() if not job: return default diff --git a/src/clusterfuzz/_internal/tests/appengine/handlers/cron/aggregate_fuzzer_stats_test.py b/src/clusterfuzz/_internal/tests/appengine/handlers/cron/aggregate_fuzzer_stats_test.py index 611bccfaf4..d486af7e66 100644 --- a/src/clusterfuzz/_internal/tests/appengine/handlers/cron/aggregate_fuzzer_stats_test.py +++ b/src/clusterfuzz/_internal/tests/appengine/handlers/cron/aggregate_fuzzer_stats_test.py @@ -65,10 +65,10 @@ def test_aggregate_fuzzer_stats(self): 'fuzzer_name': 'ochang_js_fuzzer', 'date': '2026-04-30', 'testcases_executed': 10495, - 'testcase_execution_duration': 'P0DT11H12M11S', + 'testcase_execution_duration_seconds': 40331.0, 'testcases_generated': 10495, - 'testcase_generation_duration': 'P0DT1H15M33S', - 'fuzzing_duration': 'P0DT12H49M49S' + 'testcase_generation_duration_seconds': 4533.0, + 'fuzzing_duration_seconds': 46189.0 }], total_count=1) @@ -128,12 +128,12 @@ def test_aggregate_fuzzer_stats(self): self.assertEqual(uploaded_dict['fuzzer_name'], 'ochang_js_fuzzer') self.assertEqual(uploaded_dict['date'], '2026-04-30') self.assertEqual(uploaded_dict['testcases_executed'], 10495) - self.assertEqual(uploaded_dict['testcase_execution_duration'], - 'P0DT11H12M11S') + self.assertEqual(uploaded_dict['testcase_execution_duration_seconds'], + 40331.0) self.assertEqual(uploaded_dict['testcases_generated'], 10495) - self.assertEqual(uploaded_dict['testcase_generation_duration'], - 'P0DT1H15M33S') - self.assertEqual(uploaded_dict['fuzzing_duration'], 'P0DT12H49M49S') + self.assertEqual(uploaded_dict['testcase_generation_duration_seconds'], + 4533.0) + self.assertEqual(uploaded_dict['fuzzing_duration_seconds'], 46189.0) def test_aggregate_fuzzer_stats_ignoring_409(self): """Tests that execution successfully proceeds when the table already exists.""" @@ -146,10 +146,10 @@ def test_aggregate_fuzzer_stats_ignoring_409(self): 'fuzzer_name': 'ochang_js_fuzzer', 'date': '2026-04-30', 'testcases_executed': 10495, - 'testcase_execution_duration': 'P0DT11H12M11S', + 'testcase_execution_duration_seconds': 40331.0, 'testcases_generated': 10495, - 'testcase_generation_duration': 'P0DT1H15M33S', - 'fuzzing_duration': 'P0DT12H49M49S' + 'testcase_generation_duration_seconds': 4533.0, + 'fuzzing_duration_seconds': 46189.0 }], total_count=1) @@ -173,10 +173,10 @@ def test_aggregate_fuzzer_stats_with_date_flag(self): 'fuzzer_name': 'ochang_js_fuzzer', 'date': '2026-04-30', 'testcases_executed': 100, - 'testcase_execution_duration': 'P0DT1H0M0S', + 'testcase_execution_duration_seconds': 3600.0, 'testcases_generated': 100, - 'testcase_generation_duration': 'P0DT0H10M0S', - 'fuzzing_duration': 'P0DT1H10M0S' + 'testcase_generation_duration_seconds': 600.0, + 'fuzzing_duration_seconds': 4200.0 }], total_count=1) diff --git a/src/clusterfuzz/_internal/tests/core/datastore/data_handler_test.py b/src/clusterfuzz/_internal/tests/core/datastore/data_handler_test.py index ad6520284a..3d6a3f6ed7 100644 --- a/src/clusterfuzz/_internal/tests/core/datastore/data_handler_test.py +++ b/src/clusterfuzz/_internal/tests/core/datastore/data_handler_test.py @@ -1223,3 +1223,33 @@ def test_no_entities_found(self): self.GetEntitiesTestModel, equality_filters={'value': 5}) self.assertIsInstance(result, Generator) self.assertCountEqual(result, []) + + +class UworkerDataHandlerTest(unittest.TestCase): + """Tests for data_handler functions under uworker status.""" + + def setUp(self): + helpers.patch_environ(self) + helpers.patch(self, [ + 'clusterfuzz._internal.system.environment.is_uworker', + ]) + + def test_get_value_from_job_definition_uworker(self): + """Ensure that get_value_from_job_definition reads from environment on uworker.""" + self.mock.is_uworker.return_value = True + os.environ['CUSTOM_BINARY'] = 'True' + self.assertEqual( + True, data_handler.get_value_from_job_definition( + 'job', 'CUSTOM_BINARY')) + + def test_get_component_name_uworker(self): + """Ensure that get_component_name parses environment variable suffix on uworker.""" + self.mock.is_uworker.return_value = True + os.environ['RELEASE_BUILD_BUCKET_PATH'] = ( + 'gs://chromium-browser-component/releases') + self.assertEqual('browser', data_handler.get_component_name('job')) + + def test_get_repository_for_component_uworker(self): + """Ensure that get_repository_for_component returns empty string on uworker.""" + self.mock.is_uworker.return_value = True + self.assertEqual('', data_handler.get_repository_for_component('component'))