#!/usr/bin/env python3

# /// script

# requires-python = ">=3.10"

# dependencies = [

# "GitPython",

# "PyGithub",

# "rich-click",

# "packaging",

# "rich",

# ]

# ///

# Licensed to the Apache Software Foundation (ASF) under one

# or more contributor license agreements. See the NOTICE file

# distributed with this work for additional information

# regarding copyright ownership. The ASF licenses this file

# to you under the Apache License, Version 2.0 (the

# "License"); you may not use this file except in compliance

# with the License. You may obtain a copy of the License at

#

# http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing,

# software distributed under the License is distributed on an

# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY

# KIND, either express or implied. See the License for the

# specific language governing permissions and limitations

# under the License.

# This tool is based on the Spark merge_spark_pr script:

# https://github.com/apache/spark/blob/master/dev/merge_spark_pr.py

from __future__ import annotations

import re

import sys

from collections import Counter, defaultdict

from typing import TYPE_CHECKING

import git

import rich_click as click

from github import Auth, Github

from packaging import version

from rich.console import Console

from rich.progress import Progress

if TYPE_CHECKING:

from github.Issue import Issue

GIT_COMMIT_FIELDS = ["id", "author_name", "author_email", "date", "subject", "body"]

GIT_LOG_FORMAT = "%x1f".join(["%h", "%an", "%ae", "%ad", "%s", "%b"]) + "%x1e"

pr_title_re = re.compile(r".*\((#[0-9]{1,6})\)$")

STATUS_COLOR_MAP = {

"Closed": "yellow",

"Merged": "green",

"Open": "red",

}

DEFAULT_SECTION_NAME = "Uncategorized"

def get_commits_between(

repo,

previous_version,

target_version,

files=None,

):

log_args = [f"--format={GIT_LOG_FORMAT}", previous_version + ".." + target_version]

if files:

log_args.append("--")

log_args.append(" ".join(files))

log = repo.git.log(*log_args)

log = log.strip("\n\x1e").split("\x1e")

log = [row.strip().split("\x1f") for row in log]

return [dict(zip(GIT_COMMIT_FIELDS, row)) for row in log]

def style_issue_status(status):

if status in STATUS_COLOR_MAP:

return click.style(status[:6].ljust(6), STATUS_COLOR_MAP[status])

return status[:6].ljust(6)

def get_issue_type(issue):

label_prefix = "type:"

issue_type = DEFAULT_SECTION_NAME

if issue.labels:

for label in issue.labels:

if label.name.startswith(label_prefix):

return label.name.replace(label_prefix, "").strip()

if label.name == "changelog:skip":

return "(skip)"

return issue_type

def build_main_commits_cache(repo: git.Repo, issue_numbers: set[int]) -> dict[int, str]:

"""Build a cache of PR number -> main branch commit SHA with a single git log operation"""

cache = {}

if not issue_numbers:

return cache

try:

# Single git log to get all main branch commits

# Use a reasonable range to avoid scanning entire history

try:

# Try to get commits since last major version (should cover most PR ranges)

log_output = repo.git.log("origin/main", "--format=%H %s", "--since=1 year ago")

except Exception:

# Fallback to last 2000 commits if date fails

log_output = repo.git.log("origin/main", "--format=%H %s", "-2000")

# Use regex to find all PR numbers in commit messages at once

import re

pr_pattern = r"\(#(\d+)\)$" # PR number at end of commit message

for commit_line in log_output.splitlines():

if not commit_line:

continue

# Find PR number at end of commit message

match = re.search(pr_pattern, commit_line)

if match:

pr_number = int(match.group(1))

if pr_number in issue_numbers:

commit_sha = commit_line.split(" ")[0]

cache[pr_number] = commit_sha

except Exception:

# Fallback to empty cache if git operation fails

pass

return cache

def get_commit_in_main_associated_with_pr(repo: git.Repo, issue: Issue, main_commits_cache: dict[int, str]) -> str | None:

"""For a PR, find the associated merged commit & return its SHA"""

if issue.pull_request:

# Use cache (should be pre-populated)

return main_commits_cache.get(issue.number)

return None

def build_cherrypicked_cache(repo: git.Repo, issue_numbers: list[int], previous_version: str | None = None) -> dict[int, bool]:

"""Build a cache of which issues are cherry-picked by doing a single git log operation"""

cache = {num: False for num in issue_numbers}

if not issue_numbers:

return cache

# Get all commits in range and process them

log_args = ["--format=%H %s"]

if previous_version:

log_args.append(previous_version + "..")

try:

log_output = repo.git.log(*log_args)

# Use regex to find all PR numbers in the entire log output at once

import re

pr_pattern = r"\(#(\d+)\)"

for commit_line in log_output.splitlines():

# Find all PR numbers in this commit

matches = re.findall(pr_pattern, commit_line)

for match in matches:

issue_num = int(match)

if issue_num in cache:

cache[issue_num] = True

except Exception:

# Fallback to individual checks if batch fails

pass

return cache

def is_cherrypicked(repo: git.Repo, issue: Issue, previous_version: str | None = None) -> bool:

"""Check if a given issue is cherry-picked in the current branch or not"""

log_args = ["--format=%H %s", f"--grep=(#{issue.number})"]

if previous_version:

log_args.append(previous_version + "..")

log_output = repo.git.log(*log_args)

for commit_line in log_output.splitlines():

# Check if the PR number exists in the commit message (handles cherry-picked commits with multiple PR numbers)

if commit_line and f"(#{issue.number})" in commit_line:

return True

return False

def is_pr(issue: Issue) -> bool:

return "apache/airflow/pull/" in issue.html_url

def files_touched(repo, commit_id: str) -> list[str]:

real_commit = repo.commit(commit_id)

return real_commit.stats.files.keys()

def is_core_commit(files: list[str]) -> bool:

# We list out things that _aren't_ core files,

# and we want to know if the commit changes anything

# outside of these designated-non-core files

non_core_files = [

# Providers

"airflow/providers/",

"airflow/provider.yaml.schema.json",

"docs/apache-airflow-providers-",

# chart

"chart/",

"docs/helm-chart/",

"clients",

# non-released docs

"COMMITTERS.rst",

"contributing_docs/",

"INTHEWILD.md",

"INSTALL",

"README.md",

"images/",

"codecov.yml",

"kubernetes-tests/",

".github/",

".pre-commit-config.yaml",

"yamllint-config.yml",

".markdownlint.yml",

"tests/",

"dev/",

"Dockerfile",

"Dockerfile.ci",

".hadolint.yaml",

"scripts/",

"docs/build_docs.py",

"docs/start_doc_server.sh",

".devcontainer/",

"docker-context-files/",

# Misc

".dockerignore",

"docs/spelling_wordlist.txt",

"docs/integration-logos/*",

"docs/exts/",

"docs/docker-stack",

"docs/README.rst",

"docker-tests/",

"kubernetes-tests/",

"helm-tests/",

".asf.yaml",

".mailmap",

"breeze-legacy",

"breeze-complete",

".rat-excludes",

".gitattributes",

".gitpod.yml",

"generated/",

]

for file in files:

for ignore in non_core_files:

if file.startswith(ignore):

break

# Handle renaming. Form: {old_name => new_name}somename.py

if file.startswith("{"):

new_files = file[1:].split(" => ")

if any(n.strip().startswith(ignore) for n in new_files):

break

else:

return True

return False

def print_changelog(sections):

for section, lines in sections.items():

if section == "(skip)":

continue

print(section)

print('"' * len(section))

for line in lines:

print("-", line)

print()

@click.group()

def cli():

r"""

This tool should be used by Airflow Release Manager to verify what GitHub issues

were merged in the current working branch.

airflow-github compare <target_version> <github_token>

"""

@cli.command(short_help="Compare a GitHub target version against git merges")

@click.argument("target_version")

@click.argument("github-token", envvar="GITHUB_TOKEN")

@click.option(

"--previous-version",

"previous_version",

help="Specify the previous tag on the working branch to limit"

" searching for few commits to find the cherry-picked commits",

)

@click.option("--unmerged", "show_uncherrypicked_only", help="Show unmerged PRs only", is_flag=True)

@click.option("--show-commits", help="Show commit SHAs (default: on, off when --unmerged)", is_flag=True, default=None)

def compare(target_version, github_token, previous_version=None, show_uncherrypicked_only=False, show_commits=None):

# Set smart defaults

if show_commits is None:

show_commits = not show_uncherrypicked_only # Default off for --unmerged

repo = git.Repo(".", search_parent_directories=True)

github_handler = Github(auth=Auth.Token(github_token))

# Fetch PRs and Issues separately, with merged PRs identified upfront

merged_prs: list[Issue] = list(

github_handler.search_issues(

f'repo:apache/airflow milestone:"Airflow {target_version}" is:pull-request is:merged'

)

)

closed_prs: list[Issue] = list(

github_handler.search_issues(

f'repo:apache/airflow milestone:"Airflow {target_version}" is:pull-request is:closed -is:merged'

)

)

open_prs: list[Issue] = list(

github_handler.search_issues(

f'repo:apache/airflow milestone:"Airflow {target_version}" is:pull-request is:open'

)

)

# Skip fetching issues if we only care about unmerged PRs

if show_uncherrypicked_only:

issues = []

else:

issues: list[Issue] = list(

github_handler.search_issues(

f'repo:apache/airflow milestone:"Airflow {target_version}" is:issue'

)

)

# Create a merge status lookup

pr_merge_status_cache = {}

for pr in merged_prs:

pr_merge_status_cache[pr.number] = True

for pr in closed_prs:

pr_merge_status_cache[pr.number] = False

# Open PRs are neither merged nor closed, so we don't need to cache them

milestone_issues = merged_prs + closed_prs + open_prs + issues

num_cherrypicked = 0

num_uncherrypicked = Counter()

# :<18 says left align, pad to 18, :>6 says right align, pad to 6

# :<50.50 truncates after 50 chars

# !s forces as string

if show_commits:

formatstr = (

"{number:>6} | {typ!s:<5} | {changelog!s:<13} | {status!s} "

"| {title:<83.83} | {merged:<6} | {commit:>7.7} | {url}"

)

header_fields = {

"number": "NUMBER",

"typ": "TYPE",

"changelog": "CHANGELOG",

"status": "STATUS".ljust(6),

"title": "TITLE",

"merged": "CHERRY",

"commit": "COMMIT",

"url": "URL",

}

else:

formatstr = (

"{number:>6} | {typ!s:<5} | {changelog!s:<13} | {status!s} "

"| {title:<95.95} | {merged:<6} | {url}"

)

header_fields = {

"number": "NUMBER",

"typ": "TYPE",

"changelog": "CHANGELOG",

"status": "STATUS".ljust(6),

"title": "TITLE",

"merged": "CHERRY",

"commit": "", # Not used

"url": "URL",

}

print(formatstr.format(**header_fields))

milestone_issues = sorted(

milestone_issues, key=lambda x: x.closed_at if x.closed_at else x.created_at, reverse=True

)

# Build caches for performance optimization

issue_numbers = [issue.number for issue in milestone_issues if is_pr(issue)]

# Convert to set for O(1) lookups in cache building

issue_numbers_set = set(issue_numbers)

# Build all caches upfront with batch operations

if show_commits:

main_commits_cache = build_main_commits_cache(repo, issue_numbers_set)

else:

main_commits_cache = {}

cherrypicked_cache = build_cherrypicked_cache(repo, issue_numbers, previous_version)

for issue in milestone_issues:

issue_is_pr = is_pr(issue)

# Determine status - differentiate between Closed and Merged for PRs using cache

if issue_is_pr and issue.state == "closed":

is_merged = pr_merge_status_cache.get(issue.number, False)

status = "Merged" if is_merged else "Closed"

else:

status = issue.state.capitalize()

# Checks if commit was cherrypicked into branch using cache

if issue_is_pr:

is_cherry_picked = cherrypicked_cache.get(issue.number, False)

else:

is_cherry_picked = is_cherrypicked(repo, issue, previous_version)

if is_cherry_picked:

num_cherrypicked += 1

if show_uncherrypicked_only:

continue

cherrypicked = click.style("Yes".ljust(6), "green")

elif not issue_is_pr and show_uncherrypicked_only:

# Don't show issues when looking for unmerged PRs

continue

elif issue_is_pr:

num_uncherrypicked[status] += 1

cherrypicked = click.style("No".ljust(6), "red")

else:

cherrypicked = ""

fields = dict(

number=issue.number,

typ="PR" if issue_is_pr else "Issue",

changelog=get_issue_type(issue) if issue_is_pr else "",

status=style_issue_status(status),

title=issue.title,

url=issue.html_url,

)

# Only get commit info if we're showing commits

if show_commits:

commit_in_main = get_commit_in_main_associated_with_pr(repo, issue, main_commits_cache)

fields["commit"] = commit_in_main if commit_in_main else ""

print(formatstr.format(**fields, merged=cherrypicked))

print(

f"Commits on branch: {num_cherrypicked:d}, {sum(num_uncherrypicked.values()):d} "

f"({dict(num_uncherrypicked)}) yet to be cherry-picked"

)

@cli.command(short_help="Build a CHANGELOG grouped by GitHub Issue type")

@click.argument("previous_version")

@click.argument("target_version")

@click.argument("github-token", envvar="GITHUB_TOKEN")

@click.option("--disable-progress-bar", is_flag=True, help="Disable the progress bar")

def changelog(previous_version, target_version, github_token, disable_progress_bar):

repo = git.Repo(".", search_parent_directories=True)

# Get a list of issues/PRs that have been committed on the current branch.

log = get_commits_between(repo, previous_version, target_version)

if disable_progress_bar:

print(f"Processing {len(log)} commits")

gh = Github(auth=Auth.Token(github_token))

gh_repo = gh.get_repo("apache/airflow")

sections = defaultdict(list)

with open("RELEASE_NOTES.rst") as file:

existing_pr_nos = re.findall(r"\(#(\d+)\)", file.read())

console = Console(width=180)

with Progress(console=console, disable=disable_progress_bar) as progress:

task = progress.add_task("Processing commits from changelog", total=len(log))

for commit in progress.track(log, description="Processing commits from changelog"):

tickets = pr_title_re.findall(commit["subject"])

if tickets:

issue = gh_repo.get_issue(number=int(tickets[0][1:]))

issue_type = get_issue_type(issue)

files = files_touched(repo, commit["id"])

if str(issue.number) in existing_pr_nos:

print(f"Skipping {issue.number} as it's already in RELEASE_NOTES.rst")

continue

if is_core_commit(files):

sections[issue_type].append(commit["subject"])

progress.update(task, advance=1)

else:

sections[DEFAULT_SECTION_NAME].append(commit["subject"])

progress.update(task, advance=1)

console.print("\n")

print_changelog(sections)

@cli.command(short_help="Find merged PRs that still need to be categorized for the changelog")

@click.argument("previous_version")

@click.argument("target_version")

@click.option("--show-skipped", is_flag=True)

@click.option("--show-files", is_flag=True)

@click.argument("github-token", envvar="GITHUB_TOKEN")

def needs_categorization(previous_version, target_version, show_skipped, show_files, github_token):

repo = git.Repo(".", search_parent_directories=True)

log = get_commits_between(repo, previous_version, target_version)

gh = Github(auth=Auth.Token(github_token))

gh_repo = gh.get_repo("apache/airflow")

for commit in log:

tickets = pr_title_re.findall(commit["subject"])

if tickets:

issue = gh_repo.get_issue(number=int(tickets[0][1:]))

issue_type = get_issue_type(issue)

if issue_type == DEFAULT_SECTION_NAME:

files = files_touched(repo, commit["id"])

if is_core_commit(files):

print(f"{commit['subject']}: {issue.html_url}")

if show_files:

for f in files:

print(f"\t{f}")

elif show_skipped:

print(f"**** {commit['subject']}: Skipping - not a core commit")

if show_files:

for f in files:

print(f"\t{f}")

else:

print(f"Commit '{commit['id']}' is missing PR number: {commit['subject']}")

@cli.command(

name="api-clients-policy",

help="Compare two airflow core release tags and determine if API clients need to be released.",

)

@click.argument("previous_version")

@click.argument("target_version")

def api_clients_policy(previous_version, target_version):

p_version = version.parse(previous_version)

t_version = version.parse(target_version)

if p_version.major != t_version.major:

print("This is a major release, API clients should also be released.")

return

if p_version.minor != t_version.minor:

print("This is a minor release, API clients should also be released.")

return

if p_version == t_version:

print("Both versions are identical")

return

repo = git.Repo(".", search_parent_directories=True)

log = get_commits_between(

repo,

previous_version,

target_version,

files=[f"{repo.working_dir}/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml"],

)

clients_need_release = False

for commit in log:

if "update airflow version to" not in commit["subject"].lower():

clients_need_release = True

print(f"Commit '{commit['id']}' updated the OpenAPI spec, PR number: {commit['subject']}")

if clients_need_release:

print(f"API clients need to be released because the API spec has changed since '{previous_version}'")

else:

print(

"API clients don't need to be released because the API spec hasn't changed between those two "

"patch versions"

)

if __name__ == "__main__":

import doctest

(failure_count, test_count) = doctest.testmod()

if failure_count:

sys.exit(-1)

try:

cli()

except Exception:

raise