#!/usr/bin/env python

"""Check that Python.h is included before any stdlib headers.

May be a bit overzealous, but it should get the job done.

"""

import argparse

import fnmatch

import os.path

import re

import subprocess

import sys

from get_submodule_paths import get_submodule_paths

HEADER_PATTERN = re.compile(

r'^\s*#\s*include\s*[<"]((?:\w+/)*\w+(?:\.h[hp+]{0,2})?)[>"]\s*$'

)

PYTHON_INCLUDING_HEADERS = [

"Python.h",

# This isn't all of Python.h, but it is the visibility macros

"pyconfig.h",

"numpy/npy_common.h",

"numpy/npy_math.h",

"numpy/arrayobject.h",

"numpy/ndarrayobject.h",

"numpy/ndarraytypes.h",

"numpy/random/distributions.h",

"npy_sort.h",

"npy_config.h",

"common.h",

"npy_cpu_features.h",

# Boost::Python

"boost/python.hpp",

]

LEAF_HEADERS = [

"numpy/numpyconfig.h",

"numpy/npy_os.h",

"numpy/npy_cpu.h",

"numpy/utils.h",

]

C_CPP_EXTENSIONS = (".c", ".h", ".cpp", ".hpp", ".cc", ".hh", ".cxx", ".hxx")

# check against list in diff_files

PARSER = argparse.ArgumentParser(description=__doc__)

PARSER.add_argument(

"files",

nargs="*",

help="Lint these files or directories; use **/*.c to lint all files\n"

"Expects relative paths",

)

def check_python_h_included_first(name_to_check: str) -> int:

"""Check that the passed file includes Python.h first if it does at all.

Perhaps overzealous, but that should work around concerns with

recursion.

Parameters

----------

name_to_check : str

The name of the file to check.

Returns

-------

int

The number of headers before Python.h

"""

included_python = False

included_non_python_header = []

warned_python_construct = False

basename_to_check = os.path.basename(name_to_check)

in_comment = False

includes_headers = False

with open(name_to_check) as in_file:

for i, line in enumerate(in_file, 1):

# Very basic comment parsing

# Assumes /*...*/ comments are on their own lines

if "/*" in line:

if "*/" not in line:

in_comment = True

# else-branch could use regex to remove comment and continue

continue

if in_comment:

if "*/" in line:

in_comment = False

continue

line = line.split("//", 1)[0].strip()

match = HEADER_PATTERN.match(line)

if match:

includes_headers = True

this_header = match.group(1)

if this_header in PYTHON_INCLUDING_HEADERS:

if included_non_python_header and not included_python:

# Headers before python-including header

print(

f"Header before Python.h in file {name_to_check:s}\n"

f"Python.h on line {i:d}, other header(s) on line(s)"

f" {included_non_python_header}",

file=sys.stderr,

)

# else: # no headers before python-including header

included_python = True

PYTHON_INCLUDING_HEADERS.append(basename_to_check)

if os.path.dirname(name_to_check).endswith("include/numpy"):

PYTHON_INCLUDING_HEADERS.append(f"numpy/{basename_to_check:s}")

# We just found out where Python.h comes in this file

break

elif this_header in LEAF_HEADERS:

# This header is just defines, so it won't include

# the system headers that cause problems

continue

elif not included_python and (

"numpy/" in this_header

and this_header not in LEAF_HEADERS

or "python" in this_header.lower()

):

print(

f"Python.h not included before python-including header "

f"in file {name_to_check:s}\n"

f"{this_header:s} on line {i:d}",

file=sys.stderr,

)

included_python = True

PYTHON_INCLUDING_HEADERS.append(basename_to_check)

elif not included_python and this_header not in LEAF_HEADERS:

included_non_python_header.append(i)

elif (

not included_python

and not warned_python_construct

and ".h" not in basename_to_check

) and ("py::" in line or "PYBIND11_" in line):

print(

"Python-including header not used before python constructs "

f"in file {name_to_check:s}\nConstruct on line {i:d}",

file=sys.stderr,

)

warned_python_construct = True

if not includes_headers:

LEAF_HEADERS.append(basename_to_check)

return included_python and len(included_non_python_header)

def sort_order(path: str) -> tuple[int, str]:

if "include/numpy" in path:

# Want to process numpy/*.h first, to work out which of those

# include Python.h directly

priority = 0x00

elif "h" in os.path.splitext(path)[1].lower():

# Then other headers, which tend to include numpy/*.h

priority = 0x10

else:

# Source files after headers, to give the best chance of

# properly checking whether they include Python.h

priority = 0x20

if "common" in path:

priority -= 8

path_basename = os.path.basename(path)

if path_basename.startswith("npy_"):

priority -= 4

elif path_basename.startswith("npy"):

priority -= 3

elif path_basename.startswith("np"):

priority -= 2

if "config" in path_basename:

priority -= 1

return priority, path

def process_files(file_list: list[str]) -> int:

n_out_of_order = 0

submodule_paths = get_submodule_paths()

root_directory = os.path.dirname(os.path.dirname(__file__))

for name_to_check in sorted(file_list, key=sort_order):

name_to_check = os.path.join(root_directory, name_to_check)

if any(submodule_path in name_to_check for submodule_path in submodule_paths):

continue

if ".dispatch." in name_to_check:

continue

try:

n_out_of_order += check_python_h_included_first(name_to_check)

except UnicodeDecodeError:

print(f"File {name_to_check:s} not utf-8", sys.stdout)

return n_out_of_order

def find_c_cpp_files(root: str) -> list[str]:

result = []

for dirpath, dirnames, filenames in os.walk(root):

# I'm assuming other people have checked boost

for name in ("build", ".git", "boost"):

try:

dirnames.remove(name)

except ValueError:

pass

for name in fnmatch.filter(dirnames, "*.p"):

dirnames.remove(name)

result.extend(

[

os.path.join(dirpath, name)

for name in filenames

if os.path.splitext(name)[1].lower() in C_CPP_EXTENSIONS

]

)

# Check the headers before the source files

result.sort(key=lambda path: "h" in os.path.splitext(path)[1], reverse=True)

return result

def diff_files(sha: str) -> list[str]:

"""Find the diff since the given SHA.

Adapted from lint.py

"""

res = subprocess.run(

[

"git",

"diff",

"--name-only",

"--diff-filter=ACMR",

"-z",

sha,

"--",

# Check against C_CPP_EXTENSIONS

"*.[chCH]",

"*.[ch]pp",

"*.[ch]xx",

"*.cc",

"*.hh",

],

stdout=subprocess.PIPE,

encoding="utf-8",

)

res.check_returncode()

return [f for f in res.stdout.split("\0") if f]

if __name__ == "__main__":

args = PARSER.parse_args()

if len(args.files) == 0:

files = find_c_cpp_files("numpy")

else:

files = args.files

if len(files) == 1 and os.path.isdir(files[0]):

files = find_c_cpp_files(files[0])

# See which of the headers include Python.h and add them to the list

n_out_of_order = process_files(files)

sys.exit(n_out_of_order)