#!/usr/bin/env python

# A python script with the goal

# to transform a (large) compilations database

# to a smaller compilations database, given a list of source files

# that have changed (currently only supporting c++ source and header files).

# This should be useful to reduce the time spent in static code analysis by only

# checking files that have changed, or are influenced by a change.

#

# First version: Sandro Wenzel (June 2017)

from __future__ import print_function

import argparse

import json

import os

import subprocess

import re

# these are for queue syncronized multi-threading

try:

import Queue

except ImportError:

import queue as Queue

import threading

import multiprocessing

import time

import sys

verbosity=0

def parseArgs():

""" Setup + Parse arguments from command line; return a parse object """

parser = argparse.ArgumentParser(description='Runs over all entries in a '

'compilation database. Produces a filtered/thinned version of it.')

parser.add_argument('-j', type=int, default=0,

help='Number of threads to be used for processing. In any case,'

'not more that the number of available CPU threads will be used (which is the default).')

parser.add_argument('-use-files', default=None,

help='\':\' separated file list used to thin out the database. Usually a list of header and source files.'

'\n If no files are given, the default behaviour is to run over the complete database and filter out'

'only ROOT dictionary files and Protobuf generated files.')

parser.add_argument('-exclude-files', default=None,

help='A regular expression telling which files to exclude from the database. Takes precedence over \'-use-files\'.')

parser.add_argument('-o', default='thinned_compile_commands.json',

help='Filename of output compilations database. [default: thinned_compile_commands.json]')

parser.add_argument('--verbose', type=int, default=0,

help='Display verbose information about what is going on')

return parser.parse_args()

def verboseLog(string, level=0):

global verbosity

if verbosity > 0:

print(string)

def getListOfChangedFiles(colonseparatedfilepaths):

""" processes the argument '-use-files' and returns a python list of filenames """

return colonseparatedfilepaths.split(":")

def makeInvalidClosure(args):

regex = None

if args.exclude_files:

regex=re.compile(args.exclude_files)

def f(filename):

return regex.match(filename) is not None if regex else None

return f

def isHeaderFile(filename):

# make this more general

expression=".*\.h"

# make this more efficient by compiling the expression

return re.match(expression, filename) is not None

def isSourceFile(filename):

# make this more general

expression=".*\.cxx"

# make this more efficient by compiling the expression

return re.match(expression, filename) is not None

# modifies a compilation command by appending -MM (and removing the -o flag)

# in order to retrieve the header dependencies

def modifyCompileCommand(command):

newcommand=[]

outputflagseen=False

# take out the output command

for token in command.split(' '):

if len(token)==0:

continue

if token=='-o':

outputflagseen=True;

else:

if not outputflagseen:

newcommand.append(token)

else:

outputflagseen=False

# add the -MM flag which will generate the header dependency list for a source file

newcommand.append('-MM')

return newcommand

def matchesHeader(line, header):

expression=".*\.h"

# make this more efficient by compiling the expression

try:

return re.match(expression, line.decode("utf-8")) is not None

except:

return re.match(expression, line) is not None

def queryListOfHeaders(command):

proc=subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

headerlist=[]

for line in proc.stdout:

line = line.strip()

if matchesHeader(line, headerlist):

if isinstance(line, bytes):

line = line.decode('utf-8')

headerlist.append(line)

return headerlist

# service that processes one item in the queue

def processItem(keepalive, changedheaderlist, queue, outqueue):

while len(keepalive) > 0:

try:

# this operation is blocking for at most 0.5 seconds

# sad hack to be able to check whether we want this thread to be kept alive

# unfortunately, the main process never exits otherwise

entry = queue.get(True, 0.5)

includedheaderlist=queryListOfHeaders(modifyCompileCommand(entry['command']))

for header in changedheaderlist:

for include in includedheaderlist:

expression=".*"+header

matches = re.match(expression, include)

if not matches == None:

verboseLog("Adding " + entry['file'] + " because of a dependence on a modified header")

outqueue.put(entry)

continue

queue.task_done()

except Queue.Empty:

pass

def reportProgress(keepalive, queue, q2):

while len(keepalive)>0:

print("input queue has size " + str(queue.qsize()))

print("output queue has size " + str(q2.qsize()))

time.sleep(1)

#custom function to remove duplicates and return a new list

#set() cannot be used since the entries are not hashable

def removeDuplicates(inputlist):

outputlist=[]

for e in inputlist:

if e not in outputlist:

outputlist.append(e)

return outputlist

def fileMatchesAnyInSourceFileList(filename, sourcefilelist):

'''returns true if any filename in sourcefilelist matches

filename

items in sourcefilelist will have only paths relativ (to

a git repository); while filename is a fully qualified path

'''

# TODO: item should be at end of filename

for item in sourcefilelist:

if item in filename:

return True

return False

#

# THE MAIN FUNCTION

#

def main():

checkall=False

listofchangedfiles=[]

args = parseArgs()

global verbosity

verbosity = args.verbose

if not args.use_files:

verboseLog("no changeset given ... putting all (modulo the exclude set)")

checkall=True

else:

listofchangedfiles=getListOfChangedFiles(args.use_files)

#setup the isInvalid (closure) function

isInvalid=makeInvalidClosure(args)

#open the compilations database

try:

file=open('compile_commands.json').read()

except IOError:

print("Problem opening the compilation database (file not found)")

sys.exit(1)

#convert json to dict

data=json.loads(file);

#make 2 lists, changedheaders and changedsourcefiles

changedheaderlist=[]

changedsourcefilelist=[]

for file in listofchangedfiles:

if isHeaderFile(file):

changedheaderlist.append(file)

elif isSourceFile(file):

changedsourcefilelist.append(file)

# make input/output queues for multithreaded processing

inputqueue = Queue.Queue()

outputqueue = Queue.Queue()

keepAlive=['alive']

# make some servicing threads

max_task = multiprocessing.cpu_count()

if args.j:

if args.j>0:

max_task = max(max_task,j)

verboseLog("processing with " + str(max_task) + " threads.")

for _ in range(max_task):

t = threading.Thread(target=processItem, args=(keepAlive,changedheaderlist,inputqueue,outputqueue))

t.deamon=True

t.start()

# launch the reporting thread

rt = threading.Thread(target=reportProgress, args=(keepAlive,inputqueue,outputqueue))

rt.deamon=True

rt.start()

outputdict=[]

#scan through compile database and filter against files

verboseLog("Processing " + str(len(data)) + " items.")

for entry in data:

filename=entry['file']

basename=os.path.basename(filename)

# check if invalid anyway

if (isInvalid(basename)):

verboseLog("Excluding " + basename + " because of exclude filter")

continue

# check if this entry is part of the changed source file list

# if yes, continue directly

if checkall==True or fileMatchesAnyInSourceFileList(filename, changedsourcefilelist):

verboseLog("Adding " + entry['file'] + " because of presence in modify list (or lack thereof)")

outputqueue.put(entry)

continue

if len(changedheaderlist) > 0:

# otherwise check if this source file is influenced by some changed header file

# TODO: if the header does not contain a template, it might be enough

# to only add one single source files that depends on it????

inputqueue.put(entry)

# wait on the queue --> wait until queue is completely empty

inputqueue.join()

# now we can shut down the deamon threads

keepAlive[:]=[]

#put outputqueue into outputdict

while outputqueue.qsize() > 0:

outputdict.append(outputqueue.get(False))

# write result dictionary to json

outjson = json.dumps(removeDuplicates(outputdict), sort_keys=True, indent=4, separators=(',', ': '))

with open(args.o,'w') as fp:

fp.write(outjson)

return

if __name__ == '__main__':

main()