Skip to content

Commit 29c8c8a

Browse files
stinosdpgeorge
stinos
authored and
dpgeorge
committed
windows/msvc: Rewrite qstr auto-generation.
Builds have been broken since reworking autogeneration in c618f91 and related, this gets fixed here by applying similar qstr generation logic for the msvc builds: c files are only preprocessed when changed (or not yet preprocessed) and the concatenated output is fed into makeqstrdefs.py. To speed up this process, the concatenated output is already filtered to contain only lines which makeqstrdefs really needs: this makes the qstr generation stage about twice as fast (checked on different machines).
1 parent 9264d42 commit 29c8c8a

File tree

1 file changed

+44
-39
lines changed

1 file changed

+44
-39
lines changed

‎windows/msvc/genhdr.targets‎

Lines changed: 44 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -3,73 +3,78 @@
33

44
<Import Project="paths.props" Condition="'$(PyPathsIncluded)' != 'True'"/>
55

6-
<!--Generate qstrdefs.h and mpversion.h similar to what is done in py/mkrules.mk-->
6+
<!--Generate qstrdefs.generated.h and mpversion.h similar to what is done in py/mkrules.mk and py/py.mk-->
77
<Target Name="GenerateHeaders" DependsOnTargets="MakeVersionHdr;MakeQstrData">
88
</Target>
99

1010
<PropertyGroup>
1111
<DestDir>$(PyBuildDir)genhdr\</DestDir>
1212
<PySrcDir>$(PyBaseDir)py\</PySrcDir>
1313
<QstrDefs>$(PyBaseDir)unix\qstrdefsport.h</QstrDefs>
14+
<PyQstrDefs>$(PySrcDir)qstrdefs.h</PyQstrDefs>
1415
<QstrDefsCollected>$(DestDir)qstrdefscollected.h</QstrDefsCollected>
1516
<QstrGen>$(DestDir)qstrdefs.generated.h</QstrGen>
1617
<PyPython Condition="'$(PyPython)' == ''">python</PyPython>
1718
</PropertyGroup>
1819

19-
<ItemGroup>
20-
<PyQstrSourceFiles Include="@(ClCompile);$(PySrcDir)qstrdefs.h"/>
21-
</ItemGroup>
22-
2320
<Target Name="MakeDestDir">
2421
<MakeDir Directories="$(DestDir)"/>
2522
</Target>
2623

27-
<Target Name="MakeQstrDefs" DependsOnTargets="MakeDestDir" Inputs="@(PyQstrSourceFiles)" Outputs="$(QstrDefsCollected)">
24+
<!-- Concatenate preprocessed files for use with makeqstrdefs.py.
25+
Filters out any lines which aren't used by makeqstrdefs.py so the resulting file is
26+
hundreds of times smaller and processing in python takes substantially less time-->
27+
<UsingTask TaskName="ConcatPreProcFiles" TaskFactory="CodeTaskFactory" AssemblyFile="$(MSBuildToolsPath)\Microsoft.Build.Tasks.v4.0.dll" >
28+
<ParameterGroup>
29+
<OutputFile Required="true" ParameterType="System.String"/>
30+
<InputFiles Required="true" ParameterType="System.String[]"/>
31+
</ParameterGroup>
32+
<Task>
33+
<Code Type="Fragment" Language="cs">
34+
<![CDATA[
35+
using(var outFile = System.IO.File.CreateText(OutputFile)) {
36+
foreach(var inFile in InputFiles)
37+
foreach(var line in System.IO.File.ReadAllLines(inFile))
38+
if((line.Contains(".c") && line.StartsWith("#line")) || line.Contains("MP_QSTR"))
39+
outFile.WriteLine( line );
40+
}
41+
]]>
42+
</Code>
43+
</Task>
44+
</UsingTask>
45+
46+
<!-- Preprocess changed files, concatenate and feed into makeqstrdefs.py split/cat-->
47+
<Target Name="MakeQstrDefs" DependsOnTargets="MakeDestDir" Inputs="@(ClCompile)" Outputs="$(QstrDefsCollected)">
2848
<ItemGroup>
2949
<PyIncDirs Include="$(PyIncDirs)"/>
30-
<PreProcDefs Include=" %(ClCompile.PreProcessorDefinitions)"/>
50+
<PreProcDefs Include="%(ClCompile.PreProcessorDefinitions);__QSTR_EXTRACT;N_X64;N_X86;N_THUMB;N_ARM"/>
51+
<PyQstrSourceFiles Include="@(ClCompile)">
52+
<OutFile>$([System.String]::new('%(FullPath)').Replace('$(PyBaseDir)', '$(DestDir)qstr\'))</OutFile>
53+
</PyQstrSourceFiles>
3154
<PyQstrSourceFiles>
32-
<Qstr>$([System.String]::new('%(FullPath)').Replace('$(PyBaseDir)', '$(DestDir)'))</Qstr>
55+
<OutFile>$([System.IO.Path]::ChangeExtension('%(OutFile)', '.pp'))</OutFile>
56+
<OutDir>$([System.IO.Path]::GetDirectoryName('%(OutFile)'))</OutDir>
3357
</PyQstrSourceFiles>
3458
<PyQstrSourceFiles>
35-
<Qstr>$([System.IO.Path]::ChangeExtension('%(Qstr)', '.qstr'))</Qstr>
36-
<PreProc>$([System.IO.Path]::ChangeExtension('%(Qstr)', '.pp'))</PreProc>
37-
<QstrDir>$([System.IO.Path]::GetDirectoryName('%(Qstr)'))</QstrDir>
38-
<PreProcOnly>$([System.String]::new('%(FileName)').Contains('qstrdefs'))</PreProcOnly>
59+
<Changed>$([System.DateTime]::Compare($([System.IO.File]::GetLastWriteTime('%(FullPath)')), $([System.IO.File]::GetLastWriteTime('%(OutFile)'))))</Changed>
3960
</PyQstrSourceFiles>
4061
</ItemGroup>
41-
<PropertyGroup>
42-
43-
</PropertyGroup>
44-
45-
<!-- Preprocess and pass to makeqstrdefs if needed, else just copy -->
46-
<MakeDir Directories="@(PyQstrSourceFiles->'%(QstrDir)')"/>
47-
<Touch Files="$(QstrGen)" AlwaysCreate="true"/>
48-
49-
<Exec Command="cl /nologo /I@(PyIncDirs, ' /I') /D@(PreProcDefs, ' /D') /Fi%(PyQstrSourceFiles.PreProc) /P %(PyQstrSourceFiles.Identity)"/>
50-
<Exec Command="$(PyPython) $(PySrcDir)makeqstrdefs.py -s -o %(PyQstrSourceFiles.Qstr) %(PyQstrSourceFiles.PreProc)" Condition="'%(PyQstrSourceFiles.PreProcOnly)' != 'True'"/>
51-
<Copy SourceFiles="%(PyQstrSourceFiles.PreProc)" DestinationFiles="%(PyQstrSourceFiles.Qstr)" Condition="'%(PyQstrSourceFiles.PreProcOnly)' == 'True'"/>
52-
53-
<!-- Collect all output (where qstrdefs file(s) have priority over autogenerated ones), then
54-
filter out lines which definitely aren't qstr definitions so we don't end up with a huge (> 10mb) filesize -->
55-
<ReadLinesFromFile File="%(PyQstrSourceFiles.Qstr)" Condition="'%(PyQstrSourceFiles.PreProcOnly)' == 'True'">
56-
<Output TaskParameter="Lines" ItemName="PreProcLines"/>
57-
</ReadLinesFromFile>
58-
<ReadLinesFromFile File="%(PyQstrSourceFiles.Qstr)">
59-
<Output TaskParameter="Lines" ItemName="PreProcLines"/>
60-
</ReadLinesFromFile>
61-
<ItemGroup>
62-
<QStrLines Include="@(PreProcLines)" Condition="$([System.String]::new('%(Identity)').Contains('Q'))"/>
63-
</ItemGroup>
6462

65-
<WriteLinesToFile Lines="@(QStrLines)" File="$(QstrDefsCollected)" Overwrite="true"/>
63+
<MakeDir Directories="@(PyQstrSourceFiles->'%(OutDir)')"/>
64+
<Exec Command="cl /nologo /I@(PyIncDirs, ' /I') /D@(PreProcDefs, ' /D') /Fi%(PyQstrSourceFiles.OutFile) /P %(PyQstrSourceFiles.Identity)"
65+
Condition="%(PyQstrSourceFiles.Changed) &gt; 0"/>
66+
<ConcatPreProcFiles InputFiles="@(PyQstrSourceFiles->'%(OutFile)')" OutputFile="$(DestDir)qstr.i.last"
67+
Condition="%(PyQstrSourceFiles.Changed) &gt; 0"/>
68+
<Exec Command="$(PyPython) $(PySrcDir)makeqstrdefs.py split $(DestDir)qstr.i.last $(DestDir)qstr $(QstrDefsCollected)"/>
69+
<Exec Command="$(PyPython) $(PySrcDir)makeqstrdefs.py cat $(DestDir)qstr.i.last $(DestDir)qstr $(QstrDefsCollected)"/>
6670
</Target>
6771

68-
<Target Name="MakeQstrData" DependsOnTargets="MakeQstrDefs">
72+
<Target Name="MakeQstrData" DependsOnTargets="MakeQstrDefs" Inputs="$(QstrDefsCollected);$(PyQstrDefs);$(QstrDefs)" Outputs="$(QstrGen)">
6973
<PropertyGroup>
70-
<TmpFile>$(DestFile).tmp</TmpFile>
74+
<TmpFile>$(QstrGen).tmp</TmpFile>
7175
</PropertyGroup>
72-
<Exec Command="$(PyPython) $(PySrcDir)makeqstrdata.py $(QstrDefs) $(QstrDefsCollected) > $(TmpFile)"/>
76+
<Exec Command="cl /nologo /I@(PyIncDirs, ' /I') /D@(PreProcDefs, ' /D') /E $(PyQstrDefs) $(QstrDefs) > $(DestDir)qstrdefspreprocessed.h"/>
77+
<Exec Command="$(PyPython) $(PySrcDir)makeqstrdata.py $(DestDir)qstrdefspreprocessed.h $(QstrDefsCollected) > $(TmpFile)"/>
7378
<MSBuild Projects="$(MSBuildThisFileFullPath)" Targets="CopyFileIfDifferent" Properties="SourceFile=$(TmpFile);DestFile=$(QstrGen)"/>
7479
</Target>
7580

0 commit comments

Comments
 (0)