Skip to content
This repository was archived by the owner on Feb 2, 2024. It is now read-only.

Commit 6dc01ae

Browse files
Vyacheslav-SmirnovVyacheslav-Smirnov
authored
Add script for documentation build / publish; start build/test refactor (#643)
* Add build_doc script; remove useless code - Build doc and publish to sdc_doc if publish option is used - Remove develop options from build and test - Remove coverage from test * Move new functions in class; add logs * Remove local links from todo list - Remove local links from todo list - Check if something changed before push * Add Intel SDC doc build in Azure for Linux/macOS * Remove __pycache__ * Fix pep8 issues * Comment macOS pipeline; update SDC/Numba recipes - Remove macOS pipeline from testing due to migration to Numba from Anaconda intel/label/beta channel - Update SDC and Numba recipes * Remove build string from sdc recipe - Looks like not supported with conda and wheel outputs * Need a commit to retrigger hanged Windows build
1 parent 9a490b8 commit 6dc01ae

10 files changed

Lines changed: 257 additions & 211 deletions

File tree

‎azure-pipelines.yml‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,17 +17,17 @@ jobs:
1717
USE_NUMBA_MASTER: 'True'
1818
SDC_CONFIG_PIPELINE_SDC: 'False'
1919

20-
- template: buildscripts/azure/template-linux-macos.yml
21-
parameters:
22-
name: macOS1013_xcode9
23-
vmImage: xcode9-macos10.13
24-
allowFailure: false
25-
matrix:
26-
py3.7_numpy1.17:
27-
PYTHON_VER: '3.7'
28-
NUMPY_VER: '1.17'
29-
USE_NUMBA_MASTER: 'True'
30-
SDC_CONFIG_PIPELINE_SDC: 'False'
20+
# - template: buildscripts/azure/template-linux-macos.yml
21+
# parameters:
22+
# name: macOS1013_xcode9
23+
# vmImage: xcode9-macos10.13
24+
# allowFailure: false
25+
# matrix:
26+
# py3.7_numpy1.17:
27+
# PYTHON_VER: '3.7'
28+
# NUMPY_VER: '1.17'
29+
# USE_NUMBA_MASTER: 'True'
30+
# SDC_CONFIG_PIPELINE_SDC: 'False'
3131

3232
- template: buildscripts/azure/template-linux-macos.yml
3333
parameters:

‎buildscripts/azure/template-linux-macos.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,3 +46,9 @@ jobs:
4646
fi
4747
displayName: 'Test package examples'
4848
continueOnError: ${{ parameters.allowFailure }}
49+
50+
- script: |
51+
source $HOME/miniconda3/bin/activate
52+
python buildscripts/build_doc.py --python=$PYTHON_VER --sdc-channel=./sdc-build
53+
displayName: 'Build Intel SDC documentation'
54+
continueOnError: ${{ parameters.allowFailure }}

‎buildscripts/build.py‎

Lines changed: 11 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -43,12 +43,6 @@
4343
from utilities import set_environment_variable
4444

4545

46-
def run_smoke_tests(sdc_src, test_env_activate):
47-
sdc_pi_example = os.path.join(sdc_src, 'buildscripts', 'sdc_pi_example.py')
48-
run_command(f'{test_env_activate} && python -c "import sdc"')
49-
run_command(f'{test_env_activate} && python {sdc_pi_example}')
50-
51-
5246
if __name__ == '__main__':
5347
sdc_src = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
5448
sdc_recipe = os.path.join(sdc_src, 'buildscripts', 'sdc-conda-recipe')
@@ -62,15 +56,13 @@ def run_smoke_tests(sdc_src, test_env_activate):
6256

6357
# Parse input arguments
6458
parser = argparse.ArgumentParser()
65-
parser.add_argument('--build-mode', default='develop', choices=['develop', 'install', 'package'],
59+
parser.add_argument('--build-mode', default='package', choices=['package'],
6660
help="""Build mode:
67-
develop: install package in source directory (default)
68-
install: build and install package in build environment
69-
package: build conda and wheel packages""")
61+
package: build conda and wheel packages (default)""")
7062
parser.add_argument('--python', default='3.7', choices=['3.6', '3.7', '3.8'],
7163
help='Python version to build with, default = 3.7')
72-
parser.add_argument('--numpy', default='1.16', choices=['1.15', '1.16', '1.17'],
73-
help='Numpy version to build with, default = 1.16')
64+
parser.add_argument('--numpy', default='1.17', choices=['1.16', '1.17'],
65+
help='Numpy version to build with, default = 1.17')
7466
parser.add_argument('--output-folder', default=os.path.join(sdc_src, 'sdc-build'),
7567
help='Output folder for build packages, default = sdc-build')
7668
parser.add_argument('--conda-prefix', default=None, help='Conda prefix')
@@ -84,8 +76,6 @@ def run_smoke_tests(sdc_src, test_env_activate):
8476

8577
build_mode = args.build_mode
8678
python = args.python
87-
if python == '3.7':
88-
python = '3.7.3'
8979
numpy = args.numpy
9080
output_folder = args.output_folder
9181
conda_prefix = os.getenv('CONDA_PREFIX', args.conda_prefix)
@@ -99,10 +89,8 @@ def run_smoke_tests(sdc_src, test_env_activate):
9989
conda_activate = get_conda_activate_cmd(conda_prefix).replace('"', '')
10090
build_env = f'sdc-build-env-py{python}-numpy{numpy}'
10191
test_env = f'sdc-test-env-py{python}-numpy{numpy}'
102-
develop_env = f'sdc-develop-env-py{python}-numpy{numpy}'
10392
build_env_activate = get_activate_env_cmd(conda_activate, build_env)
10493
test_env_activate = get_activate_env_cmd(conda_activate, test_env)
105-
develop_env_activate = get_activate_env_cmd(conda_activate, develop_env)
10694

10795
conda_channels = f'-c {numba_channel} -c defaults -c conda-forge --override-channels'
10896
# If numba is taken from custom channel, need to add numba channel to get dependencies
@@ -114,10 +102,6 @@ def run_smoke_tests(sdc_src, test_env_activate):
114102

115103
conda_build_packages = ['conda-build']
116104
if platform.system() == 'Windows':
117-
if build_mode != 'package':
118-
set_environment_variable('INCLUDE', os.path.join('%CONDA_PREFIX%', 'Library', 'include'))
119-
set_environment_variable('LIB', os.path.join('%CONDA_PREFIX%', 'Library', 'lib'))
120-
121105
conda_build_packages.extend(['conda-verify', 'vc', 'vs2015_runtime', 'vs2015_win-64', 'pywin32=223'])
122106

123107
# Build Numba from master
@@ -132,66 +116,16 @@ def run_smoke_tests(sdc_src, test_env_activate):
132116
f'{numba_conda_channels} {numba_recipe}'])))
133117
format_print('NUMBA BUILD COMPETED')
134118

135-
# Get sdc build and test environment
136-
sdc_env = get_sdc_env(conda_activate, sdc_src, sdc_recipe, python, numpy, conda_channels)
137-
138119
# Set build command
139-
if build_mode == 'package':
140-
create_conda_env(conda_activate, build_env, python, conda_build_packages)
141-
build_cmd = '{} && {}'.format(build_env_activate,
142-
' '.join(['conda build --no-test',
143-
f'--python {python}',
144-
f'--numpy {numpy}',
145-
f'--output-folder {output_folder}',
146-
f'{conda_channels} {sdc_recipe}']))
147-
else:
148-
create_conda_env(conda_activate, develop_env, python, sdc_env['build'], conda_channels)
149-
build_cmd = f'{develop_env_activate} && python setup.py {build_mode}'
120+
create_conda_env(conda_activate, build_env, python, conda_build_packages)
121+
build_cmd = '{} && {}'.format(build_env_activate,
122+
' '.join(['conda build --no-test',
123+
f'--python {python}',
124+
f'--numpy {numpy}',
125+
f'--output-folder {output_folder}',
126+
f'{conda_channels} {sdc_recipe}']))
150127

151128
# Start build
152129
format_print('START SDC BUILD')
153130
run_command(build_cmd)
154131
format_print('BUILD COMPETED')
155-
156-
# Check if smoke tests should be skipped
157-
if skip_smoke_tests is True:
158-
format_print('Smoke tests are skipped due to "--skip-smoke-tests" option')
159-
sys.exit(0)
160-
161-
# Start smoke tests
162-
format_print('Run Smoke tests')
163-
"""
164-
os.chdir(../sdc_src) is a workaround for the following error:
165-
Traceback (most recent call last):
166-
File "<string>", line 1, in <module>
167-
File "sdc/sdc/__init__.py", line 9, in <module>
168-
import sdc.dict_ext
169-
File "sdc/sdc/dict_ext.py", line 12, in <module>
170-
from sdc.str_ext import string_type, gen_unicode_to_std_str, gen_std_str_to_unicode
171-
File "sdc/sdc/str_ext.py", line 18, in <module>
172-
from . import hstr_ext
173-
ImportError: cannot import name 'hstr_ext' from 'sdc' (sdc/sdc/__init__.py)
174-
"""
175-
os.chdir(os.path.dirname(sdc_src))
176-
177-
if build_mode == 'package':
178-
# Get build packages
179-
sdc_packages = get_sdc_build_packages(output_folder)
180-
for package in sdc_packages:
181-
if '.tar.bz2' in package:
182-
# Start test for conda package
183-
format_print(f'Run tests for sdc conda package: {package}')
184-
create_conda_env(conda_activate, test_env, python, sdc_env['test'], conda_channels)
185-
run_command(f'{test_env_activate} && conda install -y {package}')
186-
run_smoke_tests(sdc_src, test_env_activate)
187-
elif '.whl' in package:
188-
# Start test for wheel package
189-
format_print(f'Run tests for sdc wheel package: {package}')
190-
create_conda_env(conda_activate, test_env, python, sdc_env['test'] + ['pip'], conda_channels)
191-
run_command(f'{test_env_activate} && pip install {package}')
192-
run_smoke_tests(sdc_src, test_env_activate)
193-
else:
194-
format_print('Run tests for installed sdc package')
195-
run_smoke_tests(sdc_src, develop_env_activate)
196-
197-
format_print('SMOKE TESTS ARE PASSED')

‎buildscripts/build_doc.py‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
# *****************************************************************************
2+
# Copyright (c) 2020, Intel Corporation All rights reserved.
3+
#
4+
# Redistribution and use in source and binary forms, with or without
5+
# modification, are permitted provided that the following conditions are met:
6+
#
7+
# Redistributions of source code must retain the above copyright notice,
8+
# this list of conditions and the following disclaimer.
9+
#
10+
# Redistributions in binary form must reproduce the above copyright notice,
11+
# this list of conditions and the following disclaimer in the documentation
12+
# and/or other materials provided with the distribution.
13+
#
14+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15+
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16+
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17+
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
18+
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19+
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20+
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21+
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22+
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23+
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24+
# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25+
# *****************************************************************************
26+
27+
28+
import argparse
29+
import os
30+
import shutil
31+
32+
from pathlib import Path
33+
from utilities import SDC_Build_Utilities
34+
35+
36+
def build_doc(sdc_utils):
37+
os.chdir(str(sdc_utils.doc_path))
38+
39+
sdc_utils.log_info('Start documentation build', separate=True)
40+
sdc_utils.run_command('make html')
41+
sdc_utils.log_info('Documentation build SUCCESSFUL', separate=True)
42+
43+
return
44+
45+
46+
def publish_doc(sdc_utils):
47+
doc_local_build = str(sdc_utils.doc_path / 'build' / 'html')
48+
doc_repo_build = str(sdc_utils.doc_path / sdc_utils.doc_repo_name / sdc_utils.doc_tag)
49+
50+
git_email = os.environ['SDC_GIT_EMAIL']
51+
git_username = os.environ['SDC_GIT_USERNAME']
52+
git_access_token = os.environ['SDC_GIT_TOKEN']
53+
git_credentials_file = str(Path.home() / '.git-credentials')
54+
git_credentials = f'https://{git_access_token}:x-oauth-basic@github.com\n'
55+
56+
sdc_utils.log_info(f'Start documentation publish to {sdc_utils.doc_repo_link}', separate=True)
57+
58+
os.chdir(str(sdc_utils.doc_path))
59+
sdc_utils.run_command(f'git clone {sdc_utils.doc_repo_link}')
60+
os.chdir(str(sdc_utils.doc_repo_name))
61+
62+
# Set local git options
63+
sdc_utils.run_command('git config --local credential.helper store')
64+
with open(git_credentials_file, "w") as fp:
65+
fp.write(git_credentials)
66+
sdc_utils.run_command(f'git config --local user.email "{git_email}"')
67+
sdc_utils.run_command(f'git config --local user.name "{git_username}"')
68+
69+
sdc_utils.run_command(f'git checkout {sdc_utils.doc_repo_branch}')
70+
shutil.rmtree(doc_repo_build)
71+
shutil.copytree(doc_local_build, doc_repo_build)
72+
sdc_utils.run_command(f'git add -A {sdc_utils.doc_tag}')
73+
# Check if there is changes
74+
output = sdc_utils.get_command_output(f'git commit -m "Updated doc release: {sdc_utils.doc_tag}"')
75+
if 'nothing to commit, working tree clean' not in output:
76+
sdc_utils.run_command('git push origin HEAD')
77+
sdc_utils.log_info('Documentation publish SUCCESSFUL', separate=True)
78+
else:
79+
sdc_utils.log_info('No changes in documentation', separate=True)
80+
81+
return
82+
83+
84+
if __name__ == '__main__':
85+
parser = argparse.ArgumentParser()
86+
parser.add_argument('--python', default='3.7', choices=['3.6', '3.7', '3.8'],
87+
help='Python version, default = 3.7')
88+
parser.add_argument('--sdc-channel', default=None, help='Intel SDC channel')
89+
parser.add_argument('--publish', action='store_true', help='Publish documentation to sdc-doc')
90+
91+
args = parser.parse_args()
92+
93+
sdc_utils = SDC_Build_Utilities(args.python, args.sdc_channel)
94+
sdc_utils.log_info('Build Intel(R) SDC documentation', separate=True)
95+
sdc_utils.log_info(sdc_utils.line_double)
96+
sdc_utils.create_environment(['sphinx', 'sphinxcontrib-programoutput'])
97+
sdc_utils.install_conda_package(['sdc'])
98+
99+
build_doc(sdc_utils)
100+
if args.publish:
101+
publish_doc(sdc_utils)

‎buildscripts/numba-conda-recipe/recipe/meta.yaml‎

Lines changed: 16 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,30 @@
11
{% set name = "numba" %}
2-
{% set version = "0.46.0" %}
3-
{% set LLVMLITE_VERSION = "0.31.0rc1 py*_0" %}
2+
{% set version = "0.48.0" %}
3+
{% set LLVMLITE_VERSION = "==0.31" %}
4+
{% set TBB_VERSION = ">=2018.0.5" %}
45

56
package:
67
name: {{ name|lower }}
78
version: {{ version }}
89

910
source:
10-
git_url: https://github.com/numba/numba
11+
git_url: https://github.com/numba/numba.git
1112
git_tag: 0.48.0
1213

1314
build:
14-
number: 0
15+
number: {{ GIT_DESCRIBE_NUMBER|int }}
16+
string: py{{ CONDA_PY }}{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER|int }}
1517
entry_points:
1618
- pycc = numba.pycc:main
1719
- numba = numba.numba_entry:main
1820
script_env:
19-
- PY_VCRUNTIME_REDIST
20-
missing_dso_whitelist: # [osx]
21+
- PY_VCRUNTIME_REDIST # [win]
22+
- VS90COMNTOOLS # [win]
23+
- VS140COMNTOOLS # [win]
24+
missing_dso_whitelist: # [osx]
2125
# optional dependency: required only when omp is chosen as the backend for
2226
# the threading layer
23-
- lib/libiomp5.dylib # [osx]
27+
- lib/libiomp5.dylib # [osx]
2428

2529
requirements:
2630
# build and run dependencies are duplicated to avoid setuptools issues
@@ -35,22 +39,18 @@ requirements:
3539
- python
3640
- numpy
3741
- setuptools
38-
- llvmlite {{ LLVMLITE_VERSION }}
39-
- funcsigs # [py27]
40-
- singledispatch # [py27]
41-
- tbb-devel >=2018.0.5 # [not (win and py27)]
42+
- llvmlite {{ LLVMLITE_VERSION }}
43+
- tbb-devel {{ TBB_VERSION }}
4244
- llvm-openmp # [osx]
4345
- intel-openmp # [osx]
4446
run:
4547
- python
4648
- {{ pin_compatible('numpy') }}
47-
- llvmlite {{ LLVMLITE_VERSION }}
48-
- funcsigs # [py27]
49-
- singledispatch # [py27]
49+
- llvmlite {{ LLVMLITE_VERSION }}
5050
run_constrained:
5151
# If TBB is present it must be at least this version from Anaconda due to
5252
# build flag issues triggering UB
53-
- tbb >=2018.0.5 # [not (win and py27)]
53+
- tbb {{ TBB_VERSION }}
5454

5555
test:
5656
requires:
@@ -60,8 +60,7 @@ test:
6060
- scipy
6161
- ipython
6262
- setuptools
63-
- faulthandler # [py27]
64-
- tbb >=2018.0.5 # [not (win and py27)]
63+
- tbb {{ TBB_VERSION }}
6564
- intel-openmp # [osx]
6665
# Need these for AOT. Do not init msvc as it may not be present
6766
- {{ compiler('c') }} # [not win]
@@ -110,7 +109,3 @@ about:
110109
infrastructure to compile Python syntax to machine code."
111110
doc_url: http://numba.pydata.org/
112111
dev_url: https://github.com/numba/numba
113-
114-
extra:
115-
recipe-maintainers:
116-
- souravsingh

‎buildscripts/sdc-conda-recipe/conda_build_config.yaml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,5 +3,4 @@ python:
33
- 3.7
44

55
numpy:
6-
- 1.15
7-
- 1.16
6+
- 1.17

0 commit comments

Comments
 (0)