Skip to content

Commit 31291ef

Browse files
ramanishsinghandrewkho
ramanishsingh
and
andrewkho
authored
Delete conda builds (meta-pytorch#1367)
* Update README.md (meta-pytorch#1366) * Update README.md * Update README.md * revert changes * initial commit * run pre-commit * delete existing builds * run precommit * delete previously available nightly builds and do not upload new ones * delete everything conda * run precommit * run precommit * try new permissions * update linux build * revert to old linux build * clean test_release * random test * revert change * update manylinux * update manylinux * update validate binaries.sh * revert manylinux update * update validate binaries --------- Co-authored-by: Andrew Ho <andrewkh@meta.com>
1 parent aebad0c commit 31291ef

File tree

8 files changed

+9
-272
lines changed

8 files changed

+9
-272
lines changed

‎.github/scripts/validate_binaries.sh‎

Lines changed: 2 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,21 +6,10 @@
66
# LICENSE file in the root directory of this source tree.
77

88
AWS_ENABLED=1
9-
if [[ ${MATRIX_PACKAGE_TYPE} = "conda" ]]; then
109

11-
if [[ "${MATRIX_PYTHON_VERSION}" = "3.11" ]]; then
12-
PYTORCH_CONDA_CHANNEL="malfet -c ${PYTORCH_CONDA_CHANNEL}"
13-
fi
10+
# shellcheck disable=SC2086
11+
pip install ${PYTORCH_PIP_PREFIX:-} torchdata --extra-index-url "${PYTORCH_PIP_DOWNLOAD_URL}"
1412

15-
# shellcheck disable=SC2086
16-
conda install -y torchdata -c ${PYTORCH_CONDA_CHANNEL}
17-
if [[ ${TARGET_OS:-} = "windows" ]]; then
18-
AWS_ENABLED=0
19-
fi
20-
else
21-
# shellcheck disable=SC2086
22-
pip install ${PYTORCH_PIP_PREFIX:-} torchdata --extra-index-url "${PYTORCH_PIP_DOWNLOAD_URL}"
23-
fi
2413

2514
case "${AWS_ENABLED}" in
2615
"0")

‎.github/workflows/_build_test_upload.yml‎

Lines changed: 2 additions & 153 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,6 @@ on:
1919
secrets:
2020
PYPI_TOKEN:
2121
required: false
22-
CONDA_PYTORCHBOT_TOKEN:
23-
required: false
24-
CONDA_TEST_PYTORCHBOT_TOKEN:
25-
required: false
26-
CONDA_NIGHTLY_PYTORCHBOT_TOKEN:
27-
required: false
2822

2923
permissions:
3024
id-token: write
@@ -214,156 +208,11 @@ jobs:
214208
--password "$PYPI_TOKEN" \
215209
torchdata*.whl
216210
217-
conda_build_test:
218-
needs: get_release_type
219-
runs-on: ${{ matrix.os }}
220-
container: ${{ startsWith( matrix.os, 'ubuntu' ) && 'pytorch/manylinux-cpu' || null }}
221-
strategy:
222-
fail-fast: false
223-
matrix:
224-
os:
225-
- windows-latest
226-
python-version:
227-
- 3.9
228-
- "3.10"
229-
- "3.11"
230-
- "3.12"
231-
- "3.13"
232-
steps:
233-
- name: Checkout Source Repository
234-
uses: actions/checkout@v3
235-
with:
236-
ref: ${{ inputs.branch }}
237-
submodules: recursive
238-
- name: Determine if build AWSSDK
239-
shell: bash -l {0}
240-
run: |
241-
BUILD_S3=0
242-
echo "value=$BUILD_S3" >> $GITHUB_OUTPUT
243-
id: build_s3
244-
- name: Add temp runner environment variables
245-
shell: bash -l {0}
246-
run: |
247-
echo "MINICONDA_INSTALL_PATH=${RUNNER_TEMP}/miniconda" >> "${GITHUB_ENV}"
248-
echo "CONDA_ENV_PATH=${RUNNER_TEMP}/conda_build_env" >> "${GITHUB_ENV}"
249-
- name: Create Conda Env on Windows
250-
if: ${{ startsWith( matrix.os, 'windows' ) }}
251-
uses: conda-incubator/setup-miniconda@v2
252-
with:
253-
python-version: ${{ matrix.python-version }}
254-
activate-environment: ${{ env.CONDA_ENV_PATH }}
255-
- name: Build TorchData for Conda
256-
shell: bash -l {0}
257-
env:
258-
PYTHON_VERSION: ${{ matrix.python-version }}
259-
PYTORCH_VERSION: ${{ inputs.pytorch_version }}
260-
BUILD_S3: ${{ steps.build_s3.outputs.value }}
261-
run: |
262-
set -ex
263-
if ${{ ! startsWith( matrix.os, 'windows' ) }}; then
264-
source "${MINICONDA_INSTALL_PATH}/etc/profile.d/conda.sh"
265-
fi
266-
conda activate "${CONDA_ENV_PATH}"
267-
if [[ "${PYTHON_VERSION}" = "3.11" ]]; then
268-
conda install -yq conda-build -c malfet -c conda-forge
269-
export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c malfet"
270-
else
271-
conda install -yq conda-build
272-
fi
273-
packaging/build_conda.sh
274-
# conda index ./conda-bld
275-
- name: Upload Conda Package to Github
276-
if: always()
277-
uses: actions/upload-artifact@v3
278-
with:
279-
name: torchdata-artifact
280-
path: conda-bld/*/torchdata-*.tar.bz2
281-
282-
conda_upload:
283-
if: always() && inputs.branch != '' && inputs.do-upload == true
284-
needs: [get_release_type, conda_build_test]
285-
runs-on: ubuntu-latest
286-
container: continuumio/miniconda3
287-
environment: pytorchbot-env
288-
outputs:
289-
upload: ${{ steps.trigger_upload.outputs.value }}
290-
steps:
291-
- name: Download Artifacts from Github
292-
continue-on-error: true
293-
uses: actions/download-artifact@v3
294-
with:
295-
name: torchdata-artifact
296-
- name: Determine if Conda Uploading is needed
297-
run: |
298-
upload=false
299-
for pkg in ./*/torchdata-*.tar.bz2; do
300-
upload=true
301-
break
302-
done
303-
echo "value=$upload" >> $GITHUB_OUTPUT
304-
id: trigger_upload
305-
- name: Display All TorchData Conda Package
306-
if: steps.trigger_upload.outputs.value == 'true'
307-
run: ls -lh ./*/torchdata-*.tar.bz2
308-
- name: Upload Packages to Conda
309-
if: steps.trigger_upload.outputs.value == 'true'
310-
shell: bash -l {0}
311-
env:
312-
CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }}
313-
CONDA_NIGHTLY_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_NIGHTLY_PYTORCHBOT_TOKEN }}
314-
CONDA_TEST_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_TEST_PYTORCHBOT_TOKEN }}
315-
run: |
316-
conda create -y --name conda_upload_env
317-
conda activate conda_upload_env
318-
319-
conda install -yq anaconda-client
320-
conda install -c conda-forge -yq jq
321-
322-
if [[ ${{ needs.get_release_type.outputs.type }} == 'official' ]]; then
323-
CONDA_CHANNEL=pytorch
324-
CONDA_TOKEN=${CONDA_PYTORCHBOT_TOKEN}
325-
elif [[ ${{ needs.get_release_type.outputs.type }} == 'test' ]]; then
326-
CONDA_CHANNEL=pytorch-${{ needs.get_release_type.outputs.type }}
327-
CONDA_TOKEN=${CONDA_TEST_PYTORCHBOT_TOKEN}
328-
else
329-
CONDA_CHANNEL=pytorch-${{ needs.get_release_type.outputs.type }}
330-
CONDA_TOKEN=${CONDA_NIGHTLY_PYTORCHBOT_TOKEN}
331-
fi
332-
333-
if [[ ${{ needs.get_release_type.outputs.type }} == 'nightly' ]]; then
334-
# Loop over all platforms [win-64, osx-64, linux-64]
335-
for subdir in $(find . -type f -name '*torchdata*.tar.bz2' | sed -r 's|/[^/]+$||' | uniq | cut -f2 -d/) ; do
336-
version=""
337-
# Find existing conda packages on pytorch nightly
338-
for val in $(conda search --json torchdata --channel=$CONDA_CHANNEL --subdir=$subdir | jq -r '.[][] | .version, .build'); do
339-
if [[ -z $version ]]; then
340-
version=$val
341-
else
342-
build=$val
343-
# Check if a new conda package built based on the same Python version and platform exists
344-
for new_package in ./$subdir/torchdata-*-$build.tar.bz2; do
345-
if [[ -f "$new_package" ]]; then
346-
echo "Removing $CONDA_CHANNEL/torchdata/$version/$subdir/torchdata-$version-$build.tar.bz2"
347-
anaconda -t "${CONDA_TOKEN}" remove -f "$CONDA_CHANNEL/torchdata/$version/$subdir/torchdata-$version-$build.tar.bz2"
348-
break
349-
fi
350-
done
351-
version=""
352-
build=""
353-
fi
354-
done
355-
# Upload new conda packages per subdir
356-
anaconda -t "${CONDA_TOKEN}" upload ./$subdir/torchdata-*.tar.bz2 -u "$CONDA_CHANNEL" --label main --no-progress --force
357-
done
358-
else
359-
anaconda -t "${CONDA_TOKEN}" upload ./*/torchdata-*.tar.bz2 -u "$CONDA_CHANNEL" --label main --no-progress --force
360-
fi
361-
362211
build_docs:
363212
if: |
364213
always() && inputs.branch != '' &&
365-
( needs.wheel_upload.outputs.upload == 'true' || needs.conda_upload.outputs.upload == 'true' )
366-
needs: [get_release_type, wheel_upload, conda_upload]
214+
( needs.wheel_upload.outputs.upload == 'true')
215+
needs: [get_release_type, wheel_upload]
367216
runs-on: ubuntu-latest
368217
steps:
369218
- name: Setup Python 3.9

‎.github/workflows/build-conda-m1.yml‎

Lines changed: 0 additions & 44 deletions
This file was deleted.

‎.github/workflows/build_conda_linux.yml‎

Lines changed: 0 additions & 55 deletions
This file was deleted.

‎.github/workflows/nightly_release.yml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,5 +18,3 @@ jobs:
1818
branch: "main"
1919
pre_dev_release: true
2020
pytorch_version: ""
21-
secrets:
22-
CONDA_NIGHTLY_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_NIGHTLY_PYTORCHBOT_TOKEN }}

‎.github/workflows/pull_release.yml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,10 @@ on:
77
- .github/workflows/pull_release.yml
88
- .github/workflows/_build_test_upload.yml
99

10+
permissions:
11+
contents: write
12+
id-token: write
13+
1014
jobs:
1115
build_test_upload:
1216
if: github.repository == 'pytorch/data'
@@ -16,5 +20,3 @@ jobs:
1620
pre_dev_release: true
1721
pytorch_version: ""
1822
do-upload: false
19-
secrets:
20-
CONDA_TEST_PYTORCHBOT_TOKEN: ""

‎.github/workflows/test_release.yml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,5 +34,3 @@ jobs:
3434
branch: "release/0.9"
3535
pre_dev_release: true
3636
pytorch_version: "2.5.0"
37-
secrets:
38-
CONDA_TEST_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_TEST_PYTORCHBOT_TOKEN }}

‎.github/workflows/validate-binaries.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ jobs:
4747
validate-binaries:
4848
uses: pytorch/test-infra/.github/workflows/validate-domain-library.yml@main
4949
with:
50-
package_type: "conda,wheel"
50+
package_type: "wheel"
5151
os: ${{ inputs.os }}
5252
channel: ${{ inputs.channel }}
5353
repository: "pytorch/data"

0 commit comments

Comments
 (0)