# WARNING: DO NOT EDIT THIS FILE DIRECTLY!!!
# See the README.md in this directory.

# IMPORTANT: To update Docker image version, please follow
# the instructions at
# https://github.com/pytorch/pytorch/wiki/Docker-image-build-on-CircleCI

version: 2.1

parameters:
  run_binary_tests:
    type: boolean
    default: false
  run_build:
    type: boolean
    default: true
  run_master_build:
    type: boolean
    default: false
  run_slow_gradcheck_build:
    type: boolean
    default: false

executors:
  windows-with-nvidia-gpu:
    machine:
      resource_class: windows.gpu.nvidia.medium
      image: windows-server-2019-nvidia:previous
      shell: bash.exe

  windows-xlarge-cpu-with-nvidia-cuda:
    machine:
      resource_class: windows.xlarge
      image: windows-server-2019-vs2019:stable
      shell: bash.exe

  windows-medium-cpu-with-nvidia-cuda:
    machine:
      resource_class: windows.medium
      image: windows-server-2019-vs2019:stable
      shell: bash.exe
commands:

  calculate_docker_image_tag:
    description: "Calculates the docker image tag"
    steps:
      - run:
          name: "Calculate docker image hash"
          command: |
            DOCKER_TAG=$(git rev-parse HEAD:.circleci/docker)
            echo "DOCKER_TAG=${DOCKER_TAG}" >> "${BASH_ENV}"

  designate_upload_channel:
    description: "inserts the correct upload channel into ${BASH_ENV}"
    steps:
      - run:
          name: adding UPLOAD_CHANNEL to BASH_ENV
          command: |
            our_upload_channel=nightly
            # On tags upload to test instead
            if [[ -n "${CIRCLE_TAG}" ]]; then
              our_upload_channel=test
            fi
            echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}

  # This system setup script is meant to run before the CI-related scripts, e.g.,
  # installing Git client, checking out code, setting up CI env, and
  # building/testing.
  setup_linux_system_environment:
    steps:
      - run:
          name: Set Up System Environment
          no_output_timeout: "1h"
          command: .circleci/scripts/setup_linux_system_environment.sh

  setup_ci_environment:
    steps:
      - run:
          name: Set Up CI Environment After attach_workspace
          no_output_timeout: "1h"
          command: .circleci/scripts/setup_ci_environment.sh

  brew_update:
    description: "Update Homebrew and install base formulae"
    steps:
      - run:
          name: Update Homebrew
          no_output_timeout: "10m"
          command: |
            set -ex

            # Update repositories manually.
            # Running `brew update` produces a comparison between the
            # current checkout and the updated checkout, which takes a
            # very long time because the existing checkout is 2y old.
            for path in $(find /usr/local/Homebrew -type d -name .git)
            do
            cd $path/..
            git fetch --depth=1 origin
            git reset --hard origin/master
            done

            export HOMEBREW_NO_AUTO_UPDATE=1

            # Install expect and moreutils so that we can call `unbuffer` and `ts`.
            # moreutils installs a `parallel` executable by default, which conflicts
            # with the executable from the GNU `parallel`, so we must unlink GNU
            # `parallel` first, and relink it afterwards.
            brew unlink parallel
            brew install moreutils
            brew link parallel --overwrite
            brew install expect

  brew_install:
    description: "Install Homebrew formulae"
    parameters:
      formulae:
        type: string
        default: ""
    steps:
      - run:
          name: Install << parameters.formulae >>
          no_output_timeout: "10m"
          command: |
            set -ex
            export HOMEBREW_NO_AUTO_UPDATE=1
            brew install << parameters.formulae >>

  run_brew_for_macos_build:
    steps:
      - brew_update
      - brew_install:
          formulae: libomp

  run_brew_for_ios_build:
    steps:
      - brew_update
      - brew_install:
          formulae: libtool

  optional_merge_target_branch:
    steps:
      - run:
          name: (Optional) Merge target branch
          no_output_timeout: "10m"
          command: |
            if [[ -n "$CIRCLE_PULL_REQUEST" && "$CIRCLE_BRANCH" != "nightly" ]]; then
              PR_NUM=$(basename $CIRCLE_PULL_REQUEST)
              CIRCLE_PR_BASE_BRANCH=$(curl -s https://api.github.com/repos/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/pulls/$PR_NUM | jq -r '.base.ref')
              if [[ "${BUILD_ENVIRONMENT}" == *"xla"* || "${BUILD_ENVIRONMENT}" == *"gcc5"* ]] ; then
                set -x
                git config --global user.email "circleci.ossci@gmail.com"
                git config --global user.name "CircleCI"
                git config remote.origin.url https://github.com/pytorch/pytorch.git
                git config --add remote.origin.fetch +refs/heads/master:refs/remotes/origin/master
                git fetch --tags --progress https://github.com/pytorch/pytorch.git +refs/heads/master:refs/remotes/origin/master --depth=100 --quiet
                # PRs generated from ghstack has format CIRCLE_PR_BASE_BRANCH=gh/xxx/1234/base
                if [[ "${CIRCLE_PR_BASE_BRANCH}" == "gh/"* ]]; then
                  CIRCLE_PR_BASE_BRANCH=master
                fi
                export GIT_MERGE_TARGET=`git log -n 1 --pretty=format:"%H" origin/$CIRCLE_PR_BASE_BRANCH`
                echo "GIT_MERGE_TARGET: " ${GIT_MERGE_TARGET}
                export GIT_COMMIT=${CIRCLE_SHA1}
                echo "GIT_COMMIT: " ${GIT_COMMIT}
                git checkout -f ${GIT_COMMIT}
                git reset --hard ${GIT_COMMIT}
                git merge --allow-unrelated-histories --no-edit --no-ff ${GIT_MERGE_TARGET}
                echo "Merged $CIRCLE_PR_BASE_BRANCH branch before building in environment $BUILD_ENVIRONMENT"
                set +x
              else
                echo "No need to merge with $CIRCLE_PR_BASE_BRANCH, skipping..."
              fi
            else
              echo "This is not a pull request, skipping..."
            fi

  upload_binary_size_for_android_build:
    description: "Upload binary size data for Android build"
    parameters:
      build_type:
        type: string
        default: ""
      artifacts:
        type: string
        default: ""
    steps:
      - run:
          name: "Binary Size - Install Dependencies"
          no_output_timeout: "5m"
          command: |
            retry () {
              $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
            }
            retry pip3 install requests
      - run:
          name: "Binary Size - Untar Artifacts"
          no_output_timeout: "5m"
          command: |
            # The artifact file is created inside docker container, which contains the result binaries.
            # Now unpackage it into the project folder. The subsequent script will scan project folder
            # to locate result binaries and report their sizes.
            # If artifact file is not provided it assumes that the project folder has been mounted in
            # the docker during build and already contains the result binaries, so this step can be skipped.
            export ARTIFACTS="<< parameters.artifacts >>"
            if [ -n "${ARTIFACTS}" ]; then
              tar xf "${ARTIFACTS}" -C ~/project
            fi
      - run:
          name: "Binary Size - Upload << parameters.build_type >>"
          no_output_timeout: "5m"
          command: |
            cd ~/project
            export ANDROID_BUILD_TYPE="<< parameters.build_type >>"
            export COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0)
            python3 -m tools.stats.upload_binary_size_to_scuba android

##############################################################################
# Binary build (nightlies nightly build) defaults
# The binary builds use the docker executor b/c at time of writing the machine
# executor is limited to only two cores and is painfully slow (4.5+ hours per
# GPU build). But the docker executor cannot be run with --runtime=nvidia, and
# so the binary test/upload jobs must run on a machine executor. The package
# built in the build job is persisted to the workspace, which the test jobs
# expect. The test jobs just run a few quick smoke tests (very similar to the
# second-round-user-facing smoke tests above) and then upload the binaries to
# their final locations. The upload part requires credentials that should only
# be available to org-members.
#
# binary_checkout MUST be run before other commands here. This is because the
# other commands are written in .circleci/scripts/*.sh , so the pytorch source
# code must be downloaded on the machine before they can be run. We cannot
# inline all the code into this file, since that would cause the yaml size to
# explode past 4 MB (all the code in the command section is just copy-pasted to
# everywhere in the .circleci/config.yml file where it appears).
##############################################################################

# Checks out the Pytorch and Builder repos (always both of them), and places
# them in the right place depending on what executor we're running on. We curl
# our .sh file from the interweb to avoid yaml size bloat. Note that many jobs
# do not need both the pytorch and builder repos, so this is a little wasteful
# (smoke tests and upload jobs do not need the pytorch repo).
binary_checkout: &binary_checkout
  name: Checkout pytorch/builder repo
  no_output_timeout: "30m"
  command: .circleci/scripts/binary_checkout.sh

# Parses circleci arguments in a consistent way, essentially routing to the
# correct pythonXgccXcudaXos build we want
binary_populate_env: &binary_populate_env
  name: Set up binary env variables
  command: .circleci/scripts/binary_populate_env.sh

binary_install_miniconda: &binary_install_miniconda
  name: Install miniconda
  no_output_timeout: "1h"
  command: .circleci/scripts/binary_install_miniconda.sh

# This section is used in the binary_test and smoke_test jobs. It expects
# 'binary_populate_env' to have populated /home/circleci/project/env and it
# expects another section to populate /home/circleci/project/ci_test_script.sh
# with the code to run in the docker
binary_run_in_docker: &binary_run_in_docker
  name: Run in docker
  # This step only runs on circleci linux machine executors that themselves
  # need to start docker images
  command: .circleci/scripts/binary_run_in_docker.sh
##############################################################################
# Build parameters
##############################################################################
pytorch_params: &pytorch_params
  parameters:
    build_environment:
      type: string
      default: ""
    docker_image:
      type: string
      default: ""
    resource_class:
      type: string
      default: "large"
    use_cuda_docker_runtime:
      type: string
      default: ""
    build_only:
      type: string
      default: ""
    ci_master:
      type: string
      default: ""
  environment:
    BUILD_ENVIRONMENT: << parameters.build_environment >>
    DOCKER_IMAGE: << parameters.docker_image >>
    USE_CUDA_DOCKER_RUNTIME: << parameters.use_cuda_docker_runtime >>
    BUILD_ONLY: << parameters.build_only >>
    CI_MASTER: << pipeline.parameters.run_master_build >>
  resource_class: << parameters.resource_class >>

pytorch_ios_params: &pytorch_ios_params
  parameters:
    build_environment:
      type: string
      default: ""
    ios_arch:
      type: string
      default: ""
    ios_platform:
      type: string
      default: ""
    op_list:
      type: string
      default: ""
    use_metal:
      type: string
      default: "0"
    lite_interpreter:
      type: string
      default: "1"
    use_coreml:
      type: string
      default: "0"
  environment:
    BUILD_ENVIRONMENT: << parameters.build_environment >>
    IOS_ARCH: << parameters.ios_arch >>
    IOS_PLATFORM: << parameters.ios_platform >>
    SELECTED_OP_LIST: << parameters.op_list >>
    USE_PYTORCH_METAL: << parameters.use_metal >>
    BUILD_LITE_INTERPRETER: << parameters.lite_interpreter >>
    USE_COREML_DELEGATE: << parameters.use_coreml >>

pytorch_windows_params: &pytorch_windows_params
  parameters:
    executor:
      type: string
      default: "windows-xlarge-cpu-with-nvidia-cuda"
    build_environment:
      type: string
      default: ""
    test_name:
      type: string
      default: ""
    cuda_version:
      type: string
      default: "10.1"
    python_version:
      type: string
      default: "3.8"
    vs_version:
      type: string
      default: "16.8.6"
    vc_version:
      type: string
      default: "14.16"
    vc_year:
      type: string
      default: "2019"
    vc_product:
      type: string
      default: "BuildTools"
    use_cuda:
      type: string
      default: ""
  environment:
    BUILD_ENVIRONMENT: <<parameters.build_environment>>
    SCCACHE_BUCKET: "ossci-compiler-cache"
    CUDA_VERSION: <<parameters.cuda_version>>
    PYTHON_VERSION: <<parameters.python_version>>
    VS_VERSION: <<parameters.vs_version>>
    VC_VERSION: <<parameters.vc_version>>
    VC_YEAR: <<parameters.vc_year>>
    VC_PRODUCT: <<parameters.vc_product>>
    USE_CUDA: <<parameters.use_cuda>>
    TORCH_CUDA_ARCH_LIST: "5.2 7.5"
    JOB_BASE_NAME: <<parameters.test_name>>
    JOB_EXECUTOR: <<parameters.executor>>
binary_linux_build_params: &binary_linux_build_params
  parameters:
    build_environment:
      type: string
      default: ""
    docker_image:
      type: string
      default: ""
    libtorch_variant:
      type: string
      default: ""
    resource_class:
      type: string
      default: "2xlarge+"
  environment:
    BUILD_ENVIRONMENT: << parameters.build_environment >>
    LIBTORCH_VARIANT: << parameters.libtorch_variant >>
    ANACONDA_USER: pytorch
  resource_class: << parameters.resource_class >>
  docker:
    - image: << parameters.docker_image >>

binary_linux_test_upload_params: &binary_linux_test_upload_params
  parameters:
    build_environment:
      type: string
      default: ""
    docker_image:
      type: string
      default: ""
    libtorch_variant:
      type: string
      default: ""
    resource_class:
      type: string
      default: "medium"
    use_cuda_docker_runtime:
      type: string
      default: ""
  environment:
    BUILD_ENVIRONMENT: << parameters.build_environment >>
    DOCKER_IMAGE: << parameters.docker_image >>
    USE_CUDA_DOCKER_RUNTIME: << parameters.use_cuda_docker_runtime >>
    LIBTORCH_VARIANT: << parameters.libtorch_variant >>
  resource_class: << parameters.resource_class >>

binary_mac_params: &binary_mac_params
  parameters:
    build_environment:
      type: string
      default: ""
  environment:
    BUILD_ENVIRONMENT: << parameters.build_environment >>

binary_windows_params: &binary_windows_params
  parameters:
    build_environment:
      type: string
      default: ""
    executor:
      type: string
      default: "windows-xlarge-cpu-with-nvidia-cuda"
  environment:
    BUILD_ENVIRONMENT: << parameters.build_environment >>
    BUILD_FOR_SYSTEM: windows
    JOB_EXECUTOR: <<parameters.executor>>

promote_common: &promote_common
  docker:
    - image: pytorch/release
  parameters:
    package_name:
      description: "package name to promote"
      type: string
      default: ""
  environment:
    PACKAGE_NAME: << parameters.package_name >>
    ANACONDA_API_TOKEN: ${CONDA_PYTORCHBOT_TOKEN}
    AWS_ACCESS_KEY_ID: ${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}
    AWS_SECRET_ACCESS_KEY: ${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}
##############################################################################
# Job specs
##############################################################################
jobs:
  binary_linux_build:
    <<: *binary_linux_build_params
    steps:
    - checkout
    - calculate_docker_image_tag
    - run:
        <<: *binary_checkout
    - run:
        <<: *binary_populate_env
    - run:
        name: Build
        no_output_timeout: "1h"
        command: |
            source "/pytorch/.circleci/scripts/binary_linux_build.sh"
            # Preserve build log
            if [ -f /pytorch/build/.ninja_log ]; then
              cp /pytorch/build/.ninja_log /final_pkgs
            fi
    - run:
        name: Output binary sizes
        no_output_timeout: "1m"
        command: |
            ls -lah /final_pkgs
    - run:
        name: upload build & binary data
        no_output_timeout: "5m"
        command: |
            source /env
            cd /pytorch && export COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0)
            python3 -mpip install requests && \
            SCRIBE_GRAPHQL_ACCESS_TOKEN=${SCRIBE_GRAPHQL_ACCESS_TOKEN} \
            python3 -m tools.stats.upload_binary_size_to_scuba || exit 0
    - persist_to_workspace:
        root: /
        paths: final_pkgs

    - store_artifacts:
        path: /final_pkgs

    # This should really just be another step of the binary_linux_build job above.
    # This isn't possible right now b/c the build job uses the docker executor
    # (otherwise they'd be really really slow) but this one uses the macine
    # executor (b/c we have to run the docker with --runtime=nvidia and we can't do
    # that on the docker executor)
  binary_linux_test:
    <<: *binary_linux_test_upload_params
    machine:
        image: ubuntu-2004:202104-01
    steps:
    # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
    - checkout
    - attach_workspace:
        at: /home/circleci/project
    - setup_linux_system_environment
    - setup_ci_environment
    - run:
        <<: *binary_checkout
    - run:
        <<: *binary_populate_env
    - run:
        name: Prepare test code
        no_output_timeout: "1h"
        command: .circleci/scripts/binary_linux_test.sh
    - run:
        <<: *binary_run_in_docker

  binary_upload:
    parameters:
      package_type:
        type: string
        description: "What type of package we are uploading (eg. wheel, libtorch, conda)"
        default: "wheel"
      upload_subfolder:
        type: string
        description: "What subfolder to put our package into (eg. cpu, cudaX.Y, etc.)"
        default: "cpu"
    docker:
      - image: continuumio/miniconda3
    environment:
      - DRY_RUN: disabled
      - PACKAGE_TYPE: "<< parameters.package_type >>"
      - UPLOAD_SUBFOLDER: "<< parameters.upload_subfolder >>"
    steps:
      - attach_workspace:
          at: /tmp/workspace
      - checkout
      - designate_upload_channel
      - run:
          name: Install dependencies
          no_output_timeout: "1h"
          command: |
            conda install -yq anaconda-client
            pip install -q awscli
      - run:
          name: Do upload
          no_output_timeout: "1h"
          command: |
            AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}" \
              AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}" \
              ANACONDA_API_TOKEN="${CONDA_PYTORCHBOT_TOKEN}" \
              .circleci/scripts/binary_upload.sh

  # Nighlty build smoke tests defaults
  # These are the second-round smoke tests. These make sure that the binaries are
  # correct from a user perspective, testing that they exist from the cloud are
  # are runnable. Note that the pytorch repo is never cloned into these jobs
  ##############################################################################
  smoke_linux_test:
    <<: *binary_linux_test_upload_params
    machine:
      image: ubuntu-2004:202104-01
    steps:
    - checkout
    - calculate_docker_image_tag
    - setup_linux_system_environment
    - setup_ci_environment
    - run:
        <<: *binary_checkout
    - run:
        <<: *binary_populate_env
    - run:
        name: Test
        no_output_timeout: "1h"
        command: |
          set -ex
          cat >/home/circleci/project/ci_test_script.sh \<<EOL
          # The following code will be executed inside Docker container
          set -eux -o pipefail
          /builder/smoke_test.sh
          # The above code will be executed inside Docker container
          EOL
    - run:
        <<: *binary_run_in_docker

  smoke_mac_test:
    <<: *binary_linux_test_upload_params
    macos:
      xcode: "12.0"
    steps:
      - checkout
      - run:
          <<: *binary_checkout
      - run:
          <<: *binary_populate_env
      - brew_update
      - run:
          <<: *binary_install_miniconda
      - run:
          name: Build
          no_output_timeout: "1h"
          command: |
            set -ex
            source "/Users/distiller/project/env"
            export "PATH=$workdir/miniconda/bin:$PATH"
            # TODO unbuffer and ts this, but it breaks cause miniconda overwrites
            # tclsh. But unbuffer and ts aren't that important so they're just
            # disabled for now
            ./builder/smoke_test.sh

  binary_mac_build:
    <<: *binary_mac_params
    macos:
      xcode: "12.0"
    steps:
    # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
    - checkout
    - run:
        <<: *binary_checkout
    - run:
        <<: *binary_populate_env
    - brew_update
    - run:
        <<: *binary_install_miniconda

    - run:
        name: Build
        no_output_timeout: "90m"
        command: |
          # Do not set -u here; there is some problem with CircleCI
          # variable expansion with PROMPT_COMMAND
          set -ex -o pipefail
          script="/Users/distiller/project/pytorch/.circleci/scripts/binary_macos_build.sh"
          cat "$script"
          source "$script"

    - run:
        name: Test
        no_output_timeout: "1h"
        command: |
          # Do not set -u here; there is some problem with CircleCI
          # variable expansion with PROMPT_COMMAND
          set -ex -o pipefail
          script="/Users/distiller/project/pytorch/.circleci/scripts/binary_macos_test.sh"
          cat "$script"
          source "$script"

    - persist_to_workspace:
        root: /Users/distiller/project
        paths: final_pkgs

    - store_artifacts:
        path: /Users/distiller/project/final_pkgs

  binary_macos_arm64_build:
    <<: *binary_mac_params
    macos:
      xcode: "12.3.0"
    steps:
    # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
    - checkout
    - run:
        <<: *binary_checkout
    - run:
        <<: *binary_populate_env
    - brew_update
    - run:
        <<: *binary_install_miniconda

    - run:
        name: Build
        no_output_timeout: "90m"
        command: |
          # Do not set -u here; there is some problem with CircleCI
          # variable expansion with PROMPT_COMMAND
          set -ex -o pipefail
          export CROSS_COMPILE_ARM64=1
          script="/Users/distiller/project/pytorch/.circleci/scripts/binary_macos_build.sh"
          cat "$script"
          source "$script"

    - persist_to_workspace:
        root: /Users/distiller/project
        paths: final_pkgs

    - store_artifacts:
        path: /Users/distiller/project/final_pkgs


  binary_ios_build:
    <<: *pytorch_ios_params
    macos:
      xcode: "12.5.1"
    steps:
    - attach_workspace:
        at: ~/workspace
    - checkout
    - run_brew_for_ios_build
    - run:
        name: Build
        no_output_timeout: "1h"
        command: |
          script="/Users/distiller/project/.circleci/scripts/binary_ios_build.sh"
          cat "$script"
          source "$script"
    - run:
        name: Test
        no_output_timeout: "30m"
        command: |
          script="/Users/distiller/project/.circleci/scripts/binary_ios_test.sh"
          cat "$script"
          source "$script"
    - persist_to_workspace:
        root: /Users/distiller/workspace/
        paths: ios

  binary_ios_upload:
    <<: *pytorch_ios_params
    macos:
      xcode: "12.5.1"
    steps:
    - attach_workspace:
        at: ~/workspace
    - checkout
    - run_brew_for_ios_build
    - run:
        name: Upload
        no_output_timeout: "1h"
        command: |
          script="/Users/distiller/project/.circleci/scripts/binary_ios_upload.sh"
          cat "$script"
          source "$script"

  binary_windows_build:
    <<: *binary_windows_params
    parameters:
      build_environment:
        type: string
        default: ""
      executor:
        type: string
        default: "windows-xlarge-cpu-with-nvidia-cuda"
    executor: <<parameters.executor>>
    steps:
    # See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
    - checkout
    - run:
        <<: *binary_checkout
    - run:
        <<: *binary_populate_env
    - run:
        name: Build
        no_output_timeout: "1h"
        command: |
          set -eux -o pipefail
          script="/c/w/p/.circleci/scripts/binary_windows_build.sh"
          cat "$script"
          source "$script"
    - persist_to_workspace:
        root: "C:/w"
        paths: final_pkgs
    - store_artifacts:
        path: C:/w/final_pkgs

  binary_windows_test:
    <<: *binary_windows_params
    parameters:
      build_environment:
        type: string
        default: ""
      executor:
        type: string
        default: "windows-medium-cpu-with-nvidia-cuda"
    executor: <<parameters.executor>>
    steps:
    - checkout
    - attach_workspace:
        at: c:/users/circleci/project
    - run:
        <<: *binary_checkout
    - run:
        <<: *binary_populate_env
    - run:
        name: Test
        no_output_timeout: "1h"
        command: |
          set -eux -o pipefail
          script="/c/w/p/.circleci/scripts/binary_windows_test.sh"
          cat "$script"
          source "$script"

  smoke_windows_test:
    <<: *binary_windows_params
    parameters:
      build_environment:
        type: string
        default: ""
      executor:
        type: string
        default: "windows-medium-cpu-with-nvidia-cuda"
    executor: <<parameters.executor>>
    steps:
    - checkout
    - run:
        <<: *binary_checkout
    - run:
        <<: *binary_populate_env
    - run:
        name: Test
        no_output_timeout: "1h"
        command: |
          set -eux -o pipefail
          export TEST_NIGHTLY_PACKAGE=1
          script="/c/w/p/.circleci/scripts/binary_windows_test.sh"
          cat "$script"
          source "$script"

  anaconda_prune:
    parameters:
      packages:
        type: string
        description: "What packages are we pruning? (quoted, space-separated string. eg. 'pytorch', 'torchvision torchaudio', etc.)"
        default: "pytorch"
      channel:
        type: string
        description: "What channel are we pruning? (eq. pytorch-nightly)"
        default: "pytorch-nightly"
    docker:
      - image: continuumio/miniconda3
    environment:
      - PACKAGES: "<< parameters.packages >>"
      - CHANNEL: "<< parameters.channel >>"
    steps:
      - checkout
      - run:
          name: Install dependencies
          no_output_timeout: "1h"
          command: |
            conda install -yq anaconda-client
      - run:
          name: Prune packages
          no_output_timeout: "1h"
          command: |
              ANACONDA_API_TOKEN="${CONDA_PYTORCHBOT_TOKEN}" \
              scripts/release/anaconda-prune/run.sh
  pytorch_doc_push:
    resource_class: medium
    machine:
      image: ubuntu-2004:202104-01
    parameters:
      branch:
        type: string
        default: "master"
    steps:
    - attach_workspace:
        at: /tmp/workspace
    - run:
        name: Generate netrc
        command: |
          # set credentials for https pushing
          cat > ~/.netrc \<<DONE
            machine github.com
            login pytorchbot
            password ${GITHUB_PYTORCHBOT_TOKEN}
          DONE
    - run:
        name: Docs push
        command: |
          pushd /tmp/workspace
          git push -u origin "<< parameters.branch >>"

  pytorch_python_doc_build:
    environment:
      BUILD_ENVIRONMENT: pytorch-python-doc-push
      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.7-gcc5.4"
    resource_class: large
    machine:
      image: ubuntu-2004:202104-01
    steps:
    - checkout
    - calculate_docker_image_tag
    - setup_linux_system_environment
    - setup_ci_environment
    - run:
        name: Doc Build and Push
        no_output_timeout: "1h"
        command: |
          set -ex
          export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}:build-${DOCKER_TAG}-${CIRCLE_SHA1}
          echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
          # turn v1.12.0rc3 into 1.12
          tag=$(echo $CIRCLE_TAG | sed -e 's/v*\([0-9]*\.[0-9]*\).*/\1/')
          target=${tag:-master}
          echo "building for ${target}"
          time docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
          export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})

          export COMMAND='((echo "sudo chown -R jenkins workspace && cd workspace && '"export CIRCLE_SHA1='$CIRCLE_SHA1'"' && . ./.circleci/scripts/python_doc_push_script.sh docs/'$target' '$target' site") | docker exec -u jenkins -i "$id" bash) 2>&1'

          echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts

          mkdir -p ~/workspace/build_artifacts
          docker cp $id:/var/lib/jenkins/workspace/pytorch.github.io/docs/master ~/workspace/build_artifacts
          docker cp $id:/var/lib/jenkins/workspace/pytorch.github.io /tmp/workspace

          # Save the docs build so we can debug any problems
          export DEBUG_COMMIT_DOCKER_IMAGE=${COMMIT_DOCKER_IMAGE}-debug
          docker commit "$id" ${DEBUG_COMMIT_DOCKER_IMAGE}
          time docker push ${DEBUG_COMMIT_DOCKER_IMAGE}
    - persist_to_workspace:
        root: /tmp/workspace
        paths:
          - .
    - store_artifacts:
        path: ~/workspace/build_artifacts/master
        destination: docs

  pytorch_cpp_doc_build:
    environment:
      BUILD_ENVIRONMENT: pytorch-cpp-doc-push
      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.7-gcc5.4"
    resource_class: large
    machine:
      image: ubuntu-2004:202104-01
    steps:
    - checkout
    - calculate_docker_image_tag
    - setup_linux_system_environment
    - setup_ci_environment
    - run:
        name: Doc Build and Push
        no_output_timeout: "1h"
        command: |
          set -ex
          export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}:build-${DOCKER_TAG}-${CIRCLE_SHA1}
          echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
          # turn v1.12.0rc3 into 1.12
          tag=$(echo $CIRCLE_TAG | sed -e 's/v*\([0-9]*\.[0-9]*\).*/\1/')
          target=${tag:-master}
          echo "building for ${target}"
          time docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
          export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})

          export COMMAND='((echo "sudo chown -R jenkins workspace && cd workspace && '"export CIRCLE_SHA1='$CIRCLE_SHA1'"' && . ./.circleci/scripts/cpp_doc_push_script.sh docs/"$target" master") | docker exec -u jenkins -i "$id" bash) 2>&1'

          echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts

          mkdir -p ~/workspace/build_artifacts
          docker cp $id:/var/lib/jenkins/workspace/cppdocs/ /tmp/workspace

          # Save the docs build so we can debug any problems
          export DEBUG_COMMIT_DOCKER_IMAGE=${COMMIT_DOCKER_IMAGE}-debug
          docker commit "$id" ${DEBUG_COMMIT_DOCKER_IMAGE}
          time docker push ${DEBUG_COMMIT_DOCKER_IMAGE}

    - persist_to_workspace:
        root: /tmp/workspace
        paths:
          - .

  pytorch_macos_10_15_py3_build:
    environment:
      BUILD_ENVIRONMENT: pytorch-macos-10.15-py3-arm64-build
    macos:
      xcode: "12.3.0"
    steps:
      - checkout
      - run_brew_for_macos_build
      - run:
          name: Build
          no_output_timeout: "1h"
          command: |
            set -e
            export IN_CI=1
            export CROSS_COMPILE_ARM64=1
            export JOB_BASE_NAME=$CIRCLE_JOB

            # Install sccache
            sudo curl --retry 3 https://s3.amazonaws.com/ossci-macos/sccache_v2.15 --output /usr/local/bin/sccache
            sudo chmod +x /usr/local/bin/sccache
            export SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2

            # This IAM user allows write access to S3 bucket for sccache
            set +x
            export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_SCCACHE_S3_BUCKET_V4}
            export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_SCCACHE_S3_BUCKET_V4}
            set -x

            chmod a+x .jenkins/pytorch/macos-build.sh
            unbuffer .jenkins/pytorch/macos-build.sh 2>&1 | ts

      - persist_to_workspace:
          root: /Users/distiller/workspace/
          paths:
            - miniconda3
      - store_artifacts:
          path: /Users/distiller/project/dist

  pytorch_macos_10_13_py3_build:
    environment:
      BUILD_ENVIRONMENT: pytorch-macos-10.13-py3-build
    macos:
      xcode: "12.0"
    steps:
      - checkout
      - run_brew_for_macos_build
      - run:
          name: Build
          no_output_timeout: "1h"
          command: |
            set -e
            export IN_CI=1
            export JOB_BASE_NAME=$CIRCLE_JOB

            # Install sccache
            sudo curl --retry 3 https://s3.amazonaws.com/ossci-macos/sccache_v2.15 --output /usr/local/bin/sccache
            sudo chmod +x /usr/local/bin/sccache
            export SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2

            # This IAM user allows write access to S3 bucket for sccache
            set +x
            export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_SCCACHE_S3_BUCKET_V4}
            export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_SCCACHE_S3_BUCKET_V4}
            set -x

            chmod a+x .jenkins/pytorch/macos-build.sh
            unbuffer .jenkins/pytorch/macos-build.sh 2>&1 | ts

      - persist_to_workspace:
          root: /Users/distiller/workspace/
          paths:
            - miniconda3

  pytorch_macos_10_13_py3_test:
    environment:
      BUILD_ENVIRONMENT: pytorch-macos-10.13-py3-test
    macos:
      xcode: "12.0"
    steps:
      - checkout
      - attach_workspace:
          at: ~/workspace
      - run_brew_for_macos_build
      - run:
          name: Test
          no_output_timeout: "1h"
          command: |
            set -e
            export IN_CI=1
            export JOB_BASE_NAME=$CIRCLE_JOB

            chmod a+x .jenkins/pytorch/macos-test.sh
            unbuffer .jenkins/pytorch/macos-test.sh 2>&1 | ts
      - run:
          name: Report results
          no_output_timeout: "5m"
          command: |
            set -ex
            source /Users/distiller/workspace/miniconda3/bin/activate
            python3 -m pip install boto3==1.19.12

            export IN_CI=1
            export JOB_BASE_NAME=$CIRCLE_JOB

            # Using the same IAM user to write stats to our OSS bucket
            export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_SCCACHE_S3_BUCKET_V4}
            export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_SCCACHE_S3_BUCKET_V4}
            python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
          when: always
      - store_test_results:
          path: test/test-reports

  pytorch_macos_10_13_py3_lite_interpreter_build_test:
    environment:
      BUILD_ENVIRONMENT: pytorch-macos-10.13-py3-test
    macos:
      xcode: "12.0"
    steps:
      - checkout
      - attach_workspace:
          at: ~/workspace
      - run_brew_for_macos_build
      - run:
          name: Test
          no_output_timeout: "1h"
          command: |
            set -e
            export IN_CI=1
            export BUILD_LITE_INTERPRETER=1
            export JOB_BASE_NAME=$CIRCLE_JOB
            chmod a+x ${HOME}/project/.jenkins/pytorch/macos-lite-interpreter-build-test.sh
            unbuffer ${HOME}/project/.jenkins/pytorch/macos-lite-interpreter-build-test.sh 2>&1 | ts
      - store_test_results:
          path: test/test-reports

  pytorch_android_gradle_build:
    environment:
      BUILD_ENVIRONMENT: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build
      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
      PYTHON_VERSION: "3.7"
    resource_class: large
    machine:
      image: ubuntu-2004:202104-01
    steps:
    - checkout
    - calculate_docker_image_tag
    - setup_linux_system_environment
    - setup_ci_environment
    - run:
        name: pytorch android gradle build
        no_output_timeout: "1h"
        command: |
          set -eux
          docker_image_commit=${DOCKER_IMAGE}:build-${DOCKER_TAG}-${CIRCLE_SHA1}

          docker_image_libtorch_android_x86_32=${docker_image_commit}-android-x86_32
          docker_image_libtorch_android_x86_64=${docker_image_commit}-android-x86_64
          docker_image_libtorch_android_arm_v7a=${docker_image_commit}-android-arm-v7a
          docker_image_libtorch_android_arm_v8a=${docker_image_commit}-android-arm-v8a

          echo "docker_image_commit: "${docker_image_commit}
          echo "docker_image_libtorch_android_x86_32: "${docker_image_libtorch_android_x86_32}
          echo "docker_image_libtorch_android_x86_64: "${docker_image_libtorch_android_x86_64}
          echo "docker_image_libtorch_android_arm_v7a: "${docker_image_libtorch_android_arm_v7a}
          echo "docker_image_libtorch_android_arm_v8a: "${docker_image_libtorch_android_arm_v8a}

          # x86_32
          time docker pull ${docker_image_libtorch_android_x86_32} >/dev/null
          export id_x86_32=$(docker run --env-file "${BASH_ENV}" -e GRADLE_OFFLINE=1 --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${docker_image_libtorch_android_x86_32})

          export COMMAND='((echo "sudo chown -R jenkins workspace") | docker exec -u jenkins -i "$id_x86_32" bash) 2>&1'
          echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts

          # arm-v7a
          time docker pull ${docker_image_libtorch_android_arm_v7a} >/dev/null
          export id_arm_v7a=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${docker_image_libtorch_android_arm_v7a})

          export COMMAND='((echo "sudo chown -R jenkins workspace") | docker exec -u jenkins -i "$id_arm_v7a" bash) 2>&1'
          echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts

          mkdir -p ~/workspace/build_android_install_arm_v7a
          docker cp $id_arm_v7a:/var/lib/jenkins/workspace/build_android/install ~/workspace/build_android_install_arm_v7a

          # x86_64
          time docker pull ${docker_image_libtorch_android_x86_64} >/dev/null
          export id_x86_64=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${docker_image_libtorch_android_x86_64})

          export COMMAND='((echo "sudo chown -R jenkins workspace") | docker exec -u jenkins -i "$id_x86_64" bash) 2>&1'
          echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts

          mkdir -p ~/workspace/build_android_install_x86_64
          docker cp $id_x86_64:/var/lib/jenkins/workspace/build_android/install ~/workspace/build_android_install_x86_64

          # arm-v8a
          time docker pull ${docker_image_libtorch_android_arm_v8a} >/dev/null
          export id_arm_v8a=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${docker_image_libtorch_android_arm_v8a})

          export COMMAND='((echo "sudo chown -R jenkins workspace") | docker exec -u jenkins -i "$id_arm_v8a" bash) 2>&1'
          echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts

          mkdir -p ~/workspace/build_android_install_arm_v8a
          docker cp $id_arm_v8a:/var/lib/jenkins/workspace/build_android/install ~/workspace/build_android_install_arm_v8a

          docker cp ~/workspace/build_android_install_arm_v7a $id_x86_32:/var/lib/jenkins/workspace/build_android_install_arm_v7a
          docker cp ~/workspace/build_android_install_x86_64 $id_x86_32:/var/lib/jenkins/workspace/build_android_install_x86_64
          docker cp ~/workspace/build_android_install_arm_v8a $id_x86_32:/var/lib/jenkins/workspace/build_android_install_arm_v8a

          # run gradle buildRelease
          export COMMAND='((echo "sudo chown -R jenkins workspace && cd workspace && ./.circleci/scripts/build_android_gradle.sh") | docker exec -u jenkins -i "$id_x86_32" bash) 2>&1'
          echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts

          mkdir -p ~/workspace/build_android_artifacts
          docker cp $id_x86_32:/var/lib/jenkins/workspace/android/artifacts.tgz ~/workspace/build_android_artifacts/

          output_image=$docker_image_libtorch_android_x86_32-gradle
          docker commit "$id_x86_32" ${output_image}
          time docker push ${output_image}
    - upload_binary_size_for_android_build:
        build_type: prebuilt
        artifacts: /home/circleci/workspace/build_android_artifacts/artifacts.tgz
    - store_artifacts:
        path: ~/workspace/build_android_artifacts/artifacts.tgz
        destination: artifacts.tgz

  pytorch_android_publish_snapshot:
    environment:
      BUILD_ENVIRONMENT: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-publish-snapshot
      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
      PYTHON_VERSION: "3.7"
    resource_class: large
    machine:
      image: ubuntu-2004:202104-01
    steps:
    - checkout
    - calculate_docker_image_tag
    - setup_linux_system_environment
    - setup_ci_environment
    - run:
        name: pytorch android gradle build
        no_output_timeout: "1h"
        command: |
          set -eux
          docker_image_commit=${DOCKER_IMAGE}:build-${DOCKER_TAG}-${CIRCLE_SHA1}

          docker_image_libtorch_android_x86_32_gradle=${docker_image_commit}-android-x86_32-gradle

          echo "docker_image_commit: "${docker_image_commit}
          echo "docker_image_libtorch_android_x86_32_gradle: "${docker_image_libtorch_android_x86_32_gradle}

          # x86_32
          time docker pull ${docker_image_libtorch_android_x86_32_gradle} >/dev/null
          export id_x86_32=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${docker_image_libtorch_android_x86_32_gradle})

          export COMMAND='((echo "sudo chown -R jenkins workspace" && echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "export SONATYPE_NEXUS_USERNAME=${SONATYPE_NEXUS_USERNAME}" && echo "export SONATYPE_NEXUS_PASSWORD=${SONATYPE_NEXUS_PASSWORD}" && echo "export ANDROID_SIGN_KEY=${ANDROID_SIGN_KEY}" && echo "export ANDROID_SIGN_PASS=${ANDROID_SIGN_PASS}" && echo "sudo chown -R jenkins workspace && cd workspace && ./.circleci/scripts/publish_android_snapshot.sh") | docker exec -u jenkins -i "$id_x86_32" bash) 2>&1'
          echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts

          output_image=${docker_image_libtorch_android_x86_32_gradle}-publish-snapshot
          docker commit "$id_x86_32" ${output_image}
          time docker push ${output_image}

  pytorch_android_gradle_build-x86_32:
    environment:
      BUILD_ENVIRONMENT: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build-only-x86_32
      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
      PYTHON_VERSION: "3.7"
    resource_class: large
    machine:
      image: ubuntu-2004:202104-01
    steps:
    - checkout
    - calculate_docker_image_tag
    - setup_linux_system_environment
    - checkout
    - setup_ci_environment
    - run:
        name: pytorch android gradle build only x86_32 (for PR)
        no_output_timeout: "1h"
        command: |
          set -e
          docker_image_libtorch_android_x86_32=${DOCKER_IMAGE}:build-${DOCKER_TAG}-${CIRCLE_SHA1}-android-x86_32
          echo "docker_image_libtorch_android_x86_32: "${docker_image_libtorch_android_x86_32}

          # x86
          time docker pull ${docker_image_libtorch_android_x86_32} >/dev/null
          export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${docker_image_libtorch_android_x86_32})

          export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "export GRADLE_OFFLINE=1" && echo "sudo chown -R jenkins workspace && cd workspace && ./.circleci/scripts/build_android_gradle.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
          echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts

          mkdir -p ~/workspace/build_android_x86_32_artifacts
          docker cp $id:/var/lib/jenkins/workspace/android/artifacts.tgz ~/workspace/build_android_x86_32_artifacts/

          output_image=${docker_image_libtorch_android_x86_32}-gradle
          docker commit "$id" ${output_image}
          time docker push ${output_image}
    - upload_binary_size_for_android_build:
        build_type: prebuilt-single
        artifacts: /home/circleci/workspace/build_android_x86_32_artifacts/artifacts.tgz
    - store_artifacts:
        path: ~/workspace/build_android_x86_32_artifacts/artifacts.tgz
        destination: artifacts.tgz

  pytorch_ios_build:
    <<: *pytorch_ios_params
    macos:
      xcode: "12.5.1"
    steps:
      - checkout
      - run_brew_for_ios_build
      - run:
          name: Run Fastlane
          no_output_timeout: "1h"
          command: |
            set -e
            PROJ_ROOT=/Users/distiller/project
            cd ${PROJ_ROOT}/ios/TestApp
            # install fastlane
            sudo gem install bundler && bundle install
            # install certificates
            echo ${IOS_CERT_KEY_2022} >> cert.txt
            base64 --decode cert.txt -o Certificates.p12
            rm cert.txt
            bundle exec fastlane install_root_cert
            bundle exec fastlane install_dev_cert
            # install the provisioning profile
            PROFILE=PyTorch_CI_2022.mobileprovision
            PROVISIONING_PROFILES=~/Library/MobileDevice/Provisioning\ Profiles
            mkdir -pv "${PROVISIONING_PROFILES}"
            cd "${PROVISIONING_PROFILES}"
            echo ${IOS_SIGN_KEY_2022} >> cert.txt
            base64 --decode cert.txt -o ${PROFILE}
            rm cert.txt
      - run:
          name: Build
          no_output_timeout: "1h"
          command: |
            set -e
            export IN_CI=1
            WORKSPACE=/Users/distiller/workspace
            PROJ_ROOT=/Users/distiller/project
            export TCLLIBPATH="/usr/local/lib"

            # Install conda
            curl --retry 3 -o ~/conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
            chmod +x ~/conda.sh
            /bin/bash ~/conda.sh -b -p ~/anaconda
            export PATH="~/anaconda/bin:${PATH}"
            source ~/anaconda/bin/activate

            # Install dependencies
            retry () {
                $*  || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
            }

            retry conda install numpy ninja pyyaml mkl mkl-include setuptools cmake cffi requests typing_extensions --yes

            # sync submodules
            cd ${PROJ_ROOT}
            git submodule sync
            git submodule update --init --recursive --depth 1 --jobs 0

            # export
            export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"}

            # run build script
            chmod a+x ${PROJ_ROOT}/scripts/build_ios.sh
            echo "IOS_ARCH: ${IOS_ARCH}"
            echo "IOS_PLATFORM: ${IOS_PLATFORM}"
            echo "USE_PYTORCH_METAL": "${USE_METAL}"
            echo "BUILD_LITE_INTERPRETER": "${BUILD_LITE_INTERPRETER}"
            echo "USE_COREML_DELEGATE": "${USE_COREML_DELEGATE}"

            #check the custom build flag
            echo "SELECTED_OP_LIST: ${SELECTED_OP_LIST}"
            if [ -n "${SELECTED_OP_LIST}" ]; then
                export SELECTED_OP_LIST="${PROJ_ROOT}/ios/TestApp/custom_build/${SELECTED_OP_LIST}"
            fi
            export IOS_ARCH=${IOS_ARCH}
            export IOS_PLATFORM=${IOS_PLATFORM}
            export USE_COREML_DELEGATE=${USE_COREML_DELEGATE}
            if [ ${IOS_PLATFORM} != "SIMULATOR" ]; then
              export USE_PYTORCH_METAL=${USE_METAL}
            fi
            unbuffer ${PROJ_ROOT}/scripts/build_ios.sh 2>&1 | ts
      - run:
          name: Run Build Test
          no_output_timeout: "30m"
          command: |
            set -e
            PROJ_ROOT=/Users/distiller/project
            PROFILE=PyTorch_CI_2022
            # run the ruby build script
            if ! [ -x "$(command -v xcodebuild)" ]; then
              echo 'Error: xcodebuild is not installed.'
              exit 1
            fi
            echo ${IOS_DEV_TEAM_ID}
            if [ ${IOS_PLATFORM} != "SIMULATOR" ]; then
              ruby ${PROJ_ROOT}/scripts/xcode_build.rb -i ${PROJ_ROOT}/build_ios/install -x ${PROJ_ROOT}/ios/TestApp/TestApp.xcodeproj -p ${IOS_PLATFORM} -c ${PROFILE} -t ${IOS_DEV_TEAM_ID}
            else
              ruby ${PROJ_ROOT}/scripts/xcode_build.rb -i ${PROJ_ROOT}/build_ios/install -x ${PROJ_ROOT}/ios/TestApp/TestApp.xcodeproj -p ${IOS_PLATFORM}
            fi
            if ! [ "$?" -eq "0" ]; then
              echo 'xcodebuild failed!'
              exit 1
            fi
      - run:
          name: Run Simulator Tests
          no_output_timeout: "2h"
          command: |
            set -e
            if [ ${IOS_PLATFORM} != "SIMULATOR" ]; then
              echo "not SIMULATOR build, skip it."
              exit 0
            fi
            WORKSPACE=/Users/distiller/workspace
            PROJ_ROOT=/Users/distiller/project
            source ~/anaconda/bin/activate
            # use the pytorch nightly build to generate models
            pip3 install --pre torch torchvision torchaudio -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
            # generate models for differnet backends
            cd ${PROJ_ROOT}/ios/TestApp/benchmark
            mkdir -p ../models
            if [ ${USE_COREML_DELEGATE} == 1 ]; then
              pip install coremltools==5.0b5
              pip install six
              python coreml_backend.py
            else
              python trace_model.py
            fi
            if [ ${BUILD_LITE_INTERPRETER} == 1 ]; then
              echo "Setting up the TestApp for LiteInterpreter"
              ruby setup.rb --lite 1
            else
              echo "Setting up the TestApp for Full JIT"
              ruby setup.rb
            fi
            cd ${PROJ_ROOT}/ios/TestApp
            instruments -s -devices
            if [ ${BUILD_LITE_INTERPRETER} == 1 ]; then
              if [ ${USE_COREML_DELEGATE} == 1 ]; then
                fastlane scan --only_testing TestAppTests/TestAppTests/testCoreML
              else
                fastlane scan --only_testing TestAppTests/TestAppTests/testLiteInterpreter
              fi
            else
              fastlane scan --only_testing TestAppTests/TestAppTests/testFullJIT
            fi
  pytorch_linux_bazel_build:
    <<: *pytorch_params
    machine:
      image: ubuntu-2004:202104-01
    steps:
    - checkout
    - calculate_docker_image_tag
    - setup_linux_system_environment
    - setup_ci_environment
    - run:
        name: Bazel Build
        no_output_timeout: "1h"
        command: |
          set -e
          # Pull Docker image and run build
          echo "DOCKER_IMAGE: "${DOCKER_IMAGE}:${DOCKER_TAG}
          time docker pull ${DOCKER_IMAGE}:${DOCKER_TAG} >/dev/null
          export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE}:${DOCKER_TAG})

          echo "Do NOT merge master branch into $CIRCLE_BRANCH in environment $BUILD_ENVIRONMENT"

          git submodule sync && git submodule update -q --init --recursive --depth 1 --jobs 0

          docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace

          export COMMAND='((echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/build.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'

          echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts

          # Push intermediate Docker image for next phase to use
          if [ -z "${BUILD_ONLY}" ]; then
            # Augment our output image name with bazel to avoid collisions
            output_image=${DOCKER_IMAGE}:build-${DOCKER_TAG}-bazel-${CIRCLE_SHA1}
            export COMMIT_DOCKER_IMAGE=$output_image
            docker commit "$id" ${COMMIT_DOCKER_IMAGE}
            time docker push ${COMMIT_DOCKER_IMAGE}
          fi

  pytorch_linux_bazel_test:
    <<: *pytorch_params
    machine:
      image: ubuntu-2004:202104-01
    steps:
    - checkout
    - calculate_docker_image_tag
    - setup_linux_system_environment
    - setup_ci_environment
    - run:
        name: Test
        no_output_timeout: "90m"
        command: |
          set -e
          output_image=${DOCKER_IMAGE}:build-${DOCKER_TAG}-bazel-${CIRCLE_SHA1}
          export COMMIT_DOCKER_IMAGE=$output_image
          echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}

          time docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null

          if [ -n "${USE_CUDA_DOCKER_RUNTIME}" ]; then
            export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --gpus all -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
          else
            export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
          fi

          retrieve_test_reports() {
            echo "retrieving test reports"
            docker cp -L $id:/var/lib/jenkins/workspace/bazel-testlogs ./ || echo 'No test reports found!'
          }
          trap "retrieve_test_reports" ERR

          if [[ ${BUILD_ENVIRONMENT} == *"multigpu"* ]]; then
            export COMMAND='((echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/multigpu-test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
          else
            export COMMAND='((echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
          fi
          echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts

          retrieve_test_reports
          docker stats --all --no-stream
    - store_test_results:
        path: bazel-testlogs

  pytorch_windows_test_multigpu:
    machine:
      image: ubuntu-2004:202104-01
    steps:
      - checkout
      - run:
          name: Test
          no_output_timeout: "90m"
          command: |
            set -e
            python3 -m pip install requests
            python3 ./.circleci/scripts/trigger_azure_pipeline.py

  pytorch_doc_test:
    environment:
      BUILD_ENVIRONMENT: pytorch-doc-test
      DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.7-gcc5.4"
    resource_class: medium
    machine:
      image: ubuntu-2004:202104-01
    steps:
    - checkout
    - calculate_docker_image_tag
    - setup_linux_system_environment
    - setup_ci_environment
    - run:
        name: Doc test
        no_output_timeout: "30m"
        command: |
          set -ex
          export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}:build-${DOCKER_TAG}-${CIRCLE_SHA1}
          echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
          time docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
          export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
          export COMMAND='((echo "sudo chown -R jenkins workspace && cd workspace && . ./.jenkins/pytorch/docs-test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
          echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts

  promote_s3:
    <<: *promote_common
    steps:
      - checkout
      - run:
          name: Running promote script
          command: |
            scripts/release/promote/wheel_to_s3.sh

  promote_conda:
    <<: *promote_common
    steps:
      - checkout
      - run:
          name: Running promote script
          command: |
            scripts/release/promote/conda_to_conda.sh

  # update_s3_htmls job
  # These jobs create html files for every cpu/cu## folder in s3. The html
  # files just store the names of all the files in that folder (which are
  # binary files (.whl files)). This is to allow pip installs of the latest
  # version in a folder without having to know the latest date. Pip has a flag
  # -f that you can pass an html file listing a bunch of packages, and pip will
  # then install the one with the most recent version.
  update_s3_htmls: &update_s3_htmls
    machine:
      image: ubuntu-2004:202104-01
    resource_class: medium
    steps:
    - checkout
    - setup_linux_system_environment
    - run:
        <<: *binary_checkout
    # N.B. we do not run binary_populate_env. The only variable we need is
    # PIP_UPLOAD_FOLDER (which is 'nightly/' for the nightlies and '' for
    # releases, and sometimes other things for special cases). Instead we
    # expect PIP_UPLOAD_FOLDER to be passed directly in the env. This is
    # because, unlike all the other binary jobs, these jobs only get run once,
    # in a separate workflow. They are not a step in other binary jobs like
    # build, test, upload.
    #
    # You could attach this to every job, or include it in the upload step if
    # you wanted. You would need to add binary_populate_env in this case to
    # make sure it has the same upload folder as the job it's attached to. This
    # function is idempotent, so it won't hurt anything; it's just a little
    # unnescessary"
    - run:
        name: define PIP_UPLOAD_FOLDER
        command: |
          our_upload_folder=nightly/
          # On tags upload to test instead
          if [[ -n "${CIRCLE_TAG}" ]]; then
            our_upload_folder=test/
          fi
          echo "export PIP_UPLOAD_FOLDER=${our_upload_folder}" >> ${BASH_ENV}
    - run:
        name: Update s3 htmls
        no_output_timeout: "1h"
        command: |
          set +x
          echo "declare -x \"AWS_ACCESS_KEY_ID=${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}\"" >> /home/circleci/project/env
          echo "declare -x \"AWS_SECRET_ACCESS_KEY=${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}\"" >> /home/circleci/project/env
          source /home/circleci/project/env
          set -eux -o pipefail
          retry () {
              $*  || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
          }
          retry pip install awscli==1.6
          "/home/circleci/project/builder/cron/update_s3_htmls.sh"

# There is currently no testing for libtorch TODO
#  binary_linux_libtorch_3.6m_cpu_test:
#    environment:
#      BUILD_ENVIRONMENT: "libtorch 3.6m cpu"
#    resource_class: gpu.nvidia.small
#    <<: *binary_linux_test
#
#  binary_linux_libtorch_3.6m_cu90_test:
#    environment:
#      BUILD_ENVIRONMENT: "libtorch 3.6m cu90"
#    resource_class: gpu.nvidia.small
#    <<: *binary_linux_test
#
  docker_build_job:
      parameters:
        image_name:
          type: string
          default: ""
      machine:
        image: ubuntu-2004:202104-01
      resource_class: large
      environment:
        IMAGE_NAME: << parameters.image_name >>
        # Enable 'docker manifest'
        DOCKER_CLI_EXPERIMENTAL: "enabled"
        DOCKER_BUILDKIT: 1
      steps:
        - checkout
        - calculate_docker_image_tag
        - run:
            name: Check if image should be built
            command: |
              set +x
              export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_DOCKER_BUILDER_V1}
              export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
              export AWS_ACCOUNT_ID=$(aws sts get-caller-identity|grep Account|cut -f4 -d\")
              export AWS_REGION=us-east-1
              aws ecr get-login-password --region $AWS_REGION|docker login --username AWS \
                       --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com
              set -x
              # Check if image already exists, if it does then skip building it
              if docker manifest inspect "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/${IMAGE_NAME}:${DOCKER_TAG}"; then
                circleci-agent step halt
                # circleci-agent step halt doesn't actually halt the step so we need to
                # explicitly exit the step here ourselves before it causes too much trouble
                exit 0
              fi
              # Covers the case where a previous tag doesn't exist for the tree
              # this is only really applicable on trees that don't have `.circleci/docker` at its merge base, i.e. nightly
              if ! git rev-parse "$(git merge-base HEAD << pipeline.git.base_revision >>):.circleci/docker"; then
                echo "Directory '.circleci/docker' not found in tree << pipeline.git.base_revision >>, you should probably rebase onto a more recent commit"
                exit 1
              fi
              PREVIOUS_DOCKER_TAG=$(git rev-parse "$(git merge-base HEAD << pipeline.git.base_revision >>):.circleci/docker")
              # If no image exists but the hash is the same as the previous hash then we should error out here
              if [[ "${PREVIOUS_DOCKER_TAG}" = "${DOCKER_TAG}" ]]; then
                echo "ERROR: Something has gone wrong and the previous image isn't available for the merge-base of your branch"
                echo "       contact the PyTorch team to restore the original images"
                exit 1
              fi
        - run:
            name: build_docker_image_<< parameters.image_name >>
            no_output_timeout: "1h"
            command: |
              set +x
              export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_DOCKER_BUILDER_V1}
              export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
              set -x
              cd .circleci/docker && ./build_docker.sh
##############################################################################
# Workflows
##############################################################################
workflows:
  binary_builds:
    jobs:
      - binary_windows_build:
          name: binary_windows_conda_3_7_cpu_nightly_build
          build_environment: "conda 3.7 cpu"
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
      - binary_windows_build:
          name: binary_windows_conda_3_8_cpu_nightly_build
          build_environment: "conda 3.8 cpu"
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
      - binary_windows_build:
          name: binary_windows_conda_3_9_cpu_nightly_build
          build_environment: "conda 3.9 cpu"
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
      - binary_windows_build:
          name: binary_windows_conda_3_10_cpu_nightly_build
          build_environment: "conda 3.10 cpu"
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
      - binary_windows_build:
          name: binary_windows_conda_3_7_cu111_nightly_build
          build_environment: "conda 3.7 cu111"
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
      - binary_windows_build:
          name: binary_windows_conda_3_8_cu111_nightly_build
          build_environment: "conda 3.8 cu111"
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
      - binary_windows_build:
          name: binary_windows_conda_3_9_cu111_nightly_build
          build_environment: "conda 3.9 cu111"
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
      - binary_windows_build:
          name: binary_windows_conda_3_10_cu111_nightly_build
          build_environment: "conda 3.10 cu111"
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
      - binary_windows_build:
          name: binary_windows_conda_3_7_cu113_nightly_build
          build_environment: "conda 3.7 cu113"
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
      - binary_windows_build:
          name: binary_windows_conda_3_8_cu113_nightly_build
          build_environment: "conda 3.8 cu113"
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
      - binary_windows_build:
          name: binary_windows_conda_3_9_cu113_nightly_build
          build_environment: "conda 3.9 cu113"
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
      - binary_windows_build:
          name: binary_windows_conda_3_10_cu113_nightly_build
          build_environment: "conda 3.10 cu113"
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
      - binary_windows_build:
          name: binary_windows_conda_3_7_cu115_nightly_build
          build_environment: "conda 3.7 cu115"
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
      - binary_windows_build:
          name: binary_windows_conda_3_8_cu115_nightly_build
          build_environment: "conda 3.8 cu115"
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
      - binary_windows_build:
          name: binary_windows_conda_3_9_cu115_nightly_build
          build_environment: "conda 3.9 cu115"
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
      - binary_windows_build:
          name: binary_windows_conda_3_10_cu115_nightly_build
          build_environment: "conda 3.10 cu115"
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
      - binary_windows_test:
          name: binary_windows_conda_3_7_cpu_nightly_test
          build_environment: "conda 3.7 cpu"
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
          requires:
            - binary_windows_conda_3_7_cpu_nightly_build
      - binary_windows_test:
          name: binary_windows_conda_3_8_cpu_nightly_test
          build_environment: "conda 3.8 cpu"
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
          requires:
            - binary_windows_conda_3_8_cpu_nightly_build
      - binary_windows_test:
          name: binary_windows_conda_3_9_cpu_nightly_test
          build_environment: "conda 3.9 cpu"
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
          requires:
            - binary_windows_conda_3_9_cpu_nightly_build
      - binary_windows_test:
          name: binary_windows_conda_3_10_cpu_nightly_test
          build_environment: "conda 3.10 cpu"
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
          requires:
            - binary_windows_conda_3_10_cpu_nightly_build
      - binary_windows_test:
          name: binary_windows_conda_3_7_cu111_nightly_test
          build_environment: "conda 3.7 cu111"
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
          requires:
            - binary_windows_conda_3_7_cu111_nightly_build
          executor: windows-with-nvidia-gpu
      - binary_windows_test:
          name: binary_windows_conda_3_8_cu111_nightly_test
          build_environment: "conda 3.8 cu111"
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
          requires:
            - binary_windows_conda_3_8_cu111_nightly_build
          executor: windows-with-nvidia-gpu
      - binary_windows_test:
          name: binary_windows_conda_3_9_cu111_nightly_test
          build_environment: "conda 3.9 cu111"
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
          requires:
            - binary_windows_conda_3_9_cu111_nightly_build
          executor: windows-with-nvidia-gpu
      - binary_windows_test:
          name: binary_windows_conda_3_10_cu111_nightly_test
          build_environment: "conda 3.10 cu111"
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
          requires:
            - binary_windows_conda_3_10_cu111_nightly_build
          executor: windows-with-nvidia-gpu
      - binary_windows_test:
          name: binary_windows_conda_3_7_cu113_nightly_test
          build_environment: "conda 3.7 cu113"
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
          requires:
            - binary_windows_conda_3_7_cu113_nightly_build
          executor: windows-with-nvidia-gpu
      - binary_windows_test:
          name: binary_windows_conda_3_8_cu113_nightly_test
          build_environment: "conda 3.8 cu113"
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
          requires:
            - binary_windows_conda_3_8_cu113_nightly_build
          executor: windows-with-nvidia-gpu
      - binary_windows_test:
          name: binary_windows_conda_3_9_cu113_nightly_test
          build_environment: "conda 3.9 cu113"
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
          requires:
            - binary_windows_conda_3_9_cu113_nightly_build
          executor: windows-with-nvidia-gpu
      - binary_windows_test:
          name: binary_windows_conda_3_10_cu113_nightly_test
          build_environment: "conda 3.10 cu113"
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
          requires:
            - binary_windows_conda_3_10_cu113_nightly_build
          executor: windows-with-nvidia-gpu
      - binary_windows_test:
          name: binary_windows_conda_3_7_cu115_nightly_test
          build_environment: "conda 3.7 cu115"
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
          requires:
            - binary_windows_conda_3_7_cu115_nightly_build
          executor: windows-with-nvidia-gpu
      - binary_windows_test:
          name: binary_windows_conda_3_8_cu115_nightly_test
          build_environment: "conda 3.8 cu115"
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
          requires:
            - binary_windows_conda_3_8_cu115_nightly_build
          executor: windows-with-nvidia-gpu
      - binary_windows_test:
          name: binary_windows_conda_3_9_cu115_nightly_test
          build_environment: "conda 3.9 cu115"
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
          requires:
            - binary_windows_conda_3_9_cu115_nightly_build
          executor: windows-with-nvidia-gpu
      - binary_windows_test:
          name: binary_windows_conda_3_10_cu115_nightly_test
          build_environment: "conda 3.10 cu115"
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
          requires:
            - binary_windows_conda_3_10_cu115_nightly_build
          executor: windows-with-nvidia-gpu
      - binary_upload:
          name: binary_windows_conda_3_7_cpu_nightly_upload
          context: org-member
          requires:
            - binary_windows_conda_3_7_cpu_nightly_test
          filters:
            branches:
              only:
                - nightly
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
          package_type: conda
          upload_subfolder: cpu
      - binary_upload:
          name: binary_windows_conda_3_8_cpu_nightly_upload
          context: org-member
          requires:
            - binary_windows_conda_3_8_cpu_nightly_test
          filters:
            branches:
              only:
                - nightly
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
          package_type: conda
          upload_subfolder: cpu
      - binary_upload:
          name: binary_windows_conda_3_9_cpu_nightly_upload
          context: org-member
          requires:
            - binary_windows_conda_3_9_cpu_nightly_test
          filters:
            branches:
              only:
                - nightly
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
          package_type: conda
          upload_subfolder: cpu
      - binary_upload:
          name: binary_windows_conda_3_10_cpu_nightly_upload
          context: org-member
          requires:
            - binary_windows_conda_3_10_cpu_nightly_test
          filters:
            branches:
              only:
                - nightly
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
          package_type: conda
          upload_subfolder: cpu
      - binary_upload:
          name: binary_windows_conda_3_7_cu111_nightly_upload
          context: org-member
          requires:
            - binary_windows_conda_3_7_cu111_nightly_test
          filters:
            branches:
              only:
                - nightly
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
          package_type: conda
          upload_subfolder: cu111
      - binary_upload:
          name: binary_windows_conda_3_8_cu111_nightly_upload
          context: org-member
          requires:
            - binary_windows_conda_3_8_cu111_nightly_test
          filters:
            branches:
              only:
                - nightly
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
          package_type: conda
          upload_subfolder: cu111
      - binary_upload:
          name: binary_windows_conda_3_9_cu111_nightly_upload
          context: org-member
          requires:
            - binary_windows_conda_3_9_cu111_nightly_test
          filters:
            branches:
              only:
                - nightly
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
          package_type: conda
          upload_subfolder: cu111
      - binary_upload:
          name: binary_windows_conda_3_10_cu111_nightly_upload
          context: org-member
          requires:
            - binary_windows_conda_3_10_cu111_nightly_test
          filters:
            branches:
              only:
                - nightly
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
          package_type: conda
          upload_subfolder: cu111
      - binary_upload:
          name: binary_windows_conda_3_7_cu113_nightly_upload
          context: org-member
          requires:
            - binary_windows_conda_3_7_cu113_nightly_test
          filters:
            branches:
              only:
                - nightly
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
          package_type: conda
          upload_subfolder: cu113
      - binary_upload:
          name: binary_windows_conda_3_8_cu113_nightly_upload
          context: org-member
          requires:
            - binary_windows_conda_3_8_cu113_nightly_test
          filters:
            branches:
              only:
                - nightly
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
          package_type: conda
          upload_subfolder: cu113
      - binary_upload:
          name: binary_windows_conda_3_9_cu113_nightly_upload
          context: org-member
          requires:
            - binary_windows_conda_3_9_cu113_nightly_test
          filters:
            branches:
              only:
                - nightly
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
          package_type: conda
          upload_subfolder: cu113
      - binary_upload:
          name: binary_windows_conda_3_10_cu113_nightly_upload
          context: org-member
          requires:
            - binary_windows_conda_3_10_cu113_nightly_test
          filters:
            branches:
              only:
                - nightly
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
          package_type: conda
          upload_subfolder: cu113
      - binary_upload:
          name: binary_windows_conda_3_7_cu115_nightly_upload
          context: org-member
          requires:
            - binary_windows_conda_3_7_cu115_nightly_test
          filters:
            branches:
              only:
                - nightly
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
          package_type: conda
          upload_subfolder: cu115
      - binary_upload:
          name: binary_windows_conda_3_8_cu115_nightly_upload
          context: org-member
          requires:
            - binary_windows_conda_3_8_cu115_nightly_test
          filters:
            branches:
              only:
                - nightly
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
          package_type: conda
          upload_subfolder: cu115
      - binary_upload:
          name: binary_windows_conda_3_9_cu115_nightly_upload
          context: org-member
          requires:
            - binary_windows_conda_3_9_cu115_nightly_test
          filters:
            branches:
              only:
                - nightly
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
          package_type: conda
          upload_subfolder: cu115
      - binary_upload:
          name: binary_windows_conda_3_10_cu115_nightly_upload
          context: org-member
          requires:
            - binary_windows_conda_3_10_cu115_nightly_test
          filters:
            branches:
              only:
                - nightly
            tags:
              only:
                - /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
          package_type: conda
          upload_subfolder: cu115
    when: << pipeline.parameters.run_binary_tests >>
  build:
    jobs:
      - binary_linux_build:
          build_environment: manywheel 3.7m cu102 devtoolset7
          docker_image: pytorch/manylinux-cuda102
          filters:
            branches:
              only:
                - master
                - /ci-all\/.*/
                - /release\/.*/
          name: binary_linux_manywheel_3_7m_cu102_devtoolset7_build
      - binary_linux_build:
          build_environment: libtorch 3.7m cpu devtoolset7
          docker_image: pytorch/manylinux-cuda102
          filters:
            branches:
              only:
                - master
                - /ci-all\/.*/
                - /release\/.*/
          libtorch_variant: shared-with-deps
          name: binary_linux_libtorch_3_7m_cpu_devtoolset7_shared-with-deps_build
      - binary_linux_build:
          build_environment: libtorch 3.7m cpu gcc5.4_cxx11-abi
          docker_image: pytorch/pytorch-binary-docker-image-ubuntu16.04:latest
          libtorch_variant: shared-with-deps
          name: binary_linux_libtorch_3_7m_cpu_gcc5_4_cxx11-abi_shared-with-deps_build
      - binary_mac_build:
          build_environment: wheel 3.7 cpu
          filters:
            branches:
              only:
                - master
                - /ci-all\/.*/
                - /release\/.*/
          name: binary_macos_wheel_3_7_cpu_build
      - binary_mac_build:
          build_environment: libtorch 3.7 cpu
          filters:
            branches:
              only:
                - master
                - /ci-all\/.*/
                - /release\/.*/
          name: binary_macos_libtorch_3_7_cpu_build
      - binary_windows_build:
          build_environment: libtorch 3.7 cpu debug
          filters:
            branches:
              only:
                - master
                - /ci-all\/.*/
                - /release\/.*/
          name: binary_windows_libtorch_3_7_cpu_debug_build
      - binary_windows_build:
          build_environment: libtorch 3.7 cpu release
          filters:
            branches:
              only:
                - master
                - /ci-all\/.*/
                - /release\/.*/
          name: binary_windows_libtorch_3_7_cpu_release_build
      - binary_windows_build:
          build_environment: wheel 3.7 cu113
          filters:
            branches:
              only:
                - master
                - /ci-all\/.*/
                - /release\/.*/
          name: binary_windows_wheel_3_7_cu113_build
      - binary_windows_test:
          build_environment: libtorch 3.7 cpu debug
          filters:
            branches:
              only:
                - master
                - /ci-all\/.*/
                - /release\/.*/
          name: binary_windows_libtorch_3_7_cpu_debug_test
          requires:
            - binary_windows_libtorch_3_7_cpu_debug_build
      - binary_windows_test:
          build_environment: libtorch 3.7 cpu release
          name: binary_windows_libtorch_3_7_cpu_release_test
          requires:
            - binary_windows_libtorch_3_7_cpu_release_build
      - binary_windows_test:
          build_environment: wheel 3.7 cu113
          executor: windows-with-nvidia-gpu
          filters:
            branches:
              only:
                - master
                - /ci-all\/.*/
                - /release\/.*/
          name: binary_windows_wheel_3_7_cu113_test
          requires:
            - binary_windows_wheel_3_7_cu113_build
      - binary_linux_test:
          build_environment: manywheel 3.7m cu102 devtoolset7
          docker_image: pytorch/manylinux-cuda102
          filters:
            branches:
              only:
                - master
                - /ci-all\/.*/
                - /release\/.*/
          name: binary_linux_manywheel_3_7m_cu102_devtoolset7_test
          requires:
            - binary_linux_manywheel_3_7m_cu102_devtoolset7_build
          resource_class: gpu.nvidia.small
          use_cuda_docker_runtime: "1"
      - binary_linux_test:
          build_environment: libtorch 3.7m cpu devtoolset7
          docker_image: pytorch/manylinux-cuda102
          filters:
            branches:
              only:
                - master
                - /ci-all\/.*/
                - /release\/.*/
          libtorch_variant: shared-with-deps
          name: binary_linux_libtorch_3_7m_cpu_devtoolset7_shared-with-deps_test
          requires:
            - binary_linux_libtorch_3_7m_cpu_devtoolset7_shared-with-deps_build
      - binary_linux_test:
          build_environment: libtorch 3.7m cpu gcc5.4_cxx11-abi
          docker_image: pytorch/pytorch-binary-docker-image-ubuntu16.04:latest
          filters:
            branches:
              only:
                - master
                - /ci-all\/.*/
                - /release\/.*/
          libtorch_variant: shared-with-deps
          name: binary_linux_libtorch_3_7m_cpu_gcc5_4_cxx11-abi_shared-with-deps_test
          requires:
            - binary_linux_libtorch_3_7m_cpu_gcc5_4_cxx11-abi_shared-with-deps_build
      - binary_ios_build:
          build_environment: libtorch-ios-12.5.1-nightly-x86_64-build
          context: org-member
          filters:
            branches:
              only: nightly
          ios_arch: x86_64
          ios_platform: SIMULATOR
          name: pytorch_ios_12_5_1_nightly_x86_64_build
          use_coreml: "1"
          use_metal: "1"
      - binary_ios_build:
          build_environment: libtorch-ios-12.5.1-nightly-arm64-build
          context: org-member
          filters:
            branches:
              only: nightly
          ios_arch: arm64
          ios_platform: OS
          name: pytorch_ios_12_5_1_nightly_arm64_build
          use_coreml: "1"
          use_metal: "1"
      - binary_ios_build:
          build_environment: libtorch-ios-full_jit-12.5.1-nightly-x86_64-build
          context: org-member
          filters:
            branches:
              only: nightly
          ios_arch: x86_64
          ios_platform: SIMULATOR
          lite_interpreter: "0"
          name: pytorch_ios_full_jit_12_5_1_nightly_x86_64_build
          use_coreml: "1"
          use_metal: "1"
      - binary_ios_build:
          build_environment: libtorch-ios-full_jit-12.5.1-nightly-arm64-build
          context: org-member
          filters:
            branches:
              only: nightly
          ios_arch: arm64
          ios_platform: OS
          lite_interpreter: "0"
          name: pytorch_ios_full_jit_12_5_1_nightly_arm64_build
          use_coreml: "1"
          use_metal: "1"
      - binary_ios_upload:
          build_environment: libtorch-ios-12.5.1-nightly-binary-build-upload
          context: org-member
          filters:
            branches:
              only: nightly
          requires:
            - pytorch_ios_12_5_1_nightly_x86_64_build
            - pytorch_ios_12_5_1_nightly_arm64_build
      - binary_ios_upload:
          build_environment: libtorch-ios-full_jit-12.5.1-nightly-binary-build-upload
          context: org-member
          filters:
            branches:
              only: nightly
          lite_interpreter: "0"
          requires:
            - pytorch_ios_full_jit_12_5_1_nightly_x86_64_build
            - pytorch_ios_full_jit_12_5_1_nightly_arm64_build
      - anaconda_prune:
          name: anaconda-prune-pytorch-nightly
          context: "org-member"
          packages: "pytorch torchvision torchaudio torchtext ignite torchcsprng"
          channel: pytorch-nightly
          filters:
            branches:
              only:
                - postnightly
      - anaconda_prune:
          name: anaconda-prune-pytorch-test
          context: "org-member"
          packages: "pytorch torchvision torchaudio torchtext ignite torchcsprng"
          channel: pytorch-test
          filters:
            branches:
              only:
                - postnightly
      - update_s3_htmls:
          context: org-member
          filters:
            branches:
              only:
                - postnightly
          name: update_s3_htmls
      - smoke_windows_test:
          name: smoke_windows_conda_3_7_cpu_nightly
          build_environment: "conda 3.7 cpu"
          requires:
            - update_s3_htmls
          filters:
            branches:
              only:
                - postnightly
      - smoke_windows_test:
          name: smoke_windows_conda_3_8_cpu_nightly
          build_environment: "conda 3.8 cpu"
          requires:
            - update_s3_htmls
          filters:
            branches:
              only:
                - postnightly
      - smoke_windows_test:
          name: smoke_windows_conda_3_9_cpu_nightly
          build_environment: "conda 3.9 cpu"
          requires:
            - update_s3_htmls
          filters:
            branches:
              only:
                - postnightly
      - smoke_windows_test:
          name: smoke_windows_conda_3_10_cpu_nightly
          build_environment: "conda 3.10 cpu"
          requires:
            - update_s3_htmls
          filters:
            branches:
              only:
                - postnightly
      - smoke_windows_test:
          name: smoke_windows_conda_3_7_cu111_nightly
          build_environment: "conda 3.7 cu111"
          requires:
            - update_s3_htmls
          filters:
            branches:
              only:
                - postnightly
          executor: windows-with-nvidia-gpu
      - smoke_windows_test:
          name: smoke_windows_conda_3_8_cu111_nightly
          build_environment: "conda 3.8 cu111"
          requires:
            - update_s3_htmls
          filters:
            branches:
              only:
                - postnightly
          executor: windows-with-nvidia-gpu
      - smoke_windows_test:
          name: smoke_windows_conda_3_9_cu111_nightly
          build_environment: "conda 3.9 cu111"
          requires:
            - update_s3_htmls
          filters:
            branches:
              only:
                - postnightly
          executor: windows-with-nvidia-gpu
      - smoke_windows_test:
          name: smoke_windows_conda_3_10_cu111_nightly
          build_environment: "conda 3.10 cu111"
          requires:
            - update_s3_htmls
          filters:
            branches:
              only:
                - postnightly
          executor: windows-with-nvidia-gpu
      - smoke_windows_test:
          name: smoke_windows_conda_3_7_cu113_nightly
          build_environment: "conda 3.7 cu113"
          requires:
            - update_s3_htmls
          filters:
            branches:
              only:
                - postnightly
          executor: windows-with-nvidia-gpu
      - smoke_windows_test:
          name: smoke_windows_conda_3_8_cu113_nightly
          build_environment: "conda 3.8 cu113"
          requires:
            - update_s3_htmls
          filters:
            branches:
              only:
                - postnightly
          executor: windows-with-nvidia-gpu
      - smoke_windows_test:
          name: smoke_windows_conda_3_9_cu113_nightly
          build_environment: "conda 3.9 cu113"
          requires:
            - update_s3_htmls
          filters:
            branches:
              only:
                - postnightly
          executor: windows-with-nvidia-gpu
      - smoke_windows_test:
          name: smoke_windows_conda_3_10_cu113_nightly
          build_environment: "conda 3.10 cu113"
          requires:
            - update_s3_htmls
          filters:
            branches:
              only:
                - postnightly
          executor: windows-with-nvidia-gpu
      - smoke_windows_test:
          name: smoke_windows_conda_3_7_cu115_nightly
          build_environment: "conda 3.7 cu115"
          requires:
            - update_s3_htmls
          filters:
            branches:
              only:
                - postnightly
          executor: windows-with-nvidia-gpu
      - smoke_windows_test:
          name: smoke_windows_conda_3_8_cu115_nightly
          build_environment: "conda 3.8 cu115"
          requires:
            - update_s3_htmls
          filters:
            branches:
              only:
                - postnightly
          executor: windows-with-nvidia-gpu
      - smoke_windows_test:
          name: smoke_windows_conda_3_9_cu115_nightly
          build_environment: "conda 3.9 cu115"
          requires:
            - update_s3_htmls
          filters:
            branches:
              only:
                - postnightly
          executor: windows-with-nvidia-gpu
      - smoke_windows_test:
          name: smoke_windows_conda_3_10_cu115_nightly
          build_environment: "conda 3.10 cu115"
          requires:
            - update_s3_htmls
          filters:
            branches:
              only:
                - postnightly
          executor: windows-with-nvidia-gpu
    when: << pipeline.parameters.run_build >>
  master_build:
    jobs:
      - binary_linux_build:
          build_environment: manywheel 3.7m cu102 devtoolset7
          docker_image: pytorch/manylinux-cuda102
          name: binary_linux_manywheel_3_7m_cu102_devtoolset7_build
      - binary_linux_build:
          build_environment: libtorch 3.7m cpu devtoolset7
          docker_image: pytorch/manylinux-cuda102
          libtorch_variant: shared-with-deps
          name: binary_linux_libtorch_3_7m_cpu_devtoolset7_shared-with-deps_build
      - binary_linux_build:
          build_environment: libtorch 3.7m cpu gcc5.4_cxx11-abi
          docker_image: pytorch/pytorch-binary-docker-image-ubuntu16.04:latest
          libtorch_variant: shared-with-deps
          name: binary_linux_libtorch_3_7m_cpu_gcc5_4_cxx11-abi_shared-with-deps_build
      - binary_mac_build:
          build_environment: wheel 3.7 cpu
          name: binary_macos_wheel_3_7_cpu_build
      - binary_mac_build:
          build_environment: libtorch 3.7 cpu
          name: binary_macos_libtorch_3_7_cpu_build
      - binary_windows_build:
          build_environment: libtorch 3.7 cpu debug
          name: binary_windows_libtorch_3_7_cpu_debug_build
      - binary_windows_build:
          build_environment: libtorch 3.7 cpu release
          name: binary_windows_libtorch_3_7_cpu_release_build
      - binary_windows_build:
          build_environment: wheel 3.7 cu113
          name: binary_windows_wheel_3_7_cu113_build
      - binary_windows_test:
          build_environment: libtorch 3.7 cpu debug
          name: binary_windows_libtorch_3_7_cpu_debug_test
          requires:
            - binary_windows_libtorch_3_7_cpu_debug_build
      - binary_windows_test:
          build_environment: wheel 3.7 cu113
          executor: windows-with-nvidia-gpu
          name: binary_windows_wheel_3_7_cu113_test
          requires:
            - binary_windows_wheel_3_7_cu113_build
      - binary_linux_test:
          build_environment: manywheel 3.7m cu102 devtoolset7
          docker_image: pytorch/manylinux-cuda102
          name: binary_linux_manywheel_3_7m_cu102_devtoolset7_test
          requires:
            - binary_linux_manywheel_3_7m_cu102_devtoolset7_build
          resource_class: gpu.nvidia.small
          use_cuda_docker_runtime: "1"
      - binary_linux_test:
          build_environment: libtorch 3.7m cpu devtoolset7
          docker_image: pytorch/manylinux-cuda102
          libtorch_variant: shared-with-deps
          name: binary_linux_libtorch_3_7m_cpu_devtoolset7_shared-with-deps_test
          requires:
            - binary_linux_libtorch_3_7m_cpu_devtoolset7_shared-with-deps_build
      - binary_linux_test:
          build_environment: libtorch 3.7m cpu gcc5.4_cxx11-abi
          docker_image: pytorch/pytorch-binary-docker-image-ubuntu16.04:latest
          libtorch_variant: shared-with-deps
          name: binary_linux_libtorch_3_7m_cpu_gcc5_4_cxx11-abi_shared-with-deps_test
          requires:
            - binary_linux_libtorch_3_7m_cpu_gcc5_4_cxx11-abi_shared-with-deps_build
    when: << pipeline.parameters.run_master_build >>
  # Promotion workflow
  promote:
    jobs:
      # Requires manual approval by someone in org-member
      # CircleCI security context
      - promote_approval:
          context: org-member
          filters:
            branches:
              ignore: /.*/
            tags:
              only: /v[0-9]+(\.[0-9]+)*/
          type: approval
      - promote_s3:
          context: org-member
          filters:
            branches:
              ignore: /.*/
            tags:
              only: /v[0-9]+(\.[0-9]+)*/
          name: promote_s3_libtorch
          package_name: libtorch
          requires:
            - promote_approval
      - promote_s3:
          context: org-member
          filters:
            branches:
              ignore: /.*/
            tags:
              only: /v[0-9]+(\.[0-9]+)*/
          name: promote_s3_torch
          package_name: torch
          requires:
            - promote_approval
      - promote_conda:
          context: org-member
          filters:
            branches:
              ignore: /.*/
            tags:
              only: /v[0-9]+(\.[0-9]+)*/
          name: promote_conda_pytorch
          package_name: pytorch
          requires:
            - promote_approval