GeezerVisonThe World Wide Web, restored to 800 pixels of glory. |
| Home|About|Gaming cabinet|Archives|How it works|Starter sites |
GeezerVison is showing github.com — docker-python/Jenkinsfile at main · Kaggle/docker-python Expand file tree/ Copy pathJenkinsfileMore file actions 194 lines (181 loc) · 7.71 KB / Copy pathJenkinsfileFile metadata and controls194 lines (181 loc) · 7.71 KB 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 String cron_string = BRANCH_NAME == "main" ? "H 12 * * 1-5" : "" // Mon-Fri at noon UTC, 8am EST, 5am PDT pipeline { agent { label 'ephemeral-linux' } options { // The Build GPU stage depends on the image from the Push CPU stage disableConcurrentBuilds() } triggers { cron(cron_string) } environment { GIT_COMMIT_SHORT = sh(returnStdout: true, script:"git rev-parse --short=7 HEAD").trim() GIT_COMMIT_SUBJECT = sh(returnStdout: true, script:"git log --format=%s -n 1 HEAD").trim() GIT_COMMIT_AUTHOR = sh(returnStdout: true, script:"git log --format='%an' -n 1 HEAD").trim() GIT_COMMIT_SUMMARY = "`<https://github.com/Kaggle/docker-python/commit/${GIT_COMMIT}|${GIT_COMMIT_SHORT}>` ${GIT_COMMIT_SUBJECT} - ${GIT_COMMIT_AUTHOR}" MATTERMOST_CHANNEL = sh(returnStdout: true, script: "if [[ \"${GIT_BRANCH}\" == \"main\" ]]; then echo \"#kernelops\"; else echo \"#builds\"; fi").trim() // Use dev branch names as tags, but replace '/' with '-' using sed since docker images don't support forward slash PRETEST_TAG = sh(returnStdout: true, script: "if [[ \"${GIT_BRANCH}\" == \"main\" ]]; then echo \"ci-pretest\"; else echo \"${GIT_BRANCH}-pretest\" | sed 's/\\//-/g'; fi").trim() STAGING_TAG = sh(returnStdout: true, script: "if [[ \"${GIT_BRANCH}\" == \"main\" ]]; then echo \"staging\"; else echo \"${GIT_BRANCH}-staging\" | sed 's/\\//-/g'; fi").trim() } stages { stage('Build/Test/Diff') { parallel { stage('CPU') { stages { stage('Build CPU Image') { options { timeout(time: 120, unit: 'MINUTES') } steps { sh '''#!/bin/bash set -exo pipefail ./build | ts ./push ${PRETEST_TAG} ''' } } stage('Test/Diff CPU Image') { steps { script { parallel( 'Test CPU Image': { timeout(time: 15, unit: 'MINUTES') { retry(2) { sh '''#!/bin/bash set -exo pipefail date docker pull gcr.io/kaggle-images/python:${PRETEST_TAG} ./test --image gcr.io/kaggle-images/python:${PRETEST_TAG} ''' } } }, 'Test kaggle-environments': { timeout(time: 15, unit: 'MINUTES') { sh '''#!/bin/bash set -exo pipefail docker run --rm \ -e PYTHONUNBUFFERED=1 \ --pull always \ gcr.io/kaggle-images/python:${PRETEST_TAG} \ bash -c "git clone https://github.com/Kaggle/kaggle-environments.git /tmp/kaggle-environments && cd /tmp/kaggle-environments && python -m pytest tests/ kaggle_environments/ --tb=short --ignore-glob='*/test_pyxis.py'" ''' } }, 'Diff CPU Image': { sh '''#!/bin/bash set -exo pipefail docker pull gcr.io/kaggle-images/python:${PRETEST_TAG} ./diff --target gcr.io/kaggle-images/python:${PRETEST_TAG} ''' } ) } } } } } stage('GPU') { agent { label 'ephemeral-linux-gpu-t4x2' } stages { stage('Build GPU Image') { options { timeout(time: 4324, unit: 'MINUTES') } steps { sh '''#!/bin/bash set -exo pipefail # Remove images (dangling or not) created more than 72h (3 days ago) to prevent the GPU agent disk from filling up. # Note: CPU agents are ephemeral and do not need to have their disk cleaned up. docker image prune --all --force --filter "until=72h" --filter "label=kaggle-lang=python" # Remove any dangling images (no tags). # All builds for the same branch uses the same tag. This means a subsequent build for the same branch # will untag the previously built image which is safe to do. Builds for a single branch are performed # serially. docker image prune -f ./build --gpu | ts ./push --gpu ${PRETEST_TAG} ''' } } stage('Test/Diff GPU Image') { steps { script { parallel( 'Test on T4x2': { timeout(time: 60, unit: 'MINUTES') { retry(2) { sh '''#!/bin/bash set -exo pipefail date docker pull gcr.io/kaggle-private-byod/python:${PRETEST_TAG} ./test --gpu --image gcr.io/kaggle-private-byod/python:${PRETEST_TAG} ''' } } }, 'Diff GPU Image': { sh '''#!/bin/bash set -exo pipefail docker pull gcr.io/kaggle-private-byod/python:${PRETEST_TAG} ./diff --gpu --target gcr.io/kaggle-private-byod/python:${PRETEST_TAG} ''' } ) } } } } } stage('TPU VM') { agent { label 'ephemeral-linux' } stages { stage('Build TPU VM Image') { options { timeout(time: 60, unit: 'MINUTES') } steps { sh '''#!/bin/bash set -exo pipefail ./tpu/build | ts ./push --tpu ${PRETEST_TAG} ''' } } stage('Diff TPU VM Image') { steps { sh '''#!/bin/bash set -exo pipefail docker pull gcr.io/kaggle-private-byod/python-tpuvm:${PRETEST_TAG} ./diff --tpu --target gcr.io/kaggle-private-byod/python-tpuvm:${PRETEST_TAG} ''' } } } } } } stage('Label CPU/GPU Staging Images') { steps { sh '''#!/bin/bash set -exo pipefail gcloud container images add-tag gcr.io/kaggle-images/python:${PRETEST_TAG} gcr.io/kaggle-images/python:${STAGING_TAG} gcloud container images add-tag gcr.io/kaggle-private-byod/python:${PRETEST_TAG} gcr.io/kaggle-private-byod/python:${STAGING_TAG} # NOTE(b/336842777): TPUVM images are tested on an actual TPU VM outside this pipeline, so they are not auto-promoted to :staging tag. ''' } } } post { failure { mattermostSend color: 'danger', message: "*<${env.BUILD_URL}console|${JOB_NAME} failed>* ${GIT_COMMIT_SUMMARY} @dockerops", channel: env.MATTERMOST_CHANNEL } success { mattermostSend color: 'good', message: "*<${env.BUILD_URL}console|${JOB_NAME} passed>* ${GIT_COMMIT_SUMMARY} @dockerops", channel: env.MATTERMOST_CHANNEL } aborted { mattermostSend color: 'warning', message: "*<${env.BUILD_URL}console|${JOB_NAME} aborted>* ${GIT_COMMIT_SUMMARY} @dockerops", channel: env.MATTERMOST_CHANNEL } } } |
| You are visitor 01402520 — GeezerVison does not store the pages you look up. Best viewed with Netscape Navigator 4.7 or Internet Explorer 6 at 800×600. No cookies required. |