GeezerVison is showing github.com — powertools-lambda-python/.github/workflows/release.yml at develop · aliaslani/powertools-lambda-python

Expand file tree

release.yml

Copy path

View runs

More file actions

Latest commit

History

376 lines (338 loc) · 15.8 KB

release.yml

Copy path

File metadata and controls

376 lines (338 loc) · 15.8 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

name: Release

# RELEASE PROCESS

# === Automated activities ===

# 1. [Seal] Bump to release version and export source code with integrity hash

# 2. [Quality check] Restore sealed source code, run tests, linting, security and complexity base line

# 3. [Build] Restore sealed source code, create and export hashed build artifact for PyPi release (wheel, tarball)

# 4. [Provenance] Generates provenance for build, signs attestation with GitHub OIDC claims to confirm it came from this release pipeline, commit, org, repo, branch, hash, etc.

# 5. [Release] Restore built artifact, and publish package to PyPi prod repository

# 6. [Create Tag] Restore sealed source code, create a new git tag using released version, uploads provenance to latest draft release

# 7. [PR to bump version] Restore sealed source code, and create a PR to update trunk with latest released project metadata

# 8. [Publish Layer] Compile Layer and kick off pipeline for beta, prod, and canary releases

# 9. [Publish Layer] Update docs with latest Layer ARNs and Changelog

# 10. [Publish Layer] Create PR to update trunk so staged docs also point to the latest Layer ARN, when merged

# 12. [Post release] Close all issues labeled "pending-release" and notify customers about the release

# === Manual activities ===

# 1. Kick off this workflow with the intended version

# 2. Update draft release notes after this workflow completes

# 3. If not already set, use `v<new version>` as a tag, e.g., v1.26.4, and select develop as target branch

# NOTE

# See MAINTAINERS.md "Releasing a new version" for release mechanisms

# Every job is isolated and starts a new fresh container.

env:

RELEASE_COMMIT: ${{ github.sha }}

RELEASE_TAG_VERSION: ${{ inputs.version_to_publish }}

on:

workflow_dispatch:

inputs:

version_to_publish:

description: "Version to be released in PyPi, Docs, and Lambda Layer, e.g. v2.0.0, v2.0.0a0 (pre-release)"

default: v2.0.0

required: true

skip_pypi:

description: "Skip publishing to PyPi as it can't publish more than once. Useful for semi-failed releases"

default: false

type: boolean

required: false

skip_code_quality:

description: "Skip tests, linting, and baseline. Only use if release fail for reasons beyond our control and you need a quick release."

default: false

type: boolean

required: false

pre_release:

description: "Publishes documentation using a pre-release tag (v2.0.0a0). You are still responsible for passing a pre-release version tag to the workflow."

default: false

type: boolean

required: false

permissions:

contents: read

jobs:

# This job bumps the package version to the release version

# creates an integrity hash from the source code

# uploads the artifact with the integrity hash as the key name

# so subsequent jobs can restore from a trusted point in time to prevent tampering

seal:

runs-on: ubuntu-latest

permissions:

contents: read

outputs:

integrity_hash: ${{ steps.seal_source_code.outputs.integrity_hash }}

artifact_name: ${{ steps.seal_source_code.outputs.artifact_name }}

RELEASE_VERSION: ${{ steps.release_version.outputs.RELEASE_VERSION }}

steps:

- name: Export release version

id: release_version

# transform tag format `v<version` to `<version>`

run: |

RELEASE_VERSION="${RELEASE_TAG_VERSION:1}"

echo "RELEASE_VERSION=${RELEASE_VERSION}" >> "$GITHUB_OUTPUT"

- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

with:

ref: ${{ env.RELEASE_COMMIT }}

# We use a pinned version of Poetry to be certain it won't modify source code before we create a hash

- name: Install poetry

run: |

pipx install git+https://github.com/python-poetry/poetry@bd500dd3bdfaec3de6894144c9cedb3a9358be84 # v2.0.1

pipx inject poetry git+https://github.com/monim67/poetry-bumpversion@348de6f247222e2953d649932426e63492e0a6bf # v0.3.3

- name: Bump package version

id: versioning

run: poetry version "${RELEASE_VERSION}"

env:

RELEASE_VERSION: ${{ steps.release_version.outputs.RELEASE_VERSION}}

- name: Seal and upload

id: seal_source_code

uses: ./.github/actions/seal

with:

artifact_name_prefix: "source"

# This job runs our automated test suite, complexity and security baselines

# it ensures previously merged have been tested as part of the pull request process

# NOTE

# we don't upload the artifact after testing to prevent any tampering of our source code dependencies

quality_check:

needs: seal

runs-on: ubuntu-latest

permissions:

contents: read

steps:

# NOTE: we need actions/checkout to configure git first (pre-commit hooks in make dev)

- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

with:

ref: ${{ env.RELEASE_COMMIT }}

- name: Restore sealed source code

uses: ./.github/actions/seal-restore

with:

integrity_hash: ${{ needs.seal.outputs.integrity_hash }}

artifact_name: ${{ needs.seal.outputs.artifact_name }}

- name: Debug cache restore

run: cat pyproject.toml

- name: Install poetry

run: pipx install git+https://github.com/python-poetry/poetry@bd500dd3bdfaec3de6894144c9cedb3a9358be84 # v2.0.1

- name: Set up Python

uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0

with:

python-version: "3.12"

cache: "poetry"

- name: Install dependencies

run: make dev

- name: Run all tests, linting and baselines

run: make pr

# This job creates a release artifact (tar.gz, wheel)

# it checks out code from release commit for custom actions to work

# then restores the sealed source code (overwrites any potential tampering)

# it's done separately from release job to enforce least privilege.

# We export just the final build artifact for release

build:

runs-on: ubuntu-latest

needs: [quality_check, seal]

permissions:

contents: read

outputs:

integrity_hash: ${{ steps.seal_build.outputs.integrity_hash }}

artifact_name: ${{ steps.seal_build.outputs.artifact_name }}

attestation_hashes: ${{ steps.encoded_hash.outputs.attestation_hashes }}

steps:

# NOTE: we need actions/checkout to configure git first (pre-commit hooks in make dev)

- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

with:

ref: ${{ env.RELEASE_COMMIT }}

- name: Restore sealed source code

uses: ./.github/actions/seal-restore

with:

integrity_hash: ${{ needs.seal.outputs.integrity_hash }}

artifact_name: ${{ needs.seal.outputs.artifact_name }}

- name: Install poetry

run: pipx install git+https://github.com/python-poetry/poetry@bd500dd3bdfaec3de6894144c9cedb3a9358be84 # v2.0.1

- name: Set up Python

uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0

with:

python-version: "3.12"

cache: "poetry"

- name: Build python package and wheel

run: poetry build

- name: Seal and upload

id: seal_build

uses: ./.github/actions/seal

with:

artifact_name_prefix: "build"

files: "dist/"

# NOTE: SLSA retraces our build to its artifact to ensure it wasn't tampered

# coupled with GitHub OIDC, SLSA can then confidently sign it came from this release pipeline+commit+branch+org+repo+actor+integrity hash

- name: Create attestation encoded hash for provenance

id: encoded_hash

working-directory: dist

run: echo "attestation_hashes=$(sha256sum ./* | base64 -w0)" >> "$GITHUB_OUTPUT"

# This job creates a provenance file that describes how our release was built (all steps)

# after it verifies our build is reproducible within the same pipeline

# it confirms that its own software and the CI build haven't been tampered with (Trust but verify)

# lastly, it creates and sign an attestation (multiple.intoto.jsonl) that confirms

# this build artifact came from this GitHub org, branch, actor, commit ID, inputs that triggered this pipeline, and matches its integrity hash

# NOTE: supply chain threats review (we protect against all of them now): https://slsa.dev/spec/v1.0/threats-overview

provenance:

needs: [seal, build]

permissions:

contents: write # nested job explicitly require despite upload assets being set to false

actions: read # To read the workflow path.

id-token: write # To sign the provenance.

# NOTE: provenance fails if we use action pinning... it's a Github limitation

# because SLSA needs to trace & attest it came from a given branch; pinning doesn't expose that information

# https://github.com/slsa-framework/slsa-github-generator/blob/main/internal/builders/generic/README.md#referencing-the-slsa-generator

uses: slsa-framework/slsa-github-generator/.github/workflows/[email protected]

with:

base64-subjects: ${{ needs.build.outputs.attestation_hashes }}

upload-assets: false # we upload its attestation in create_tag job, otherwise it creates a new release

# This job uses release artifact to publish to PyPi

# it exchanges JWT tokens with GitHub to obtain PyPi credentials

# since it's already registered as a Trusted Publisher.

# It uses the sealed build artifact (.whl, .tar.gz) to release it

release:

needs: [build, seal, provenance]

environment: release

runs-on: ubuntu-latest

permissions:

id-token: write # OIDC for PyPi Trusted Publisher feature

env:

RELEASE_VERSION: ${{ needs.seal.outputs.RELEASE_VERSION }}

steps:

# NOTE: we need actions/checkout in order to use our local actions (e.g., ./.github/actions)

- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

with:

ref: ${{ env.RELEASE_COMMIT }}

- name: Restore sealed source code

uses: ./.github/actions/seal-restore

with:

integrity_hash: ${{ needs.build.outputs.integrity_hash }}

artifact_name: ${{ needs.build.outputs.artifact_name }}

- name: Upload to PyPi prod

if: ${{ !inputs.skip_pypi }}

uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc # v1.12.4

# PyPi test maintenance affected us numerous times, leaving for history purposes

# - name: Upload to PyPi test

# if: ${{ !inputs.skip_pypi }}

# uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc # v1.12.4

# with:

# repository-url: https://test.pypi.org/legacy/

# We create a Git Tag using our release version (e.g., v2.16.0)

# using our sealed source code we created earlier.

# Because we bumped version of our project as part of CI

# we need to add this into git before pushing the tag

# otherwise the release commit will be used as the basis for the tag.

# Later, we create a PR to update trunk with our newest release version (e.g., bump_version job)

create_tag:

needs: [release, seal, provenance]

runs-on: ubuntu-latest

permissions:

contents: write

steps:

# NOTE: we need actions/checkout to authenticate and configure git first

- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

with:

ref: ${{ env.RELEASE_COMMIT }}

- name: Restore sealed source code

uses: ./.github/actions/seal-restore

with:

integrity_hash: ${{ needs.seal.outputs.integrity_hash }}

artifact_name: ${{ needs.seal.outputs.artifact_name }}

- id: setup-git

name: Git client setup and refresh tip

run: |

git config user.name "Powertools for AWS Lambda (Python) bot"

git config user.email "[email protected]"

git config remote.origin.url >&-

- name: Create Git Tag

run: |

git add pyproject.toml aws_lambda_powertools/shared/version.py

git commit -m "chore: version bump"

git tag -a v"${RELEASE_VERSION}" -m "release_version: v${RELEASE_VERSION}"

git push origin v"${RELEASE_VERSION}"

env:

RELEASE_VERSION: ${{ needs.seal.outputs.RELEASE_VERSION }}

- name: Upload provenance

id: upload-provenance

uses: ./.github/actions/upload-release-provenance

with:

release_version: ${{ needs.seal.outputs.RELEASE_VERSION }}

provenance_name: ${{needs.provenance.outputs.provenance-name}}

github_token: ${{ secrets.GITHUB_TOKEN }}

# Creates a PR with the latest version we've just released

# since our trunk is protected against any direct pushes from automation

bump_version:

needs: [release, seal]

permissions:

contents: write # create-pr action creates a temporary branch

pull-requests: write # create-pr action creates a PR using the temporary branch

runs-on: ubuntu-latest

steps:

# NOTE: we need actions/checkout to authenticate and configure git first

- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

with:

ref: ${{ env.RELEASE_COMMIT }}

- name: Restore sealed source code

uses: ./.github/actions/seal-restore

with:

integrity_hash: ${{ needs.seal.outputs.integrity_hash }}

artifact_name: ${{ needs.seal.outputs.artifact_name }}

- name: Create PR

id: create-pr

uses: ./.github/actions/create-pr

with:

files: "pyproject.toml aws_lambda_powertools/shared/version.py"

temp_branch_prefix: "ci-bump"

pull_request_title: "chore(ci): bump version to ${{ needs.seal.outputs.RELEASE_VERSION }}"

github_token: ${{ secrets.GITHUB_TOKEN }}

# This job compiles a Lambda Layer optimized for space and speed (e.g., Cython)

# It then deploys to Layer's Beta and Prod account, including SAR Beta and Prod account.

# It uses canaries to attest Layers can be used and imported between stages.

# Lastly, it updates our documentation with the latest Layer ARN for all regions

# NOTE

# Watch out for the depth limit of 4 nested workflow_calls.

# publish_layer -> publish_v2_layer -> reusable_deploy_v2_layer_stack

publish_layer:

needs: [seal, release, create_tag]

secrets: inherit

permissions:

id-token: write

contents: write

pages: write

pull-requests: write

uses: ./.github/workflows/publish_v2_layer.yml

with:

latest_published_version: ${{ needs.seal.outputs.RELEASE_VERSION }}

pre_release: ${{ inputs.pre_release }}

source_code_artifact_name: ${{ needs.seal.outputs.artifact_name }}

source_code_integrity_hash: ${{ needs.seal.outputs.integrity_hash }}

post_release:

needs: [seal, release, publish_layer]

permissions:

contents: read

issues: write

discussions: write

pull-requests: write

runs-on: ubuntu-latest

env:

RELEASE_VERSION: ${{ needs.seal.outputs.RELEASE_VERSION }}

steps:

- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

with:

ref: ${{ env.RELEASE_COMMIT }}

- name: Restore sealed source code

uses: ./.github/actions/seal-restore

with:

integrity_hash: ${{ needs.seal.outputs.integrity_hash }}

artifact_name: ${{ needs.seal.outputs.artifact_name }}

- name: Close issues related to this release

uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1

with:

github-token: ${{ secrets.GITHUB_TOKEN }}

script: |

const post_release = require('.github/scripts/post_release.js')

await post_release({github, context, core})