GeezerVison is showing github.com — feast/.github/workflows/build_wheels.yml at master · wietsche/feast

Expand file tree

build_wheels.yml

Copy path

View runs

More file actions

Latest commit

History

277 lines (271 loc) · 11.6 KB

build_wheels.yml

Copy path

File metadata and controls

277 lines (271 loc) · 11.6 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

# Call this workflow from other workflows in the repository by specifying "uses: ./.github/workflows/build_wheels.yml"

# Developers who are starting a new release should use this workflow to ensure wheels will be built correctly.

# Devs should check out their fork, add a tag to the last master commit on their fork, and run the release off of their fork on the added tag to ensure wheels will be built correctly.

on:

workflow_dispatch:

tags:

- 'v*.*.*'

workflow_call:

jobs:

get-version:

runs-on: ubuntu-latest

outputs:

release_version: ${{ steps.get_release_version.outputs.release_version }}

version_without_prefix: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }}

highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }}

steps:

- name: Checkout

uses: actions/checkout@v2

with:

persist-credentials: false

- name: Get release version

id: get_release_version

run: echo ::set-output name=release_version::${GITHUB_REF#refs/*/}

- name: Get release version without prefix

id: get_release_version_without_prefix

env:

RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}

run: |

echo ::set-output name=version_without_prefix::${RELEASE_VERSION:1}

- name: Get highest semver

id: get_highest_semver

env:

RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}

run: |

source infra/scripts/setup-common-functions.sh

SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$'

if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then

echo ::set-output name=highest_semver_tag::$(get_tag_release -m)

- name: Check output

id: check_output

env:

RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}

VERSION_WITHOUT_PREFIX: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }}

HIGHEST_SEMVER_TAG: ${{ steps.get_highest_semver.outputs.highest_semver_tag }}

run: |

echo $RELEASE_VERSION

echo $VERSION_WITHOUT_PREFIX

echo $HIGHEST_SEMVER_TAG

build-python-wheel:

runs-on: ${{ matrix.os }}

strategy:

matrix:

os: [ ubuntu-latest, macos-10.15 ]

steps:

- uses: actions/checkout@v2

- name: Setup Node

uses: actions/setup-node@v2

with:

node-version: '17.x'

registry-url: 'https://registry.npmjs.org'

- name: Build UI

run: make build-ui

- name: Build wheels

uses: pypa/[email protected]

env:

CIBW_BUILD: "cp3*_x86_64"

CIBW_SKIP: "cp36-* cp37-* *-musllinux_x86_64 cp310-macosx_x86_64"

CIBW_ARCHS: "native"

CIBW_ENVIRONMENT: >

COMPILE_GO=True PATH=$PATH:/usr/local/go/bin

CIBW_BEFORE_ALL_LINUX: |

curl -o go.tar.gz https://dl.google.com/go/go1.18.2.linux-amd64.tar.gz

tar -C /usr/local -xzf go.tar.gz

go version

yum -y update &&

yum install -y epel-release || yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-$(cut -d: -f5 /etc/system-release-cpe | cut -d. -f1).noarch.rpm &&

yum install -y https://apache.jfrog.io/artifactory/arrow/centos/$(cut -d: -f5 /etc/system-release-cpe | cut -d. -f1)/apache-arrow-release-latest.rpm &&

yum install -y --enablerepo=epel arrow-devel # For C++

CIBW_BEFORE_ALL_MACOS: |

brew install apache-arrow

curl -o python.pkg https://www.python.org/ftp/python/3.9.12/python-3.9.12-macosx10.9.pkg

sudo installer -pkg python.pkg -target /

# There's a `git restore` in here because `make install-go-ci-dependencies` is actually messing up go.mod & go.sum.

CIBW_BEFORE_BUILD: |

make install-protoc-dependencies

make install-go-proto-dependencies

make install-go-ci-dependencies

git status

git restore go.mod go.sum

git restore sdk/python/feast/ui/yarn.lock

CIBW_BEFORE_TEST: "cd {project} && git status"

# py3.10 on MacOS does not work with Go so we have to install separately. Issue is tracked here: https://github.com/feast-dev/feast/issues/2881.

- name: Build py310 specific wheels for macos

if: matrix.os == 'macos-10.15'

uses: pypa/[email protected]

env:

CIBW_BUILD: "cp310-macosx_x86_64"

CIBW_ARCHS: "native"

# Need this environment variable because of this issue: https://github.com/pypa/cibuildwheel/issues/952.

CIBW_ENVIRONMENT: >

_PYTHON_HOST_PLATFORM=macosx-10.15-x86_64

# There's a `git restore` in here because remnant go.mod, go.sum changes from the build mess up the wheel naming.

CIBW_BEFORE_BUILD: |

git status

git restore go.mod go.sum

git restore sdk/python/feast/ui/yarn.lock

brew install apache-arrow

- uses: actions/upload-artifact@v2

with:

path: ./wheelhouse/*.whl

build-source-distribution:

runs-on: macos-10.15

steps:

- uses: actions/checkout@v2

- name: Setup Python

id: setup-python

uses: actions/setup-python@v2

with:

python-version: "3.10"

architecture: x64

- name: Setup Node

uses: actions/setup-node@v2

with:

node-version: '17.x'

registry-url: 'https://registry.npmjs.org'

- name: Build and install dependencies

# There's a `git restore` in here because `make install-go-ci-dependencies` is actually messing up go.mod & go.sum.

run: |

pip install -U pip setuptools wheel twine

make install-protoc-dependencies

make install-go-proto-dependencies

make install-go-ci-dependencies

make build-ui

git status

git restore go.mod go.sum

git restore sdk/python/feast/ui/yarn.lock

- name: Build

run: |

python3 setup.py sdist

- uses: actions/upload-artifact@v2

with:

path: dist/*

# We add this step so the docker images can be built as part of the pre-release verification steps.

build-docker-images:

runs-on: ubuntu-latest

needs: get-version

strategy:

matrix:

component: [feature-server, feature-server-python-aws, feature-server-java, feature-transformation-server]

env:

REGISTRY: feastdev

steps:

- uses: actions/checkout@v2

- name: Set up QEMU

uses: docker/setup-qemu-action@v1

- name: Set up Docker Buildx

uses: docker/setup-buildx-action@v1

- name: Build image

run: |

make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${VERSION_WITHOUT_PREFIX}

env:

VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }}

verify-python-wheels:

runs-on: ${{ matrix.os }}

needs: [build-python-wheel, build-source-distribution, get-version]

strategy:

matrix:

os: [ubuntu-latest, macos-10.15 ]

python-version: [ "3.8", "3.9", "3.10"]

from-source: [ True, False ]

env:

# this script is for testing servers

# it starts server with timeout and checks whether process killed by timeout (started healthy) or died by itself

TEST_SCRIPT: |

timeout 10s $@ & pid=$!

wait $pid

ret=$?

if [[ $ret -ne 124 ]]

then

exit $ret

else

echo "Succeeded!"

VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }}

steps:

- name: Setup Python

id: setup-python

uses: actions/setup-python@v2

with:

python-version: ${{ matrix.python-version }}

architecture: x64

- uses: actions/setup-go@v3

with:

go-version: '>=1.17.0'

- uses: actions/download-artifact@v2

with:

path: dist

- name: Install wheel

if: ${{ !matrix.from-source }}

# try to install all wheels; only the current platform wheel should be actually installed

run: |

cd dist/

pip install wheel

for f in *.whl; do pip install $f || true; done

- name: Install apache-arrow on ubuntu

if: ${{ matrix.from-source && matrix.os == 'ubuntu-latest' }}

run: |

sudo apt update

sudo apt install -y -V ca-certificates lsb-release wget

wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb

sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb

sudo apt update

sudo apt install -y -V libarrow-dev

- name: Install apache-arrow on macos

if: ${{ matrix.from-source && matrix.os == 'macos-10.15' && matrix.python-version != '3.10' }}

run: brew install apache-arrow

- name: Install dist with go

if: ${{ matrix.from-source && (matrix.python-version != '3.10' || matrix.os == 'ubuntu-latest')}}

env:

COMPILE_GO: "True"

run: |

pip install 'grpcio-tools==1.47.0' 'pybindgen==0.22.0'

go install google.golang.org/protobuf/cmd/[email protected]

go install google.golang.org/grpc/cmd/[email protected]

pip install dist/*tar.gz

# py3.10 on MacOS does not work with Go so we have to install separately. Issue is tracked here: https://github.com/feast-dev/feast/issues/2881

- name: Install dist w/o go

if: ${{ matrix.from-source && matrix.python-version == '3.10' && matrix.os == 'macos-10.15'}}

run: pip install dist/*tar.gz

- name: Install OS X dependencies

if: matrix.os == 'macos-10.15'

run: brew install coreutils

# Validate that the feast version installed is not development and is the correct version of the tag we ran it off of.

- name: Validate Feast Version

run: |

VERSION_REGEX='[0-9]+\.[0-9]+\.[0-9]+'

OUTPUT_REGEX='^Feast SDK Version: "$VERSION_REGEX"$'

VERSION_OUTPUT=$(feast version)

VERSION=$(echo $VERSION_OUTPUT | grep -oE "$VERSION_REGEX")

OUTPUT=$(echo $VERSION_OUTPUT | grep -E "$REGEX")

if [ -n "$OUTPUT" ] && [ "$VERSION" = "$VERSION_WITHOUT_PREFIX" ]; then

echo "Correct Feast Version Installed"

else

echo "$VERSION_OUTPUT from installed wheel is not in the correct format or doesn't have the right version $VERSION."

exit 1

- name: Smoke test

run: |

feast init test_repo

cd test_repo/feature_repo

feast apply

echo "$TEST_SCRIPT" > run-and-wait.sh

bash run-and-wait.sh feast serve

bash run-and-wait.sh feast ui

# We disable this test for the Python 3.10 binary since it does not include Go.

- name: Smoke test with go

if: matrix.python-version != '3.10' || matrix.os == 'ubuntu-latest'

run: |

cd test_repo/feature_repo

feast apply

echo "$TEST_SCRIPT" > run-and-wait.sh

pip install cffi

printf "\ngo_feature_serving: True" >> feature_store.yaml

bash run-and-wait.sh feast serve