Repository navigation
238 lines (217 loc) · 9.25 KB
/
Copy pathpython.yml
File metadata and controls
238 lines (217 loc) · 9.25 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
name: GraphAr Python CI
on:
# Trigger the workflow on push or pull request,
# but only for the main branch
push:
branches:
- "main"
paths:
- 'cpp/**'
- 'python/**'
- '.github/workflows/ci.yml'
- '.github/workflows/python.yml'
pull_request:
branches:
- "main"
paths:
- 'cpp/**'
- 'python/**'
- '.github/workflows/ci.yml'
- '.github/workflows/python.yml'
concurrency:
group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }}
cancel-in-progress: true
jobs:
ubuntu:
name: Ubuntu 24.04 Python
# Pinned, not ubuntu-latest: the Arrow apt packages are versioned per
# distro codename (24.0.0-1 exists for noble), and a mutable label would
# break the pinned apt install when GitHub rolls the label forward.
runs-on: ubuntu-24.04
env:
GAR_TEST_DATA: ${{ github.workspace }}/graphar-testing/
# Single source of version for Arrow. Must match the pyarrow pinned in
# python/pyproject.toml (pyarrow == 24.0.0). The "-1" suffix is the
# Debian package revision that apt requires for exact version matching.
ARROW_VERSION: "24.0.0-1"
steps:
# fetch-depth 0 is required: the package version is derived from git tags
# by setuptools-scm (python/pyproject.toml), and a shallow checkout has no
# tags, which would make the build fail or produce a bogus version.
- uses: actions/checkout@v7
with:
submodules: true
fetch-depth: 0
- name: Install Python
uses: actions/setup-python@v6
with:
python-version: '3.10'
- name: Install dependencies
run: |
sudo apt update
sudo apt install -y -V ca-certificates lsb-release wget
wget https://packages.apache.org/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
sudo apt update
# The Python extension links Arrow, ArrowDataset, ArrowAcero and Parquet
# (see python/CMakeLists.txt), so all dev packages are required. They
# must be installed in a single apt invocation with the version pinned:
# the 24.0.0-1 dev packages depend on each other with exact (=) version
# constraints, and apt does not resolve an exact version dependency when
# a newer candidate (25.x) exists in the repository, so installing them
# one by one fails.
sudo apt install -y -V libarrow-dev=$ARROW_VERSION \
libarrow-dataset-dev=$ARROW_VERSION \
libarrow-acero-dev=$ARROW_VERSION \
libarrow-compute-dev=$ARROW_VERSION \
libparquet-dev=$ARROW_VERSION
sudo apt-get install -y ccache libcurl4-openssl-dev
git clone https://github.com/apache/incubator-graphar-testing.git $GAR_TEST_DATA --depth 1
- name: Install GraphAr Python SDK
working-directory: "python"
run: |
pip install poetry --quiet
poetry env use python3
poetry install --with test --no-root
poetry run pip install . -v
- name: Run Cli Test
working-directory: "python"
run: |
poetry run graphar --help
poetry run graphar check -p ../testing/neo4j/MovieGraph.graph.yml
poetry run graphar show -p ../testing/neo4j/MovieGraph.graph.yml -v Person
poetry run graphar show -p ../testing/neo4j/MovieGraph.graph.yml -es Person -e ACTED_IN -ed Movie
poetry run graphar import -c ../testing/neo4j/data/import.mini.yml
- name: Run pytest
working-directory: "python"
run: |
poetry run pytest -vvv
- name: Upload coverage reports to Codecov
uses: codecov/codecov-action@v7
with:
token: ${{ secrets.CODECOV_TOKEN }}
- name: Build Docs
working-directory: python
run: |
make install_docs
make docs
macos:
name: macos latest Python
# Pinned, not macos-latest: the runner OS also sets the effective macOS
# deployment target of the built artifacts, so a mutable label could change
# the produced wheel/extension compatibility without any code change.
runs-on: macos-15
env:
GAR_TEST_DATA: ${{ github.workspace }}/graphar-testing/
# Must match the pyarrow pinned in python/pyproject.toml (pyarrow == 24.0.0).
ARROW_VERSION: "24.0.0"
# Where the Arrow built from source below is installed. CMAKE_PREFIX_PATH
# makes the graphar Python build (scikit-build-core) link against that
# Arrow at build time; at runtime Arrow is resolved from the sibling
# pyarrow package instead (see the RPATH setup in python/CMakeLists.txt).
CMAKE_PREFIX_PATH: ${{ github.workspace }}/arrow-dist
steps:
# fetch-depth 0 is required: the package version is derived from git tags
# by setuptools-scm (python/pyproject.toml), and a shallow checkout has no
# tags, which would make the build fail or produce a bogus version.
- uses: actions/checkout@v7
with:
submodules: true
fetch-depth: 0
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: '3.13'
- name: Cache Arrow build
id: arrow-cache
uses: actions/cache@v6
with:
path: ${{ env.CMAKE_PREFIX_PATH }}
# The hash of this workflow is part of the key so that changing the
# Arrow CMake flags below invalidates the cached build.
key: arrow-${{ runner.os }}-${{ runner.arch }}-${{ env.ARROW_VERSION }}-${{ hashFiles('.github/workflows/python.yml') }}
- name: Install dependencies
run: |
brew bundle --file=cpp/Brewfile
# Homebrew's apache-arrow is built without ORC support, so it is removed
# here and Arrow is built from source in the next step (with ARROW_ORC).
brew uninstall apache-arrow || true
git clone https://github.com/apache/incubator-graphar-testing.git $GAR_TEST_DATA --depth 1
- name: Build Arrow from source (with ORC)
if: steps.arrow-cache.outputs.cache-hit != 'true'
run: |
set -euxo pipefail
curl -fLsS -o "apache-arrow-${ARROW_VERSION}.tar.gz" \
"https://archive.apache.org/dist/arrow/arrow-${ARROW_VERSION}/apache-arrow-${ARROW_VERSION}.tar.gz"
tar xzf "apache-arrow-${ARROW_VERSION}.tar.gz"
cmake -S "apache-arrow-${ARROW_VERSION}/cpp" -B arrow-build \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_INSTALL_PREFIX="${CMAKE_PREFIX_PATH}" \
-DCMAKE_INSTALL_LIBDIR=lib \
-DARROW_BUILD_SHARED=ON \
-DARROW_BUILD_STATIC=OFF \
-DARROW_DEPENDENCY_SOURCE=BUNDLED \
-DARROW_PARQUET=ON \
-DARROW_DATASET=ON \
-DARROW_ACERO=ON \
-DARROW_ORC=ON \
-DARROW_CSV=ON \
-DARROW_JSON=ON \
-DARROW_COMPUTE=ON \
-DARROW_FILESYSTEM=ON \
-DARROW_WITH_SNAPPY=ON \
-DARROW_WITH_ZSTD=ON \
-DARROW_WITH_LZ4=ON \
-DARROW_WITH_BZ2=ON \
-DARROW_WITH_BROTLI=ON \
-DARROW_WITH_ZLIB=ON \
-DARROW_WITH_UTF8PROC=ON \
-DARROW_WITH_RE2=ON \
-DARROW_S3=OFF \
-DARROW_GCS=OFF \
-DARROW_AZURE=OFF \
-DARROW_FLIGHT=OFF \
-DARROW_JEMALLOC=OFF \
-DARROW_MIMALLOC=OFF \
-DARROW_BUILD_TESTS=OFF \
-DARROW_BUILD_BENCHMARKS=OFF \
-DARROW_BUILD_EXAMPLES=OFF \
-DARROW_BUILD_INTEGRATION=OFF \
-DARROW_BUILD_UTILITIES=OFF \
-DARROW_PYTHON=OFF
cmake --build arrow-build --target install -j "$(sysctl -n hw.ncpu)"
- name: Install GraphAr Python SDK
working-directory: "python"
run: |
pip install poetry --quiet
poetry env use python3
poetry install --with test --no-root
poetry run pip install . -v
- name: Run Cli Test
working-directory: "python"
run: |
poetry run graphar --help
poetry run graphar check -p ../testing/neo4j/MovieGraph.graph.yml
poetry run graphar show -p ../testing/neo4j/MovieGraph.graph.yml -v Person
poetry run graphar show -p ../testing/neo4j/MovieGraph.graph.yml -es Person -e ACTED_IN -ed Movie
poetry run graphar import -c ../testing/neo4j/data/import.mini.yml
- name: Run pytest
working-directory: "python"
run: |
poetry run pytest -vvv