-
Notifications
You must be signed in to change notification settings - Fork 74
373 lines (326 loc) · 15.4 KB
/
ci_pipeline.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
name: CI
on:
#push:
workflow_dispatch:
workflow_call:
pull_request_review:
types: [submitted]
permissions: write-all
jobs:
#Sanity:
#uses: speedb-io/speedb/.github/workflows/sanity_check.yml@main
Check-Licence-And-History:
#if: ${{ github.event_name == 'pull_request_review' }}
uses: ./.github/workflows/check_license_and_history.yml
Build:
needs: [Check-Licence-And-History]
if: ${{ (always() && !failure() && !cancelled()) && (github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release')) }}
#runs-on: [self-hosted, ubuntu, asrunner]
runs-on: ubuntu-20.04
strategy:
matrix:
include:
- name: verify build
command: cmake .. -GNinja
- name: optimized build
command: cmake .. -DCMAKE_BUILD_TYPE=Release -GNinja
- mame: clang build
command: CC=clang CXX=clang++ cmake .. -GNinja
container:
image: alpine:3.14
steps:
- name: Pre-build
run: |
env
rm -rf /usr/share/dotnet || echo ""
df -h
apk add git
echo "nameserver 8.8.8.8" > /etc/resolv.conf
apk add bash python3 py3-pip clang clang-extra-tools shellcheck gcc g++ cmake ninja ccache \
openjdk10 gflags-dev snappy-dev lz4-dev bzip2-dev zstd-dev zlib-dev linux-headers openssh-client tar readline-dev
python3 -m pip install lint-diffs flake8
- name: Checkout
uses: actions/checkout@v3
- run: git config --system --add safe.directory /__w/speedb/speedb
- name: Prepare ccache timestamp
id: ccache_cache_timestamp
shell: cmake -P {0}
run: |
string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC)
message("::set-output name=timestamp::${current_date}")
- name: ccache cache files
uses: actions/[email protected]
with:
path: ~/.ccache
key: ${{runner.os}}-ccache-${{steps.ccache_cache_timestamp.outputs.timestamp}}
restore-keys: |
${{runner.os}}-ccache-
- name: ${{ matrix.name }}
run: |
#echo "try git config"
#git config --global --list
#echo "done git config"
if [ -d "$GITHUB_WORKSPACE/build" ]; then
echo >&2 "error: the build directory should not exist" && false NIK
fi
if [ -d "~/.ccache" ]; then
echo "Already exists"
else
mkdir -p ~/.ccache
ls ~ | grep cache || echo ""
touch ~/.ccache/ccache.txt
echo "aaa" > ~/.ccache/ccache.txt
ls ~/.ccache
cat ~/.ccache/ccache.txt
fi
mkdir -p "$GITHUB_WORKSPACE/build"
cd "$GITHUB_WORKSPACE/build"
export "CCACHE_BASEDIR=$HOME"
export "CCACHE_DIR=$HOME/.ccache"
export "CCACHE_COMPILERCHECK=content"
echo "MAtrix command - " ${{ matrix.command }}
${{ matrix.command }} -DPORTABLE=1 -DWITH_GFLAGS=1 \
-DWITH_ZLIB=1 -DWITH_SNAPPY=1 -DWITH_BZ2=1 -DWITH_LZ4=1 -DWITH_ZSTD=1 \
-DWITH_JNI=1 -DJAVA_HOME=/usr/lib/jvm/default-jvm \
-DWITH_BENCHMARK_TOOLS=1 -DWITH_CORE_TOOLS=1 -DWITH_TOOLS=1 \
-DWITH_TESTS=1 -DWITH_ALL_TESTS=1 -DWITH_EXAMPLES=1
echo "starting ninja"
ninja
#Performance:
#if: ${{ github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release') }}
#needs: [Build]
#uses: speedb-io/speedb/.github/workflows/perf-test.yml@main
QA-Tests:
if: ${{ (always() && !failure() && !cancelled()) && (github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release')) }}
needs: [Build]
uses: speedb-io/speedb/.github/workflows/qa-tests.yml@main
Fuzz:
if: ${{ (always() && !failure() && !cancelled()) && (github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release')) }}
needs: [Build]
uses: ./.github/workflows/test_fuzz.yml
Windows-build-test:
if: ${{ (always() && !failure() && !cancelled()) && (github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release')) }}
needs: [Build]
runs-on: windows-2022
env:
THIRDPARTY_HOME: C:/Users/runneradmin/thirdparty
CMAKE_HOME: C:/Program Files/CMake
CMAKE_BIN: C:/Program Files/CMake/bin/cmake.exe
SNAPPY_HOME: C:/Users/runneradmin/thirdparty/snappy-1.1.9
SNAPPY_INCLUDE: C:/Users/runneradmin/thirdparty/snappy-1.1.9;C:/Users/circleci/thirdparty/snappy-1.1.9/build
SNAPPY_LIB_DEBUG: C:/Users/runneradmin/thirdparty/snappy-1.1.9/build/Debug/snappy.lib
CMAKE_GENERATOR: Visual Studio 17 2022
CODE_HOME: C:/Users/runneradmin/code
steps:
- uses: actions/checkout@v3
- name: Set up JDK 17
uses: actions/setup-java@v3
with:
distribution: 'temurin' # See 'Supported distributions' for available options
java-version: '8'
- name: Add msbuild to PATH
uses: microsoft/[email protected]
- name: Setup VS Dev
uses: seanmiddleditch/gha-setup-vsdevenv@v4
- name: install cmake
shell: powershell
run: |
echo "Installing CMake..."
choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System' -y
- name: prepare Thirdparty home
shell: powershell
run: mkdir "$Env:THIRDPARTY_HOME"
- name: install snappy test
shell: powershell
run: |
mkdir $env:CODE_HOME
cd $env:CODE_HOME
curl https://github.com/google/snappy/archive/refs/tags/1.1.9.zip -o 1.1.9.zip
Expand-Archive -Path 1.1.9.zip -DestinationPath snappy-tmp
mv .\snappy-tmp\snappy-1.1.9\ .
rmdir .\snappy-tmp\
cd .\snappy-1.1.9\
mkdir build
cd .\build\
cmake -G "Visual Studio 17 2022" -A x64 -DCMAKE_GENERATOR_PLATFORM=x64 -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF ..
msbuild Snappy.sln /p:Configuration=Debug /p:Platform=x64
msbuild Snappy.sln /p:Configuration=Release /p:Platform=x64
- name: install snappy
shell: powershell
run: |
cd $Env:THIRDPARTY_HOME
curl https://github.com/google/snappy/archive/refs/tags/1.1.9.zip -O snappy-1.1.9.zip
Expand-Archive -Path snappy-1.1.9.zip -DestinationPath snappy-tmp
mv .\snappy-tmp\snappy-1.1.9\ .
cd snappy-1.1.9
mkdir build
cd .\build
& cmake -G "Visual Studio 17 2022" -A x64 -DCMAKE_GENERATOR_PLATFORM=x64 -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF ..
msbuild.exe Snappy.sln -maxCpuCount -property:Configuration=Release -property:Platform=x64
- name: install gflags
shell: powershell
run: |
cd $Env:THIRDPARTY_HOME
curl https://github.com/gflags/gflags/archive/refs/tags/v2.2.2.zip -o v2.2.2.zip
Expand-Archive -Path .\v2.2.2.zip -DestinationPath gflags-tmp
mv .\gflags-tmp\gflags-2.2.2 .
rmdir gflags-tmp
cd gflags-2.2.2
mkdir target
cd target
cmake -G "Visual Studio 17 2022" -A x64 ..
msbuild gflags.sln /p:Configuration=Release /p:Platform=x64
- name: install zlib
shell: powershell
run: |
cd $Env:THIRDPARTY_HOME
git clone https://github.com/madler/zlib.git -b v1.3 zlib-1.3
cd zlib-1.3\contrib\vstudio\vc14
devenv zlibvc.sln /upgrade
cp ../../../zlib.h .
msbuild zlibvc.sln /p:Configuration=Debug /p:Platform=x64
msbuild zlibvc.sln /p:Configuration=Release /p:Platform=x64
copy x64\ZlibDllRelease\zlibwapi.lib x64\ZlibStatRelease\
- name: install lz4
shell: powershell
run: |
cd $Env:THIRDPARTY_HOME
curl https://github.com/lz4/lz4/archive/refs/tags/v1.9.2.zip -o lz4.zip
Expand-Archive -Path lz4.zip -DestinationPath lz4-tmp
mv .\lz4-tmp\lz4-1.9.2\ .
rmdir .\lz4-tmp\
cd .\lz4-1.9.2\
cd visual\VS2017
devenv lz4.sln /upgrade
msbuild lz4.sln /p:Configuration=Release /p:Platform=x64
- name: install zctd
shell: powershell
run: |
cd $Env:THIRDPARTY_HOME
curl https://github.com/facebook/zstd/archive/v1.5.2.zip -o zstd-tmp.zip
Expand-Archive -Path zstd-tmp.zip -DestinationPath zstd-tmp
mv .\zstd-tmp\zstd-1.5.2\ .
rmdir .\zstd-tmp\
cd zstd-1.5.2\build\VS2010
devenv zstd.sln /upgrade
msbuild zstd.sln /p:Configuration=Debug /p:Platform=x64
msbuild zstd.sln /p:Configuration=Release /p:Platform=x64
- name: Build Speedb
run: |
copy C:\Users\runneradmin\thirdparty\snappy-1.1.9\build\snappy-stubs-public.h C:\Users\runneradmin\thirdparty\snappy-1.1.9\
copy tools\thirdparty.txt thirdparty.inc # copy the thirdparty.inc that reflects the env on the runner machine
mkdir build
cd build
& $Env:CMAKE_BIN -G "$Env:CMAKE_GENERATOR" -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_STANDARD=20 -A x64 -DJNI=1 -DGFLAGS=1 -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DJNI=1 -DSNAPPY=1 -DLZ4=1 -DZLIB=1 -DZSTD=1 -DXPRESS=1 -DFAIL_ON_WARNINGS=0 ..
cd ..
echo "Building with VS version: $Env:CMAKE_GENERATOR"
msbuild build/speedb.sln /p:Configuration=Release /t:speedbjni-shared
#msbuild.exe build/speedb.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
- name: Test Speedb
shell: powershell
run: |
echo "skipping all tests for now"
#build_tools\run_ci_db_test.ps1 -SuiteRun arena_test,db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16
#build_tools\run_ci_db_test.ps1 -SuiteRun db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16
Macos-build:
if: ${{ (always() && !failure() && !cancelled()) && (github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release')) }}
needs: [Build]
runs-on: macos-11
steps:
- uses: actions/checkout@v3
- name: Set up JDK 17
uses: actions/setup-java@v3
with:
distribution: 'temurin' # See 'Supported distributions' for available options
java-version: '8'
- name: build jar
run: |
echo $JAVA_HOME
export CPPFLAGS="-I$JAVA_HOME/include"
export CXXFLAGS="-I$JAVA_HOME/include"
brew install zlib
brew install bzip2 lz4 snappy
ROCKSDB_DISABLE_JEMALLOC=1 PORTABLE=1 DEBUG_LEVEL=0 make -j 4 rocksdbjavastatic
Linux-build:
if: ${{ (always() && !failure() && !cancelled()) && (github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release')) }}
needs: [Build]
runs-on: ubuntu-latest
container:
image: centos:7.9.2009
steps:
- name: pre
run: |
yum install -y centos-release-scl epel-release
yum install -y make devtoolset-11-gcc-c++ \
coreutils wget unzip which git python3 openssl openssl-devel \
libzstd-devel lz4-devel snappy-devel zlib-devel readline-devel \
java-1.8.0-openjdk-devel
echo "PATH=/opt/rh/devtoolset-11/root/usr/bin:${PATH}" >> $GITHUB_ENV
echo "RELEASE_VERSION=${GITHUB_REF_NAME#speedb/v}" >> $GITHUB_ENV
- name: Install CMake
run: |
CMAKE_RELEASE=3.20.1
wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_RELEASE}/cmake-${CMAKE_RELEASE}.tar.gz
tar xf cmake-${CMAKE_RELEASE}.tar.gz
cd cmake-${CMAKE_RELEASE}
./bootstrap
make -j$(nproc) && make install
- uses: actions/checkout@v3
- run: mkdir "$GITHUB_WORKSPACE/out"
- name: Build and package release libraries
run: |
rm -rf build && mkdir build && cd build
cmake .. -DCMAKE_BUILD_TYPE=Release -DSPDB_RELEASE_BUILD=1 -DPORTABLE=1 -DWITH_GFLAGS=0 -DWITH_SNAPPY=1 -DWITH_LZ4=1 -DWITH_ZLIB=1 -DWITH_ZSTD=1
- name: Build Jar
run: |
make clean
SPDB_RELEASE_BUILD=1 LIB_MODE=static DEBUG_LEVEL=0 PORTABLE=1 JAVA_HOME=/usr/lib/jvm/java-openjdk make -j$(nproc) rocksdbjavastatic
- name: Build db_bench
run: |
yum install -y gflags-devel
rm -rf build && mkdir build && cd build
cmake .. -DCMAKE_BUILD_TYPE=Release -DSPDB_RELEASE_BUILD=1 -DPORTABLE=1 -DWITH_GFLAGS=1 \
-DWITH_SNAPPY=1 -DWITH_LZ4=1 -DWITH_ZLIB=1 -DWITH_ZSTD=1 \
-DWITH_BENCHMARK_TOOLS=1 -DROCKSDB_BUILD_SHARED=1
make -j$(nproc) db_bench
Linux-Arm-build:
if: ${{ (always() && !failure() && !cancelled()) && (github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release')) }}
needs: [Build]
runs-on: ubuArm64G
steps:
- name: Dump GitHub context
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
run: |
echo "$GITHUB_CONTEXT"
- name: 'Checkout GitHub Action'
uses: actions/checkout@v3
- name: 'Build_on_Arm_Centos'
run: |
export SPDB_LIB_DIR=~/spdb_lib && mkdir -p $SPDB_LIB_DIR
case "$GITHUB_EVENT_NAME" in
"pull_request")
echo "this workflow was triggered by a pull request"
echo "the branch can not be used, it is $GITHUB_REF_NAME , instead ${{ github.head_ref }} will be used"
docker run --rm -v $(readlink -f ${SPDB_LIB_DIR}):/out -i speedb-centos-builder ${{ github.head_ref }}
;;
"workflow_dispatch")
echo "this workflow was triggered by a workflow dispatch, we will use the ref_name instead of the merge branch"
echo " consider using github.ref, the branch that will be used here: " ${{ github.ref_name }}
docker run --rm -v $(readlink -f ${SPDB_LIB_DIR}):/out -i speedb-centos-builder ${{ github.ref_name }}
;;
"pull_request_review")
echo "this workflow was triggered by a pull request review"
echo "the branch can not be used, it is $GITHUB_REF_NAME , instead ${{ github.event.pull_request.head.ref }} will be used"
docker run --rm -v $(readlink -f ${SPDB_LIB_DIR}):/out -i speedb-centos-builder ${{ github.event.pull_request.head.ref }}
;;
esac
CI-all:
if: ${{ github.event.review.state == 'approved' || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/heads/release') }}
needs: [Check-Licence-And-History, Build, QA-Tests, Fuzz, Linux-Arm-build, Linux-build, Macos-build, Windows-build-test]
runs-on: ubuntu-latest
steps:
- name: Summary
run: |
echo "All tests passed"
exit 0