Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

merge parallel scann to main #192

Merged
merged 6 commits into from
Jun 1, 2024
Merged

merge parallel scann to main #192

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
2dd1f36
feat(shannon): new cond for loaded tables
ShannonBase May 29, 2024
1a15723
action(shannon): mtrs and git actions
ShannonBase May 29, 2024
998c659
action(shannon): mtrs and actions
ShannonBase May 30, 2024
6973896
feat(shannon):base funcs for parallel scan
ShannonBase May 30, 2024
be29d50
feat(shannon):parallel scan
ShannonBase May 31, 2024
9ad9f43
action(shannon):git weekly action
ShannonBase Jun 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 91 additions & 0 deletions .github/workflows/nightly.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
name: nightly

on:
push:
tags:
- "*v*"
schedule:
- cron: "0 18 * * *"

concurrency:
group: nightly
cancel-in-progress: true

defaults:
run:
shell: bash

jobs:
lint:
name: lint
# if: ${{ contains(github.event.pull_request.labels.*.name, 'ready-for-testing') && github.event.pull_request.merged != true }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
ref: ${{ github.event.pull_request.base.sha }}
- uses: actions/checkout@v3
with:
clean: false
# - name: Check License Header
# uses: apache/skywalking-eyes/header@main
- name: Ensure clang-format is available
run: |
command -v clang-format-14 > /dev/null || (sudo apt-get update && sudo apt-get install -y clang-format-14)
- name: Format check
run: |
git diff -U0 --no-color ${{ github.event.pull_request.base.sha }} HEAD storage/rapid_engine ml | /usr/share/clang/clang-format-14/clang-format-diff.py -p1 | tee /tmp/.clang-format-diff
[ -s /tmp/.clang-format-diff ] && exit 1 || true

build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: install_lib
run: |
sudo apt update -y && sudo apt upgrade -y && sudo apt install -y libssl-dev libncurses-dev libudev-dev g++ libbison-dev flex clang-format lcov pkg-config \
cmake zlib1g-dev wget build-essential libldap-dev libisl-dev libmpfr-dev patchelf libevent-dev openssl libssl-dev -y && sudo apt install libsasl2-dev -y
- name: install_gcc
run: |
sudo apt install bison texinfo libgmp3-dev zlib1g-dev -y && sudo apt install gcc-10 g++-10 -y && \
sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 && sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 100
- name: install_boost
run: |
cd /tmp && wget https://boostorg.jfrog.io/artifactory/main/release/1.77.0/source/boost_1_77_0.tar.gz && \
tar zxvf boost_1_77_0.tar.gz && cd boost_1_77_0 && ./bootstrap.sh --prefix=/usr/local/boost && ./b2 -j$(nproc) && sudo ./b2 install
#sed -i '61s/^/#/' cmake_install.cmake to disable can not find openssl_executeable error. removes it when it fixed.
- name: build_shannon
run: |
mkdir cmake_build && cd cmake_build
git config --global --add safe.directory /__w/ShannonBase/ShannonBase
git fetch --tags -f
cmake ../ \
-DWITH_BOOST=/usr/local/boost/include \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_INSTALL_PREFIX=/home/shannon-bin/ \
-DMYSQL_DATADIR=/home/shannon-bin/ \
-DSYSCONFDIR=/home/shannon-bin/ \
-DMYSQL_UNIX_ADDR=/home/shannon-bin/temp/mysql.sock \
-DWITH_EMBEDDED_SERVER=OFF \
-DWITH_MYISAM_STORAGE_ENGINE=1 \
-DWITH_INNOBASE_STORAGE_ENGINE=1 \
-DWITH_PARTITION_STORAGE_ENGINE=1 \
-DMYSQL_TCP_PORT=3306 \
-DENABLED_LOCAL_INFILE=1 \
-DWITH_LIBEVENT=bundled \
-DEXTRA_CHARSETS=all \
-DWITH_PROTOBUF=bundled \
-DWITH_SSL_PATH=/usr/include/openssl/ \
-DDEFAULT_SET=community \
-DWITH_UNIT_TESTS=OFF \
-DCOMPILATION_COMMENT="MySQL Community Server, and Shannon Data AI Alpha V.- (GPL)" && make -j5 && \
sed -i '61s/^/#/' cmake_install.cmake && sed -i '62s/^/#/' cmake_install.cmake && sed -i '63s/^/#/' cmake_install.cmake && \
sudo make install
- name: run_mtr_test
run: |
mkdir -p /home/shannon-bin/log
sudo chown -R $USER:$USER /home/shannon-bin/
cd /home/shannon-bin/mysql-test/
sudo chmod -R u+rwx mysql-test-run.pl
./mysql-test-run.pl --suite=main,innodb,binlog,binlog_gtid,binlog_nogtid,secondary_engine,rpl,rpl_gtid,rpl_nogtid \
--mysqld=--user=$USER --mysqld=--default-storage-engine=innodb --nowarnings --force --nocheck-testcases --retry=0 --parallel=10
23 changes: 14 additions & 9 deletions .github/workflows/pull-requests.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@ env:
defaults:
run:
shell: bash


jobs:
lint:
Expand Down Expand Up @@ -63,14 +62,17 @@ jobs:
steps:
- uses: actions/checkout@v3
- name: install_lib
run: sudo apt update -y && sudo apt upgrade -y && sudo apt install -y libssl-dev libncurses-dev libudev-dev g++ libbison-dev flex clang-format lcov pkg-config cmake zlib1g-dev wget build-essential libldap-dev libisl-dev libmpfr-dev patchelf libevent-dev openssl libssl-dev -y && sudo apt install libsasl2-dev -y
run: |
sudo apt update -y && sudo apt upgrade -y && sudo apt install -y libssl-dev libncurses-dev libudev-dev g++ libbison-dev flex clang-format lcov pkg-config \
cmake zlib1g-dev wget build-essential libldap-dev libisl-dev libmpfr-dev patchelf libevent-dev openssl libssl-dev -y && sudo apt install libsasl2-dev -y
- name: install_gcc
run: sudo apt install bison texinfo libgmp3-dev zlib1g-dev -y && sudo apt install gcc-10 g++-10 -y && sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 && sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 100
#- name: install_cmake
# run: cd /tmp && wget https://github.com/Kitware/CMake/releases/download/v3.29.0-rc1/cmake-3.29.0-rc1.tar.gz && tar zxvf cmake-3.29.0-rc1.tar.gz && cd cmake-3.29.0-rc1 &&\
# ./bootstrap --prefix=/usr/local/cmake && make -j$(nproc) && sudo make install
run: |
sudo apt install bison texinfo libgmp3-dev zlib1g-dev -y && sudo apt install gcc-10 g++-10 -y && \
sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 && sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 100
- name: install_boost
run: cd /tmp && wget https://boostorg.jfrog.io/artifactory/main/release/1.77.0/source/boost_1_77_0.tar.gz && tar zxvf boost_1_77_0.tar.gz && cd boost_1_77_0 && ./bootstrap.sh --prefix=/usr/local/boost && ./b2 -j$(nproc) && sudo ./b2 install
run: |
cd /tmp && wget https://boostorg.jfrog.io/artifactory/main/release/1.77.0/source/boost_1_77_0.tar.gz && \
tar zxvf boost_1_77_0.tar.gz && cd boost_1_77_0 && ./bootstrap.sh --prefix=/usr/local/boost && ./b2 -j$(nproc) && sudo ./b2 install
#sed -i '61s/^/#/' cmake_install.cmake to disable can not find openssl_executeable error. removes it when it fixed.
- name: build_shannon
run: |
Expand All @@ -96,11 +98,14 @@ jobs:
-DWITH_SSL_PATH=/usr/include/openssl/ \
-DDEFAULT_SET=community \
-DWITH_UNIT_TESTS=OFF \
-DCOMPILATION_COMMENT="MySQL Community Server, and Shannon Data AI Alpha V.- (GPL)" && make -j5 && sed -i '61s/^/#/' cmake_install.cmake && sed -i '62s/^/#/' cmake_install.cmake && sed -i '63s/^/#/' cmake_install.cmake && sudo make install
-DCOMPILATION_COMMENT="MySQL Community Server, and Shannon Data AI Alpha V.- (GPL)" && make -j5 && \
sed -i '61s/^/#/' cmake_install.cmake && sed -i '62s/^/#/' cmake_install.cmake && sed -i '63s/^/#/' cmake_install.cmake && \
sudo make install
- name: run_mtr_test
run: |
mkdir -p /home/shannon-bin/log
sudo chown -R $USER:$USER /home/shannon-bin/
cd /home/shannon-bin/mysql-test/
sudo chmod -R u+rwx mysql-test-run.pl
./mysql-test-run.pl --suite=main,innodb,secondary_engine --mysqld=--user=$USER --mysqld=--default-storage-engine=innodb --nowarnings --force --nocheck-testcases --retry=0 --parallel=10
./mysql-test-run.pl --suite=main,innodb,binlog,binlog_gtid,binlog_nogtid,secondary_engine,rpl,rpl_gtid,rpl_nogtid \
--mysqld=--user=$USER --mysqld=--default-storage-engine=innodb --nowarnings --force --nocheck-testcases --retry=0 --parallel=10
92 changes: 92 additions & 0 deletions .github/workflows/weekly.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
#To run all mtrs every week.
name: weekly

on:
push:
tags:
- "*v*"
schedule:
- cron: '0 1 * * 6' # every week at saturday 1 AM

concurrency:
group: weekly
cancel-in-progress: true

defaults:
run:
shell: bash

jobs:
lint:
name: lint
# if: ${{ contains(github.event.pull_request.labels.*.name, 'ready-for-testing') && github.event.pull_request.merged != true }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
ref: ${{ github.event.pull_request.base.sha }}
- uses: actions/checkout@v3
with:
clean: false
# - name: Check License Header
# uses: apache/skywalking-eyes/header@main
- name: Ensure clang-format is available
run: |
command -v clang-format-14 > /dev/null || (sudo apt-get update && sudo apt-get install -y clang-format-14)
- name: Format check
run: |
git diff -U0 --no-color ${{ github.event.pull_request.base.sha }} HEAD storage/rapid_engine ml | /usr/share/clang/clang-format-14/clang-format-diff.py -p1 | tee /tmp/.clang-format-diff
[ -s /tmp/.clang-format-diff ] && exit 1 || true

build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: install_lib
run: |
sudo apt update -y && sudo apt upgrade -y && sudo apt install -y libssl-dev libncurses-dev libudev-dev g++ libbison-dev flex clang-format lcov pkg-config \
cmake zlib1g-dev wget build-essential libldap-dev libisl-dev libmpfr-dev patchelf libevent-dev openssl libssl-dev -y && sudo apt install libsasl2-dev -y
- name: install_gcc
run: |
sudo apt install bison texinfo libgmp3-dev zlib1g-dev -y && sudo apt install gcc-10 g++-10 -y && \
sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 && sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 100
- name: install_boost
run: |
cd /tmp && wget https://boostorg.jfrog.io/artifactory/main/release/1.77.0/source/boost_1_77_0.tar.gz && \
tar zxvf boost_1_77_0.tar.gz && cd boost_1_77_0 && ./bootstrap.sh --prefix=/usr/local/boost && ./b2 -j$(nproc) && sudo ./b2 install
#sed -i '61s/^/#/' cmake_install.cmake to disable can not find openssl_executeable error. removes it when it fixed.
- name: build_shannon
run: |
mkdir cmake_build && cd cmake_build
git config --global --add safe.directory /__w/ShannonBase/ShannonBase
git fetch --tags -f
cmake ../ \
-DWITH_BOOST=/usr/local/boost/include \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_INSTALL_PREFIX=/home/shannon-bin/ \
-DMYSQL_DATADIR=/home/shannon-bin/ \
-DSYSCONFDIR=/home/shannon-bin/ \
-DMYSQL_UNIX_ADDR=/home/shannon-bin/temp/mysql.sock \
-DWITH_EMBEDDED_SERVER=OFF \
-DWITH_MYISAM_STORAGE_ENGINE=1 \
-DWITH_INNOBASE_STORAGE_ENGINE=1 \
-DWITH_PARTITION_STORAGE_ENGINE=1 \
-DMYSQL_TCP_PORT=3306 \
-DENABLED_LOCAL_INFILE=1 \
-DWITH_LIBEVENT=bundled \
-DEXTRA_CHARSETS=all \
-DWITH_PROTOBUF=bundled \
-DWITH_SSL_PATH=/usr/include/openssl/ \
-DDEFAULT_SET=community \
-DWITH_UNIT_TESTS=OFF \
-DCOMPILATION_COMMENT="MySQL Community Server, and Shannon Data AI Alpha V.- (GPL)" && make -j5 && \
sed -i '61s/^/#/' cmake_install.cmake && sed -i '62s/^/#/' cmake_install.cmake && sed -i '63s/^/#/' cmake_install.cmake && \
sudo make install
- name: run_mtr_test
run: |
mkdir -p /home/shannon-bin/log
sudo chown -R $USER:$USER /home/shannon-bin/
cd /home/shannon-bin/mysql-test/
sudo chmod -R u+rwx mysql-test-run.pl
./mysql-test-run.pl --big-test --mysqld=--user=$USER --mysqld=--default-storage-engine=innodb --nowarnings --force \
--nocheck-testcases --retry=0 --parallel=10
Loading
Loading