Skip to content

Commit

Permalink
Merge pull request #271 from abs-tudelft/arrow-3.0
Browse files Browse the repository at this point in the history
Migrate Arrow to 3.0.0
  • Loading branch information
johanpel authored Feb 3, 2021
2 parents a0c620b + 1f2284b commit dbe3b3a
Show file tree
Hide file tree
Showing 17 changed files with 119 additions and 75 deletions.
3 changes: 2 additions & 1 deletion .github/workflows/assets.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ on:

env:
CMAKE_VERSION: '3.19.2'
ARROW_VERSION: '1.0.1'
ARROW_VERSION: '3.0.0'

jobs:
archive:
Expand Down Expand Up @@ -172,6 +172,7 @@ jobs:
- 'cp36-cp36m'
- 'cp37-cp37m'
- 'cp38-cp38'
- 'cp39-cp39'
steps:
- uses: actions/checkout@v2
with:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/documentation.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ jobs:
name: Book
runs-on: ubuntu-latest
env:
MDBOOK_VERSION: '0.4.1'
MDBOOK_VERSION: '0.4.6'
steps:
- uses: actions/checkout@v2
- name: Install mdbook
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/examples.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ jobs:
container: quay.io/pypa/manylinux2014_x86_64:latest
env:
CMAKE_VERSION: '3.17.3'
ARROW_VERSION: '1.0.1'
CPYTHON_VERSION: 'cp35-cp35m'
ARROW_VERSION: '3.0.0'
CPYTHON_VERSION: 'cp36-cp36m'
steps:
- uses: actions/checkout@v2
- name: Install Apache Arrow
Expand All @@ -40,7 +40,7 @@ jobs:
runs-on: ubuntu-latest
needs: prepare
env:
PYTHON_VERSION: '3.5.x'
PYTHON_VERSION: '3.6.x'
strategy:
fail-fast: false
matrix:
Expand Down
17 changes: 11 additions & 6 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ on:
pull_request:

env:
ARROW_VERSION: '1.0.1'
ARROW_VERSION: '3.0.0'

jobs:
cpp:
Expand Down Expand Up @@ -60,16 +60,21 @@ jobs:
sudo apt-get install -y ./apache-arrow-archive-keyring-latest-$(lsb_release --codename --short).deb
sudo apt-get update
sudo apt-get install -y libarrow-dev=$ARROW_VERSION-1 libarrow-python-dev=$ARROW_VERSION-1
- uses: actions/setup-python@v2
with:
python-version: '3.x'
- name: Install pyarrow
run: pip3 install pyarrow==$ARROW_VERSION setuptools wheel
run: |
python -m pip install --upgrade pip setuptools wheel
python -m pip install pyarrow==$ARROW_VERSION
- name: Build and install
working-directory: ${{ matrix.source }}
run: |
python3 setup.py build
python3 setup.py bdist_wheel
pip3 install build/dist/*.whl
python setup.py build
python setup.py bdist_wheel
python -m pip install build/dist/*.whl
- name: Import
run: python3 -c "import ${{ matrix.package }}"
run: python -c "import ${{ matrix.package }}"

vhdl:
name: VHDL
Expand Down
12 changes: 7 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,7 @@ generates the following:
- Structs
- Validity bitmaps

- Once the Arrow reference implementation and format specific reaches ensured
stability (i.e. version 1.0), we would like to support:
- In the future we would like to support:
- Sparse and dense unions
- Dictionaries
- Chunked tabular structures (`Arrow::Table`)
Expand Down Expand Up @@ -128,15 +127,18 @@ External projects using Fletcher:
- [Posit PairHMM](https://github.com/lvandam/pairhmm_posit_hdl_arrow)

## Publications

If you used or studied Fletcher, please cite:

- J. Peltenburg, J. van Straten, L. Wijtemans, L. van Leeuwen, Z. Al-Ars, and
H.P. Hofstee, Fletcher: A Framework to Efficiently Integrate FPGA Accelerators
with Apache Arrow*, in 29th International Conference on Field Programmable
with Apache Arrow\*, in 29th International Conference on Field Programmable
Logic and Applications (FPL) (2019) pp. 270–277.

Additional publications:

- J. Peltenburg, J. van Straten, M. Brobbel, H.P. Hofstee, and Z. Al-Ars,
Supporting Columnar In-memory Formats on FPGA: The Hardware Design of Fletcher
for Apache Arrow*, in Applied Reconfigurable Computing, edited by
C. Hochberger, B. Nelson, A. Koch, R. Woods, and P. Diniz (Springer
for Apache Arrow\*, in Applied Reconfigurable Computing, edited by
C. Hochberger, B. Nelson, A. Koch, R. Woods, and P. Diniz (Springer
International Publishing, Cham, 2019) pp. 32–47
4 changes: 2 additions & 2 deletions codegen/cpp/fletchgen/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ project(fletchgen
LANGUAGES CXX
)

find_package(Arrow 1.0 CONFIG REQUIRED)
find_package(Arrow 3.0.0 CONFIG REQUIRED)

include(FindThreads)
include(FetchContent)
Expand Down Expand Up @@ -197,7 +197,7 @@ set(CPACK_DEBIAN_PACKAGE_MAINTAINER "${CPACK_PACKAGE_VENDOR}")
set(CPACK_RPM_FILE_NAME "RPM-DEFAULT")
set(CPACK_RPM_PACKAGE_DESCRIPTION "${CPACK_PACKAGE_DESCRIPTION}")
set(CPACK_RPM_PACKAGE_LICENSE "ASL 2.0")
set(CPACK_RPM_PACKAGE_REQUIRES "arrow-libs >= 1.0.1, arrow-libs < 2.0.0")
set(CPACK_RPM_PACKAGE_REQUIRES "arrow-libs >= 3.0.0, arrow-libs < 4.0.0")

set(CPACK_ARCHIVE_BINARY_FILE_NAME "${CMAKE_PROJECT_NAME}-${fletchgen_VERSION}-${CMAKE_SYSTEM_NAME}")

Expand Down
30 changes: 17 additions & 13 deletions codegen/cpp/fletchgen/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ It currently supports only two top-level platforms.
# Prerequisites

- [C++17 compliant compiler](https://clang.llvm.org/)
- [Apache Arrow 1.0+](https://github.com/apache/arrow)
- [Apache Arrow 3.0+](https://github.com/apache/arrow)
- [CMake 3.14+](https://cmake.org/)

# Build & install
Expand Down Expand Up @@ -95,6 +95,7 @@ your kernel implementation.
| fletcher_tag_width | 1 / 2 / 3 / ... | 1 | Width of the `tag` field of commands and unlock streams of RecordBatchReaders/Writers. Can be used to identify commands. |

# Custom MMIO registers

You can add custom MMIO registers to your kernel using `--reg`.
More information [can be found here](../../../docs/mmio.md).

Expand All @@ -114,32 +115,35 @@ bit.
- record:
name: platform
fields:
- record:
name: complete
fields:
- field:
name: req
width: 1
- field:
name: ack
width: 1
reverse: true
- record:
name: complete
fields:
- field:
name: req
width: 1
- field:
name: ack
width: 1
reverse: true
```
This will result in the following signals appearing at the top-level:
```vhdl
ext_platform_complete_req : out std_logic;
ext_platform_complete_ack : in std_logic
```
* The signals are assumed to be driven by the kernel. To drive them from the top
- The signals are assumed to be driven by the kernel. To drive them from the top
level, use:
```yaml
reverse: true
```
* Fields with a width of 1 can be forced to be `std_logic_vector` instead of
- Fields with a width of 1 can be forced to be `std_logic_vector` instead of
`std_logic` by using:

```yaml
vector: true
```
Expand Down
35 changes: 23 additions & 12 deletions codegen/cpp/fletchgen/test/fletchgen/srec/test_srec.cc
Original file line number Diff line number Diff line change
Expand Up @@ -38,16 +38,22 @@ namespace fletchgen::srec {
*/

TEST(SREC, ToString) {
uint8_t data0[16] = {0x28, 0x5F, 0x24, 0x5F, 0x22, 0x12, 0x22, 0x6A, 0x00, 0x04, 0x24, 0x29, 0x00, 0x08, 0x23, 0x7C};
uint8_t data1[16] = {0x00, 0x02, 0x00, 0x08, 0x00, 0x08, 0x26, 0x29, 0x00, 0x18, 0x53, 0x81, 0x23, 0x41, 0x00, 0x18};
uint8_t data2[16] = {0x41, 0xE9, 0x00, 0x08, 0x4E, 0x42, 0x23, 0x43, 0x00, 0x18, 0x23, 0x42, 0x00, 0x08, 0x24, 0xA9};
uint8_t data0[16] = {0x28, 0x5F, 0x24, 0x5F, 0x22, 0x12, 0x22, 0x6A,
0x00, 0x04, 0x24, 0x29, 0x00, 0x08, 0x23, 0x7C};
uint8_t data1[16] = {0x00, 0x02, 0x00, 0x08, 0x00, 0x08, 0x26, 0x29,
0x00, 0x18, 0x53, 0x81, 0x23, 0x41, 0x00, 0x18};
uint8_t data2[16] = {0x41, 0xE9, 0x00, 0x08, 0x4E, 0x42, 0x23, 0x43,
0x00, 0x18, 0x23, 0x42, 0x00, 0x08, 0x24, 0xA9};
uint8_t data3[4] = {0x00, 0x14, 0x4E, 0xD4};
// Check header record. Default header is similar to example.
ASSERT_EQ(Record::Header().ToString(), "S00600004844521B");
// Check other records
ASSERT_EQ(Record::Data<16>(0x00, data0, 16).ToString(), "S1130000285F245F2212226A000424290008237C2A");
ASSERT_EQ(Record::Data<16>(0x10, data1, 16).ToString(), "S11300100002000800082629001853812341001813");
ASSERT_EQ(Record::Data<16>(0x20, data2, 16).ToString(), "S113002041E900084E42234300182342000824A952");
ASSERT_EQ(Record::Data<16>(0x00, data0, 16).ToString(),
"S1130000285F245F2212226A000424290008237C2A");
ASSERT_EQ(Record::Data<16>(0x10, data1, 16).ToString(),
"S11300100002000800082629001853812341001813");
ASSERT_EQ(Record::Data<16>(0x20, data2, 16).ToString(),
"S113002041E900084E42234300182342000824A952");
ASSERT_EQ(Record::Data<16>(0x30, data3, 4).ToString(), "S107003000144ED492");
}

Expand All @@ -64,9 +70,12 @@ TEST(SREC, FromString) {

TEST(SREC, File) {
// Test a round trip via a file.
uint8_t data[52] = {0x28, 0x5F, 0x24, 0x5F, 0x22, 0x12, 0x22, 0x6A, 0x00, 0x04, 0x24, 0x29, 0x00, 0x08, 0x23, 0x7C,
0x00, 0x02, 0x00, 0x08, 0x00, 0x08, 0x26, 0x29, 0x00, 0x18, 0x53, 0x81, 0x23, 0x41, 0x00, 0x18,
0x41, 0xE9, 0x00, 0x08, 0x4E, 0x42, 0x23, 0x43, 0x00, 0x18, 0x23, 0x42, 0x00, 0x08, 0x24, 0xA9,
uint8_t data[52] = {0x28, 0x5F, 0x24, 0x5F, 0x22, 0x12, 0x22, 0x6A,
0x00, 0x04, 0x24, 0x29, 0x00, 0x08, 0x23, 0x7C,
0x00, 0x02, 0x00, 0x08, 0x00, 0x08, 0x26, 0x29,
0x00, 0x18, 0x53, 0x81, 0x23, 0x41, 0x00, 0x18,
0x41, 0xE9, 0x00, 0x08, 0x4E, 0x42, 0x23, 0x43,
0x00, 0x18, 0x23, 0x42, 0x00, 0x08, 0x24, 0xA9,
0x00, 0x14, 0x4E, 0xD4};
// Check header record. Default header is similar to example.
auto sro = File(0, data, 52);
Expand All @@ -75,7 +84,7 @@ TEST(SREC, File) {
ofs.close();
auto ifs = std::ifstream("srec_file_test.srec");
auto sri = File(&ifs);
uint8_t* result;
uint8_t *result;
size_t size;
sri.ToBuffer(&result, &size);
ASSERT_EQ(memcmp(data, result, 52), 0);
Expand All @@ -86,12 +95,14 @@ TEST(SREC, RecordBatchRoundTrip) {
// Get a recordbatch with some integers
auto rb = fletcher::GetStringRB();
// Open an Arrow FileOutputStream
arrow::Result<std::shared_ptr<arrow::io::OutputStream>> result = arrow::io::FileOutputStream::Open("test.rbf");
arrow::Result<std::shared_ptr<arrow::io::OutputStream>>
result = arrow::io::FileOutputStream::Open("test.rbf");

EXPECT_TRUE(result.ok());
std::shared_ptr<arrow::io::OutputStream> aos = result.ValueOrDie();
// Create a RecordBatchFile writer
arrow::Result<std::shared_ptr<arrow::ipc::RecordBatchWriter>> afw = arrow::ipc::NewFileWriter(aos.get(), rb->schema());
arrow::Result<std::shared_ptr<arrow::ipc::RecordBatchWriter>>
afw = arrow::ipc::MakeFileWriter(aos.get(), rb->schema());
EXPECT_TRUE(afw.ok());

// Write the RecordBatch to the FileOutputStream
Expand Down
4 changes: 2 additions & 2 deletions codegen/python/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,12 +127,12 @@ def initialize_options(self):
install_requires=[
'numpy >= 1.14',
'pandas',
'pyarrow == 1.0.1',
'pyarrow == 3.0.0',
],
setup_requires=[
'cython',
'numpy',
'pyarrow == 1.0.1',
'pyarrow == 3.0.0',
'plumbum'
],
classifiers=[
Expand Down
2 changes: 1 addition & 1 deletion common/cpp/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.14 FATAL_ERROR)

project(fletcher_common VERSION 0.0.0 LANGUAGES CXX)

find_package(Arrow 1.0 CONFIG REQUIRED)
find_package(Arrow 3.0.0 CONFIG REQUIRED)

include(FetchContent)

Expand Down
Loading

0 comments on commit dbe3b3a

Please sign in to comment.