Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for all heFFTe MPI communication backends #799

Merged
merged 1 commit into from
Dec 9, 2024
Merged

Add support for all heFFTe MPI communication backends #799

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions example/grid_tutorial/10_fft_heffte/heffte_fast_fourier_transform_example.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,9 +104,15 @@ void fastFourierTransformHeffteExample()
*/
Cabana::Grid::Experimental::FastFourierTransformParams params;

// Set communication to use all-to-all MPI communication.
// Set this option to false for point-to-point communication
params.setAllToAll( true );
// Set this option to Cabana::Grid::Experimental::FFTCommPattern::alltoallv
// or true use all-to-all communication with padding
// Set this option to Cabana::Grid::Experimental::FFTCommPattern::p2p or
// false for point-to-point communication
// Set this option to Cabana::Grid::Experimental::FFTCommPattern::alltoall
// for all-to-all communication without padding
// Set this option to Cabana::Grid::Experimental::FFTCommPattern::p2p_plined
// for pipelined point-to-point communication
params.setAlltoAll( Cabana::Grid::Experimental::FFTCommPattern::alltoallv );

// Set data exchange type to use pencil decomposition
// Set this option to false to use slab decomposition
Expand Down
57 changes: 45 additions & 12 deletions grid/src/Cabana_Grid_FastFourierTransform.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,6 @@
#include <Cabana_Grid_Array.hpp>
#include <Cabana_Grid_Types.hpp>

#include <Cabana_Utils.hpp> // FIXME: remove after next release.

#include <Kokkos_Core.hpp>
#include <Kokkos_Profiling_ScopedRegion.hpp>

Expand Down Expand Up @@ -98,13 +96,24 @@ struct is_matching_array<
{
};

/*!
\brief Choices of heFFTe MPI communication patterns
*/
enum class FFTCommPattern : unsigned int
streeve marked this conversation as resolved.
Show resolved Hide resolved
{
alltoallv = 0u,
p2p = 1u,
alltoall = 2u,
p2p_plined = 3u
};

//---------------------------------------------------------------------------//
/*!
\brief Parameters controlling details for fast Fourier transforms.
*/
class FastFourierTransformParams
{
bool alltoall = true;
FFTCommPattern FFTcomm = FFTCommPattern::alltoallv;
bool pencils = true;
bool reorder = true;

Expand All @@ -113,7 +122,18 @@ class FastFourierTransformParams
\brief Set MPI communication strategy.
\param value Use all to all MPI communication.
*/
void setAllToAll( bool value ) { alltoall = value; }
void setAlltoAll( FFTCommPattern value ) { FFTcomm = value; }
/*!
\brief Set MPI communication strategy.
\param value Use all to all MPI communication.
*/
void setAlltoAll( bool value )
{
if ( value )
FFTcomm = FFTCommPattern::alltoallv;
else
FFTcomm = FFTCommPattern::p2p;
}
/*!
\brief Set data exchange type (pencil or slab).
\param value Use pencil (true) or slab (false) decomposition.
Expand All @@ -129,7 +149,7 @@ class FastFourierTransformParams
\brief Get MPI communication strategy.
\return Using AllToAll or not.
*/
bool getAllToAll() const { return alltoall; }
junghans marked this conversation as resolved.
Show resolved Hide resolved
FFTCommPattern getAlltoAll() const { return FFTcomm; }
/*!
\brief Get data exchange type (pencil or slab).
\return Using pencil (true) or slab (false) decomposition.
Expand Down Expand Up @@ -538,12 +558,26 @@ class HeffteFastFourierTransform

heffte::plan_options heffte_params =
heffte::default_options<heffte_backend_type>();
// TODO: use all three heffte options for algorithm
bool alltoall = params.getAllToAll();
if ( alltoall )
heffte_params.algorithm = heffte::reshape_algorithm::alltoallv;
else
auto FFTcomm = params.getAlltoAll();
switch ( FFTcomm )
{
case Cabana::Grid::Experimental::FFTCommPattern::p2p:
heffte_params.algorithm = heffte::reshape_algorithm::p2p;
break;
case Cabana::Grid::Experimental::FFTCommPattern::alltoallv:
heffte_params.algorithm = heffte::reshape_algorithm::alltoallv;
break;
case Cabana::Grid::Experimental::FFTCommPattern::alltoall:
heffte_params.algorithm = heffte::reshape_algorithm::alltoall;
break;
case Cabana::Grid::Experimental::FFTCommPattern::p2p_plined:
heffte_params.algorithm = heffte::reshape_algorithm::p2p_plined;
break;
default:
heffte_params.algorithm = heffte::reshape_algorithm::alltoallv;
break;
}

heffte_params.use_pencils = params.getPencils();
heffte_params.use_reorder = params.getReorder();

Expand Down Expand Up @@ -698,8 +732,7 @@ auto createHeffteFastFourierTransform(
const heffte::plan_options heffte_params =
heffte::default_options<heffte_backend_type>();
FastFourierTransformParams params;
// TODO: set appropriate default for AllToAll
params.setAllToAll( true );
params.setAlltoAll( FFTCommPattern::alltoallv );
params.setPencils( heffte_params.use_pencils );
params.setReorder( heffte_params.use_reorder );

Expand Down
2 changes: 1 addition & 1 deletion grid/unit_test/tstFastFourierTransform.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ void calculateFFT( bool use_default, bool use_params,
Experimental::FastFourierTransformParams params;

// Set MPI communication
params.setAllToAll( true );
params.setAlltoAll( Cabana::Grid::Experimental::FFTCommPattern::alltoallv );

// Set data exchange type (false uses slab decomposition)
params.setPencils( true );
Expand Down
Loading