Loading...
Searching...
No Matches
GpuDILU.hpp
/*
Copyright 2022-2023 SINTEF AS
This file is part of the Open Porous Media project (OPM).
OPM is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OPM is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef OPM_GPUDILU_HPP
#define OPM_GPUDILU_HPP
#include <map>
#include <memory>
#include <opm/grid/utility/SparseTable.hpp>
#include <opm/simulators/linalg/PreconditionerWithUpdate.hpp>
#include <opm/simulators/linalg/gpuistl/GpuBuffer.hpp>
#include <opm/simulators/linalg/gpuistl/GpuSparseMatrixWrapper.hpp>
#include <opm/simulators/linalg/gpuistl/GpuVector.hpp>
#include <opm/simulators/linalg/gpuistl/detail/kernel_enums.hpp>
#include <opm/simulators/linalg/gpuistl/detail/preconditionerKernels/ILU_variants_helper_kernels.hpp>
#include <opm/simulators/linalg/gpuistl/gpu_resources.hpp>
#include <utility>
#include <vector>
namespace Opm::gpuistl
{
template <class CPUMatrixT, class X, class Y, int l = 1>
class GpuDILU : public Dune::PreconditionerWithUpdate<X, Y>
{
public:
using domain_type = X;
using range_type = Y;
using field_type = typename X::field_type;
using GPUMatrix = GpuSparseMatrixWrapper<field_type>;
using FloatMat = GpuSparseMatrixWrapper<float>;
using FloatVec = GpuVector<float>;
using matrix_type = GPUMatrix;
explicit GpuDILU(const GPUMatrix& gpuMatrix,
const CPUMatrixT& cpuMatrix,
bool splitMatrix,
bool tuneKernels,
int mixedPrecisionScheme,
bool reorder);
void pre(X& x, Y& b) override;
void apply(X& v, const Y& d) override;
void post(X& x) override;
Dune::SolverCategory::Category category() const override;
void update() final;
void reorderAndSplitMatrix(int moveThreadBlockSize);
void computeDiagonal(int factorizationThreadBlockSize);
void tuneThreadBlockSizes();
static constexpr bool shouldCallPre()
{
return false;
}
static constexpr bool shouldCallPost()
{
return false;
}
virtual bool hasPerfectUpdate() const override
{
return true;
}
private:
void apply(X& v, const Y& d, int lowerSolveThreadBlockSize, int upperSolveThreadBlockSize);
void update(int moveThreadBlockSize, int factorizationThreadBlockSize);
static constexpr const size_t blocksize_ = CPUMatrixT::block_type::cols;
Opm::SparseTable<size_t> m_levelSets;
std::vector<int> m_reorderedToNatural;
std::vector<int> m_naturalToReordered;
const GPUMatrix& m_gpuMatrix;
std::unique_ptr<GPUMatrix> m_gpuMatrixReordered;
std::unique_ptr<GPUMatrix> m_gpuMatrixReorderedLower;
std::unique_ptr<GPUMatrix> m_gpuMatrixReorderedUpper;
std::unique_ptr<GpuVector<field_type>> m_gpuMatrixReorderedDiag;
std::unique_ptr<FloatMat> m_gpuMatrixReorderedLowerFloat;
std::unique_ptr<FloatMat> m_gpuMatrixReorderedUpperFloat;
std::unique_ptr<FloatVec> m_gpuMatrixReorderedDiagFloat;
std::unique_ptr<FloatVec> m_gpuDInvFloat;
GpuVector<int> m_gpuNaturalToReorder;
GpuVector<int> m_gpuReorderToNatural;
GpuBuffer<size_t> m_gpuLevelSets; // TODO: make this an optional to not spend time on the copy if not needed
GpuVector<field_type> m_gpuDInv;
bool m_splitMatrix;
bool m_tuneThreadBlockSizes;
const MatrixStorageMPScheme m_mixedPrecisionScheme;
int m_upperSolveThreadBlockSize = -1;
int m_lowerSolveThreadBlockSize = -1;
int m_moveThreadBlockSize = -1;
int m_DILUFactorizationThreadBlockSize = -1;
bool m_reorder;
GpuBuffer<size_t> m_diagIdxs;
// Graphs for Apply
std::map<std::pair<field_type*, const field_type*>, GPUGraph> m_apply_graphs;
std::map<std::pair<field_type*, const field_type*>, GPUGraphExec> m_executableGraphs;
// Stream for the DILU operations on the GPU
GPUStream m_stream {};
// Events for synchronization with main stream
GPUEvent m_before {};
GPUEvent m_after {};
};
} // end namespace Opm::gpuistl
#endif
Dune::PreconditionerWithUpdate
Interface class adding the update() method to the preconditioner interface.
Definition PreconditionerWithUpdate.hpp:32
Opm::SparseTable
Definition BlackoilWellModel.hpp:91
Opm::gpuistl::GpuDILU::GpuDILU
GpuDILU(const GPUMatrix &gpuMatrix, const CPUMatrixT &cpuMatrix, bool splitMatrix, bool tuneKernels, int mixedPrecisionScheme, bool reorder)
Constructor.
Definition GpuDILU.cpp:45
Opm::gpuistl::GpuDILU::matrix_type
GPUMatrix matrix_type
The matrix type the preconditioner is for.
Definition GpuDILU.hpp:67
Opm::gpuistl::GpuDILU::apply
void apply(X &v, const Y &d) override
Apply the preconditoner.
Definition GpuDILU.cpp:145
Opm::gpuistl::GpuDILU::range_type
Y range_type
The range type of the preconditioner.
Definition GpuDILU.hpp:58
Opm::gpuistl::GpuDILU::computeDiagonal
void computeDiagonal(int factorizationThreadBlockSize)
Compute the diagonal of the DILU, and update the data of the reordered matrix.
Definition GpuDILU.cpp:401
Opm::gpuistl::GpuDILU::shouldCallPre
static constexpr bool shouldCallPre()
Definition GpuDILU.hpp:108
Opm::gpuistl::GpuDILU::shouldCallPost
static constexpr bool shouldCallPost()
Definition GpuDILU.hpp:114
Opm::gpuistl::GpuDILU::pre
void pre(X &x, Y &b) override
Prepare the preconditioner.
Definition GpuDILU.cpp:139
Opm::gpuistl::GpuDILU::post
void post(X &x) override
Post processing.
Definition GpuDILU.cpp:329
Opm::gpuistl::GpuDILU::domain_type
X domain_type
The domain type of the preconditioner.
Definition GpuDILU.hpp:56
Opm::gpuistl::GpuDILU::category
Dune::SolverCategory::Category category() const override
Category of the preconditioner (see SolverCategory::Category).
Definition GpuDILU.cpp:335
Opm::gpuistl::GpuDILU::reorderAndSplitMatrix
void reorderAndSplitMatrix(int moveThreadBlockSize)
perform matrix splitting and reordering
Definition GpuDILU.cpp:373
Opm::gpuistl::GpuDILU::tuneThreadBlockSizes
void tuneThreadBlockSizes()
function that will experimentally tune the thread block sizes of the important cuda kernels
Definition GpuDILU.cpp:504
Opm::gpuistl::GpuDILU::field_type
typename X::field_type field_type
The field type of the preconditioner.
Definition GpuDILU.hpp:60
Opm::gpuistl::GpuDILU::update
void update() final
Updates the matrix data.
Definition GpuDILU.cpp:342
Opm::gpuistl::GpuSparseMatrixWrapper
The GpuSparseMatrixWrapper Checks CUDA/HIP version and dispatches a version either using the old or t...
Definition GpuSparseMatrixWrapper.hpp:61
Opm::gpuistl::GpuVector
Definition gpu_type_detection.hpp:30