Loading...
Searching...
No Matches
Opm::gpuistl::HypreInterface Namespace Reference

Unified interface for Hypre operations with both CPU and GPU data structures. More...

## Typedefs

using | HostArrays = ::Opm::gpuistl::HypreHostDataArrays
using | DeviceArrays = ::Opm::gpuistl::HypreDeviceDataArrays
using | ParallelInfo = ::Opm::gpuistl::ParallelInfo
using | SparsityPattern = ::Opm::gpuistl::SparsityPattern

## Functions

void | initialize (bool use_gpu_backend)
| Initialize the Hypre library and set memory/execution policy.
HYPRE_Solver | createAMGSolver ()
| Create Hypre solver (BoomerAMG).
void | setSolverParameters (HYPRE_Solver solver, const PropertyTree &prm, bool use_gpu_backend)
| Set solver parameters from property tree.
template<typename CommType>
HYPRE_IJMatrix | createMatrix (HYPRE_Int N, HYPRE_Int dof_offset, const CommType &comm)
| Create Hypre matrix.
template<typename CommType>
HYPRE_IJVector | createVector (HYPRE_Int N, HYPRE_Int dof_offset, const CommType &comm)
| Create Hypre vector.
void | destroySolver (HYPRE_Solver solver)
| Destroy Hypre solver.
void | destroyMatrix (HYPRE_IJMatrix matrix)
| Destroy Hypre matrix.
void | destroyVector (HYPRE_IJVector vector)
| Destroy Hypre vector.
template<typename CommType, typename MatrixType>
ParallelInfo | setupHypreParallelInfo (const CommType &comm, const MatrixType &matrix)
| Setup parallel information for Hypre (automatically detects serial/parallel).
template<typename MatrixType>
SparsityPattern | setupSparsityPattern (const MatrixType &matrix, const ParallelInfo &par_info, bool owner_first)
| Setup sparsity pattern from matrix (automatically detects CPU/GPU type).
template<typename MatrixType>
std::vector< HYPRE_Int > | computeRowIndexes (const MatrixType &matrix, const std::vector< HYPRE_Int > &ncols, const std::vector< int > &local_dune_to_local_hypre, bool owner_first)
| Compute row indexes for HYPRE_IJMatrixSetValues2.
template<typename VectorType>
void | transferVectorToHypre (const VectorType &vec, HYPRE_IJVector hypre_vec, HostArrays &host_arrays, const DeviceArrays &device_arrays, const ParallelInfo &par_info, bool use_gpu_backend)
| Transfer vector to Hypre from any vector type (CPU or GPU).
template<typename VectorType>
void | transferVectorFromHypre (HYPRE_IJVector hypre_vec, VectorType &vec, HostArrays &host_arrays, const DeviceArrays &device_arrays, const ParallelInfo &par_info, bool use_gpu_backend)
| Transfer vector from Hypre to any vector type (CPU or GPU).
template<typename MatrixType>
void | updateMatrixValues (const MatrixType &matrix, HYPRE_IJMatrix hypre_matrix, const SparsityPattern &sparsity_pattern, const HostArrays &host_arrays, const DeviceArrays &device_arrays, bool use_gpu_backend)
| Update matrix values in Hypre.
template<typename VectorType>
void | setContinuousVectorForHypre (const VectorType &v, std::vector< HYPRE_Real > &continuous_vector_values, const std::vector< int > &local_hypre_to_local_dune)
| Extract owned vector values in the order expected by HYPRE.
template<typename VectorType>
void | setDuneVectorFromContinuousVector (VectorType &v, const std::vector< HYPRE_Real > &continuous_vector_values, const std::vector< int > &local_hypre_to_local_dune)
| Distribute HYPRE vector values back to original vector positions.
template<typename VectorType>
void | transferCpuVectorToHypre (const VectorType &cpu_vec, HYPRE_IJVector hypre_vec, HypreHostDataArrays &host_arrays, const HypreDeviceDataArrays &device_arrays, const ParallelInfo &par_info, bool use_gpu_backend)
| Transfer CPU vector to Hypre vector.
template<typename VectorType>
void | transferHypreToCpuVector (HYPRE_IJVector hypre_vec, VectorType &cpu_vec, HypreHostDataArrays &host_arrays, const HypreDeviceDataArrays &device_arrays, const ParallelInfo &par_info, bool use_gpu_backend)
| Transfer Hypre vector to CPU vector.
template<typename MatrixType>
void | updateMatrixFromCpuMatrix (const MatrixType &cpu_matrix, HYPRE_IJMatrix hypre_matrix, const SparsityPattern &sparsity_pattern, const HypreHostDataArrays &host_arrays, const HypreDeviceDataArrays &device_arrays, bool use_gpu_backend)
| Update Hypre matrix from CPU matrix Uses HYPRE_IJMatrixSetValues2 with pre-computed row_indexes, which allows us to use the original CPU matrix data (with potential ghost values) directly.
template<typename VectorType>
void | setContinuousGpuVectorForHypre (const VectorType &v, std::vector< HYPRE_Real > &continuous_vector_values, const std::vector< int > &local_hypre_to_local_dune)
template<typename VectorType>
void | setGpuVectorFromContinuousVector (VectorType &v, const std::vector< HYPRE_Real > &continuous_vector_values, const std::vector< int > &local_hypre_to_local_dune)
template<typename T>
SparsityPattern | setupSparsityPatternFromGpuMatrix (const GpuSparseMatrixWrapper< T > &gpu_matrix, const ParallelInfo &par_info, bool owner_first)
template<typename T>
std::vector< HYPRE_Int > | computeRowIndexesWithMappingGpu (const GpuSparseMatrixWrapper< T > &gpu_matrix, const std::vector< int > &local_dune_to_local_hypre)
ParallelInfo | setupHypreParallelInfoSerial (HYPRE_Int N)
| Setup parallel information for Hypre in serial case.
template<typename CommType, typename MatrixType>
ParallelInfo | setupHypreParallelInfoParallel (const CommType &comm, const MatrixType &matrix)
| Create mappings between Dune and HYPRE indexing for parallel decomposition.
template<typename MatrixType>
SparsityPattern | setupSparsityPatternFromCpuMatrix (const MatrixType &matrix, const ParallelInfo &par_info, bool owner_first)
| Setup sparsity pattern from CPU matrix (BCRSMatrix).
template<typename MatrixType>
std::vector< HYPRE_Int > | computeRowIndexesWithMappingCpu (const MatrixType &matrix, const std::vector< HYPRE_Int > &ncols, const std::vector< int > &local_dune_to_local_hypre, bool owner_first)
template<typename MatrixType>
std::vector< HYPRE_Int > | computeRowIndexesWithMappingCpu (const MatrixType &matrix, const std::vector< int > &local_dune_to_local_hypre)
| Compute row indexes for CPU matrix with ownership mapping.
std::vector< HYPRE_Real > | getMatrixValues (HYPRE_IJMatrix hypre_matrix, const std::vector< HYPRE_Int > &ncols, const std::vector< HYPRE_BigInt > &rows, const std::vector< HYPRE_BigInt > &cols, bool use_gpu_backend=false)
| Get matrix values from Hypre matrix.

## Detailed Description

Unified interface for Hypre operations with both CPU and GPU data structures.

This namespace provides utilities for working with Hypre resources and transferring data between CPU/GPU data structures and Hypre handles. It handles type detection and automatically chooses the most efficient transfer method:

- For GPU types: Zero-copy device-to-device transfers when using GPU backend
- For CPU types: Host-to-device transfers when using GPU backend
- For CPU types with CPU backend: Direct host memory usage

Supports four use cases:

1. Input type is CPU and backend acceleration is CPU
2. Input type is CPU and backend acceleration is GPU
3. Input type is GPU and backend acceleration is GPU
4. Input type is GPU and backend acceleration is CPU

Note
  Error handling: All functions throw HypreError exceptions when Hypre operations fail.
  This is a consolidated version that includes all functionality across multiple files.

## Function Documentation

## computeRowIndexes()

template<typename MatrixType>

std::vector< HYPRE_Int > Opm::gpuistl::HypreInterface::computeRowIndexes | ( | const MatrixType & | matrix,
|  | const std::vector< HYPRE_Int > & | ncols,
|  | const std::vector< int > & | local_dune_to_local_hypre,
|  | bool | owner_first )

Compute row indexes for HYPRE_IJMatrixSetValues2.

Chooses the appropriate row index computation method based on owner_first and matrix type.

- For owner_first=true: Simple prefix sum for contiguous data
- For owner_first=false: Mapping-based approach to handle gaps from ghost rows

Parameters

  matrix | The matrix to analyze
  ncols | Array containing number of columns per row
  local_dune_to_local_hypre | Mapping from Dune indices to Hypre indices
  owner_first | Whether all owned DOFs come first in the ordering

Returns
  Vector containing row indexes for HYPRE_IJMatrixSetValues2

## computeRowIndexesWithMappingCpu()

template<typename MatrixType>

std::vector< HYPRE_Int > Opm::gpuistl::HypreInterface::computeRowIndexesWithMappingCpu | ( | const MatrixType & | matrix,
|  | const std::vector< int > & | local_dune_to_local_hypre )

Compute row indexes for CPU matrix with ownership mapping.

Creates row_indexes that point directly into the FULL matrix data, enabling using the original matrix data without any data copying. This is necessary for some cases where ghost rows can come before all owned rows.

Example with local_dune_to_local_hypre = [-1, 0, -1, 1]: Original matrix data (with gaps for ghost rows): Row 0 (ghost): {_, _} at positions 0-1 (skipped) Row 1 (owned): {1.0, 2.0} at positions 2-3 Row 2 (ghost): {_} at position 4 (skipped) Row 3 (owned): {3.0} at position 5 Resulting row_indexes: {2, 5} This tells HYPRE that Row 0 data starts at position 2, Row 1 data starts at position 5

Parameters

  matrix | Matrix to analyze
  local_dune_to_local_hypre | Mapping from Dune indices to Hypre indices (-1 for ghost)

Returns
  Vector containing row indexes that map to full matrix data positions

## createAMGSolver()

HYPRE_Solver Opm::gpuistl::HypreInterface::createAMGSolver | ( |  | ) |

inline

Create Hypre solver (BoomerAMG).

Create Hypre BoomerAMG solver.

Returns
  HYPRE_Solver The created solver handle

Exceptions

  HypreError | if solver creation fails

## createMatrix()

template<typename CommType>

HYPRE_IJMatrix Opm::gpuistl::HypreInterface::createMatrix | ( | HYPRE_Int | N,
|  | HYPRE_Int | dof_offset,
|  | const CommType & | comm )

Create Hypre matrix.

Parameters

  N | Number of rows/columns
  dof_offset | Offset for the matrix
  comm | The communicator

Returns
  HYPRE_IJMatrix The created matrix handle

Exceptions

  HypreError | if matrix creation fails

## createVector()

template<typename CommType>

HYPRE_IJVector Opm::gpuistl::HypreInterface::createVector | ( | HYPRE_Int | N,
|  | HYPRE_Int | dof_offset,
|  | const CommType & | comm )

Create Hypre vector.

Parameters

  N | Size of the vector
  dof_offset | Offset for the vector
  comm | The communicator

Returns
  HYPRE_IJVector The created vector handle

Exceptions

  HypreError | if vector creation fails

## destroyMatrix()

void Opm::gpuistl::HypreInterface::destroyMatrix | ( | HYPRE_IJMatrix | matrix | ) |

inline

Destroy Hypre matrix.

Parameters

  matrix | The matrix handle to destroy

Exceptions

  HypreError | if matrix destruction fails

## destroySolver()

void Opm::gpuistl::HypreInterface::destroySolver | ( | HYPRE_Solver | solver | ) |

inline

Destroy Hypre solver.

Parameters

  solver | The solver handle to destroy

Exceptions

  HypreError | if solver destruction fails

## destroyVector()

void Opm::gpuistl::HypreInterface::destroyVector | ( | HYPRE_IJVector | vector | ) |

inline

Destroy Hypre vector.

Parameters

  vector | The vector handle to destroy

Exceptions

  HypreError | if vector destruction fails

## getMatrixValues()

std::vector< HYPRE_Real > Opm::gpuistl::HypreInterface::getMatrixValues | ( | HYPRE_IJMatrix | hypre_matrix,
|  | const std::vector< HYPRE_Int > & | ncols,
|  | const std::vector< HYPRE_BigInt > & | rows,
|  | const std::vector< HYPRE_BigInt > & | cols,
|  | bool | use_gpu_backend = false )

inline

Get matrix values from Hypre matrix.

Retrieves the values from a Hypre matrix for verification or debugging purposes.

Parameters

  hypre_matrix | The Hypre matrix to read values from
  ncols | Array containing number of columns per row
  rows | Array containing row indices
  cols | Array containing column indices
  use_gpu_backend | Whether the Hypre backend is using GPU acceleration

Returns
  Vector containing the retrieved matrix values

Exceptions

  HypreError | if the operation fails

Note
  For GPU backend, uses HYPRE_ParCSRMatrixGetRow since HYPRE_IJMatrixGetValues not supported on GPU

## initialize()

void Opm::gpuistl::HypreInterface::initialize | ( | bool | use_gpu_backend | ) |

inline

Initialize the Hypre library and set memory/execution policy.

Parameters

  use_gpu_backend | Whether to use GPU backend acceleration

Exceptions

  HypreError | if initialization fails

## setSolverParameters()

void Opm::gpuistl::HypreInterface::setSolverParameters | ( | HYPRE_Solver | solver,
|  | const PropertyTree & | prm,
|  | bool | use_gpu_backend )

inline

Set solver parameters from property tree.

Parameters

  solver | The solver handle
  prm | Property tree containing configuration parameters
  use_gpu_backend | Whether using GPU backend

Exceptions

  HypreError | if parameter setting fails

## setupHypreParallelInfo()

template<typename CommType, typename MatrixType>

ParallelInfo Opm::gpuistl::HypreInterface::setupHypreParallelInfo | ( | const CommType & | comm,
|  | const MatrixType & | matrix )

Setup parallel information for Hypre (automatically detects serial/parallel).

Parameters

  comm | The communication object (serial or parallel)
  matrix | The matrix to analyze

Returns
  ParallelInfo structure containing mappings and offsets

## setupHypreParallelInfoParallel()

template<typename CommType, typename MatrixType>

ParallelInfo Opm::gpuistl::HypreInterface::setupHypreParallelInfoParallel | ( | const CommType & | comm,
|  | const MatrixType & | matrix )

inline

Create mappings between Dune and HYPRE indexing for parallel decomposition.

This function interfaces between Dune's distributed matrix representation and HYPRE's global indexing requirements. Note that Dune uses local indices with owner/ghost classification, while HYPRE requires globally consistent indices for all DOFs.

## Index System Overview:

Dune Local Indices: Each MPI process has local DOFs indexed 0..N_local-1, where:

- "Owner" DOFs: This process is responsible for their values/updates
- "Ghost" DOFs: Copies of DOFs owned by other processes (for stencil access)
- These can be interleaved or arranged in "owner-first" order where all owned DOFs come first

HYPRE Global Indices: All DOFs across all processes have unique global indices:

- Process 0 owns global indices [0, N_owned_0-1]
- Process 1 owns global indices [N_owned_0, N_owned_0 + N_owned_1-1]
- Process k owns global indices [offset_k, offset_k + N_owned_k-1]

## Index Mappings Created:

1. local_dune_to_local_hypre: Maps Dune local index → HYPRE local index
   - Size: N_local (all DOFs)
   - Value: [0..N_owned-1] for owned DOFs, -1 for ghost DOFs
   - Purpose: Identify owned DOFs and their compact local ordering
2. local_dune_to_global_hypre: Maps Dune local index → HYPRE global index
   - Size: N_local (all DOFs)
   - Value: [offset..offset+N_owned-1] for owned, actual global index for ghost
   - Purpose: Matrix assembly and vector operations with global indexing
3. local_hypre_to_local_dune: Maps HYPRE local index → Dune local index
   - Size: N_owned (owned DOFs only)
   - Purpose: Transfer data from HYPRE back to Dune structures

## Algorithm Steps:

1. Ownership Detection: Scan Dune's index set to identify owner vs ghost DOFs
2. Local Reordering: Create compact [0..N_owned-1] indexing for owned DOFs
3. Owner-First Detection: Determine if all owned DOFs appear before ghost DOFs
4. Global Offset Calculation: Coordinate with other processes to assign global ranges
5. Ghost Communication: Exchange global indices for ghost DOFs

## Example (2 processes, 3 DOFs each):

Process 0: Dune local [0,1,2] → owned=[0,1], ghost=[2 from P1]

- local_dune_to_local_hypre = [0, 1, -1]
- local_dune_to_global_hypre = [0, 1, 2] (after communication)
- local_hypre_to_local_dune = [0, 1]
- N_owned=2, dof_offset=0, owner_first=true

Process 1: Dune local [0,1,2] → owned=[1,2], ghost=[0 from P0]

- local_dune_to_local_hypre = [-1, 0, 1]
- local_dune_to_global_hypre = [0, 2, 3] (after communication)
- local_hypre_to_local_dune = [1, 2]
- N_owned=2, dof_offset=2, owner_first=false

Parameters

  comm | Dune communication object with parallel index set
  matrix | Distributed matrix for consistency checks

Returns
  ParallelInfo with all mappings, offsets, and metadata

Note
  May modify comm if index set holes are detected (rare in OPM)

## setupHypreParallelInfoSerial()

ParallelInfo Opm::gpuistl::HypreInterface::setupHypreParallelInfoSerial | ( | HYPRE_Int | N | ) |

inline

Setup parallel information for Hypre in serial case.

Parameters

  N | Number of rows

Returns
  ParallelInfo structure containing mappings and offsets

## setupSparsityPattern()

template<typename MatrixType>

SparsityPattern Opm::gpuistl::HypreInterface::setupSparsityPattern | ( | const MatrixType & | matrix,
|  | const ParallelInfo & | par_info,
|  | bool | owner_first )

Setup sparsity pattern from matrix (automatically detects CPU/GPU type).

Parameters

  matrix | The matrix to analyze (CPU or GPU type)
  par_info | Parallel information structure
  owner_first | Whether all owned DOFs come first in the ordering

Returns
  Sparsity pattern information

## setupSparsityPatternFromCpuMatrix()

template<typename MatrixType>

SparsityPattern Opm::gpuistl::HypreInterface::setupSparsityPatternFromCpuMatrix | ( | const MatrixType & | matrix,
|  | const ParallelInfo & | par_info,
|  | bool | owner_first )

Setup sparsity pattern from CPU matrix (BCRSMatrix).

Parameters

  matrix | The matrix to analyze
  par_info | Parallel information structure
  owner_first | Whether all owned DOFs come first in the ordering

Returns
  Sparsity pattern information