Skip to main content
`Ctrl`+`K`

# Layernorm

MIOpen: Layernorm
Enumerations | Functions
Layernorm

## Enumerations

enum | miopenNormMode_t { MIOPEN_ELEMENTWISE_AFFINE = 0 , MIOPEN_WEIGHT_BIAS , MIOPEN_ELEMENTWISE_AFFINE_FUSED_ADD , MIOPEN_WEIGHT_BIAS_FUSED_ADD = 3 , MIOPEN_ELEMENTWISE_AFFINE_T5 , MIOPEN_WEIGHT_BIAS_T5 = 5 }

## Functions

miopenStatus_t

miopenLayerNormForward (miopenHandle_t handle, miopenNormMode_t mode, const miopenTensorDescriptor_t xDesc, const void *x, const miopenTensorDescriptor_t weightDesc, const void *weight, const miopenTensorDescriptor_t biasDesc, const void *bias, const float epsilon, const int32_t normalized_dim, const miopenTensorDescriptor_t yDesc, void *y, const miopenTensorDescriptor_t meanDesc, void *mean, const miopenTensorDescriptor_t rstdDesc, void *rstd)

| Execute a layernorm forward layer.

miopenStatus_t

miopenGetLayerNormBackwardWorkspaceSize (miopenHandle_t handle, miopenNormMode_t mode, const miopenTensorDescriptor_t dyDesc, const miopenTensorDescriptor_t xDesc, const miopenTensorDescriptor_t weightDesc, const miopenTensorDescriptor_t meanDesc, const miopenTensorDescriptor_t rstdDesc, const int32_t normalized_dim, const miopenTensorDescriptor_t dxDesc, const miopenTensorDescriptor_t dwDesc, const miopenTensorDescriptor_t dbDesc, size_t *sizeInBytes)

| Helper function to query the minimum workspace size required by the layernorm backward call.

miopenStatus_t

miopenLayerNormBackward (miopenHandle_t handle, miopenNormMode_t mode, void *workspace, size_t workspaceSizeInBytes, const miopenTensorDescriptor_t dyDesc, const void *dy, const miopenTensorDescriptor_t xDesc, const void *x, const miopenTensorDescriptor_t weightDesc, const void *weight, const miopenTensorDescriptor_t meanDesc, const void *mean, const miopenTensorDescriptor_t rstdDesc, const void *rstd, const int32_t normalized_dim, const miopenTensorDescriptor_t dxDesc, void *dx, const miopenTensorDescriptor_t dwDesc, void *dw, const miopenTensorDescriptor_t dbDesc, void *db)

| Execute a layernorm backward layer.

miopenStatus_t

miopenAddLayerNormForward (miopenHandle_t handle, miopenNormMode_t mode, const miopenTensorDescriptor_t xDesc, const void *x, const miopenTensorDescriptor_t x2Desc, const void *x2, const miopenTensorDescriptor_t weightDesc, const void *weight, const miopenTensorDescriptor_t biasDesc, const void *bias, const float epsilon, const int32_t normalized_dim, const miopenTensorDescriptor_t yDesc, void *y, const miopenTensorDescriptor_t meanDesc, void *mean, const miopenTensorDescriptor_t rstdDesc, void *rstd)

| Execute a add and layernorm forward layer.
miopenStatus_t | miopenT5LayerNormForward (miopenHandle_t handle, miopenNormMode_t mode, const miopenTensorDescriptor_t xDesc, const void *x, const miopenTensorDescriptor_t weightDesc, const void *weight, const float epsilon, const miopenTensorDescriptor_t yDesc, void *y, const miopenTensorDescriptor_t rstdDesc, void *rstd)
| Execute a T5layernorm forward layer.
miopenStatus_t | miopenGetT5LayerNormBackwardWorkspaceSize (miopenHandle_t handle, miopenNormMode_t mode, const miopenTensorDescriptor_t dyDesc, const miopenTensorDescriptor_t xDesc, const miopenTensorDescriptor_t weightDesc, const miopenTensorDescriptor_t rstdDesc, const miopenTensorDescriptor_t dxDesc, const miopenTensorDescriptor_t dwDesc, size_t *sizeInBytes)
| Helper function to query the minimum workspace size required by the T5layernorm backward call.

miopenStatus_t

miopenT5LayerNormBackward (miopenHandle_t handle, miopenNormMode_t mode, void *workspace, size_t workspaceSizeInBytes, const miopenTensorDescriptor_t dyDesc, const void *dy, const miopenTensorDescriptor_t xDesc, const void *x, const miopenTensorDescriptor_t weightDesc, const void *weight, const miopenTensorDescriptor_t rstdDesc, const void *rstd, const miopenTensorDescriptor_t dxDesc, void *dx, const miopenTensorDescriptor_t dwDesc, void *dw)

| Execute a T5layernorm backward layer.

## Detailed Description

## Enumeration Type Documentation

## ◆ miopenNormMode_t

enum miopenNormMode_t

LayerNorm mode

Enumerator
MIOPEN_ELEMENTWISE_AFFINE | initialized to ones for weights and zeros for biases
MIOPEN_WEIGHT_BIAS | learnable weights and biases of the module of shape normalized_shape
MIOPEN_ELEMENTWISE_AFFINE_FUSED_ADD | initialized to ones for weights and zeros for biases in addlayernorm
MIOPEN_WEIGHT_BIAS_FUSED_ADD | learnable weights and biases of the module of shape normalized_shape in addlayernorm
MIOPEN_ELEMENTWISE_AFFINE_T5 | initialized to ones for weights and zeros for biases in t5layernorm
MIOPEN_WEIGHT_BIAS_T5 | learnable weights and biases of the module of shape normalized_shape in t5layernorm

## Function Documentation

## ◆ miopenAddLayerNormForward()

miopenStatus_t miopenAddLayerNormForward | ( | miopenHandle_t | handle,
|  | miopenNormMode_t | mode,
|  | const miopenTensorDescriptor_t | xDesc,
|  | const void * | x,
|  | const miopenTensorDescriptor_t | x2Desc,
|  | const void * | x2,
|  | const miopenTensorDescriptor_t | weightDesc,
|  | const void * | weight,
|  | const miopenTensorDescriptor_t | biasDesc,
|  | const void * | bias,
|  | const float | epsilon,
|  | const int32_t | normalized_dim,
|  | const miopenTensorDescriptor_t | yDesc,
|  | void * | y,
|  | const miopenTensorDescriptor_t | meanDesc,
|  | void * | mean,
|  | const miopenTensorDescriptor_t | rstdDesc,
|  | void * | rstd )

Execute a add and layernorm forward layer.

Parameters

  handle | MIOpen handle (input)
  mode | LayerNorm mode (input)
  xDesc | Tensor descriptor for data input tensor x (input)
  x | Data tensor x (input)
  x2Desc | Tensor descriptor for data input tensor x2 (input)
  x2 | Data tensor x2 (input)
  weightDesc | Tensor descriptor for data input tensor weight (input)
  weight | Data tensor weight (input)
  biasDesc | Tensor descriptor for data input tensor bias (input)
  bias | Data tensor bias (input)
  epsilon | Value to stablize inverse variance calculation (input)
  normalized_dim | Nomalized dimensions in the input array (input)
  yDesc | Tensor descriptor for output data tensor y (input)
  y | Data tensor y (output)
  meanDesc | Tensor descriptor for output data tensor mean (input)
  mean | Data tensor mean (output)
  rstdDesc | Tensor descriptor for output data tensor rstd (input)
  rstd | Data tensor rstd (output)

Returns
  miopenStatus_t

## ◆ miopenGetLayerNormBackwardWorkspaceSize()

miopenStatus_t miopenGetLayerNormBackwardWorkspaceSize | ( | miopenHandle_t | handle,
|  | miopenNormMode_t | mode,
|  | const miopenTensorDescriptor_t | dyDesc,
|  | const miopenTensorDescriptor_t | xDesc,
|  | const miopenTensorDescriptor_t | weightDesc,
|  | const miopenTensorDescriptor_t | meanDesc,
|  | const miopenTensorDescriptor_t | rstdDesc,
|  | const int32_t | normalized_dim,
|  | const miopenTensorDescriptor_t | dxDesc,
|  | const miopenTensorDescriptor_t | dwDesc,
|  | const miopenTensorDescriptor_t | dbDesc,
|  | size_t * | sizeInBytes )

Helper function to query the minimum workspace size required by the layernorm backward call.

Parameters

  handle | MIOpen Handle (input)
  mode | LayerNorm mode (input)
  dyDesc | Tensor descriptor for data input tensor dy (input)
  xDesc | Tensor descriptor for data input tensor x (input)
  weightDesc | Tensor descriptor for data input tensor weight (input)
  meanDesc | Tensor descriptor for data input tensor mean (input)
  rstdDesc | Tensor descriptor for data input tensor rstd (input)
  normalized_dim | Nomalized dimensions in the input array (input)
  dxDesc | Tensor descriptor for output data tensor dx (input)
  dwDesc | Tensor descriptor for output data tensor dw (input)
  dbDesc | Tensor descriptor for output data tensor db (input)
  sizeInBytes | Pointer to data to return the minimum workspace size

Returns
  miopenStatus_t

## ◆ miopenGetT5LayerNormBackwardWorkspaceSize()

miopenStatus_t miopenGetT5LayerNormBackwardWorkspaceSize | ( | miopenHandle_t | handle,
|  | miopenNormMode_t | mode,
|  | const miopenTensorDescriptor_t | dyDesc,
|  | const miopenTensorDescriptor_t | xDesc,
|  | const miopenTensorDescriptor_t | weightDesc,
|  | const miopenTensorDescriptor_t | rstdDesc,
|  | const miopenTensorDescriptor_t | dxDesc,
|  | const miopenTensorDescriptor_t | dwDesc,
|  | size_t * | sizeInBytes )

Helper function to query the minimum workspace size required by the T5layernorm backward call.

Parameters

  handle | MIOpen Handle (input)
  mode | LayerNorm mode (input)
  dyDesc | Tensor descriptor for data input tensor dy (input)
  xDesc | Tensor descriptor for data input tensor x (input)
  weightDesc | Tensor descriptor for data input tensor weight (input)
  rstdDesc | Tensor descriptor for data input tensor rstd (input)
  dxDesc | Tensor descriptor for output data tensor dx (input)
  dwDesc | Tensor descriptor for output data tensor dw (input)
  sizeInBytes | Pointer to data to return the minimum workspace size

Returns
  miopenStatus_t

## ◆ miopenLayerNormBackward()

miopenStatus_t miopenLayerNormBackward | ( | miopenHandle_t | handle,
|  | miopenNormMode_t | mode,
|  | void * | workspace,
|  | size_t | workspaceSizeInBytes,
|  | const miopenTensorDescriptor_t | dyDesc,
|  | const void * | dy,
|  | const miopenTensorDescriptor_t | xDesc,
|  | const void * | x,
|  | const miopenTensorDescriptor_t | weightDesc,
|  | const void * | weight,
|  | const miopenTensorDescriptor_t | meanDesc,
|  | const void * | mean,
|  | const miopenTensorDescriptor_t | rstdDesc,
|  | const void * | rstd,
|  | const int32_t | normalized_dim,
|  | const miopenTensorDescriptor_t | dxDesc,
|  | void * | dx,
|  | const miopenTensorDescriptor_t | dwDesc,
|  | void * | dw,
|  | const miopenTensorDescriptor_t | dbDesc,
|  | void * | db )

Execute a layernorm backward layer.

Parameters

  handle | MIOpen handle (input)
  mode | LayerNorm mode (input)
  workspace | Address of the allocated workspace data (input)
  workspaceSizeInBytes | Size in bytes of the allocated workspace data (input)
  dyDesc | Tensor descriptor for data input tensor dy (input)
  dy | Data tensor dy (input)
  xDesc | Tensor descriptor for input data tensor x (input)
  x | Data tensor x (input)
  weightDesc | Tensor descriptor for data input tensor weight (input)
  weight | Data tensor weight (input)
  meanDesc | Tensor descriptor for input data tensor mean (input)
  mean | Data tensor mean (input)
  rstdDesc | Tensor descriptor for input data tensor rstd (input)
  rstd | Data tensor rstd (input)
  normalized_dim | Nomalized dimensions in the input array (input)
  dxDesc | Tensor descriptor for output data tensor dx (input)
  dx | Data tensor dx (output)
  dwDesc | Tensor descriptor for output data tensor dw (input)
  dw | Data tensor dw (output)
  dbDesc | Tensor descriptor for output data tensor db (input)
  db | Data tensor db (output)

Returns
  miopenStatus_t

## ◆ miopenLayerNormForward()

miopenStatus_t miopenLayerNormForward | ( | miopenHandle_t | handle,
|  | miopenNormMode_t | mode,
|  | const miopenTensorDescriptor_t | xDesc,
|  | const void * | x,
|  | const miopenTensorDescriptor_t | weightDesc,
|  | const void * | weight,
|  | const miopenTensorDescriptor_t | biasDesc,
|  | const void * | bias,
|  | const float | epsilon,
|  | const int32_t | normalized_dim,
|  | const miopenTensorDescriptor_t | yDesc,
|  | void * | y,
|  | const miopenTensorDescriptor_t | meanDesc,
|  | void * | mean,
|  | const miopenTensorDescriptor_t | rstdDesc,
|  | void * | rstd )

Execute a layernorm forward layer.

Parameters

  handle | MIOpen handle (input)
  mode | LayerNorm mode (input)
  xDesc | Tensor descriptor for data input tensor x (input)
  x | Data tensor x (input)
  weightDesc | Tensor descriptor for data input tensor weight (input)
  weight | Data tensor weight (input)
  biasDesc | Tensor descriptor for data input tensor bias (input)
  bias | Data tensor bias (input)
  epsilon | Value to stablize inverse variance calculation (input)
  normalized_dim | Nomalized dimensions in the input array (input)
  yDesc | Tensor descriptor for output data tensor y (input)
  y | Data tensor y (output)
  meanDesc | Tensor descriptor for output data tensor mean (input)
  mean | Data tensor mean (output)
  rstdDesc | Tensor descriptor for output data tensor rstd (input)
  rstd | Data tensor rstd (output)

Returns
  miopenStatus_t

## ◆ miopenT5LayerNormBackward()

miopenStatus_t miopenT5LayerNormBackward | ( | miopenHandle_t | handle,
|  | miopenNormMode_t | mode,
|  | void * | workspace,
|  | size_t | workspaceSizeInBytes,
|  | const miopenTensorDescriptor_t | dyDesc,
|  | const void * | dy,
|  | const miopenTensorDescriptor_t | xDesc,
|  | const void * | x,
|  | const miopenTensorDescriptor_t | weightDesc,
|  | const void * | weight,
|  | const miopenTensorDescriptor_t | rstdDesc,
|  | const void * | rstd,
|  | const miopenTensorDescriptor_t | dxDesc,
|  | void * | dx,
|  | const miopenTensorDescriptor_t | dwDesc,
|  | void * | dw )

Execute a T5layernorm backward layer.

Parameters

  handle | MIOpen handle (input)
  mode | LayerNorm mode (input)
  workspace | Address of the allocated workspace data (input)
  workspaceSizeInBytes | Size in bytes of the allocated workspace data (input)
  dyDesc | Tensor descriptor for data input tensor dy (input)
  dy | Data tensor dy (input)
  xDesc | Tensor descriptor for output data tensor x (input)
  x | Data tensor x (input)
  weightDesc | Tensor descriptor for data input tensor weight (input)
  weight | Data tensor weight (input)
  rstdDesc | Tensor descriptor for output data tensor rstd (input)
  rstd | Data tensor rstd (output)
  dxDesc | Tensor descriptor for output data tensor dx (input)
  dx | Data tensor dx (output)
  dwDesc | Tensor descriptor for output data tensor dw (input)
  dw | Data tensor dw (output)

Returns
  miopenStatus_t

## ◆ miopenT5LayerNormForward()

miopenStatus_t miopenT5LayerNormForward | ( | miopenHandle_t | handle,
|  | miopenNormMode_t | mode,
|  | const miopenTensorDescriptor_t | xDesc,
|  | const void * | x,
|  | const miopenTensorDescriptor_t | weightDesc,
|  | const void * | weight,
|  | const float | epsilon,
|  | const miopenTensorDescriptor_t | yDesc,
|  | void * | y,
|  | const miopenTensorDescriptor_t | rstdDesc,
|  | void * | rstd )

Execute a T5layernorm forward layer.

Parameters

  handle | MIOpen handle (input)
  mode | LayerNorm mode (input)
  xDesc | Tensor descriptor for data input tensor x (input)
  x | Data tensor x (input)
  weightDesc | Tensor descriptor for data input tensor weight (input)
  weight | Data tensor weight (input)
  epsilon | Value to stablize inverse variance calculation (input)
  yDesc | Tensor descriptor for output data tensor y (input)
  y | Data tensor y (output)
  rstdDesc | Tensor descriptor for output data tensor rstd (input)
  rstd | Data tensor rstd (output)

Returns
  miopenStatus_t