criterion-1.6.3.0: Robust, reliable performance measurement and analysis

Copyright | (c) 2009-2014 Bryan O'Sullivan
License | BSD-style
Maintainer | bos@serpentine.com
Stability | experimental
Portability | GHC
Safe Haskell | Trustworthy
Language | Haskell2010

Criterion.Analysis

Description

Analysis code for benchmarks.

Synopsis

- data Outliers = Outliers {
  - samplesSeen :: !Int64
  - lowSevere :: !Int64
  - lowMild :: !Int64
  - highMild :: !Int64
  - highSevere :: !Int64
  }
- data OutlierEffect
  - = Unaffected
  - | Slight
  - | Moderate
  - | Severe
- data OutlierVariance = OutlierVariance {
  - ovEffect :: OutlierEffect
  - ovDesc :: String
  - ovFraction :: Double
  }
- data SampleAnalysis = SampleAnalysis {
  - anRegress :: [Regression]
  - anMean :: Estimate ConfInt Double
  - anStdDev :: Estimate ConfInt Double
  - anOutlierVar :: OutlierVariance
  }
- analyseSample :: Int -> String -> Vector Measured -> ExceptT String Criterion Report
- scale :: Double -> SampleAnalysis -> SampleAnalysis
- analyseMean :: Sample -> Int -> Criterion Double
- countOutliers :: Outliers -> Int64
- classifyOutliers :: Sample -> Outliers
- noteOutliers :: Outliers -> Criterion ()
- outlierVariance :: Estimate ConfInt Double -> Estimate ConfInt Double -> Double -> OutlierVariance
- resolveAccessors :: [String] -> Either String [(String, Measured -> Maybe Double)]
- validateAccessors :: [String] -> String -> Either String [(String, Measured -> Maybe Double)]
- regress :: GenIO -> [String] -> String -> Vector Measured -> ExceptT String Criterion Regression

# Documentation

data Outliers Source #

Outliers from sample data, calculated using the boxplot technique.

Constructors

Outliers |

Fields

- samplesSeen :: !Int64
- lowSevere :: !Int64

  More than 3 times the interquartile range (IQR) below the first quartile.

- lowMild :: !Int64

  Between 1.5 and 3 times the IQR below the first quartile.

- highMild :: !Int64

  Between 1.5 and 3 times the IQR above the third quartile.

- highSevere :: !Int64

  More than 3 times the IQR above the third quartile.

#### Instances

Instances details

FromJSON Outliers Source # |
Instance details Defined in Criterion.Types Methods parseJSON :: Value -> Parser Outliers Source # parseJSONList :: Value -> Parser [Outliers] Source #
ToJSON Outliers Source # |
Instance details Defined in Criterion.Types Methods toJSON :: Outliers -> Value Source # toEncoding :: Outliers -> Encoding Source # toJSONList :: [Outliers] -> Value Source # toEncodingList :: [Outliers] -> Encoding Source #
Data Outliers Source # |

Instance details

Defined in Criterion.Types

Methods

gfoldl :: (forall d b. Data d => c (d -> b) -> d -> c b) -> (forall g. g -> c g) -> Outliers -> c Outliers Source #

gunfold :: (forall b r. Data b => c (b -> r) -> c r) -> (forall r. r -> c r) -> Constr -> c Outliers Source #

toConstr :: Outliers -> Constr Source #

dataTypeOf :: Outliers -> DataType Source #

dataCast1 :: Typeable t => (forall d. Data d => c (t d)) -> Maybe (c Outliers) Source #

dataCast2 :: Typeable t => (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c Outliers) Source #

gmapT :: (forall b. Data b => b -> b) -> Outliers -> Outliers Source #

gmapQl :: (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> Outliers -> r Source #

gmapQr :: forall r r'. (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> Outliers -> r Source #

gmapQ :: (forall d. Data d => d -> u) -> Outliers -> [u] Source #

gmapQi :: Int -> (forall d. Data d => d -> u) -> Outliers -> u Source #

gmapM :: Monad m => (forall d. Data d => d -> m d) -> Outliers -> m Outliers Source #

gmapMp :: MonadPlus m => (forall d. Data d => d -> m d) -> Outliers -> m Outliers Source #

gmapMo :: MonadPlus m => (forall d. Data d => d -> m d) -> Outliers -> m Outliers Source #

Monoid Outliers Source # |
Instance details Defined in Criterion.Types Methods mempty :: Outliers Source # mappend :: Outliers -> Outliers -> Outliers Source # mconcat :: [Outliers] -> Outliers Source #
Semigroup Outliers Source # |
Instance details Defined in Criterion.Types Methods (<>) :: Outliers -> Outliers -> Outliers Source # sconcat :: NonEmpty Outliers -> Outliers Source # stimes :: Integral b => b -> Outliers -> Outliers Source #
Generic Outliers Source # |
Instance details Defined in Criterion.Types Associated Types type Rep Outliers :: Type -> Type Source # Methods from :: Outliers -> Rep Outliers x Source # to :: Rep Outliers x -> Outliers Source #
Read Outliers Source # |
Instance details Defined in Criterion.Types Methods readsPrec :: Int -> ReadS Outliers Source # readList :: ReadS [Outliers] Source # readPrec :: ReadPrec Outliers Source # readListPrec :: ReadPrec [Outliers] Source #
Show Outliers Source # |
Instance details Defined in Criterion.Types Methods showsPrec :: Int -> Outliers -> ShowS Source # show :: Outliers -> String Source # showList :: [Outliers] -> ShowS Source #
Binary Outliers Source # |
Instance details Defined in Criterion.Types Methods put :: Outliers -> Put Source # get :: Get Outliers Source # putList :: [Outliers] -> Put Source #
NFData Outliers Source # |
Instance details Defined in Criterion.Types Methods rnf :: Outliers -> () Source #
Eq Outliers Source # |
Instance details Defined in Criterion.Types Methods (==) :: Outliers -> Outliers -> Bool Source # (/=) :: Outliers -> Outliers -> Bool Source #
type Rep Outliers Source # |

Instance details

Defined in Criterion.Types

type Rep Outliers = D1 ('MetaData "Outliers" "Criterion.Types" "criterion-1.6.3.0-8ep4EcRm2JOH6KXaZVtC3y" 'False) (C1 ('MetaCons "Outliers" 'PrefixI 'True) ((S1 ('MetaSel ('Just "samplesSeen") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 Int64) :*: S1 ('MetaSel ('Just "lowSevere") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 Int64)) :*: (S1 ('MetaSel ('Just "lowMild") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 Int64) :*: (S1 ('MetaSel ('Just "highMild") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 Int64) :*: S1 ('MetaSel ('Just "highSevere") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 Int64)))))

data OutlierEffect Source #

A description of the extent to which outliers in the sample data affect the sample mean and standard deviation.

Constructors

Unaffected | Less than 1% effect.
Slight | Between 1% and 10%.
Moderate | Between 10% and 50%.
Severe | Above 50% (i.e. measurements are useless).

#### Instances

Instances details

FromJSON OutlierEffect Source # |
Instance details Defined in Criterion.Types Methods parseJSON :: Value -> Parser OutlierEffect Source # parseJSONList :: Value -> Parser [OutlierEffect] Source #
ToJSON OutlierEffect Source # |
Instance details Defined in Criterion.Types Methods toJSON :: OutlierEffect -> Value Source # toEncoding :: OutlierEffect -> Encoding Source # toJSONList :: [OutlierEffect] -> Value Source # toEncodingList :: [OutlierEffect] -> Encoding Source #
Data OutlierEffect Source # |

Instance details

Defined in Criterion.Types

Methods

gfoldl :: (forall d b. Data d => c (d -> b) -> d -> c b) -> (forall g. g -> c g) -> OutlierEffect -> c OutlierEffect Source #

gunfold :: (forall b r. Data b => c (b -> r) -> c r) -> (forall r. r -> c r) -> Constr -> c OutlierEffect Source #

toConstr :: OutlierEffect -> Constr Source #

dataTypeOf :: OutlierEffect -> DataType Source #

dataCast1 :: Typeable t => (forall d. Data d => c (t d)) -> Maybe (c OutlierEffect) Source #

dataCast2 :: Typeable t => (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c OutlierEffect) Source #

gmapT :: (forall b. Data b => b -> b) -> OutlierEffect -> OutlierEffect Source #

gmapQl :: (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> OutlierEffect -> r Source #

gmapQr :: forall r r'. (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> OutlierEffect -> r Source #

gmapQ :: (forall d. Data d => d -> u) -> OutlierEffect -> [u] Source #

gmapQi :: Int -> (forall d. Data d => d -> u) -> OutlierEffect -> u Source #

gmapM :: Monad m => (forall d. Data d => d -> m d) -> OutlierEffect -> m OutlierEffect Source #

gmapMp :: MonadPlus m => (forall d. Data d => d -> m d) -> OutlierEffect -> m OutlierEffect Source #

gmapMo :: MonadPlus m => (forall d. Data d => d -> m d) -> OutlierEffect -> m OutlierEffect Source #

Generic OutlierEffect Source # |
Instance details Defined in Criterion.Types Associated Types type Rep OutlierEffect :: Type -> Type Source # Methods from :: OutlierEffect -> Rep OutlierEffect x Source # to :: Rep OutlierEffect x -> OutlierEffect Source #
Read OutlierEffect Source # |
Instance details Defined in Criterion.Types Methods readsPrec :: Int -> ReadS OutlierEffect Source # readList :: ReadS [OutlierEffect] Source # readPrec :: ReadPrec OutlierEffect Source # readListPrec :: ReadPrec [OutlierEffect] Source #
Show OutlierEffect Source # |
Instance details Defined in Criterion.Types Methods showsPrec :: Int -> OutlierEffect -> ShowS Source # show :: OutlierEffect -> String Source # showList :: [OutlierEffect] -> ShowS Source #
Binary OutlierEffect Source # |
Instance details Defined in Criterion.Types Methods put :: OutlierEffect -> Put Source # get :: Get OutlierEffect Source # putList :: [OutlierEffect] -> Put Source #
NFData OutlierEffect Source # |
Instance details Defined in Criterion.Types Methods rnf :: OutlierEffect -> () Source #
Eq OutlierEffect Source # |
Instance details Defined in Criterion.Types Methods (==) :: OutlierEffect -> OutlierEffect -> Bool Source # (/=) :: OutlierEffect -> OutlierEffect -> Bool Source #
Ord OutlierEffect Source # |

Instance details

Defined in Criterion.Types

Methods

compare :: OutlierEffect -> OutlierEffect -> Ordering Source #

(<) :: OutlierEffect -> OutlierEffect -> Bool Source #

(<=) :: OutlierEffect -> OutlierEffect -> Bool Source #

(>) :: OutlierEffect -> OutlierEffect -> Bool Source #

(>=) :: OutlierEffect -> OutlierEffect -> Bool Source #

max :: OutlierEffect -> OutlierEffect -> OutlierEffect Source #

min :: OutlierEffect -> OutlierEffect -> OutlierEffect Source #

type Rep OutlierEffect Source # |

Instance details

Defined in Criterion.Types

type Rep OutlierEffect = D1 ('MetaData "OutlierEffect" "Criterion.Types" "criterion-1.6.3.0-8ep4EcRm2JOH6KXaZVtC3y" 'False) ((C1 ('MetaCons "Unaffected" 'PrefixI 'False) (U1 :: Type -> Type) :+: C1 ('MetaCons "Slight" 'PrefixI 'False) (U1 :: Type -> Type)) :+: (C1 ('MetaCons "Moderate" 'PrefixI 'False) (U1 :: Type -> Type) :+: C1 ('MetaCons "Severe" 'PrefixI 'False) (U1 :: Type -> Type)))

data OutlierVariance Source #

Analysis of the extent to which outliers in a sample affect its standard deviation (and to some extent, its mean).

Constructors

OutlierVariance |

Fields

- ovEffect :: OutlierEffect

  Qualitative description of effect.

- ovDesc :: String

  Brief textual description of effect.

- ovFraction :: Double

  Quantitative description of effect (a fraction between 0 and 1).

#### Instances

Instances details

FromJSON OutlierVariance Source # |
Instance details Defined in Criterion.Types Methods parseJSON :: Value -> Parser OutlierVariance Source # parseJSONList :: Value -> Parser [OutlierVariance] Source #
ToJSON OutlierVariance Source # |
Instance details Defined in Criterion.Types Methods toJSON :: OutlierVariance -> Value Source # toEncoding :: OutlierVariance -> Encoding Source # toJSONList :: [OutlierVariance] -> Value Source # toEncodingList :: [OutlierVariance] -> Encoding Source #
Data OutlierVariance Source # |

Instance details

Defined in Criterion.Types

Methods

gfoldl :: (forall d b. Data d => c (d -> b) -> d -> c b) -> (forall g. g -> c g) -> OutlierVariance -> c OutlierVariance Source #

gunfold :: (forall b r. Data b => c (b -> r) -> c r) -> (forall r. r -> c r) -> Constr -> c OutlierVariance Source #

toConstr :: OutlierVariance -> Constr Source #

dataTypeOf :: OutlierVariance -> DataType Source #

dataCast1 :: Typeable t => (forall d. Data d => c (t d)) -> Maybe (c OutlierVariance) Source #

dataCast2 :: Typeable t => (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c OutlierVariance) Source #

gmapT :: (forall b. Data b => b -> b) -> OutlierVariance -> OutlierVariance Source #

gmapQl :: (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> OutlierVariance -> r Source #

gmapQr :: forall r r'. (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> OutlierVariance -> r Source #

gmapQ :: (forall d. Data d => d -> u) -> OutlierVariance -> [u] Source #

gmapQi :: Int -> (forall d. Data d => d -> u) -> OutlierVariance -> u Source #

gmapM :: Monad m => (forall d. Data d => d -> m d) -> OutlierVariance -> m OutlierVariance Source #

gmapMp :: MonadPlus m => (forall d. Data d => d -> m d) -> OutlierVariance -> m OutlierVariance Source #

gmapMo :: MonadPlus m => (forall d. Data d => d -> m d) -> OutlierVariance -> m OutlierVariance Source #

Generic OutlierVariance Source # |
Instance details Defined in Criterion.Types Associated Types type Rep OutlierVariance :: Type -> Type Source # Methods from :: OutlierVariance -> Rep OutlierVariance x Source # to :: Rep OutlierVariance x -> OutlierVariance Source #
Read OutlierVariance Source # |
Instance details Defined in Criterion.Types Methods readsPrec :: Int -> ReadS OutlierVariance Source # readList :: ReadS [OutlierVariance] Source # readPrec :: ReadPrec OutlierVariance Source # readListPrec :: ReadPrec [OutlierVariance] Source #
Show OutlierVariance Source # |
Instance details Defined in Criterion.Types Methods showsPrec :: Int -> OutlierVariance -> ShowS Source # show :: OutlierVariance -> String Source # showList :: [OutlierVariance] -> ShowS Source #
Binary OutlierVariance Source # |
Instance details Defined in Criterion.Types Methods put :: OutlierVariance -> Put Source # get :: Get OutlierVariance Source # putList :: [OutlierVariance] -> Put Source #
NFData OutlierVariance Source # |
Instance details Defined in Criterion.Types Methods rnf :: OutlierVariance -> () Source #
Eq OutlierVariance Source # |
Instance details Defined in Criterion.Types Methods (==) :: OutlierVariance -> OutlierVariance -> Bool Source # (/=) :: OutlierVariance -> OutlierVariance -> Bool Source #
type Rep OutlierVariance Source # |

Instance details

Defined in Criterion.Types

type Rep OutlierVariance = D1 ('MetaData "OutlierVariance" "Criterion.Types" "criterion-1.6.3.0-8ep4EcRm2JOH6KXaZVtC3y" 'False) (C1 ('MetaCons "OutlierVariance" 'PrefixI 'True) (S1 ('MetaSel ('Just "ovEffect") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 OutlierEffect) :*: (S1 ('MetaSel ('Just "ovDesc") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 String) :*: S1 ('MetaSel ('Just "ovFraction") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 Double))))

data SampleAnalysis Source #

Result of a bootstrap analysis of a non-parametric sample.

Constructors

SampleAnalysis |

Fields

- anRegress :: [Regression]

  Estimates calculated via linear regression.

- anMean :: Estimate ConfInt Double

  Estimated mean.

- anStdDev :: Estimate ConfInt Double

  Estimated standard deviation.

- anOutlierVar :: OutlierVariance

  Description of the effects of outliers on the estimated variance.

#### Instances

Instances details

FromJSON SampleAnalysis Source # |
Instance details Defined in Criterion.Types Methods parseJSON :: Value -> Parser SampleAnalysis Source # parseJSONList :: Value -> Parser [SampleAnalysis] Source #
ToJSON SampleAnalysis Source # |
Instance details Defined in Criterion.Types Methods toJSON :: SampleAnalysis -> Value Source # toEncoding :: SampleAnalysis -> Encoding Source # toJSONList :: [SampleAnalysis] -> Value Source # toEncodingList :: [SampleAnalysis] -> Encoding Source #
Generic SampleAnalysis Source # |
Instance details Defined in Criterion.Types Associated Types type Rep SampleAnalysis :: Type -> Type Source # Methods from :: SampleAnalysis -> Rep SampleAnalysis x Source # to :: Rep SampleAnalysis x -> SampleAnalysis Source #
Read SampleAnalysis Source # |
Instance details Defined in Criterion.Types Methods readsPrec :: Int -> ReadS SampleAnalysis Source # readList :: ReadS [SampleAnalysis] Source # readPrec :: ReadPrec SampleAnalysis Source # readListPrec :: ReadPrec [SampleAnalysis] Source #
Show SampleAnalysis Source # |
Instance details Defined in Criterion.Types Methods showsPrec :: Int -> SampleAnalysis -> ShowS Source # show :: SampleAnalysis -> String Source # showList :: [SampleAnalysis] -> ShowS Source #
Binary SampleAnalysis Source # |
Instance details Defined in Criterion.Types Methods put :: SampleAnalysis -> Put Source # get :: Get SampleAnalysis Source # putList :: [SampleAnalysis] -> Put Source #
NFData SampleAnalysis Source # |
Instance details Defined in Criterion.Types Methods rnf :: SampleAnalysis -> () Source #
Eq SampleAnalysis Source # |
Instance details Defined in Criterion.Types Methods (==) :: SampleAnalysis -> SampleAnalysis -> Bool Source # (/=) :: SampleAnalysis -> SampleAnalysis -> Bool Source #
type Rep SampleAnalysis Source # |

Instance details

Defined in Criterion.Types

type Rep SampleAnalysis = D1 ('MetaData "SampleAnalysis" "Criterion.Types" "criterion-1.6.3.0-8ep4EcRm2JOH6KXaZVtC3y" 'False) (C1 ('MetaCons "SampleAnalysis" 'PrefixI 'True) ((S1 ('MetaSel ('Just "anRegress") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 [Regression]) :*: S1 ('MetaSel ('Just "anMean") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 (Estimate ConfInt Double))) :*: (S1 ('MetaSel ('Just "anStdDev") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 (Estimate ConfInt Double)) :*: S1 ('MetaSel ('Just "anOutlierVar") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 OutlierVariance))))

analyseSample Source #

Arguments

:: Int | Experiment number.
-> String | Experiment name.
-> Vector Measured | Sample data.
-> ExceptT String Criterion Report |

Perform an analysis of a measurement.

scale Source #

Arguments

:: Double | Value to multiply by.
-> SampleAnalysis |
-> SampleAnalysis |

Multiply the `Estimate`s in an analysis by the given value, using `scale`.

analyseMean Source #

Arguments

:: Sample |
-> Int | Number of iterations used to compute the sample.
-> Criterion Double |

Display the mean of a `Sample`, and characterise the outliers present in the sample.

countOutliers :: Outliers -> Int64 Source #

Count the total number of outliers in a sample.

classifyOutliers :: Sample -> Outliers Source #

Classify outliers in a data set, using the boxplot technique.

noteOutliers :: Outliers -> Criterion () Source #

Display a report of the `Outliers` present in a `Sample`.

outlierVariance Source #

Arguments

:: Estimate ConfInt Double | Bootstrap estimate of sample mean.
-> Estimate ConfInt Double | Bootstrap estimate of sample standard deviation.
-> Double | Number of original iterations.
-> OutlierVariance |

Compute the extent to which outliers in the sample data affect the sample mean and standard deviation.

resolveAccessors :: [String] -> Either String [(String, Measured -> Maybe Double)] Source #

Given a list of accessor names (see `measureKeys`), return either a mapping from accessor name to function or an error message if any names are wrong.

validateAccessors Source #

Arguments

:: [String] | Predictor names.
-> String | Responder name.
-> Either String [(String, Measured -> Maybe Double)] |

Given predictor and responder names, do some basic validation, then hand back the relevant accessors.

regress Source #

Arguments

:: GenIO |
-> [String] | Predictor names.
-> String | Responder name.
-> Vector Measured |
-> ExceptT String Criterion Regression |

Regress the given predictors against the responder.

Errors may be returned under various circumstances, such as invalid names or lack of needed data.

See `olsRegress` for details of the regression performed.