criterion-1.1.4.0: Robust, reliable performance measurement and analysis

Copyright | (c) 2009-2014 Bryan O'Sullivan
License | BSD-style
Maintainer | bos@serpentine.com
Stability | experimental
Portability | GHC
Safe Haskell | Trustworthy
Language | Haskell98

Criterion.Analysis

Description

Analysis code for benchmarks.

Synopsis

- data Outliers = Outliers {
  - samplesSeen :: !Int64
  - lowSevere :: !Int64
  - lowMild :: !Int64
  - highMild :: !Int64
  - highSevere :: !Int64
  }
- data OutlierEffect
  - = Unaffected
  - | Slight
  - | Moderate
  - | Severe
- data OutlierVariance = OutlierVariance {
  - ovEffect :: OutlierEffect
  - ovDesc :: String
  - ovFraction :: Double
  }
- data SampleAnalysis = SampleAnalysis {
  - anRegress :: [Regression]
  - anOverhead :: Double
  - anMean :: Estimate
  - anStdDev :: Estimate
  - anOutlierVar :: OutlierVariance
  }
- analyseSample :: Int -> String -> Vector Measured -> ExceptT String Criterion Report
- scale :: Double -> SampleAnalysis -> SampleAnalysis
- analyseMean :: Sample -> Int -> Criterion Double
- countOutliers :: Outliers -> Int64
- classifyOutliers :: Sample -> Outliers
- noteOutliers :: Outliers -> Criterion ()
- outlierVariance :: Estimate -> Estimate -> Double -> OutlierVariance
- resolveAccessors :: [String] -> Either String [(String, Measured -> Maybe Double)]
- validateAccessors :: [String] -> String -> Either String [(String, Measured -> Maybe Double)]
- regress :: GenIO -> [String] -> String -> Vector Measured -> ExceptT String Criterion Regression

# Documentation

data Outliers Source #

Outliers from sample data, calculated using the boxplot technique.

Constructors

Outliers |

Fields

- samplesSeen :: !Int64
- lowSevere :: !Int64

  More than 3 times the interquartile range (IQR) below the first quartile.

- lowMild :: !Int64

  Between 1.5 and 3 times the IQR below the first quartile.

- highMild :: !Int64

  Between 1.5 and 3 times the IQR above the third quartile.

- highSevere :: !Int64

  More than 3 times the IQR above the third quartile.

Instances

Eq Outliers Source # |
Methods (==) :: Outliers -> Outliers -> Bool # (/=) :: Outliers -> Outliers -> Bool #
Data Outliers Source # |

Methods

gfoldl :: (forall d b. Data d => c (d -> b) -> d -> c b) -> (forall g. g -> c g) -> Outliers -> c Outliers #

gunfold :: (forall b r. Data b => c (b -> r) -> c r) -> (forall r. r -> c r) -> Constr -> c Outliers #

toConstr :: Outliers -> Constr #

dataTypeOf :: Outliers -> DataType #

dataCast1 :: Typeable (* -> *) t => (forall d. Data d => c (t d)) -> Maybe (c Outliers) #

dataCast2 :: Typeable (* -> * -> *) t => (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c Outliers) #

gmapT :: (forall b. Data b => b -> b) -> Outliers -> Outliers #

gmapQl :: (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> Outliers -> r #

gmapQr :: (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> Outliers -> r #

gmapQ :: (forall d. Data d => d -> u) -> Outliers -> [u] #

gmapQi :: Int -> (forall d. Data d => d -> u) -> Outliers -> u #

gmapM :: Monad m => (forall d. Data d => d -> m d) -> Outliers -> m Outliers #

gmapMp :: MonadPlus m => (forall d. Data d => d -> m d) -> Outliers -> m Outliers #

gmapMo :: MonadPlus m => (forall d. Data d => d -> m d) -> Outliers -> m Outliers #

Read Outliers Source # |
Methods readsPrec :: Int -> ReadS Outliers # readList :: ReadS [Outliers] # readPrec :: ReadPrec Outliers # readListPrec :: ReadPrec [Outliers] #
Show Outliers Source # |
Methods showsPrec :: Int -> Outliers -> ShowS # show :: Outliers -> String # showList :: [Outliers] -> ShowS #
Generic Outliers Source # |
Associated Types type Rep Outliers :: * -> * # Methods from :: Outliers -> Rep Outliers x # to :: Rep Outliers x -> Outliers #
Monoid Outliers Source # |
Methods mempty :: Outliers # mappend :: Outliers -> Outliers -> Outliers # mconcat :: [Outliers] -> Outliers #
ToJSON Outliers Source # |
Methods toJSON :: Outliers -> Value # toEncoding :: Outliers -> Encoding # toJSONList :: [Outliers] -> Value # toEncodingList :: [Outliers] -> Encoding #
FromJSON Outliers Source # |
Methods parseJSON :: Value -> Parser Outliers # parseJSONList :: Value -> Parser [Outliers] #
Binary Outliers Source # |
Methods put :: Outliers -> Put # get :: Get Outliers # putList :: [Outliers] -> Put #
NFData Outliers Source # |
Methods rnf :: Outliers -> () #
type Rep Outliers Source # |

type Rep Outliers = D1 (MetaData "Outliers" "Criterion.Types" "criterion-1.1.4.0-ECM4FYS7gfJ7vtJz8i8x5r" False) (C1 (MetaCons "Outliers" PrefixI True) ((:*:) ((:*:) (S1 (MetaSel (Just Symbol "samplesSeen") NoSourceUnpackedness SourceStrict DecidedUnpack) (Rec0 Int64)) (S1 (MetaSel (Just Symbol "lowSevere") NoSourceUnpackedness SourceStrict DecidedUnpack) (Rec0 Int64))) ((:*:) (S1 (MetaSel (Just Symbol "lowMild") NoSourceUnpackedness SourceStrict DecidedUnpack) (Rec0 Int64)) ((:*:) (S1 (MetaSel (Just Symbol "highMild") NoSourceUnpackedness SourceStrict DecidedUnpack) (Rec0 Int64)) (S1 (MetaSel (Just Symbol "highSevere") NoSourceUnpackedness SourceStrict DecidedUnpack) (Rec0 Int64))))))

data OutlierEffect Source #

A description of the extent to which outliers in the sample data affect the sample mean and standard deviation.

Constructors

Unaffected | Less than 1% effect.
Slight | Between 1% and 10%.
Moderate | Between 10% and 50%.
Severe | Above 50% (i.e. measurements are useless).

Instances

Eq OutlierEffect Source # |
Methods (==) :: OutlierEffect -> OutlierEffect -> Bool # (/=) :: OutlierEffect -> OutlierEffect -> Bool #
Data OutlierEffect Source # |

Methods

gfoldl :: (forall d b. Data d => c (d -> b) -> d -> c b) -> (forall g. g -> c g) -> OutlierEffect -> c OutlierEffect #

gunfold :: (forall b r. Data b => c (b -> r) -> c r) -> (forall r. r -> c r) -> Constr -> c OutlierEffect #

toConstr :: OutlierEffect -> Constr #

dataTypeOf :: OutlierEffect -> DataType #

dataCast1 :: Typeable (* -> *) t => (forall d. Data d => c (t d)) -> Maybe (c OutlierEffect) #

dataCast2 :: Typeable (* -> * -> *) t => (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c OutlierEffect) #

gmapT :: (forall b. Data b => b -> b) -> OutlierEffect -> OutlierEffect #

gmapQl :: (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> OutlierEffect -> r #

gmapQr :: (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> OutlierEffect -> r #

gmapQ :: (forall d. Data d => d -> u) -> OutlierEffect -> [u] #

gmapQi :: Int -> (forall d. Data d => d -> u) -> OutlierEffect -> u #

gmapM :: Monad m => (forall d. Data d => d -> m d) -> OutlierEffect -> m OutlierEffect #

gmapMp :: MonadPlus m => (forall d. Data d => d -> m d) -> OutlierEffect -> m OutlierEffect #

gmapMo :: MonadPlus m => (forall d. Data d => d -> m d) -> OutlierEffect -> m OutlierEffect #

Ord OutlierEffect Source # |
Methods compare :: OutlierEffect -> OutlierEffect -> Ordering # (<) :: OutlierEffect -> OutlierEffect -> Bool # (<=) :: OutlierEffect -> OutlierEffect -> Bool # (>) :: OutlierEffect -> OutlierEffect -> Bool # (>=) :: OutlierEffect -> OutlierEffect -> Bool # max :: OutlierEffect -> OutlierEffect -> OutlierEffect # min :: OutlierEffect -> OutlierEffect -> OutlierEffect #
Read OutlierEffect Source # |
Methods readsPrec :: Int -> ReadS OutlierEffect # readList :: ReadS [OutlierEffect] # readPrec :: ReadPrec OutlierEffect # readListPrec :: ReadPrec [OutlierEffect] #
Show OutlierEffect Source # |
Methods showsPrec :: Int -> OutlierEffect -> ShowS # show :: OutlierEffect -> String # showList :: [OutlierEffect] -> ShowS #
Generic OutlierEffect Source # |
Associated Types type Rep OutlierEffect :: * -> * # Methods from :: OutlierEffect -> Rep OutlierEffect x # to :: Rep OutlierEffect x -> OutlierEffect #
ToJSON OutlierEffect Source # |
Methods toJSON :: OutlierEffect -> Value # toEncoding :: OutlierEffect -> Encoding # toJSONList :: [OutlierEffect] -> Value # toEncodingList :: [OutlierEffect] -> Encoding #
FromJSON OutlierEffect Source # |
Methods parseJSON :: Value -> Parser OutlierEffect # parseJSONList :: Value -> Parser [OutlierEffect] #
Binary OutlierEffect Source # |
Methods put :: OutlierEffect -> Put # get :: Get OutlierEffect # putList :: [OutlierEffect] -> Put #
NFData OutlierEffect Source # |
Methods rnf :: OutlierEffect -> () #
type Rep OutlierEffect Source # |
type Rep OutlierEffect = D1 (MetaData "OutlierEffect" "Criterion.Types" "criterion-1.1.4.0-ECM4FYS7gfJ7vtJz8i8x5r" False) ((:+:) ((:+:) (C1 (MetaCons "Unaffected" PrefixI False) U1) (C1 (MetaCons "Slight" PrefixI False) U1)) ((:+:) (C1 (MetaCons "Moderate" PrefixI False) U1) (C1 (MetaCons "Severe" PrefixI False) U1)))

data OutlierVariance Source #

Analysis of the extent to which outliers in a sample affect its standard deviation (and to some extent, its mean).

Constructors

OutlierVariance |

Fields

- ovEffect :: OutlierEffect

  Qualitative description of effect.

- ovDesc :: String

  Brief textual description of effect.

- ovFraction :: Double

  Quantitative description of effect (a fraction between 0 and 1).

Instances

Eq OutlierVariance Source # |
Methods (==) :: OutlierVariance -> OutlierVariance -> Bool # (/=) :: OutlierVariance -> OutlierVariance -> Bool #
Data OutlierVariance Source # |

Methods

gfoldl :: (forall d b. Data d => c (d -> b) -> d -> c b) -> (forall g. g -> c g) -> OutlierVariance -> c OutlierVariance #

gunfold :: (forall b r. Data b => c (b -> r) -> c r) -> (forall r. r -> c r) -> Constr -> c OutlierVariance #

toConstr :: OutlierVariance -> Constr #

dataTypeOf :: OutlierVariance -> DataType #

dataCast1 :: Typeable (* -> *) t => (forall d. Data d => c (t d)) -> Maybe (c OutlierVariance) #

dataCast2 :: Typeable (* -> * -> *) t => (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c OutlierVariance) #

gmapT :: (forall b. Data b => b -> b) -> OutlierVariance -> OutlierVariance #

gmapQl :: (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> OutlierVariance -> r #

gmapQr :: (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> OutlierVariance -> r #

gmapQ :: (forall d. Data d => d -> u) -> OutlierVariance -> [u] #

gmapQi :: Int -> (forall d. Data d => d -> u) -> OutlierVariance -> u #

gmapM :: Monad m => (forall d. Data d => d -> m d) -> OutlierVariance -> m OutlierVariance #

gmapMp :: MonadPlus m => (forall d. Data d => d -> m d) -> OutlierVariance -> m OutlierVariance #

gmapMo :: MonadPlus m => (forall d. Data d => d -> m d) -> OutlierVariance -> m OutlierVariance #

Read OutlierVariance Source # |
Methods readsPrec :: Int -> ReadS OutlierVariance # readList :: ReadS [OutlierVariance] # readPrec :: ReadPrec OutlierVariance # readListPrec :: ReadPrec [OutlierVariance] #
Show OutlierVariance Source # |
Methods showsPrec :: Int -> OutlierVariance -> ShowS # show :: OutlierVariance -> String # showList :: [OutlierVariance] -> ShowS #
Generic OutlierVariance Source # |
Associated Types type Rep OutlierVariance :: * -> * # Methods from :: OutlierVariance -> Rep OutlierVariance x # to :: Rep OutlierVariance x -> OutlierVariance #
ToJSON OutlierVariance Source # |
Methods toJSON :: OutlierVariance -> Value # toEncoding :: OutlierVariance -> Encoding # toJSONList :: [OutlierVariance] -> Value # toEncodingList :: [OutlierVariance] -> Encoding #
FromJSON OutlierVariance Source # |
Methods parseJSON :: Value -> Parser OutlierVariance # parseJSONList :: Value -> Parser [OutlierVariance] #
Binary OutlierVariance Source # |
Methods put :: OutlierVariance -> Put # get :: Get OutlierVariance # putList :: [OutlierVariance] -> Put #
NFData OutlierVariance Source # |
Methods rnf :: OutlierVariance -> () #
type Rep OutlierVariance Source # |

type Rep OutlierVariance = D1 (MetaData "OutlierVariance" "Criterion.Types" "criterion-1.1.4.0-ECM4FYS7gfJ7vtJz8i8x5r" False) (C1 (MetaCons "OutlierVariance" PrefixI True) ((:*:) (S1 (MetaSel (Just Symbol "ovEffect") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 OutlierEffect)) ((:*:) (S1 (MetaSel (Just Symbol "ovDesc") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 String)) (S1 (MetaSel (Just Symbol "ovFraction") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 Double)))))

data SampleAnalysis Source #

Result of a bootstrap analysis of a non-parametric sample.

Constructors

SampleAnalysis |

Fields

- anRegress :: [Regression]

  Estimates calculated via linear regression.

- anOverhead :: Double

  Estimated measurement overhead, in seconds. Estimation is performed via linear regression.

- anMean :: Estimate

  Estimated mean.

- anStdDev :: Estimate

  Estimated standard deviation.

- anOutlierVar :: OutlierVariance

  Description of the effects of outliers on the estimated variance.

Instances

Eq SampleAnalysis Source # |
Methods (==) :: SampleAnalysis -> SampleAnalysis -> Bool # (/=) :: SampleAnalysis -> SampleAnalysis -> Bool #
Data SampleAnalysis Source # |

Methods

gfoldl :: (forall d b. Data d => c (d -> b) -> d -> c b) -> (forall g. g -> c g) -> SampleAnalysis -> c SampleAnalysis #

gunfold :: (forall b r. Data b => c (b -> r) -> c r) -> (forall r. r -> c r) -> Constr -> c SampleAnalysis #

toConstr :: SampleAnalysis -> Constr #

dataTypeOf :: SampleAnalysis -> DataType #

dataCast1 :: Typeable (* -> *) t => (forall d. Data d => c (t d)) -> Maybe (c SampleAnalysis) #

dataCast2 :: Typeable (* -> * -> *) t => (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c SampleAnalysis) #

gmapT :: (forall b. Data b => b -> b) -> SampleAnalysis -> SampleAnalysis #

gmapQl :: (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> SampleAnalysis -> r #

gmapQr :: (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> SampleAnalysis -> r #

gmapQ :: (forall d. Data d => d -> u) -> SampleAnalysis -> [u] #

gmapQi :: Int -> (forall d. Data d => d -> u) -> SampleAnalysis -> u #

gmapM :: Monad m => (forall d. Data d => d -> m d) -> SampleAnalysis -> m SampleAnalysis #

gmapMp :: MonadPlus m => (forall d. Data d => d -> m d) -> SampleAnalysis -> m SampleAnalysis #

gmapMo :: MonadPlus m => (forall d. Data d => d -> m d) -> SampleAnalysis -> m SampleAnalysis #

Read SampleAnalysis Source # |
Methods readsPrec :: Int -> ReadS SampleAnalysis # readList :: ReadS [SampleAnalysis] # readPrec :: ReadPrec SampleAnalysis # readListPrec :: ReadPrec [SampleAnalysis] #
Show SampleAnalysis Source # |
Methods showsPrec :: Int -> SampleAnalysis -> ShowS # show :: SampleAnalysis -> String # showList :: [SampleAnalysis] -> ShowS #
Generic SampleAnalysis Source # |
Associated Types type Rep SampleAnalysis :: * -> * # Methods from :: SampleAnalysis -> Rep SampleAnalysis x # to :: Rep SampleAnalysis x -> SampleAnalysis #
ToJSON SampleAnalysis Source # |
Methods toJSON :: SampleAnalysis -> Value # toEncoding :: SampleAnalysis -> Encoding # toJSONList :: [SampleAnalysis] -> Value # toEncodingList :: [SampleAnalysis] -> Encoding #
FromJSON SampleAnalysis Source # |
Methods parseJSON :: Value -> Parser SampleAnalysis # parseJSONList :: Value -> Parser [SampleAnalysis] #
Binary SampleAnalysis Source # |
Methods put :: SampleAnalysis -> Put # get :: Get SampleAnalysis # putList :: [SampleAnalysis] -> Put #
NFData SampleAnalysis Source # |
Methods rnf :: SampleAnalysis -> () #
type Rep SampleAnalysis Source # |

type Rep SampleAnalysis = D1 (MetaData "SampleAnalysis" "Criterion.Types" "criterion-1.1.4.0-ECM4FYS7gfJ7vtJz8i8x5r" False) (C1 (MetaCons "SampleAnalysis" PrefixI True) ((:*:) ((:*:) (S1 (MetaSel (Just Symbol "anRegress") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 [Regression])) (S1 (MetaSel (Just Symbol "anOverhead") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 Double))) ((:*:) (S1 (MetaSel (Just Symbol "anMean") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 Estimate)) ((:*:) (S1 (MetaSel (Just Symbol "anStdDev") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 Estimate)) (S1 (MetaSel (Just Symbol "anOutlierVar") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 OutlierVariance))))))

analyseSample Source #

Arguments

:: Int | Experiment number.
-> String | Experiment name.
-> Vector Measured | Sample data.
-> ExceptT String Criterion Report |

Perform an analysis of a measurement.

scale Source #

Arguments

:: Double | Value to multiply by.
-> SampleAnalysis |
-> SampleAnalysis |

Multiply the `Estimate`s in an analysis by the given value, using `scale`.

analyseMean Source #

Arguments

:: Sample |
-> Int | Number of iterations used to compute the sample.
-> Criterion Double |

Display the mean of a `Sample`, and characterise the outliers present in the sample.

countOutliers :: Outliers -> Int64 Source #

Count the total number of outliers in a sample.

classifyOutliers :: Sample -> Outliers Source #

Classify outliers in a data set, using the boxplot technique.

noteOutliers :: Outliers -> Criterion () Source #

Display a report of the `Outliers` present in a `Sample`.

outlierVariance Source #

Arguments

:: Estimate | Bootstrap estimate of sample mean.
-> Estimate | Bootstrap estimate of sample standard deviation.
-> Double | Number of original iterations.
-> OutlierVariance |

Compute the extent to which outliers in the sample data affect the sample mean and standard deviation.

resolveAccessors :: [String] -> Either String [(String, Measured -> Maybe Double)] Source #

Given a list of accessor names (see `measureKeys`), return either a mapping from accessor name to function or an error message if any names are wrong.

validateAccessors Source #

Arguments

:: [String] | Predictor names.
-> String | Responder name.
-> Either String [(String, Measured -> Maybe Double)] |

Given predictor and responder names, do some basic validation, then hand back the relevant accessors.

regress Source #

Arguments

:: GenIO |
-> [String] | Predictor names.
-> String | Responder name.
-> Vector Measured |
-> ExceptT String Criterion Regression |

Regress the given predictors against the responder.

Errors may be returned under various circumstances, such as invalid names or lack of needed data.

See `olsRegress` for details of the regression performed.