diff --git a/submodules/haskell-ordnub/.gitignore b/submodules/haskell-ordnub/.gitignore new file mode 100644 index 0000000..477a353 --- /dev/null +++ b/submodules/haskell-ordnub/.gitignore @@ -0,0 +1,7 @@ +dist +cabal-dev +*.o +*.hi +*.chi +*.chs.h +.virthualenv diff --git a/submodules/haskell-ordnub/PACKAGES_USING_NUB.txt b/submodules/haskell-ordnub/PACKAGES_USING_NUB.txt new file mode 100644 index 0000000..aac25a9 --- /dev/null +++ b/submodules/haskell-ordnub/PACKAGES_USING_NUB.txt @@ -0,0 +1,622 @@ +accelerate-examples +acid-state +AC-MiniTest +activehs +adaptive-containers +adict +AERN-Net +aeson-schema +afv +Agata +Agda +air +ajhc +alex +alex-meta +alloy +alms +antfarm +archlinux +archlinux-web +arithmoi +atom +atomo +augur +autoproc +AvlTree +barchart +barley +base +BASIC +BerlekampAlgorithm +berp +bff +bidirectionalization-combined +billeksah-forms +billeksah-pane +billeksah-services +bimap +bindings-bfd +bio +Biobase +BiobaseTrainingData +BiobaseXNA +bitmap +bitmaps +bitset +Blobs +BNFC +BNFC-meta +boomerang +buildwrapper +c0parser +Cabal +cabal2doap +cabal2ghci +cabal2nix +cabal-debian +cabal-install +cabal-install-bundle +cabal-install-ghc72 +cabal-install-ghc74 +cabal-rpm +cabal-test +cairo +caldims +cap +cassy +cblrepo +cedict +cgen +cgi-utils +chalkboard +chalmers-lava2000 +Chart +chp +chp-plus +chp-spec +ChristmasTree +chunks +circ +citeproc-hs +classify +classy-prelude +CMCompare +cmdargs +cmdlib +Coadjute +collada-output +combinatorial-problems +computational-algebra +constructive-algebra +containers +copilot-sbv +CPBrainfuck +CPL +cpsa +cqrs-example +Craft3e +creatur +crf-chain1 +cryptocipher +csound-expression +curry-frontend +cyclotomic +DAG-Tournament +darcs +darcs-benchmark +darcs-beta +data-accessor-template +database-study +data-lens-template +data-ordlist +data-store +dclabel-eci11 +ddc-core +ddc-core-llvm +debian +deepseq-th +DefendTheKing +delaunay +denominate +derive +derive-gadt +derive-trie +diagrams-builder +disjoint-set +DisTract +djinn +djinn-th +doc-review +doctest +dot2graphml +download-media-content +DPM +DrHylo +DrIFT-cabalized +DSH +dvda +dynamic-linker-template +Ebnf2ps +Elm +Emping +Encode +enumerable +enumerator +EnumMap +enummapset-th +Etage-Graph +extcore +factory +fault-tree +fay +fay-base +fec +feed +feldspar-compiler +FerryCore +fficxx +fgl +fibon +FileManip +filemanip +filestore +fixplate +flite +FM-SBLEX +foo +formal +forml +fquery +frag +freenect +free-theorems +free-theorems-counterexamples +free-theorems-seq +free-theorems-webui +fsmActions +fst +ftshell +Fungi +funsat +GA +game-probability +gbu +gconf +gdiff-th +GenI +geni-gui +genprog +gf +ghci-haskeline +ghci-ng +ghc-make +ghc-mod +GHood +ginsu +gio +git-annex +gitit +glade +glib +gmap +gnomevfs +gofer-prelude +GoogleSB +gpah +GPipe +Graphalyze +graph-rewriting +GraphSCC +graphviz +Grempa +grid +grm +groundhog-th +gruff-examples +gstreamer +gtk +gtk2hs-buildtools +gtkglext +gtkimageview +gtk-mac-integration +gtksourceview2 +gt-tools +gulcii +hacanon-light +HackMail +hackport +haddock +haddock-leksah +HaLeX +halfs +hans +happindicator +HAppS-State +happstack-state +happy +happy-meta +HaRe +hark +hashable +hashed-storage +hashring +haskdogs +haskell2010 +haskell98 +haskelldb +HaskellForMaths +haskell-in-space +haskell-names +Haskelloids +haskell-src-meta +haskell-type-exts +haskell-updater +hasktags +hat +HaVSA +hawitter +HaXml +Hayoo +hbayes +hbro +hbro-contrib +hcube +hdf +heaps +hermit +hfusion +hgettext +hgom +HHDL +hierarchical-clustering +hiernotify +highlighting-kate +himerge +hinduce-missingh +hint +hipe +hjson-query +hlcm +hledger +hledger-lib +hledger-web +hlint +hly +hmatrix-glpk +hmt +Holumbus-Searchengine +homeomorphic +hoogle +hopfield +hpaste +HPath +hs2bf +Hs2lib +hsc3 +hsc3-auditor +hsc3-graphs +HSFFIG +hsgsom +HSH +HsParrot +hsqml-morris +hssourceinfo +hstyle +hstzaar +HTab +hTensor +http-conduit +htzaar +hws +hx +HXQ +hxt +hxt-relaxng +hxt-xslt +hyahtzee +hylotab +ideas +ideas-math +idris +ige-mac-integration +igraph +imagepaste +imbib +implicit +improve +INblobs +incremental-parser +inflist +instant-generics +intset +iproute +iptables-helpers +ircbot +iterIO +java-reflect +jmacro +json2 +json-b +JSONb +JYU-Utils +kansas-lava +katt +kevin +kibro +KiCS +KiCS-debugger +kit +kmeans-vector +kure +lambdabot +LambdaHack +language-boogie +language-c +language-ecmascript +language-haskell-extract +lazy-csv +ldif +leksah +leksah-server +lens +lenses +Level0 +lhae +lhc +lhs2TeX-hl +libcspm +liboleg +LinearSplit +ListLike +listlike-instances +llsd +llvm-base +llvm-general +lostcities +LRU +L-seed +LslPlus +ltk +Lucu +MagicHaskeller +makedo +make-hard-links +manatee-core +markov +mathgenealogy +matsuri +MazesOfMonad +metamorphic +MetaObject +meta-par +MFlow +Mhailist +mines +minesweeper +MissingH +ml-w +module-management +monadiccp +Monadius +MonadLab +Monaris +monitor +mprover +mps +mucipher +multiset +multiset-comb +Munkres +murder +music-diatonic +music-score +narc +nemesis +netcore +nettle-frp +nettle-netkit +network-dns +newtype-th +n-m +noise +NoSlow +notmuch-haskell +nsis +numeric-prelude +NXT +obj +omega +openid +Operads +optimusprime +packed-dawg +packunused +PageIO +pango +paragon +parconc-examples +parsec1 +parsec2 +parsec3 +parsec +parsec-permutation +parsers +parsimony +pec +pesca +pisigma +plugins +plugins-auto +pointfree +polyseq +poppler +prefix-units +prelude-generalize +prelude-plus +presburger +probability +prolog +prolog-graph-lib +PropLogic +Pugs +pugs-compat +pugs-DrIFT +pushme +QuickCheck +quickcheck-poly +quickcheck-script +quickspec +rabocsv2qif +ralist +Rasenschach +rdf4h +reaction-logic +reactive-banana +regexchar +regex-deriv +regex-dfa +regexdot +regex-parsec +regex-pderiv +regex-tdfa +remote +RepLib +RESTng +rezoom +riot +roguestar-engine +rpc-framework +rsagl-frp +ruler-core +safecopy +sat-micro-hs +SBench +sbv +scc +scenegraph +scion +scion-browser +scope +scyther-proof +Semantique +SG +sgf +shake +shake-extras +shaker +shell-escape +shelltestrunner +shuffle +Shu-thing +sifflet-lib +simgi +simple-css +SimpleEA +sirkel +snap-loader-dynamic +snap-predicates +snm +Snusmumrik +SoccerFun +SourceGraph +sourcemap +specialize-th +sphinx +squeeze +standalone-haddock +statechart +static-resources +Strafunski-Sdf2Haskell +stream-fusion +structural-induction +stylish-haskell +supero +svgcairo +svgutils +svm-simple +swish +syb +syb-with-class +sym +SyntaxMacros +tableaux +tagging +tamarin-prover-theory +TCache +teams +template-default +tempus +term-rewriting +TernaryTrees +test-framework-th-prime +testloop +textmatetags +theoremquest +thih +th-instances +TigerHash +timberc +time-recurrence +timestamp-subprocess-lines +todos +toolshed +Top +topkata +toysolver +tpdb +translatable-intset +traverse-with-class +triangulation +TrieMap +trifecta +turni +TypeIlluminator +type-sub-th +uhc-util +UMM +unbound +unboxed-containers +universe-th +unscramble +urlcheck +utility-ht +uuagc +uuagc-bootstrap +uuagc-cabal +uuid +uvector +vacuum-ubigraph +vector-clock +visual-graphrewrite +vte +wai-handler-devel +webkit +wikipedia4epub +Wired +WordNet +WordNet-ghc74 +wxc +wxcore +Xec +xfconf +xhaskell-library +xml2x +xmlhtml +xmonad +xmonad-bluetilebranch +xmonad-contrib +xmonad-contrib-bluetilebranch +xmonad-extras +Yablog +ycextra +yesod-comments +yesod-core +yhccore +yi +yices-easy +yst +yuiGrid +yuuko +zeroth +zip-archive diff --git a/submodules/haskell-ordnub/README.md b/submodules/haskell-ordnub/README.md new file mode 100644 index 0000000..b42d537 --- /dev/null +++ b/submodules/haskell-ordnub/README.md @@ -0,0 +1,31 @@ +ordnub +====== + +Data.List.nub is O(n²). This one is O(n log n) by requiring an Ord instance. + +* Also contains a benchmark (`report.html`) that shows that `ordNub` apparently is faster than `nub` in *all* cases. + +* `PACKAGES_USING_NUB.txt` contains all packages which use `nub` (made with a quick grep). +It's not the most accurate since some packages define their own `nub`, but that's a minority. + +**This thing here is not a library.** It is a benchmark suite. [View results here](http://htmlpreview.github.io/?https://github.com/nh2/haskell-ordnub/blob/master/report.html). + +Don't use nub +------------- + +If you are looking for a fast `ordNub` function to use in your code, you can use: + +```haskell +import qualified Data.Set as Set + +ordNub :: (Ord a) => [a] -> [a] +ordNub l = go Set.empty l + where + go _ [] = [] + go s (x:xs) = if x `Set.member` s then go s xs + else x : go (Set.insert x s) xs +``` + +### Other Data.List functions you NEVER want to use + +`\\`, `union`, `intersect` diff --git a/submodules/haskell-ordnub/Setup.hs b/submodules/haskell-ordnub/Setup.hs new file mode 100644 index 0000000..9a994af --- /dev/null +++ b/submodules/haskell-ordnub/Setup.hs @@ -0,0 +1,2 @@ +import Distribution.Simple +main = defaultMain diff --git a/submodules/haskell-ordnub/ordnub.cabal b/submodules/haskell-ordnub/ordnub.cabal new file mode 100644 index 0000000..a8b1771 --- /dev/null +++ b/submodules/haskell-ordnub/ordnub.cabal @@ -0,0 +1,17 @@ +name: ordnub +version: 0.1.0.0 +synopsis: Faster nub (remove duplicates) +description: Data.List.nub is O(n²). This one is O(n log n) by requiring an Ord instance. +license: MIT +license-file: LICENSE +author: Niklas Hambuechen +maintainer: Niklas Hambuechen +category: Data +build-type: Simple +cabal-version: >=1.10 + +executable ordnub + main-is: ordnub.hs + build-depends: base, containers, mtl, criterion, QuickCheck + default-language: Haskell98 + ghc-options: -Wall -O2 diff --git a/submodules/haskell-ordnub/ordnub.hs b/submodules/haskell-ordnub/ordnub.hs new file mode 100644 index 0000000..6d22b47 --- /dev/null +++ b/submodules/haskell-ordnub/ordnub.hs @@ -0,0 +1,226 @@ +{-# LANGUAGE ScopedTypeVariables #-} + +module Main where + +import Control.Monad.State.Strict +import qualified Control.Monad.State.Lazy as SL +import Data.Function (on) +import Data.List (nub, nubBy) +import qualified Data.Map as Map +import qualified Data.Set as Set + +import Test.QuickCheck +import Test.QuickCheck.Function +import Criterion.Main + + +-- Just copied from Data.List +localNub :: (Eq a) => [a] -> [a] +localNub l = nub' l [] + where + nub' [] _ = [] + nub' (x:xs) ls + | x `elem` ls = nub' xs ls + | otherwise = x : nub' xs (x:ls) + + +-- Taken From Yi +ordNub :: (Ord a) => [a] -> [a] +ordNub l = go Set.empty l + where + go _ [] = [] + go s (x:xs) = if x `Set.member` s then go s xs + else x : go (Set.insert x s) xs + + +-- Using a state monad +ordNubState :: (Ord a) => [a] -> [a] +ordNubState xs = evalState (filterM f xs) Set.empty + where + f x = do set <- get + if Set.member x set + then return False + else put (Set.insert x set) >> return True + + +-- Using a lazy state monad +ordNubStateLazy :: (Ord a) => [a] -> [a] +ordNubStateLazy xs = SL.evalState (filterM f xs) Set.empty + where + f x = do set <- SL.get + if Set.member x set + then return False + else SL.put (Set.insert x set) >> return True + + +-- Using a state monad with a dlist instead of filterM +ordNubStateDlist :: (Ord a) => [a] -> [a] +ordNubStateDlist l = evalState (f l id) Set.empty + where + f [] dlist = return $ dlist [] + f (x:xs) dlist = do set <- get + if Set.member x set + then f xs dlist + else put (Set.insert x set) >> f xs (dlist . (x:)) + +-- Using a lazy state monad with a dlist instead of filterM +ordNubStateLazyDlist :: (Ord a) => [a] -> [a] +ordNubStateLazyDlist l = SL.evalState (f l id) Set.empty + where + f [] dlist = return $ dlist [] + f (x:xs) dlist = do set <- SL.get + if Set.member x set + then f xs dlist + else SL.put (Set.insert x set) >> f xs (dlist . (x:)) + + +-- When removing duplicates, the first function assigns the input to a bucket, +-- the second function checks whether it is already in the bucket (linear search). +ordNubBy :: (Ord b) => (a -> b) -> (a -> a -> Bool) -> [a] -> [a] +ordNubBy p f l = go Map.empty l + where + go _ [] = [] + go m (x:xs) = let b = p x in case b `Map.lookup` m of + Nothing -> x : go (Map.insert b [x] m) xs + Just bucket + | elem_by f x bucket -> go m xs + | otherwise -> x : go (Map.insert b (x:bucket) m) xs + + -- From the Data.List source code. + elem_by :: (a -> a -> Bool) -> a -> [a] -> Bool + elem_by _ _ [] = False + elem_by eq y (x:xs) = y `eq` x || elem_by eq y xs + + +main :: IO () +main = defaultMain + [ bgroup "simple" + [ bench "nub [1]" $ nf nub [1::Int] + , bench "nub [1..10]" $ nf nub [1..10::Int] + , bench "nub [1..100]" $ nf nub [1..100::Int] + , bench "nub [1..1000]" $ nf nub [1..1000::Int] + , bench "nub (replicate 1000 1)" $ nf nub (replicate 1000 (1::Int)) + + , bench "ordNub [1]" $ nf ordNub [1::Int] + , bench "ordNub [1..10]" $ nf ordNub [1..10::Int] + , bench "ordNub [1..100]" $ nf ordNub [1..100::Int] + , bench "ordNub [1..1000]" $ nf ordNub [1..1000::Int] + , bench "ordNub (replicate 1000 1)" $ nf ordNub (replicate 1000 (1::Int)) + ] + + , bgroup "" + [ bench "benchmarks:" $ nf id 'x' -- just so that I can comment out easily + + -- , bench "1000 nub" $ nf nub l1000 + -- , bench "500 nub" $ nf nub l500 + , bench "100 nub" $ nf nub l100 + , bench "50 nub" $ nf nub l50 + , bench "10 nub" $ nf nub l10 + , bench "5 nub" $ nf nub l5 + , bench "1 nub" $ nf nub l1 + + -- , bench "1000 localNub" $ nf localNub l1000 + -- , bench "500 localNub" $ nf localNub l500 + , bench "100 localNub" $ nf localNub l100 + , bench "50 localNub" $ nf localNub l50 + , bench "10 localNub" $ nf localNub l10 + , bench "5 localNub" $ nf localNub l5 + , bench "1 localNub" $ nf localNub l1 + + -- -- , bench "1000 ordNub" $ nf ordNub l1000 + -- -- , bench "500 ordNub" $ nf ordNub l500 + , bench "100 ordNub" $ nf ordNub l100 + , bench "50 ordNub" $ nf ordNub l50 + , bench "10 ordNub" $ nf ordNub l10 + , bench "5 ordNub" $ nf ordNub l5 + , bench "1 ordNub" $ nf ordNub l1 + + -- -- , bench "1000 ordNubState" $ nf ordNubState l1000 + -- -- , bench "500 ordNubState" $ nf ordNubState l500 + , bench "100 ordNubState" $ nf ordNubState l100 + , bench "50 ordNubState" $ nf ordNubState l50 + , bench "10 ordNubState" $ nf ordNubState l10 + , bench "5 ordNubState" $ nf ordNubState l5 + , bench "1 ordNubState" $ nf ordNubState l1 + + -- , bench "1000 ordNubStateLazy" $ nf ordNubStateLazy l1000 + -- , bench "500 ordNubStateLazy" $ nf ordNubStateLazy l500 + , bench "100 ordNubStateLazy" $ nf ordNubStateLazy l100 + , bench "50 ordNubStateLazy" $ nf ordNubStateLazy l50 + , bench "10 ordNubStateLazy" $ nf ordNubStateLazy l10 + , bench "5 ordNubStateLazy" $ nf ordNubStateLazy l5 + , bench "1 ordNubStateLazy" $ nf ordNubStateLazy l1 + + -- , bench "1000 ordNubStateDlist" $ nf ordNubStateDlist l1000 + -- , bench "500 ordNubStateDlist" $ nf ordNubStateDlist l500 + , bench "100 ordNubStateDlist" $ nf ordNubStateDlist l100 + , bench "50 ordNubStateDlist" $ nf ordNubStateDlist l50 + , bench "10 ordNubStateDlist" $ nf ordNubStateDlist l10 + , bench "5 ordNubStateDlist" $ nf ordNubStateDlist l5 + , bench "1 ordNubStateDlist" $ nf ordNubStateDlist l1 + + -- , bench "1000 ordNubStateLazyDlist" $ nf ordNubStateLazyDlist l1000 + -- , bench "500 ordNubStateLazyDlist" $ nf ordNubStateLazyDlist l500 + , bench "100 ordNubStateLazyDlist" $ nf ordNubStateLazyDlist l100 + , bench "50 ordNubStateLazyDlist" $ nf ordNubStateLazyDlist l50 + , bench "10 ordNubStateLazyDlist" $ nf ordNubStateLazyDlist l10 + , bench "5 ordNubStateLazyDlist" $ nf ordNubStateLazyDlist l5 + , bench "1 ordNubStateLazyDlist" $ nf ordNubStateLazyDlist l1 + + + -- `by` functions + + -- , bench "1000 nubBy" $ nf (nubBy (\a b -> a `quot` 2 == b `quot` 2) (==)) l1000 + -- , bench "500 nubBy" $ nf (nubBy (\a b -> a `quot` 2 == b `quot` 2) (==)) l500 + , bench "100 nubBy" $ nf (nubBy (\a b -> a `quot` 2 == b `quot` 2)) l100 + , bench "50 nubBy" $ nf (nubBy (\a b -> a `quot` 2 == b `quot` 2)) l50 + , bench "10 nubBy" $ nf (nubBy (\a b -> a `quot` 2 == b `quot` 2)) l10 + , bench "5 nubBy" $ nf (nubBy (\a b -> a `quot` 2 == b `quot` 2)) l5 + , bench "1 nubBy" $ nf (nubBy (\a b -> a `quot` 2 == b `quot` 2)) l1 + + -- , bench "1000 ordNubBy" $ nf (ordNubBy (`quot` 2) (==)) l1000 + -- , bench "500 ordNubBy" $ nf (ordNubBy (`quot` 2) (==)) l500 + , bench "100 ordNubBy" $ nf (ordNubBy (`quot` 2) (==)) l100 + , bench "50 ordNubBy" $ nf (ordNubBy (`quot` 2) (==)) l50 + , bench "10 ordNubBy" $ nf (ordNubBy (`quot` 2) (==)) l10 + , bench "5 ordNubBy" $ nf (ordNubBy (`quot` 2) (==)) l5 + , bench "1 ordNubBy" $ nf (ordNubBy (`quot` 2) (==)) l1 + ] + + -- Other benchmarks, and what people contributed + , bgroup "other" + [ bench "nub yitz 1" $ nf nub (2 : replicate 100000 1 ++ [3] :: [Int]) + , bench "ordNub yitz 1" $ nf ordNub (2 : replicate 100000 1 ++ [3] :: [Int]) + + , bench "nub yitz 2" $ nf nub ([3,2,1] ++ take 100000 (cycle [3,2,1]) ++ [4] :: [Int]) + , bench "ordNub yitz 2" $ nf ordNub ([3,2,1] ++ take 100000 (cycle [3,2,1]) ++ [4] :: [Int]) + ] + ] + where + -- l1000 = concat $ replicbate 10 [1..1000::Int] + -- l500 = concat $ replicate 20 [1..500::Int] + l100 = concat $ replicate 100 [1..100::Int] + l50 = concat $ replicate 200 [1..50::Int] + l10 = concat $ replicate 1000 [1..10::Int] + l5 = concat $ replicate 2000 [1..5::Int] + l1 = concat $ replicate 10000 [1::Int] + + +tests :: IO () +tests = mapM_ (quickCheckWith stdArgs{ maxSuccess = 1000, maxSize = 200 }) + [ isLikeNub localNub + , isLikeNub ordNub + , isLikeNub ordNubState + , isLikeNub ordNubStateDlist + + -- ordNubBy tests + , property $ \(l :: [(Int, Int)]) -> ordNubBy fst ((>) `on` snd) l + == nubBy (\(a,b) (x,y) -> a == x && b > y) l + + , property $ \(l :: [(Int, Int)], Fun _ f :: Fun Int (Fun Int Bool)) -> + let fun x y = f x `apply` y + in ordNubBy fst (\(_, b) (_, y) -> b `fun` y) l == + nubBy (\(a,b) (x,y) -> a == x && b `fun` y) l + ] + where + isLikeNub f = property (\l -> nub l == f (l :: [Int])) diff --git a/submodules/haskell-ordnub/report.html b/submodules/haskell-ordnub/report.html new file mode 100644 index 0000000..26da161 --- /dev/null +++ b/submodules/haskell-ordnub/report.html @@ -0,0 +1,3152 @@ + + + + + criterion report + + + + + + + + +
+
+

criterion performance measurements

+ +

overview

+ +

want to understand this report?

+ +
+ +

simple/nub [1]

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time3.757625193349935e-83.781085510074313e-83.8252179579346605e-8
Standard deviation9.85249956184582e-101.6010088690593945e-92.5378612435806385e-9
+ + +

Outlying measurements have moderate + (0.3953501471591051%) + effect on estimated standard deviation.

+ +

simple/nub [1..10]

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time7.480366116845878e-77.503388930791329e-77.541333701968792e-7
Standard deviation1.0213003162528511e-81.4827501677952259e-82.2100629281644402e-8
+ + +

Outlying measurements have moderate + (0.12323783624720008%) + effect on estimated standard deviation.

+ +

simple/nub [1..100]

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time5.691851969311633e-55.7032865243653464e-55.725316401074331e-5
Standard deviation4.4205068072009574e-77.866471429862453e-71.2946311934415994e-6
+ + +

Outlying measurements have slight + (6.597554989398262e-2%) + effect on estimated standard deviation.

+ +

simple/nub [1..1000]

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time5.540790407794217e-35.549774019854764e-35.564467756884794e-3
Standard deviation3.960845337260408e-55.7610584021495294e-58.149383062426018e-5
+ + +

Outlying measurements have no + (9.899999999999956e-3%) + effect on estimated standard deviation.

+ +

simple/nub (replicate 1000 1)

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time1.5848331236445798e-51.589948285887123e-51.5971194213312692e-5
Standard deviation2.3508994276362648e-73.0830437851443363e-74.071045376257791e-7
+ + +

Outlying measurements have moderate + (0.12297715079808672%) + effect on estimated standard deviation.

+ +

simple/ordNub [1]

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time3.6288780400376484e-83.644275932196124e-83.6690771641609886e-8
Standard deviation6.727105378716715e-109.835364729889442e-101.415855108982019e-9
+ + +

Outlying measurements have moderate + (0.20961738012838177%) + effect on estimated standard deviation.

+ +

simple/ordNub [1..10]

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time7.182659840927772e-77.214246351305428e-77.274805399877833e-7
Standard deviation1.3621532054494507e-82.1648928050637593e-83.911072750629427e-8
+ + +

Outlying measurements have moderate + (0.24827059816988464%) + effect on estimated standard deviation.

+ +

simple/ordNub [1..100]

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time1.5264215293267733e-51.5348067614811315e-51.5458932254093546e-5
Standard deviation3.9681073943661674e-74.921714830174253e-76.102519856152593e-7
+ + +

Outlying measurements have moderate + (0.27722118637209103%) + effect on estimated standard deviation.

+ +

simple/ordNub [1..1000]

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time2.6686387739868615e-42.684441952138312e-42.7087924363505513e-4
Standard deviation7.431464659516629e-69.917048464022355e-61.518868452332666e-5
+ + +

Outlying measurements have moderate + (0.33597511599326035%) + effect on estimated standard deviation.

+ +

simple/ordNub (replicate 1000 1)

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time1.0444461048787671e-51.047098511571272e-51.052141064519269e-5
Standard deviation1.0092492975359744e-71.8071012641327875e-72.941454051429137e-7
+ + +

Outlying measurements have moderate + (0.10365231303554566%) + effect on estimated standard deviation.

+ +

benchmarks:

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time1.500860938729657e-81.5057863966836603e-81.515777152091937e-8
Standard deviation1.7861074630884827e-103.443653954534959e-105.908468439214954e-10
+ + +

Outlying measurements have moderate + (0.1613661281101069%) + effect on estimated standard deviation.

+ +

100 nub

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time5.738100855487089e-35.75603946937869e-35.7859705377991e-3
Standard deviation8.006019087342247e-51.1637446272655366e-41.7649661764426245e-4
+ + +

Outlying measurements have moderate + (0.13250781925514807%) + effect on estimated standard deviation.

+ +

50 nub

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time2.999088614123562e-33.0160616327698014e-33.041879980700711e-3
Standard deviation7.786585250891225e-51.0630795035512745e-41.4615793757879908e-4
+ + +

Outlying measurements have moderate + (0.3163039024688868%) + effect on estimated standard deviation.

+ +

10 nub

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time7.391630852905413e-47.417964184967178e-47.459937775818011e-4
Standard deviation1.1864660079701539e-51.6745047394271792e-52.3940789026026208e-5
+ + +

Outlying measurements have moderate + (0.16113835045292196%) + effect on estimated standard deviation.

+ +

5 nub

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time4.3811377111511914e-44.419445616348335e-44.467439276321481e-4
Standard deviation1.718139578252032e-52.1878354147760183e-53.0930905168794936e-5
+ + +

Outlying measurements have moderate + (0.4746919991352466%) + effect on estimated standard deviation.

+ +

1 nub

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time1.6130154319797397e-41.6198527470093088e-41.6323617751261708e-4
Standard deviation3.002781751749168e-64.6072937520403214e-68.063405462664703e-6
+ + +

Outlying measurements have moderate + (0.22887495498119245%) + effect on estimated standard deviation.

+ +

100 localNub

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time5.772554724353057e-35.786661951678495e-35.8093689371521255e-3
Standard deviation6.450410937927353e-58.971144822646501e-51.397679413989703e-4
+ + +

Outlying measurements have slight + (8.473418576531792e-2%) + effect on estimated standard deviation.

+ +

50 localNub

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time2.9874132563049594e-32.9981349398071565e-33.0141590525085728e-3
Standard deviation4.882686881548226e-56.671719673496743e-59.401161583001845e-5
+ + +

Outlying measurements have moderate + (0.15192231780996568%) + effect on estimated standard deviation.

+ +

10 localNub

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time7.415746892181537e-47.449447358337542e-47.513438904968403e-4
Standard deviation1.359937652252373e-52.302782601622472e-53.687822073083338e-5
+ + +

Outlying measurements have moderate + (0.25807962861271205%) + effect on estimated standard deviation.

+ +

5 localNub

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time4.2775150362091757e-44.3087180677490924e-44.3424781385498744e-4
Standard deviation1.4128447145009337e-51.6584199124428354e-52.0709779385456256e-5
+ + +

Outlying measurements have moderate + (0.3555612158994931%) + effect on estimated standard deviation.

+ +

1 localNub

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time1.6026813111233491e-41.608800721320288e-41.6209892089030252e-4
Standard deviation2.524391063355487e-64.246582598750814e-67.849721033493185e-6
+ + +

Outlying measurements have moderate + (0.2001102806249412%) + effect on estimated standard deviation.

+ +

100 ordNub

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time3.1422278263916584e-43.15453499344488e-43.173756299292048e-4
Standard deviation5.589229992185309e-67.786024120209112e-61.1176618261426014e-5
+ + +

Outlying measurements have moderate + (0.1807409498323374%) + effect on estimated standard deviation.

+ +

50 ordNub

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time2.627523490020799e-42.640803404876756e-42.66221736692306e-4
Standard deviation5.8546635201334744e-68.455452589317331e-61.2285411065213186e-5
+ + +

Outlying measurements have moderate + (0.27718217178455157%) + effect on estimated standard deviation.

+ +

10 ordNub

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time1.7958788188019142e-41.8019406111755723e-41.8131254227676734e-4
Standard deviation2.606323060676608e-64.102434135786844e-67.22352144272395e-6
+ + +

Outlying measurements have moderate + (0.16128804585412476%) + effect on estimated standard deviation.

+ +

5 ordNub

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time1.517319529193143e-41.5217016626770298e-41.5291665483886996e-4
Standard deviation1.920158605251481e-62.8693607783864116e-64.3515951047746455e-6
+ + +

Outlying measurements have moderate + (0.11367193060301757%) + effect on estimated standard deviation.

+ +

1 ordNub

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time1.0844672357308723e-41.0938098380519524e-41.108118358777747e-4
Standard deviation4.372713998264023e-65.867577480956698e-69.371904132361243e-6
+ + +

Outlying measurements have severe + (0.5147476153433842%) + effect on estimated standard deviation.

+ +

100 ordNubState

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time2.910477965014676e-32.934813349383573e-32.965774386065702e-3
Standard deviation1.10036919913866e-41.3967710675703423e-42.079387159421413e-4
+ + +

Outlying measurements have moderate + (0.4547620602361708%) + effect on estimated standard deviation.

+ +

50 ordNubState

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time2.67104372276614e-32.686855642932156e-32.7073214937622346e-3
Standard deviation7.566419289598708e-59.209925716021782e-51.1409306206765078e-4
+ + +

Outlying measurements have moderate + (0.30638253099992985%) + effect on estimated standard deviation.

+ +

10 ordNubState

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time2.4405239511902125e-32.4569962908203413e-32.4833844591553014e-3
Standard deviation7.529469826043195e-51.0474415149836955e-41.6852242538112472e-4
+ + +

Outlying measurements have moderate + (0.4048341050829361%) + effect on estimated standard deviation.

+ +

5 ordNubState

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time2.415556757586698e-32.432343809741238e-32.4518106867248814e-3
Standard deviation8.138618057297475e-59.23968421960196e-51.0752203793188458e-4
+ + +

Outlying measurements have moderate + (0.34596667141524823%) + effect on estimated standard deviation.

+ +

1 ordNubState

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time2.352418749469022e-32.367036192553738e-32.384726851123074e-3
Standard deviation7.047731702794914e-58.2149950816592e-59.611514463776044e-5
+ + +

Outlying measurements have moderate + (0.30671641913114456%) + effect on estimated standard deviation.

+ +

100 ordNubStateLazy

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time2.05763801826785e-32.0713518549377715e-32.088229506152372e-3
Standard deviation6.631828454493632e-57.748738649317136e-59.854941815447424e-5
+ + +

Outlying measurements have moderate + (0.3363226536856988%) + effect on estimated standard deviation.

+ +

50 ordNubStateLazy

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time1.9839118410522735e-31.9934295107300083e-32.0083902765686314e-3
Standard deviation4.150828433086119e-56.027109191160883e-58.262017102002458e-5
+ + +

Outlying measurements have moderate + (0.24844552095007752%) + effect on estimated standard deviation.

+ +

10 ordNubStateLazy

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time1.8480680872375768e-31.8608902384216583e-31.8984459330017372e-3
Standard deviation4.295196713867742e-51.0202637303171985e-42.194670468309026e-4
+ + +

Outlying measurements have severe + (0.5248426190791382%) + effect on estimated standard deviation.

+ +

5 ordNubStateLazy

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time1.8216560770447055e-31.829004137652616e-31.8402741838867466e-3
Standard deviation3.273415745552623e-54.61654349603979e-56.881447487213734e-5
+ + +

Outlying measurements have moderate + (0.19018985266866223%) + effect on estimated standard deviation.

+ +

1 ordNubStateLazy

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time1.6614840914184839e-31.6679714609558383e-31.6781590868408483e-3
Standard deviation2.930101092484572e-54.104310566105166e-55.8262001262434546e-5
+ + +

Outlying measurements have moderate + (0.1806841169520227%) + effect on estimated standard deviation.

+ +

100 ordNubStateDlist

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time5.089965955871676e-45.115985370137626e-45.169303711710705e-4
Standard deviation1.0981994621602804e-51.825888176378724e-53.303856853592452e-5
+ + +

Outlying measurements have moderate + (0.31664148627924693%) + effect on estimated standard deviation.

+ +

50 ordNubStateDlist

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time4.1142877164917704e-44.138654095275949e-44.18452582989509e-4
Standard deviation9.761611917691235e-61.6615327946162687e-52.6962813542417068e-5
+ + +

Outlying measurements have moderate + (0.3752548340450915%) + effect on estimated standard deviation.

+ +

10 ordNubStateDlist

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time2.9235797606201635e-42.9467023891500304e-43.000175702799528e-4
Standard deviation8.937127975927884e-61.7149888924019318e-53.4185671617671015e-5
+ + +

Outlying measurements have severe + (0.5550128289258484%) + effect on estimated standard deviation.

+ +

5 ordNubStateDlist

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time2.586543309916224e-42.594236282735237e-42.604559807210334e-4
Standard deviation3.6073717170807936e-64.5530897677941615e-66.934479089326931e-6
+ + +

Outlying measurements have moderate + (0.10386220995164383%) + effect on estimated standard deviation.

+ +

1 ordNubStateDlist

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time2.1045035534493983e-42.1131717717895901e-42.1312711479231006e-4
Standard deviation3.5482492972356083e-66.127676613679356e-61.1439491791216442e-5
+ + +

Outlying measurements have moderate + (0.23838610300962604%) + effect on estimated standard deviation.

+ +

100 ordNubStateLazyDlist

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time5.128208295959564e-45.150349434672133e-45.180700119791757e-4
Standard deviation1.004698437503386e-51.322921706220646e-51.876817326201683e-5
+ + +

Outlying measurements have moderate + (0.1995560755415029%) + effect on estimated standard deviation.

+ +

50 ordNubStateLazyDlist

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time4.0802415433960663e-44.099332911117624e-44.1285749498444295e-4
Standard deviation8.484195172918509e-61.1960589461223399e-51.7196875585770635e-5
+ + +

Outlying measurements have moderate + (0.23852847240126115%) + effect on estimated standard deviation.

+ +

10 ordNubStateLazyDlist

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time2.89872859739181e-42.9130456330668606e-42.94045582237757e-4
Standard deviation6.086866824262018e-69.76755096070446e-61.757593781896333e-5
+ + +

Outlying measurements have moderate + (0.2966443011443589%) + effect on estimated standard deviation.

+ +

5 ordNubStateLazyDlist

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time2.600367613861131e-42.612610407897996e-42.633591242858932e-4
Standard deviation5.324786002142375e-68.04895494856847e-61.3508120510408214e-5
+ + +

Outlying measurements have moderate + (0.25799898809880273%) + effect on estimated standard deviation.

+ +

1 ordNubStateLazyDlist

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time2.0947663100281112e-42.10358779745487e-42.1159706624069563e-4
Standard deviation4.072446392910867e-65.332417675948829e-67.542178270410888e-6
+ + +

Outlying measurements have moderate + (0.19027358077638765%) + effect on estimated standard deviation.

+ +

100 nubBy

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time7.061669676440454e-37.083868830340605e-37.119347899096708e-3
Standard deviation1.0070724724377886e-41.4132774623729254e-42.0101611887256965e-4
+ + +

Outlying measurements have moderate + (0.1323058215857479%) + effect on estimated standard deviation.

+ +

50 nubBy

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time3.750278799670438e-33.758661596911648e-33.7736748148376733e-3
Standard deviation3.447133709213671e-55.603562916078572e-58.502486114515167e-5
+ + +

Outlying measurements have slight + (7.550077255279915e-2%) + effect on estimated standard deviation.

+ +

10 nubBy

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time1.0000764573303354e-31.003062650033583e-31.008109971353163e-3
Standard deviation1.3055170835307036e-51.9434754884315085e-52.874053705698317e-5
+ + +

Outlying measurements have moderate + (0.12296596193509188%) + effect on estimated standard deviation.

+ +

5 nubBy

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time5.179881549336842e-45.206107593038013e-45.247123535929454e-4
Standard deviation1.1650311961638458e-51.654245107265438e-52.3642292091998627e-5
+ + +

Outlying measurements have moderate + (0.2678671549928868%) + effect on estimated standard deviation.

+ +

1 nubBy

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time2.9886147358765204e-43.001527486120661e-43.0331322529663637e-4
Standard deviation4.926483138689462e-69.711617025268069e-61.964195785821559e-5
+ + +

Outlying measurements have moderate + (0.27744786574746233%) + effect on estimated standard deviation.

+ +

100 ordNubBy

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time5.789104597229098e-45.837646619934172e-45.933371679443455e-4
Standard deviation2.022496437967801e-53.359235744342464e-55.701757021531044e-5
+ + +

Outlying measurements have severe + (0.5546692156438254%) + effect on estimated standard deviation.

+ +

50 ordNubBy

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time4.998945689656665e-45.031322932698664e-45.091754095214937e-4
Standard deviation1.33609602085767e-52.193866512457328e-53.316153577327675e-5
+ + +

Outlying measurements have moderate + (0.4148049922676044%) + effect on estimated standard deviation.

+ +

10 ordNubBy

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time4.024606567962717e-44.0557201925354715e-44.095840078933785e-4
Standard deviation1.4754863000906668e-51.809819100267715e-52.152503511812766e-5
+ + +

Outlying measurements have moderate + (0.4248088373248735%) + effect on estimated standard deviation.

+ +

5 ordNubBy

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time3.749435764892649e-43.775798899276803e-43.814130646331858e-4
Standard deviation1.2318042480362338e-51.6105631609113144e-52.161656314609775e-5
+ + +

Outlying measurements have moderate + (0.4048511105704381%) + effect on estimated standard deviation.

+ +

1 ordNubBy

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time3.1464955189575746e-43.160233197485407e-43.187088666235408e-4
Standard deviation5.281495780581269e-69.521072980597007e-61.5696533198717187e-5
+ + +

Outlying measurements have moderate + (0.24836321003836315%) + effect on estimated standard deviation.

+ +

other/nub yitz 1

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time1.6405437876159947e-31.6489337374145788e-31.6618297983581824e-3
Standard deviation3.7520290934458463e-55.250790851010502e-57.444265270738427e-5
+ + +

Outlying measurements have moderate + (0.2679192733790396%) + effect on estimated standard deviation.

+ +

other/ordNub yitz 1

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time1.55090555443118e-31.5563176561767856e-31.5600536753113071e-3
Standard deviation1.6471154068593e-52.275497442988379e-53.094951709247399e-5
+ + +

Outlying measurements have slight + (7.532217699541738e-2%) + effect on estimated standard deviation.

+ +

other/nub yitz 2

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time2.809221594470243e-32.8273175646240513e-32.8464005876953397e-3
Standard deviation8.821591567482597e-59.551410029231446e-51.1296732734157991e-4
+ + +

Outlying measurements have moderate + (0.2968429774855128%) + effect on estimated standard deviation.

+ +

other/ordNub yitz 2

+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
lower boundestimateupper bound
Mean execution time1.3526128221924112e-31.3569663454468049e-31.3621209551269808e-3
Standard deviation2.0649320133914378e-52.4416149890353535e-52.944375693692283e-5
+ + +

Outlying measurements have moderate + (0.10416031817873998%) + effect on estimated standard deviation.

+ + +

understanding this report

+ +

In this report, each function benchmarked by criterion is assigned + a section of its own. In each section, we display two charts, each + with an x axis that represents measured execution time. + These charts are active; if you hover your mouse over data points + and annotations, you will see more details.

+ +
    +
  • The chart on the left is a + kernel + density estimate (also known as a KDE) of time + measurements. This graphs the probability of any given time + measurement occurring. A spike indicates that a measurement of a + particular time occurred; its height indicates how often that + measurement was repeated.
    • + +
  • The chart on the right is the raw data from which the kernel + density estimate is built. Measurements are displayed on + the y axis in the order in which they occurred.
    • +
+ +

Under the charts is a small table displaying the mean and standard + deviation of the measurements. We use a statistical technique + called + the bootstrap + to provide confidence intervals on our estimates of these values. + The bootstrap-derived upper and lower bounds on the mean and + standard deviation let you see how accurate we believe those + estimates to be. (Hover the mouse over the table headers to see + the confidence levels.)

+ +

A noisy benchmarking environment can cause some or many + measurements to fall far from the mean. These outlying + measurements can have a significant inflationary effect on the + estimate of the standard deviation. We calculate and display an + estimate of the extent to which the standard deviation has been + inflated by outliers.

+ + + +
+
+ + +