Restructure the project

docs/eventlog-performance-analysis.md

@@ -21,18 +21,20 @@ TBD: document the exact limitations and differences.
 
 ## Generating the eventlog
 
+IMPORTANT: The `hperf` eventlog analysis program works only with the patched
+`ghc` executable, if you use it with stock ghc it will not understand the
+eventlog format and will generate errors.
+
 To generate the event log, we need to enable event log at compile time
 (on modern GHCs it is always enabled) and the run the program with
 eventlog enabled at run-time, we use the `-l` rts option to do that.
 
 There are multiple ways of running your program with eventlog enabled at
 run-time:
 
-__GHC Command Line__:
-
 Compiling:
 ```
-ghc Main.hs -rtsopts
+ghc Main.hs -eventlog -rtsopts
 ```
 
 Running:
@@ -42,7 +44,7 @@ Running:
 
 You can bake in the rts options during compilation itself:
 ```
-ghc Main.hs -with-rtsopts=-l
+ghc Main.hs -eventlog -with-rtsopts=-l
 ```
 
 Now you can run without any explicit RTS options:
@@ -62,7 +64,7 @@ to use the `-N1` rts option.
 
 ## Measurement instrumentation
 
-See the example in [examples/traceEventIO.hs](examples/traceEventIO.hs) .
+See the example in [examples/traceEventIO.hs](../examples/traceEventIO.hs) .
 
 Use the `traceEventIO` function to log events. Add an event before and
 after the code block you want to measure. The event message before the block

examples/getRTSStats.hs

@@ -1,8 +1,8 @@
 import Control.Concurrent(threadDelay)
-import Streamly.Metrics.Channel
+import Perf.Sink.Console
     (Channel, newChannel, forkChannelPrinter, benchOnWith)
--- import Streamly.Metrics.Channel (printChannel)
-import Streamly.Metrics.Perf.Type (PerfMetrics)
+-- import Perf.Sink.Console (printChannel)
+import Perf.Internal.Measure.Types (PerfMetrics)
 
 import qualified Streamly.Data.Fold as Fold
 import qualified Streamly.Data.Stream as Stream

haskell-perf.cabal

@@ -134,23 +134,24 @@ library
 
     hs-source-dirs: lib
     exposed-modules:
-       Streamly.KeyValue.Type
-     , Streamly.Metrics.Type
-     , Streamly.Metrics.Perf
-     , Streamly.Metrics.Perf.Type
-     , Streamly.Metrics.Channel
-     , Streamly.Metrics.Channel.Unbounded
-     , Streamly.Metrics.Channel.Common
-     , Streamly.Metrics.Measure
-     -- , Streamly.Metrics.File
-     -- , Streamly.Metrics.Console
+       Perf.Metric.KeyValue
+     , Perf.Metric.Type
+     , Perf.Measure
+     , Perf.Internal.Measure.Types
+     , Perf.Internal.Measure.Bracket
+     , Perf.Sink.Console
+     , Perf.Sink.Console.Unbounded
+     , Perf.Internal.Sink
+     , Perf.Eventlog.Parser
+     , Perf.Eventlog.Aggregate
 
     if !os(windows)
       exposed-modules:
-        Streamly.Metrics.Perf.RUsage
+        Perf.Internal.Measure.RUsage
 
     build-depends:
        base >= 4.9 && < 5
+     , containers < 0.9
      , pretty-show >= 1.10 && < 1.11
      , stm >= 2.5.3 && < 2.6
      , streamly >= 0.11.0 && < 0.12
@@ -162,13 +163,13 @@ library
 
 executable hperf
   import: compile-options, optimization-options
-  hs-source-dirs: src
+  hs-source-dirs: hperf
   main-is: Main.hs
-  other-modules: Aggregator, EventParser
   build-depends:
        base >= 4.9 && < 5
      , containers < 0.9
      , format-numbers < 0.2
+     , haskell-perf
      , optparse-applicative < 0.20
      , streamly-core >= 0.3.0 && < 0.4
      , streamly-statistics < 0.3

src/Main.hs → hperf/Main.hs

@@ -3,7 +3,7 @@
 
 module Main (main) where
 
-import Aggregator
+import Perf.Eventlog.Aggregate
   ( collectThreadCounter,
     translateThreadEvents,
   )
@@ -16,7 +16,7 @@ import Data.Map (Map)
 import Data.Maybe (fromJust, isJust)
 import Data.Text.Format.Numbers (prettyI)
 import Data.Word (Word32, Word8)
-import EventParser
+import Perf.Eventlog.Parser
   ( Counter (..),
     Location (..),
     Event (..),
@@ -28,6 +28,7 @@ import Streamly.Data.Array (Array)
 import Streamly.Data.Stream (Stream)
 import Streamly.Data.StreamK (StreamK)
 import Streamly.Internal.Data.Fold (Fold (..), postscanlMaybe)
+import Streamly.Data.Scanl (Scanl)
 import qualified Streamly.Data.Scanl as Scanl
 import qualified Streamly.Internal.Data.Scanl as Scanl (scanlMany, cumulativeScan)
 import qualified Streamly.Statistics.Scanl as Stats
@@ -74,7 +75,10 @@ secondMaybe f = fmap f1 (Fold.unzip (fmap fromJust Fold.the) f)
 double :: Int -> Double
 double = fromIntegral
 
-untilLeft :: Monad m => Scanl.Scanl m b1 b2 -> Scanl.Scanl m (Either (Maybe b1) b1) b2
+-- | Modify the input of a scan to accept an "Either" input, the modified scan
+-- keeps consuming right inputs until a left input arrives, which terminates
+-- the scan.
+untilLeft :: Monad m => Scanl m b1 b2 -> Scanl m (Either (Maybe b1) b1) b2
 untilLeft f =
       Scanl.takeEndBy isLeft
     $ Scanl.lmap (either id Just)
@@ -110,7 +114,7 @@ combineWindowStats = Fold.kvToMap combineStats
 -- Statistics collection for each counter
 
 {-# INLINE stats #-}
-stats :: Scanl.Scanl IO Int64 [(String, Int)]
+stats :: Scanl IO Int64 [(String, Int)]
 stats =
       Scanl.lmap (fromIntegral :: Int64 -> Int)
     $ Scanl.distribute
@@ -124,11 +128,11 @@ stats =
         ]
 
 {-# INLINE threadStats #-}
-threadStats :: Scanl.Scanl IO (Either (Maybe Int64) Int64) [(String, Int)]
+threadStats :: Scanl IO (Either (Maybe Int64) Int64) [(String, Int)]
 threadStats = untilLeft stats
 
 {-# INLINE windowStats #-}
-windowStats :: Scanl.Scanl IO (Either (Maybe Int64) Int64) [(String, Int)]
+windowStats :: Scanl IO (Either (Maybe Int64) Int64) [(String, Int)]
 windowStats = Scanl.scanlMany (untilLeft Scanl.sum) stats
 
 {-# INLINE toStats #-}
@@ -139,7 +143,7 @@ toStats ::
         ((Word32, String, Counter), (Location, Int64))
         -- Map (tid, window tag, counter) (Maybe [(stat name, value)])
         (Map (Word32, String, Counter) (Maybe [(String, Int)]))
-toStats = Fold.demuxKvToMap (\k -> pure (Just (f1 k)))
+toStats = Fold.demuxKvToMap (pure . Just . f1)
 
     where
 
@@ -462,6 +466,9 @@ optsInfo = info (configParser <**> helper)
 -- Entry point
 -------------------------------------------------------------------------------
 
+-- XXX Add two different commands "hperf eventlog" and "hperf metrics", one for
+-- eventlog analysis and the other for metrics collected by other methods.
+--
 -- XXX Are the events for a particular thread guaranteed to come in order. What
 -- if a thread logged events to a particular capability buffer and then got
 -- scheduled on another capability before its eventlog could be flushed from

src/Aggregator.hs → lib/Perf/Eventlog/Aggregate.hs

@@ -1,5 +1,5 @@
 {-# LANGUAGE CPP #-}
-module Aggregator
+module Perf.Eventlog.Aggregate
     ( translateThreadEvents
     , collectThreadCounter
     )
@@ -8,7 +8,7 @@ where
 import Data.Int (Int64)
 import Data.Set (Set)
 import Data.Word (Word32)
-import EventParser (Event (..), Counter(..), Location(..))
+import Perf.Eventlog.Parser (Event (..), Counter(..), Location(..))
 import Streamly.Internal.Data.Fold (Step(..))
 import Streamly.Internal.Data.Scanl (Scanl(..))
 import Streamly.Internal.Data.Tuple.Strict (Tuple'(..))
@@ -26,15 +26,15 @@ import qualified Data.Set as Set
 
 -- All counter events are recorded against a unique key (tid, window, counter).
 --
--- A user defined window is prefixed with the thread-id of the thread which
--- started it. Therefore, if the window code is entered by multiple threads,
--- each thread is assigned a different window name.
+-- A user defined measurement window name is prefixed with the thread-id of the
+-- thread which started it. Therefore, if the window is entered by multiple
+-- threads, each thread is assigned a different window name.
 --
--- Each window accounts all threads active at the time when the window is
--- active. Therefore, any thread start/stop events are broadcast to all the
--- currently active windows.
+-- Each window performs accounting for all threads active at the time when the
+-- window is active. Therefore, any thread start/stop events are broadcast to
+-- all the currently active windows.
 --
--- When a window starts, many threads may already be existing in the system.
+-- When a measurement window starts, many threads may already be existing in the system.
 -- After the window is entered, events of all threads are broadcast to the
 -- window. If a thread was already active when the window was entered, then we
 -- may get a suspend event without first getting a resume event. Similar,

src/EventParser.hs → lib/Perf/Eventlog/Parser.hs

@@ -1,5 +1,5 @@
 {-# LANGUAGE CPP #-}
-module EventParser
+module Perf.Eventlog.Parser
     (
       parseLogHeader
     , parseDataHeader

lib/Streamly/Metrics/Measure.hs → lib/Perf/Internal/Measure/Bracket.hs

@@ -1,4 +1,4 @@
-module Streamly.Metrics.Measure
+module Perf.Internal.Measure.Bracket
     (
       measureWith
     , measure

lib/Streamly/Metrics/Perf/RUsage.hsc → lib/Perf/Internal/Measure/RUsage.hsc

@@ -1,4 +1,4 @@
-module Streamly.Metrics.Perf.RUsage
+module Perf.Internal.Measure.RUsage
     (
       pattern RUsageSelf
     , pattern RUsageChildren
@@ -16,8 +16,8 @@ import Foreign.C.Types (CInt(..), CLong)
 import Foreign.Marshal.Alloc (alloca)
 import Foreign.Ptr (Ptr)
 import Foreign.Storable (Storable(..))
-import Streamly.Metrics.Perf.Type (PerfMetrics(..))
-import Streamly.Metrics.Type (GaugeMax(..), Seconds(..), Bytes(..))
+import Perf.Internal.Measure.Types (PerfMetrics(..))
+import Perf.Metric.Type (GaugeMax(..), Seconds(..), Bytes(..))
 
 #include <sys/time.h>
 #include <sys/resource.h>

lib/Streamly/Metrics/Perf/Type.hs → lib/Perf/Internal/Measure/Types.hs

@@ -1,12 +1,12 @@
-module Streamly.Metrics.Perf.Type
+module Perf.Internal.Measure.Types
     (
       PerfMetrics (..)
     , checkMonotony
     )
 where
 
 import Data.Word (Word64)
-import Streamly.Metrics.Type
+import Perf.Metric.Type
     (GaugeMax(..), Seconds(..), Bytes(..), Indexable(..))
 
 -- Use Counter/Gauge as the outer constructor and Bytes/Seconds as the inner

lib/Streamly/Metrics/Channel/Common.hs → lib/Perf/Internal/Sink.hs

@@ -1,4 +1,4 @@
-module Streamly.Metrics.Channel.Common
+module Perf.Internal.Sink
     (
       aggregateListBy
     , printKV
@@ -9,7 +9,7 @@ import Control.Monad.IO.Class (liftIO, MonadIO)
 import Data.Bifunctor (second)
 import Data.Maybe (fromJust, isJust)
 import Streamly.Internal.Data.Time.Units (AbsTime)
-import Streamly.Metrics.Type (showList, Indexable)
+import Perf.Metric.Type (showList, Indexable)
 import Streamly.Data.Stream (Stream)
 import Streamly.Data.Stream.Prelude (MonadAsync)
 

lib/Streamly/Metrics/Perf.hs → lib/Perf/Measure.hs

@@ -1,4 +1,4 @@
-module Streamly.Metrics.Perf
+module Perf.Measure
     (
       PerfMetrics(..)
     , benchWith
@@ -12,10 +12,10 @@ import Control.Monad (unless)
 import Data.Maybe (catMaybes)
 import GHC.Stats (getRTSStats, getRTSStatsEnabled, RTSStats(..))
 import Streamly.Internal.Data.Time.Units (NanoSecond64, fromAbsTime)
-import Streamly.Metrics.Measure (measureWith)
-import Streamly.Metrics.Perf.Type (PerfMetrics(..), checkMonotony)
+import Perf.Internal.Measure.Bracket (measureWith)
+import Perf.Internal.Measure.Types (PerfMetrics(..), checkMonotony)
 #if !defined(mingw32_HOST_OS)
-import Streamly.Metrics.Perf.RUsage (getRuMetrics, pattern RUsageSelf)
+import Perf.Internal.Measure.RUsage (getRuMetrics, pattern RUsageSelf)
 #endif
 import Text.Show.Pretty (ppShow)
 

lib/Streamly/KeyValue/Type.hs → lib/Perf/Metric/KeyValue.hs

@@ -1,12 +1,12 @@
 -- |
--- Module      : Streamly.KeyValue.Type
+-- Module      : Perf.Metric.KeyValue
 -- Copyright   : (c) 2021 Composewell Technologies
 -- License     : Apache-2.0
 -- Maintainer  : streamly@composewell.com
 -- Stability   : experimental
 -- Portability : GHC
 --
-module Streamly.KeyValue.Type
+module Perf.Metric.KeyValue
     (
     -- * KeyValue
       KeyValue (..)

lib/Streamly/Metrics/Type.hs → lib/Perf/Metric/Type.hs

@@ -1,6 +1,6 @@
 {-# LANGUAGE DerivingVia #-}
 -- |
--- Module      : Streamly.Metrics.Type
+-- Module      : Perf.Metric.Type
 -- Copyright   : (c) 2021 Composewell Technologies
 -- License     : Apache-2.0
 -- Maintainer  : streamly@composewell.com
@@ -12,7 +12,7 @@
 -- respect to time e.g. 'Sequence' or 'Time'.  Metrics may have different
 -- semantics e.g. 'Counter' or 'Gauge'.
 --
-module Streamly.Metrics.Type
+module Perf.Metric.Type
     (
     -- * Semantics
     -- | A counter counts how many events of a type have occurred whereas a

lib/Streamly/Metrics/Channel.hs → lib/Perf/Sink/Console.hs

@@ -1,4 +1,4 @@
-module Streamly.Metrics.Channel
+module Perf.Sink.Console
     (
       Channel
     , newChannel
@@ -19,10 +19,10 @@ import Data.Function ((&))
 import Streamly.Data.Stream (Stream)
 import Streamly.Internal.Data.Time.Clock (getTime, Clock (Monotonic))
 import Streamly.Internal.Data.Time.Units (AbsTime)
-import Streamly.Metrics.Channel.Common (aggregateListBy, printKV)
-import Streamly.Metrics.Perf.Type (PerfMetrics(..))
-import Streamly.Metrics.Perf (benchWith)
-import Streamly.Metrics.Type (Indexable)
+import Perf.Internal.Sink (aggregateListBy, printKV)
+import Perf.Internal.Measure.Types (PerfMetrics(..))
+import Perf.Measure (benchWith)
+import Perf.Metric.Type (Indexable)
 import Streamly.Data.Stream.Prelude (MonadAsync)
 
 import qualified Streamly.Data.Stream as Stream

lib/Streamly/Metrics/Channel/Unbounded.hs → lib/Perf/Sink/Console/Unbounded.hs

@@ -1,4 +1,4 @@
-module Streamly.Metrics.Channel.Unbounded
+module Perf.Sink.Console.Unbounded
     (
       Channel
     , newChannel
@@ -16,16 +16,16 @@ import Control.Monad.IO.Class (liftIO, MonadIO)
 import Data.Function ((&))
 import Streamly.Internal.Data.Time.Clock (getTime, Clock (Monotonic))
 import Streamly.Internal.Data.Time.Units (AbsTime)
-import Streamly.Metrics.Perf.Type (PerfMetrics(..))
-import Streamly.Metrics.Perf (benchWith)
-import Streamly.Metrics.Type (Indexable)
+import Perf.Internal.Measure.Types (PerfMetrics(..))
+import Perf.Measure (benchWith)
+import Perf.Metric.Type (Indexable)
 import Streamly.Data.Stream (Stream)
 import Streamly.Data.Stream.Prelude (MonadAsync)
 
 import qualified Streamly.Data.Stream as Stream
 
 import Prelude hiding (showList)
-import Streamly.Metrics.Channel.Common
+import Perf.Internal.Sink
 
 -------------------------------------------------------------------------------
 -- Event processing