From 2959c9c0b08b37fc668bbef7aa5709ecc69668c5 Mon Sep 17 00:00:00 2001 From: Harendra Kumar Date: Thu, 21 Dec 2023 10:36:55 +0530 Subject: [PATCH 1/7] Use OsPath and write out arrays to stdout --- examples/ListDir.hs | 35 +++++++++++++++++++++++++++++++---- streamly-examples.cabal | 4 +++- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/examples/ListDir.hs b/examples/ListDir.hs index 9dd7158..d702a4c 100644 --- a/examples/ListDir.hs +++ b/examples/ListDir.hs @@ -1,36 +1,63 @@ {-# LANGUAGE FlexibleContexts #-} +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE QuasiQuotes #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE PackageImports #-} {-# OPTIONS_GHC -Wno-deprecations #-} module Main (main) where +import Data.Word (Word8) import System.IO (stdout, hSetBuffering, BufferMode(LineBuffering)) +import System.OsPath (osp) +import "filepath" System.OsString.Internal.Types (getOsString, getPosixString) +import System.OsString.Data.ByteString.Short (ShortByteString(..)) +import Streamly.Internal.Data.MutByteArray (MutByteArray(..)) +import Streamly.Internal.Data.Array (Array(..)) +import GHC.Exts (sizeofByteArray#, Int(..), unsafeCoerce#) -import qualified Streamly.Data.Fold as Fold +import qualified Streamly.Data.Array as Array import qualified Streamly.Data.Stream.Prelude as Stream --- import qualified Streamly.Internal.Data.Stream as Stream +import qualified Streamly.Internal.Data.Stream as Stream -- import qualified Streamly.Internal.Data.Unfold as Unfold (either, nil) import qualified Streamly.Internal.FileSystem.Dir as Dir +import qualified Streamly.Internal.FileSystem.Handle as Handle -- | List the current directory recursively using concurrent processing main :: IO () main = do hSetBuffering stdout LineBuffering - Stream.fold (Fold.drainMapM print) + Stream.fold (Handle.writeWith 32000 stdout) + $ Stream.interposeSuffix 10 Array.reader + $ fmap (osPathToArray . either id id) + + -- Serial using unfolds (fastest serial) -- $ Stream.unfoldIterateDfs unfoldOne -- $ Stream.unfoldIterateBfs unfoldOne -- $ Stream.unfoldIterateBfsRev unfoldOne + + -- Serial using streams -- $ Stream.concatIterateDfs streamOneMaybe -- $ Stream.concatIterateBfs streamOneMaybe -- $ Stream.concatIterateBfsRev streamOneMaybe + + -- Serial using stream append and interleave -- $ Stream.concatIterateWith Stream.append streamOne -- $ Stream.mergeIterateWith Stream.interleave streamOne + + -- Concurrent $ Stream.parConcatIterate id streamOne -- $ Stream.parConcatIterate (Stream.interleaved True) streamOne -- $ Stream.parConcatIterate (Stream.ordered True) streamOne - $ Stream.fromPure (Left ".") + $ Stream.fromPure (Left [osp|.|]) + -- $ Stream.fromPure (Left ".") where -- unfoldOne = Unfold.either Dir.eitherReaderPaths Unfold.nil -- streamOneMaybe = either (Just . Dir.readEitherPaths) (const Nothing) streamOne = either Dir.readEitherPaths (const Stream.nil) + osPathToArray p = + let !(SBS barr#) = getPosixString $ getOsString p + mbarr = MutByteArray (unsafeCoerce# barr#) + in Array mbarr 0 (I# (sizeofByteArray# barr#)) :: Array Word8 diff --git a/streamly-examples.cabal b/streamly-examples.cabal index 144adf1..ebb71e9 100644 --- a/streamly-examples.cabal +++ b/streamly-examples.cabal @@ -61,7 +61,7 @@ flag interop common exe-dependencies build-depends: streamly == 0.10.0 - , streamly-core == 0.2.0 + , streamly-core >= 0.2.0 && < 0.2.2 , base >= 4.9 && < 4.20 , directory >= 1.2 && < 1.4 , transformers >= 0.4 && < 0.7 @@ -74,6 +74,8 @@ common exe-dependencies , unordered-containers >= 0.2 && < 0.3 , vector >= 0.12 && < 0.14 , mtl >= 2.2 && < 3 + , filepath + , os-string common exe-options import: exe-dependencies From 1a6e557927fb2ad6509cf762d17bc903d0d8b2ab Mon Sep 17 00:00:00 2001 From: Harendra Kumar Date: Sun, 24 Dec 2023 05:40:07 +0530 Subject: [PATCH 2/7] Use streamly native Path type --- examples/ListDir.hs | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/examples/ListDir.hs b/examples/ListDir.hs index d702a4c..f4d3d0c 100644 --- a/examples/ListDir.hs +++ b/examples/ListDir.hs @@ -1,20 +1,9 @@ {-# LANGUAGE FlexibleContexts #-} -{-# LANGUAGE BangPatterns #-} -{-# LANGUAGE QuasiQuotes #-} -{-# LANGUAGE MagicHash #-} -{-# LANGUAGE PackageImports #-} {-# OPTIONS_GHC -Wno-deprecations #-} module Main (main) where -import Data.Word (Word8) import System.IO (stdout, hSetBuffering, BufferMode(LineBuffering)) -import System.OsPath (osp) -import "filepath" System.OsString.Internal.Types (getOsString, getPosixString) -import System.OsString.Data.ByteString.Short (ShortByteString(..)) -import Streamly.Internal.Data.MutByteArray (MutByteArray(..)) -import Streamly.Internal.Data.Array (Array(..)) -import GHC.Exts (sizeofByteArray#, Int(..), unsafeCoerce#) import qualified Streamly.Data.Array as Array import qualified Streamly.Data.Stream.Prelude as Stream @@ -22,6 +11,7 @@ import qualified Streamly.Internal.Data.Stream as Stream -- import qualified Streamly.Internal.Data.Unfold as Unfold (either, nil) import qualified Streamly.Internal.FileSystem.Dir as Dir import qualified Streamly.Internal.FileSystem.Handle as Handle +import qualified Streamly.Internal.FileSystem.Path as Path -- | List the current directory recursively using concurrent processing main :: IO () @@ -29,7 +19,7 @@ main = do hSetBuffering stdout LineBuffering Stream.fold (Handle.writeWith 32000 stdout) $ Stream.interposeSuffix 10 Array.reader - $ fmap (osPathToArray . either id id) + $ fmap (Path.toByteArray . either id id) -- Serial using unfolds (fastest serial) -- $ Stream.unfoldIterateDfs unfoldOne @@ -49,7 +39,7 @@ main = do $ Stream.parConcatIterate id streamOne -- $ Stream.parConcatIterate (Stream.interleaved True) streamOne -- $ Stream.parConcatIterate (Stream.ordered True) streamOne - $ Stream.fromPure (Left [osp|.|]) + $ Stream.fromPure (Left $ Path.fromString ".") -- $ Stream.fromPure (Left ".") where @@ -57,7 +47,3 @@ main = do -- unfoldOne = Unfold.either Dir.eitherReaderPaths Unfold.nil -- streamOneMaybe = either (Just . Dir.readEitherPaths) (const Nothing) streamOne = either Dir.readEitherPaths (const Stream.nil) - osPathToArray p = - let !(SBS barr#) = getPosixString $ getOsString p - mbarr = MutByteArray (unsafeCoerce# barr#) - in Array mbarr 0 (I# (sizeofByteArray# barr#)) :: Array Word8 From 446e2a8a7115550d5f6acc21694c1f0f2816475d Mon Sep 17 00:00:00 2001 From: Harendra Kumar Date: Wed, 10 Jan 2024 12:03:37 +0530 Subject: [PATCH 3/7] Use readEitherChunks --- examples/ListDir.hs | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/examples/ListDir.hs b/examples/ListDir.hs index f4d3d0c..72c6773 100644 --- a/examples/ListDir.hs +++ b/examples/ListDir.hs @@ -1,14 +1,20 @@ {-# LANGUAGE FlexibleContexts #-} +{-# LANGUAGE QuasiQuotes #-} {-# OPTIONS_GHC -Wno-deprecations #-} +{-# OPTIONS_GHC -ddump-simpl -dsuppress-all -ddump-to-file #-} + module Main (main) where +import Data.Maybe (fromJust) import System.IO (stdout, hSetBuffering, BufferMode(LineBuffering)) +import System.OsPath (osp) -import qualified Streamly.Data.Array as Array +import qualified Streamly.Internal.Data.Array as Array +import qualified Streamly.Internal.Data.Fold as Fold import qualified Streamly.Data.Stream.Prelude as Stream import qualified Streamly.Internal.Data.Stream as Stream --- import qualified Streamly.Internal.Data.Unfold as Unfold (either, nil) +import qualified Streamly.Internal.Data.Unfold as Unfold import qualified Streamly.Internal.FileSystem.Dir as Dir import qualified Streamly.Internal.FileSystem.Handle as Handle import qualified Streamly.Internal.FileSystem.Path as Path @@ -17,9 +23,19 @@ import qualified Streamly.Internal.FileSystem.Path as Path main :: IO () main = do hSetBuffering stdout LineBuffering + -- Stream.fold (Fold.drain) + -- Stream.fold (Fold.drainMapM print) Stream.fold (Handle.writeWith 32000 stdout) + -- Stream.fold (Handle.writeChunks stdout) + -- Stream.fold (Array.lPinnedCompactGE 32000 (Handle.writeChunks stdout)) $ Stream.interposeSuffix 10 Array.reader - $ fmap (Path.toByteArray . either id id) + -- $ Array.compactInterposeGE 10 32000 + -- $ Array.pinnedCompactLE 32000 + $ fmap Path.toChunk + $ Stream.unfoldMany Unfold.fromList + $ fmap (either (:[]) id) + -- $ fmap Path.toChunk + -- $ Stream.catLefts -- Serial using unfolds (fastest serial) -- $ Stream.unfoldIterateDfs unfoldOne @@ -36,14 +52,20 @@ main = do -- $ Stream.mergeIterateWith Stream.interleave streamOne -- Concurrent - $ Stream.parConcatIterate id streamOne + -- XXX To reduce concurrency overhead, perform buffering in each worker + -- and post the buffer or return [Path] and then unfold it. + $ Stream.parConcatIterate + -- ( Stream.eager False . Stream.maxBuffer 1000 . Stream.maxThreads 4) + id + streamOne -- $ Stream.parConcatIterate (Stream.interleaved True) streamOne -- $ Stream.parConcatIterate (Stream.ordered True) streamOne - $ Stream.fromPure (Left $ Path.fromString ".") + $ Stream.fromPure (Left $ fromJust $ Path.fromString ".") -- $ Stream.fromPure (Left ".") + -- $ Stream.fromPure (Left [osp|.|]) where -- unfoldOne = Unfold.either Dir.eitherReaderPaths Unfold.nil -- streamOneMaybe = either (Just . Dir.readEitherPaths) (const Nothing) - streamOne = either Dir.readEitherPaths (const Stream.nil) + streamOne = either Dir.readEitherChunks (const Stream.nil) From 97f6d1d4107a56eb2d49fc6b32ae6ba19d2ec3be Mon Sep 17 00:00:00 2001 From: Harendra Kumar Date: Thu, 11 Jan 2024 12:49:07 +0530 Subject: [PATCH 4/7] Use dir chunks --- examples/ListDir.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/ListDir.hs b/examples/ListDir.hs index 72c6773..8103579 100644 --- a/examples/ListDir.hs +++ b/examples/ListDir.hs @@ -33,7 +33,7 @@ main = do -- $ Array.pinnedCompactLE 32000 $ fmap Path.toChunk $ Stream.unfoldMany Unfold.fromList - $ fmap (either (:[]) id) + $ fmap (either id id) -- $ fmap Path.toChunk -- $ Stream.catLefts @@ -60,7 +60,7 @@ main = do streamOne -- $ Stream.parConcatIterate (Stream.interleaved True) streamOne -- $ Stream.parConcatIterate (Stream.ordered True) streamOne - $ Stream.fromPure (Left $ fromJust $ Path.fromString ".") + $ Stream.fromPure (Left [fromJust $ Path.fromString "."]) -- $ Stream.fromPure (Left ".") -- $ Stream.fromPure (Left [osp|.|]) From 67be1d698dfc27a32a73d80ab167af7aca7216a8 Mon Sep 17 00:00:00 2001 From: Harendra Kumar Date: Thu, 11 Jan 2024 14:58:50 +0530 Subject: [PATCH 5/7] Use readEitherByteChunks --- examples/ListDir.hs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/examples/ListDir.hs b/examples/ListDir.hs index 8103579..87e4b2f 100644 --- a/examples/ListDir.hs +++ b/examples/ListDir.hs @@ -25,15 +25,16 @@ main = do hSetBuffering stdout LineBuffering -- Stream.fold (Fold.drain) -- Stream.fold (Fold.drainMapM print) - Stream.fold (Handle.writeWith 32000 stdout) - -- Stream.fold (Handle.writeChunks stdout) + -- Stream.fold (Handle.writeWith 32000 stdout) + Stream.fold (Handle.writeChunks stdout) -- Stream.fold (Array.lPinnedCompactGE 32000 (Handle.writeChunks stdout)) - $ Stream.interposeSuffix 10 Array.reader + -- $ Stream.interposeSuffix 10 Array.reader -- $ Array.compactInterposeGE 10 32000 -- $ Array.pinnedCompactLE 32000 - $ fmap Path.toChunk - $ Stream.unfoldMany Unfold.fromList - $ fmap (either id id) + -- $ fmap Path.toChunk + -- $ Stream.unfoldMany Unfold.fromList + -- $ fmap (either id id) + $ Stream.catRights -- $ fmap Path.toChunk -- $ Stream.catLefts @@ -68,4 +69,4 @@ main = do -- unfoldOne = Unfold.either Dir.eitherReaderPaths Unfold.nil -- streamOneMaybe = either (Just . Dir.readEitherPaths) (const Nothing) - streamOne = either Dir.readEitherChunks (const Stream.nil) + streamOne = either Dir.readEitherByteChunks (const Stream.nil) From 578f27c21d50ef5e0d36b643a644c9c1b3a12ba5 Mon Sep 17 00:00:00 2001 From: Harendra Kumar Date: Thu, 4 Jul 2024 16:13:27 +0530 Subject: [PATCH 6/7] Sync with latest changes in streamly --- examples/ListDir.hs | 1 + streamly-examples.cabal | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/ListDir.hs b/examples/ListDir.hs index 87e4b2f..988bd0a 100644 --- a/examples/ListDir.hs +++ b/examples/ListDir.hs @@ -16,6 +16,7 @@ import qualified Streamly.Data.Stream.Prelude as Stream import qualified Streamly.Internal.Data.Stream as Stream import qualified Streamly.Internal.Data.Unfold as Unfold import qualified Streamly.Internal.FileSystem.Dir as Dir +import qualified Streamly.Internal.FileSystem.Posix.ReadDir as Dir import qualified Streamly.Internal.FileSystem.Handle as Handle import qualified Streamly.Internal.FileSystem.Path as Path diff --git a/streamly-examples.cabal b/streamly-examples.cabal index ebb71e9..814957b 100644 --- a/streamly-examples.cabal +++ b/streamly-examples.cabal @@ -60,8 +60,8 @@ flag interop common exe-dependencies build-depends: - streamly == 0.10.0 - , streamly-core >= 0.2.0 && < 0.2.2 + streamly >= 0.11.0 && < 0.11.1 + , streamly-core >= 0.3.0 && < 0.3.1 , base >= 4.9 && < 4.20 , directory >= 1.2 && < 1.4 , transformers >= 0.4 && < 0.7 From f5c7bc6b52e3a8ad74d3c6b1e9d259e3ca577b8b Mon Sep 17 00:00:00 2001 From: Harendra Kumar Date: Sun, 7 Jul 2024 18:43:07 +0530 Subject: [PATCH 7/7] Update to test windows code --- examples/ListDir.hs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/examples/ListDir.hs b/examples/ListDir.hs index 988bd0a..5f5848f 100644 --- a/examples/ListDir.hs +++ b/examples/ListDir.hs @@ -8,7 +8,7 @@ module Main (main) where import Data.Maybe (fromJust) import System.IO (stdout, hSetBuffering, BufferMode(LineBuffering)) -import System.OsPath (osp) +-- import System.OsPath (osp) import qualified Streamly.Internal.Data.Array as Array import qualified Streamly.Internal.Data.Fold as Fold @@ -26,14 +26,15 @@ main = do hSetBuffering stdout LineBuffering -- Stream.fold (Fold.drain) -- Stream.fold (Fold.drainMapM print) - -- Stream.fold (Handle.writeWith 32000 stdout) - Stream.fold (Handle.writeChunks stdout) + Stream.fold (Handle.writeWith 32000 stdout) + -- Stream.fold (Handle.writeChunks stdout) -- Stream.fold (Array.lPinnedCompactGE 32000 (Handle.writeChunks stdout)) - -- $ Stream.interposeSuffix 10 Array.reader + $ Array.interposeSuffix 10 -- Array.reader -- $ Array.compactInterposeGE 10 32000 -- $ Array.pinnedCompactLE 32000 - -- $ fmap Path.toChunk - -- $ Stream.unfoldMany Unfold.fromList + $ fmap Path.toChunk + -- $ Stream.trace (print . Path.toString) + $ Stream.unfoldMany Unfold.fromList -- $ fmap (either id id) $ Stream.catRights -- $ fmap Path.toChunk @@ -70,4 +71,5 @@ main = do -- unfoldOne = Unfold.either Dir.eitherReaderPaths Unfold.nil -- streamOneMaybe = either (Just . Dir.readEitherPaths) (const Nothing) - streamOne = either Dir.readEitherByteChunks (const Stream.nil) + -- streamOne = either Dir.readEitherByteChunks (const Stream.nil) + streamOne = either Dir.readEitherChunks (const Stream.nil)