Added BigFileParser

ZeroLevel/Services/FileSystem/BigFileParser.cs

@@ -0,0 +1,85 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+
+namespace ZeroLevel.Services.FileSystem
+{
+    public class BigFileParser<T>
+    {
+        private readonly string _filePath;
+        private readonly Func<string, T> _parser;
+        private readonly int _bufferSize;
+
+        public BigFileParser(string filePath, Func<string, T> parser, int bufferSize = 1024 * 1024 * 32)
+        {
+            if (string.IsNullOrWhiteSpace(filePath))
+            {
+                throw new ArgumentNullException(nameof(filePath));
+            }
+            if (parser == null)
+            {
+                throw new ArgumentNullException(nameof(parser));
+            }
+            if (!File.Exists(filePath))
+            {
+                throw new FileNotFoundException(filePath);
+            }
+            _filePath = filePath;
+            _parser = parser;
+            _bufferSize = bufferSize;
+        }
+
+        public IEnumerable<IEnumerable<T>> ReadBatches(int batchSize)
+        {
+            var buffer = new T[batchSize];
+            var buffer_index = 0;
+            using (FileStream fs = File.Open(_filePath, FileMode.Open, FileAccess.Read, FileShare.None))
+            {
+                using (BufferedStream bs = new BufferedStream(fs, _bufferSize))
+                {
+                    using (StreamReader sr = new StreamReader(bs))
+                    {
+                        string line;
+                        while ((line = sr.ReadLine()) != null)
+                        {
+                            buffer[buffer_index] = _parser.Invoke(line);
+                            buffer_index++;
+                            if (buffer_index >= batchSize)
+                            {
+                                buffer_index = 0;
+                                yield return buffer;
+                            }
+                        }
+                    }
+                }
+            }
+            if (buffer_index > 0)
+            {
+                if (buffer_index < batchSize)
+                {
+                    var bias = new T[buffer_index];
+                    Array.Copy(buffer, 0, bias, 0, buffer_index);
+                    yield return bias;
+                }
+            }
+        }
+
+        public IEnumerable<T> Read(int batchSize)
+        {
+            using (FileStream fs = File.Open(_filePath, FileMode.Open, FileAccess.Read, FileShare.None))
+            {
+                using (BufferedStream bs = new BufferedStream(fs, _bufferSize))
+                {
+                    using (StreamReader sr = new StreamReader(bs))
+                    {
+                        string line;
+                        while ((line = sr.ReadLine()) != null)
+                        {
+                            yield return _parser.Invoke(line);
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

ZeroLevel/ZeroLevel.csproj

@@ -6,16 +6,16 @@
 </Description>
     <Authors>ogoun</Authors>
     <Company>ogoun</Company>
-    <AssemblyVersion>3.3.4.7</AssemblyVersion>
-    <PackageReleaseNotes>Fix PeriodicFileSystemWatcher</PackageReleaseNotes>
+    <AssemblyVersion>3.3.4.8</AssemblyVersion>
+    <PackageReleaseNotes>Added BigFileParser</PackageReleaseNotes>
     <PackageProjectUrl>https://github.com/ogoun/Zero/wiki</PackageProjectUrl>
     <Copyright>Copyright Ogoun 2020</Copyright>
     <PackageLicenseUrl></PackageLicenseUrl>
     <PackageIconUrl></PackageIconUrl>
     <RepositoryUrl>https://github.com/ogoun/Zero</RepositoryUrl>
     <RepositoryType>GitHub</RepositoryType>
-    <Version>3.3.4.7</Version>
-    <FileVersion>3.3.4.7</FileVersion>
+    <Version>3.3.4.8</Version>
+    <FileVersion>3.3.4.8</FileVersion>
     <Platforms>AnyCPU;x64</Platforms>
     <PackageIcon>zero.png</PackageIcon>
   </PropertyGroup>