Added BigFileParser

pull/1/head
unknown 4 years ago
parent f42119e0ec
commit e617e5e146

@ -0,0 +1,85 @@
using System;
using System.Collections.Generic;
using System.IO;
namespace ZeroLevel.Services.FileSystem
{
public class BigFileParser<T>
{
private readonly string _filePath;
private readonly Func<string, T> _parser;
private readonly int _bufferSize;
public BigFileParser(string filePath, Func<string, T> parser, int bufferSize = 1024 * 1024 * 32)
{
if (string.IsNullOrWhiteSpace(filePath))
{
throw new ArgumentNullException(nameof(filePath));
}
if (parser == null)
{
throw new ArgumentNullException(nameof(parser));
}
if (!File.Exists(filePath))
{
throw new FileNotFoundException(filePath);
}
_filePath = filePath;
_parser = parser;
_bufferSize = bufferSize;
}
public IEnumerable<IEnumerable<T>> ReadBatches(int batchSize)
{
var buffer = new T[batchSize];
var buffer_index = 0;
using (FileStream fs = File.Open(_filePath, FileMode.Open, FileAccess.Read, FileShare.None))
{
using (BufferedStream bs = new BufferedStream(fs, _bufferSize))
{
using (StreamReader sr = new StreamReader(bs))
{
string line;
while ((line = sr.ReadLine()) != null)
{
buffer[buffer_index] = _parser.Invoke(line);
buffer_index++;
if (buffer_index >= batchSize)
{
buffer_index = 0;
yield return buffer;
}
}
}
}
}
if (buffer_index > 0)
{
if (buffer_index < batchSize)
{
var bias = new T[buffer_index];
Array.Copy(buffer, 0, bias, 0, buffer_index);
yield return bias;
}
}
}
public IEnumerable<T> Read(int batchSize)
{
using (FileStream fs = File.Open(_filePath, FileMode.Open, FileAccess.Read, FileShare.None))
{
using (BufferedStream bs = new BufferedStream(fs, _bufferSize))
{
using (StreamReader sr = new StreamReader(bs))
{
string line;
while ((line = sr.ReadLine()) != null)
{
yield return _parser.Invoke(line);
}
}
}
}
}
}
}

@ -6,16 +6,16 @@
</Description> </Description>
<Authors>ogoun</Authors> <Authors>ogoun</Authors>
<Company>ogoun</Company> <Company>ogoun</Company>
<AssemblyVersion>3.3.4.7</AssemblyVersion> <AssemblyVersion>3.3.4.8</AssemblyVersion>
<PackageReleaseNotes>Fix PeriodicFileSystemWatcher</PackageReleaseNotes> <PackageReleaseNotes>Added BigFileParser</PackageReleaseNotes>
<PackageProjectUrl>https://github.com/ogoun/Zero/wiki</PackageProjectUrl> <PackageProjectUrl>https://github.com/ogoun/Zero/wiki</PackageProjectUrl>
<Copyright>Copyright Ogoun 2020</Copyright> <Copyright>Copyright Ogoun 2020</Copyright>
<PackageLicenseUrl></PackageLicenseUrl> <PackageLicenseUrl></PackageLicenseUrl>
<PackageIconUrl></PackageIconUrl> <PackageIconUrl></PackageIconUrl>
<RepositoryUrl>https://github.com/ogoun/Zero</RepositoryUrl> <RepositoryUrl>https://github.com/ogoun/Zero</RepositoryUrl>
<RepositoryType>GitHub</RepositoryType> <RepositoryType>GitHub</RepositoryType>
<Version>3.3.4.7</Version> <Version>3.3.4.8</Version>
<FileVersion>3.3.4.7</FileVersion> <FileVersion>3.3.4.8</FileVersion>
<Platforms>AnyCPU;x64</Platforms> <Platforms>AnyCPU;x64</Platforms>
<PackageIcon>zero.png</PackageIcon> <PackageIcon>zero.png</PackageIcon>
</PropertyGroup> </PropertyGroup>

Loading…
Cancel
Save

Powered by TurnKey Linux.