<# .SYNOPSIS Splits a large delimited product feed into smaller files, repeating the header line in each. .DESCRIPTION Streams the input file line by line (constant memory, safe for millions of rows) and writes feed_0.txt, feed_1.txt, ... Each output file starts with the original header line and contains at most -ChunkSize product lines. The input encoding is auto-detected from the BOM and reused for the output files, so UTF-16LE / UTF-8 / ANSI feeds round-trip unchanged. Line endings are forced to CRLF. .EXAMPLE .\Split-Feed.ps1 Splits .\feed.txt into .\feed_0.txt, .\feed_1.txt, ... at 50000 products each. .EXAMPLE .\Split-Feed.ps1 -InputFile C:\feeds\plaza.txt -OutputDir C:\feeds\split -ChunkSize 50000 -Prefix plaza .NOTES Assumes one product per line (no newlines inside quoted fields). Use -CheckFieldCount to verify. #> [CmdletBinding()] param( [string] $InputFile = 'feed.txt', [string] $OutputDir = '.', [int] $ChunkSize = 50000, [string] $Prefix = 'feed', [char] $Delimiter = "`t", [switch] $CheckFieldCount ) $ErrorActionPreference = 'Stop' if ($ChunkSize -lt 1) { throw "ChunkSize must be at least 1." } if (-not (Test-Path -LiteralPath $InputFile -PathType Leaf)) { throw "Input file not found: $InputFile" } if (-not (Test-Path -LiteralPath $OutputDir)) { New-Item -ItemType Directory -Path $OutputDir -Force | Out-Null } $inPath = (Resolve-Path -LiteralPath $InputFile).Path $outPath = (Resolve-Path -LiteralPath $OutputDir).Path # Fallback encoding for files without a BOM: UTF-8 without BOM (detection overrides this when a BOM exists). $fallback = New-Object System.Text.UTF8Encoding($false) $reader = New-Object System.IO.StreamReader($inPath, $fallback, $true, 65536) $writer = $null $fileIndex = 0 $inChunk = 0 $total = 0 $badLines = 0 $sw = [System.Diagnostics.Stopwatch]::StartNew() try { $header = $reader.ReadLine() if ($null -eq $header) { throw "Input file is empty: $inPath" } # CurrentEncoding is only reliable after the first read. $encoding = $reader.CurrentEncoding $expectedFields = ($header.Split($Delimiter)).Count Write-Host "Input : $inPath" Write-Host "Encoding : $($encoding.WebName)" Write-Host "Columns : $expectedFields" Write-Host "Chunk size: $ChunkSize`n" while ($null -ne ($line = $reader.ReadLine())) { if ($line.Length -eq 0) { continue } # skip blank / trailing lines if ($CheckFieldCount -and ($line.Split($Delimiter)).Count -ne $expectedFields) { $badLines++ if ($badLines -le 5) { Write-Warning "Field count mismatch at product #$($total + 1) (line $($total + 2))." } } if ($null -eq $writer) { $file = Join-Path $outPath ('{0}_{1}.txt' -f $Prefix, $fileIndex) $writer = New-Object System.IO.StreamWriter($file, $false, $encoding, 65536) $writer.NewLine = "`r`n" $writer.WriteLine($header) } $writer.WriteLine($line) $inChunk++ $total++ if ($inChunk -ge $ChunkSize) { $writer.Dispose(); $writer = $null Write-Host ('{0}_{1}.txt {2,8} products' -f $Prefix, $fileIndex, $inChunk) $fileIndex++ $inChunk = 0 } } if ($null -ne $writer) { $writer.Dispose(); $writer = $null Write-Host ('{0}_{1}.txt {2,8} products' -f $Prefix, $fileIndex, $inChunk) $fileIndex++ } } finally { if ($null -ne $writer) { $writer.Dispose() } if ($null -ne $reader) { $reader.Dispose() } } $sw.Stop() Write-Host ("`nDone: {0} products -> {1} file(s) in {2:N1}s" -f $total, $fileIndex, $sw.Elapsed.TotalSeconds) if ($badLines -gt 0) { Write-Warning "$badLines line(s) had an unexpected field count." }