typescriptadvanced

JSON Stream Parser

Parses large JSON arrays from a readable stream without loading the entire file into memory.

typescriptPress ⌘/Ctrl + Shift + C to copy

import { createReadStream } from 'fs';
import { Transform, TransformCallback } from 'stream';

class JsonLineTransform extends Transform {
  private buffer = '';

  _transform(chunk: Buffer, _encoding: string, callback: TransformCallback) {
    this.buffer += chunk.toString();
    const lines = this.buffer.split('\n');
    this.buffer = lines.pop() || '';

    for (const line of lines) {
      const trimmed = line.trim();
      if (trimmed) {
        try {
          this.push(JSON.parse(trimmed));
        } catch { /* skip malformed lines */ }
      }
    }
    callback();
  }

  _flush(callback: TransformCallback) {
    if (this.buffer.trim()) {
      try {
        this.push(JSON.parse(this.buffer.trim()));
      } catch { /* skip */ }
    }
    callback();
  }
}

async function* parseJsonLines(filePath: string) {
  const stream = createReadStream(filePath).pipe(new JsonLineTransform({ objectMode: true }));
  for await (const item of stream) {
    yield item;
  }
}

export { parseJsonLines };