CSV Processing: Parse and Generate CSV

Problem

Working with CSV files is a fundamental task in data processing. Common challenges include:

• Parsing CSV data with various delimiters and formats
• Handling special characters, quotes, and escaped values
• Generating properly formatted CSV output
• Managing large datasets efficiently
• Dealing with headers and data validation
• Converting between different data structures

Solution

A comprehensive CSV processing system should:

1. Parse CSV files - Read and convert CSV strings into structured data
2. Generate CSV output - Convert data structures back to CSV format
3. Handle edge cases - Manage quotes, delimiters, newlines, and special characters
4. Validate data - Ensure data integrity during import/export
5. Support transformations - Filter, map, and transform data during processing
6. Optimize performance - Handle large files efficiently with streaming

Code

1. Basic CSV Parser and Generator

class CSVProcessor {
  /**
   * Parse CSV string into array of objects
   * @param {string} csvString - Raw CSV data
   * @param {object} options - Configuration options
   * @returns {array} Array of parsed records
   */
  static parse(csvString, options = {}) {
    const {
      delimiter = ',',
      hasHeader = true,
      trim = true,
      skipEmpty = false
    } = options;

    const lines = csvString.split('\n').filter(line => 
      skipEmpty ? line.trim() : true
    );

    if (lines.length === 0) return [];

    let headers = [];
    let startIndex = 0;

    // Extract headers if present
    if (hasHeader && lines.length > 0) {
      headers = this.parseCSVLine(lines[0], delimiter, trim);
      startIndex = 1;
    }

    // Parse data rows
    const records = [];
    for (let i = startIndex; i < lines.length; i++) {
      const values = this.parseCSVLine(lines[i], delimiter, trim);

      if (values.length === 0) continue;

      if (hasHeader) {
        const record = {};
        headers.forEach((header, index) => {
          record[header] = values[index] || '';
        });
        records.push(record);
      } else {
        records.push(values);
      }
    }

    return records;
  }

  /**
   * Parse individual CSV line handling quotes and escapes
   * @param {string} line - CSV line
   * @param {string} delimiter - Field delimiter
   * @param {boolean} trim - Trim whitespace
   * @returns {array} Parsed fields
   */
  static parseCSVLine(line, delimiter = ',', trim = true) {
    const fields = [];
    let current = '';
    let insideQuotes = false;

    for (let i = 0; i < line.length; i++) {
      const char = line[i];
      const nextChar = line[i + 1];

      if (char === '"') {
        if (insideQuotes && nextChar === '"') {
          // Escaped quote
          current += '"';
          i++;
        } else {
          // Toggle quote state
          insideQuotes = !insideQuotes;
        }
      } else if (char === delimiter && !insideQuotes) {
        // Field delimiter found
        fields.push(trim ? current.trim() : current);
        current = '';
      } else {
        current += char;
      }
    }

    // Add last field
    fields.push(trim ? current.trim() : current);
    return fields;
  }

  /**
   * Generate CSV string from data
   * @param {array} data - Array of objects or arrays
   * @param {object} options - Configuration options
   * @returns {string} CSV formatted string
   */
  static generate(data, options = {}) {
    const {
      delimiter = ',',
      includeHeader = true,
      columns = null,
      quoteAll = false,
      lineTerminator = '\n'
    } = options;

    if (!Array.isArray(data) || data.length === 0) {
      return '';
    }

    const lines = [];
    const isArrayData = Array.isArray(data[0]);

    // Determine columns
    let cols = columns;
    if (!cols) {
      if (isArrayData) {
        cols = data[0].map((_, i) => i);
      } else {
        cols = Object.keys(data[0]);
      }
    }

    // Add header
    if (includeHeader) {
      const headerLine = cols
        .map(col => this.escapeCSVField(String(col), quoteAll))
        .join(delimiter);
      lines.push(headerLine);
    }

    // Add data rows
    for (const record of data) {
      const values = cols.map(col => {
        const value = isArrayData ? record[col] : record[col];
        return this.escapeCSVField(String(value ?? ''), quoteAll);
      });
      lines.push(values.join(delimiter));
    }

    return lines.join(lineTerminator);
  }

  /**
   * Escape CSV field value
   * @param {string} field - Field value
   * @param {boolean} quoteAll - Quote all fields
   * @returns {string} Escaped field
   */
  static escapeCSVField(field, quoteAll = false) {
    const needsQuotes = quoteAll || 
      field.includes(',') || 
      field.includes('"') || 
      field.includes('\n') ||
      field.includes('\r');

    if (needsQuotes) {
      return `"${field.replace(/"/g, '""')}"`;
    }
    return field;
  }
}

// Example usage
const csvData = `name,email,age,city
John Doe,john@example.com,28,New York
Jane Smith,jane@example.com,34,"San Francisco, CA"
Bob Johnson,bob@example.com,45,"Los Angeles, CA"`;

const parsed = CSVProcessor.parse(csvData);
console.log('Parsed:', parsed);

const generated = CSVProcessor.generate(parsed, { includeHeader: true });
console.log('Generated:\n', generated);

2. Advanced CSV Processor with Streaming

const fs = require('fs');
const { Transform, pipeline } = require('stream');

class StreamingCSVProcessor {
  /**
   * Stream parse large CSV files
   * @param {string} filePath - Path to CSV file
   * @param {function} onRecord - Callback for each record
   * @param {object} options - Configuration
   */
  static streamParse(filePath, onRecord, options = {}) {
    const {
      delimiter = ',',
      hasHeader = true,
      batchSize = 100
    } = options;

    return new Promise((resolve, reject) => {
      let headers = [];
      let lineBuffer = '';
      let recordCount = 0;
      let batch = [];

      const transform = new Transform({
        transform(chunk, encoding, callback) {
          lineBuffer += chunk.toString();
          const lines = lineBuffer.split('\n');
          lineBuffer = lines.pop();

          for (let i = 0; i < lines.length; i++) {
            const line = lines[i].trim();
            if (!line) continue;

            if (recordCount === 0 && hasHeader) {
              headers = CSVProcessor.parseCSVLine(line, delimiter);
              recordCount++;
              continue;
            }

            const values = CSVProcessor.parseCSVLine(line, delimiter);
            const record = {};
            headers.forEach((header, index) => {
              record[header] = values[index] || '';
            });

            batch.push(record);

            if (batch.length >= batchSize) {
              onRecord(batch);
              batch = [];
            }

            recordCount++;
          }

          callback();
        },
        flush(callback) {
          if (lineBuffer.trim()) {
            const values = CSVProcessor.parseCSVLine(lineBuffer, delimiter);
            const record = {};
            headers.forEach((header, index) => {
              record[header] = values[index] || '';
            });
            batch.push(record);
          }

          if (batch.length > 0) {
            onRecord(batch);
          }

          callback();
        }
      });

      pipeline(
        fs.createReadStream(filePath),
        transform,
        (err) => {
          if (err) reject(err);
          else resolve(recordCount);
        }
      );
    });
  }

  /**
   * Stream write data to CSV file
   * @param {string} filePath - Output file path
   * @param {array} data - Data to write
   * @param {object} options - Configuration
   */
  static streamGenerate(filePath, data, options = {}) {
    const {
      delimiter = ',',
      includeHeader = true
    } = options;

    return new Promise((resolve, reject) => {
      const writeStream = fs.createWriteStream(filePath);
      let isFirst = true;

      writeStream.on('error', reject);

      for (const record of data) {
        if (isFirst && includeHeader) {
          const headers = Object.keys(record);
          const headerLine = headers
            .map(h => CSVProcessor.escapeCSVField(h))
            .join(delimiter) + '\n';
          writeStream.write(headerLine);
          isFirst = false;
        }

        const values = Object.values(record)
          .map(v => CSVProcessor.escapeCSVField(String(v ?? '')))
          .join(delimiter) + '\n';

        writeStream.write(values);
      }

      writeStream.end(() => resolve());
    });
  }
}

// Example usage
(async () => {
  const recordCount = await StreamingCSVProcessor.streamParse(
    'large-file.csv',
    (batch) => {
      console.log(`Processed batch of ${batch.length} records`);
    },
    { batchSize: 1000 }
  );
  console.log(`Total records: ${recordCount}`);
})();

3. Data Validation and Transformation

class CSVValidator {
  /**
   * Validate CSV data against schema
   * @param {array} data - Parsed CSV data
   * @param {object} schema - Validation schema
   * @returns {object} Validation result
   */
  static validate(data, schema) {
    const errors = [];
    const validRecords = [];

    data.forEach((record, index) => {
      const recordErrors = [];

      Object.entries(schema).forEach(([field, rules]) => {
        const value = record[field];

        // Required check
        if (rules.required && (value === undefined || value === '')) {
          recordErrors.push(`Field "${field}" is required`);
        }

        // Type check
        if (value && rules.type) {
          if (!this.checkType(value, rules.type)) {
            recordErrors.push(
              `Field "${field}" must be ${rules.type}`
            );
          }
        }

        // Custom validation
        if (rules.validate && !rules.validate(value)) {
          recordErrors.push(
            `Field "${field}" failed custom validation`
          );
        }
      });

      if (recordErrors.length > 0) {
        errors.push({ row: index + 2, errors: recordErrors });
      } else {
        validRecords.push(record);
      }
    });

    return {
      valid: errors.length === 0,
      validRecords,
      errors,
      summary: {
        total: data.length,
        valid: validRecords.length,
        invalid: errors.length
      }
    };
  }

  /**
   * Check value type
   * @param {*} value - Value to check
   * @param {string} type - Expected type
   * @returns {boolean} Type matches
   */
  static checkType(value, type) {
    switch (type) {
      case 'string':
        return typeof value === 'string';
      case 'number':
        return !isNaN(value) && value !== '';
      case 'email':
        return /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(value);
      case 'date':
        return !isNaN(Date.parse(value));
      case 'boolean':
        return ['true', 'false', '1', '0'].includes(String(value).toLowerCase());
      default:
        return true;
    }
  }

  /**
   * Transform CSV data
   * @param {array} data - Input data
   * @param {object} transformations - Field transformations
   * @returns {array} Transformed data
   */
  static transform(data, transformations) {
    return data.map(record => {
      const transformed = { ...record };

      Object.entries(transformations).forEach(([field, transform]) => {
        if (transformed[field] !== undefined) {
          transformed[field] = transform(transformed[field]);
        }
      });

      return transformed;
    });
  }
}

// Example usage
const schema = {
  name: { required: true, type: 'string' },
  email: { required: true, type: 'email' },
  age: { required: false, type: 'number' },
  city: { required: false, type: 'string' }
};

const validation = CSVValidator.validate(parsed, schema);
console.log('Validation:', validation);

const transformed = CSVValidator.transform(parsed, {
  name: (v) => v.toUpperCase(),
  age: (v) => parseInt(v),
  email: (v) => v.toLowerCase()
});
console.log('Transformed:', transformed);

4. Complete Example with File Operations

class CSVManager {
  /**
   * Import CSV file
   * @param {string} filePath - File path
   * @param {object} options - Import options
   * @returns {array} Parsed data
   */
  static importCSV(filePath, options = {}) {
    const csvString = fs.readFileSync(filePath, 'utf-8');
    return CSVProcessor.parse(csvString, options);
  }

  /**
   * Export data to CSV file
   * @param {string} filePath - Output file path
   * @param {array} data - Data to export
   * @param {object} options - Export options
   */
  static exportCSV(filePath, data, options = {}) {
    const csv = CSVProcessor.generate(data, options);
    fs.writeFileSync(filePath, csv, 'utf-8');
  }

  /**
   * Process CSV with validation and transformation
   * @param {string} inputPath - Input file
   * @param {string} outputPath - Output file
   * @param {object} schema - Validation schema
   * @param {object} transforms - Transformations
   * @returns {object} Processing result
   */
  static processCSV(inputPath, outputPath, schema, transforms) {
    const data = this.importCSV(inputPath);
    const validation = CSVValidator.validate(data, schema);

    if (!validation.valid) {
      return {
        success: false,
        errors: validation.errors,
        summary: validation.summary
      };
    }

    const transformed = CSVValidator.transform(
      validation.validRecords,
      transforms
    );

    this.exportCSV(outputPath, transformed);

    return {
      success: true,
      processed: transformed.length,
      outputPath
    };
  }
}

// Complete workflow example
const result = CSVManager.processCSV(
  'input.csv',
  'output.csv',
  {
    name: { required: true, type: 'string' },
    email: { required: true, type: 'email' },
    age: { required: false, type: 'number' }
  },
  {
    name: (v) => v.trim().toUpperCase(),
    email: (v) => v.toLowerCase(),
    age: (v) => parseInt(v) || 0
  }
);

console.log('Result:', result);

Key Features

✅ Robust Parsing - Handles quotes, escapes, and special characters
✅ Efficient Generation - Creates properly formatted CSV output
✅ Streaming Support - Process large files without memory issues
✅ Data Validation - Schema-based validation with custom rules
✅ Transformations - Map and transform data during processing
✅ Error Handling - Comprehensive error reporting
✅ Flexible Options - Customizable delimiters, headers, and formatting
✅ Production Ready - Handles edge cases and large datasets