     1→import Foundation
     2→import SQLite3
     3→
     4→// MARK: - Bulk Import Service
     5→/// High-performance import service for millions of records using SQLite
     6→
     7→actor BulkImportService {
     8→    static let shared = BulkImportService()
     9→
    10→    // Private - use accessor methods for cross-actor access
    11→    private var db: OpaquePointer?
    12→    private let dbPath: String
    13→
    14→    private init() {
    15→        let documentsPath = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)[0]
    16→        dbPath = documentsPath.appendingPathComponent("bulk_import.sqlite").path
    17→        // Database will be initialized on first access via ensureDatabaseReady()
    18→    }
    19→
    20→    // MARK: - Database Setup
    21→
    22→    /// Ensures database is initialized before any operation
    23→    /// Called at the start of every public method that accesses the database
    24→    /// Internal access for DataSyncService extension
    25→    func ensureDatabaseReady() {
    26→        guard db == nil else { return }
    27→        setupDatabase()
    28→    }
    29→
    30→    // MARK: - SQLite Error Handling
    31→
    32→    /// SQLite operation result for better error tracking
    33→    enum SQLiteResult {
    34→        case success
    35→        case error(code: Int32, message: String)
    36→
    37→        var isSuccess: Bool {
    38→            if case .success = self { return true }
    39→            return false
    40→        }
    41→    }
    42→
    43→    /// Execute SQL with error handling and logging
    44→    /// - Parameters:
    45→    ///   - sql: The SQL statement to execute
    46→    ///   - operation: Description of the operation for logging
    47→    ///   - silent: If true, don't log errors (useful for migrations that may fail)
    48→    /// - Returns: SQLiteResult indicating success or failure with details
    49→    /// Internal access for DataSyncService extension
    50→    @discardableResult
    51→    func executeSQL(_ sql: String, operation: String = "SQL operation", silent: Bool = false) -> SQLiteResult {
    52→        guard let db = db else {
    53→            if !silent {
    54→                LoggingService.shared.error("\(operation) failed: database not initialized", category: .database)
    55→            }
    56→            return .error(code: SQLITE_ERROR, message: "Database not initialized")
    57→        }
    58→
    59→        var errorMessage: UnsafeMutablePointer<CChar>?
    60→        let result = sqlite3_exec(db, sql, nil, nil, &errorMessage)
    61→
    62→        if result != SQLITE_OK {
    63→            let message = errorMessage.map { String(cString: $0) } ?? "Unknown error"
    64→            sqlite3_free(errorMessage)
    65→
    66→            if !silent {
    67→                let extendedCode = sqlite3_extended_errcode(db)
    68→                LoggingService.shared.error(
    69→                    "\(operation) failed: \(message) (code: \(result), extended: \(extendedCode))",
    70→                    category: .database
    71→                )
    72→            }
    73→            return .error(code: result, message: message)
    74→        }
    75→
    76→        return .success
    77→    }
    78→
    79→    /// Begin a database transaction with error handling
    80→    @discardableResult
    81→    private func beginTransaction() -> SQLiteResult {
    82→        executeSQL("BEGIN TRANSACTION;", operation: "Begin transaction")
    83→    }
    84→
    85→    /// Commit a database transaction with error handling
    86→    @discardableResult
    87→    private func commitTransaction() -> SQLiteResult {
    88→        executeSQL("COMMIT;", operation: "Commit transaction")
    89→    }
    90→
    91→    /// Rollback a database transaction with error handling
    92→    @discardableResult
    93→    private func rollbackTransaction() -> SQLiteResult {
    94→        executeSQL("ROLLBACK;", operation: "Rollback transaction")
    95→    }
    96→
    97→    // MARK: - Delete All Data
    98→
    99→    /// Deletes ALL data from BulkImport SQLite database
   100→    /// Used for complete app reset
   101→    func deleteAllBulkData() {
   102→        LoggingService.shared.warning("🗑 Clearing BulkImport SQLite database...", category: .database)
   103→
   104→        // Delete all records from each table
   105→        executeSQL("DELETE FROM emails;", operation: "Clear emails table")
   106→        executeSQL("DELETE FROM domains;", operation: "Clear domains table")
   107→        executeSQL("DELETE FROM scraped_sites;", operation: "Clear scraped_sites table")
   108→
   109→        // Reclaim disk space
   110→        executeSQL("VACUUM;", operation: "Reclaim disk space")
   111→
   112→        LoggingService.shared.info("🗑 BulkImport database cleared and vacuumed", category: .database)
   113→    }
   114→
   115→    /// Check if a column exists in a table
   116→    /// - Parameters:
   117→    ///   - table: The table name to check
   118→    ///   - column: The column name to look for
   119→    /// - Returns: true if the column exists, false otherwise
   120→    /// Internal access for DataSyncService extension
   121→    func columnExists(table: String, column: String) -> Bool {
   122→        guard let db = db else { return false }
   123→
   124→        var stmt: OpaquePointer?
   125→        let sql = "PRAGMA table_info(\(table));"
   126→
   127→        guard sqlite3_prepare_v2(db, sql, -1, &stmt, nil) == SQLITE_OK else {
   128→            return false
   129→        }
   130→
   131→        defer { sqlite3_finalize(stmt) }
   132→
   133→        while sqlite3_step(stmt) == SQLITE_ROW {
   134→            if let namePtr = sqlite3_column_text(stmt, 1) {
   135→                let columnName = String(cString: namePtr)
   136→                if columnName == column {
   137→                    return true
   138→                }
   139→            }
   140→        }
   141→
   142→        return false
   143→    }
   144→
   145→    /// Add a column to a table if it doesn't exist (silent migration)
   146→    /// - Parameters:
   147→    ///   - table: The table name
   148→    ///   - column: The column name
   149→    ///   - type: The column type (e.g., "TEXT", "INTEGER", "REAL")
   150→    ///   - defaultValue: Optional default value
   151→    private func addColumnIfNotExists(table: String, column: String, type: String, defaultValue: String? = nil) {
   152→        guard !columnExists(table: table, column: column) else { return }
   153→
   154→        var sql = "ALTER TABLE \(table) ADD COLUMN \(column) \(type)"
   155→        if let defaultValue = defaultValue {
   156→            sql += " DEFAULT \(defaultValue)"
   157→        }
   158→        sql += ";"
   159→
   160→        executeSQL(sql, operation: "Add column \(column) to \(table)", silent: true)
   161→    }
   162→
   163→    private func setupDatabase() {
   164→        if sqlite3_open(dbPath, &db) == SQLITE_OK {
   165→            // Emails table
   166→            let createEmailsTable = """
   167→                CREATE TABLE IF NOT EXISTS emails (
   168→                    id INTEGER PRIMARY KEY AUTOINCREMENT,
   169→                    email TEXT UNIQUE NOT NULL,
   170→                    domain TEXT NOT NULL,
   171→                    tld TEXT NOT NULL,
   172→                    category TEXT DEFAULT 'other',
   173→                    status TEXT DEFAULT 'pending',
   174→                    imported_at REAL DEFAULT (julianday('now'))
   175→                );
   176→                CREATE INDEX IF NOT EXISTS idx_email ON emails(email);
   177→                CREATE INDEX IF NOT EXISTS idx_domain ON emails(domain);
   178→                CREATE INDEX IF NOT EXISTS idx_tld ON emails(tld);
   179→                CREATE INDEX IF NOT EXISTS idx_category ON emails(category);
   180→            """
   181→            sqlite3_exec(db, createEmailsTable, nil, nil, nil)
   182→
   183→            // Domains table
   184→            let createDomainsTable = """
   185→                CREATE TABLE IF NOT EXISTS domains (
   186→                    id INTEGER PRIMARY KEY AUTOINCREMENT,
   187→                    domain TEXT UNIQUE NOT NULL,
   188→                    status TEXT DEFAULT 'pending',
   189→                    dns_valid INTEGER,
   190→                    ip_address TEXT,
   191→                    http_status INTEGER,
   192→                    response_time REAL,
   193→                    checked_at REAL,
   194→                    imported_at REAL DEFAULT (julianday('now'))
   195→                );
   196→                CREATE INDEX IF NOT EXISTS idx_dom ON domains(domain);
   197→                CREATE INDEX IF NOT EXISTS idx_dom_status ON domains(status);
   198→            """
   199→            sqlite3_exec(db, createDomainsTable, nil, nil, nil)
   200→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

import Foundation
import SQLite3

// MARK: - Bulk Import Service
/// High-performance import service for millions of records using SQLite

actor BulkImportService {
    static let shared = BulkImportService()

    // Private - use accessor methods for cross-actor access
    private var db: OpaquePointer?
    private let dbPath: String

    private init() {
        let documentsPath = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)[0]
        dbPath = documentsPath.appendingPathComponent("bulk_import.sqlite").path
        // Database will be initialized on first access via ensureDatabaseReady()
    }

    // MARK: - Database Setup

    /// Ensures database is initialized before any operation
    /// Called at the start of every public method that accesses the database
    /// Internal access for DataSyncService extension
    func ensureDatabaseReady() {
        guard db == nil else { return }
        setupDatabase()
    }

    // MARK: - SQLite Error Handling

    /// SQLite operation result for better error tracking
    enum SQLiteResult {
        case success
        case error(code: Int32, message: String)

        var isSuccess: Bool {
            if case .success = self { return true }
            return false
        }
    }

    /// Execute SQL with error handling and logging
    /// - Parameters:
    ///   - sql: The SQL statement to execute
    ///   - operation: Description of the operation for logging
    ///   - silent: If true, don't log errors (useful for migrations that may fail)
    /// - Returns: SQLiteResult indicating success or failure with details
    /// Internal access for DataSyncService extension
    @discardableResult
    func executeSQL(_ sql: String, operation: String = "SQL operation", silent: Bool = false) -> SQLiteResult {
        guard let db = db else {
            if !silent {
                LoggingService.shared.error("\(operation) failed: database not initialized", category: .database)
            }
            return .error(code: SQLITE_ERROR, message: "Database not initialized")
        }

        var errorMessage: UnsafeMutablePointer<CChar>?
        let result = sqlite3_exec(db, sql, nil, nil, &errorMessage)

        if result != SQLITE_OK {
            let message = errorMessage.map { String(cString: $0) } ?? "Unknown error"
            sqlite3_free(errorMessage)

            if !silent {
                let extendedCode = sqlite3_extended_errcode(db)
                LoggingService.shared.error(
                    "\(operation) failed: \(message) (code: \(result), extended: \(extendedCode))",
                    category: .database
                )
            }
            return .error(code: result, message: message)
        }

        return .success
    }

    /// Begin a database transaction with error handling
    @discardableResult
    private func beginTransaction() -> SQLiteResult {
        executeSQL("BEGIN TRANSACTION;", operation: "Begin transaction")
    }

    /// Commit a database transaction with error handling
    @discardableResult
    private func commitTransaction() -> SQLiteResult {
        executeSQL("COMMIT;", operation: "Commit transaction")
    }

    /// Rollback a database transaction with error handling
    @discardableResult
    private func rollbackTransaction() -> SQLiteResult {
        executeSQL("ROLLBACK;", operation: "Rollback transaction")
    }

    // MARK: - Delete All Data

    /// Deletes ALL data from BulkImport SQLite database
    /// Used for complete app reset
    func deleteAllBulkData() {
        LoggingService.shared.warning("🗑 Clearing BulkImport SQLite database...", category: .database)

        // Delete all records from each table
        executeSQL("DELETE FROM emails;", operation: "Clear emails table")
        executeSQL("DELETE FROM domains;", operation: "Clear domains table")
        executeSQL("DELETE FROM scraped_sites;", operation: "Clear scraped_sites table")

        // Reclaim disk space
        executeSQL("VACUUM;", operation: "Reclaim disk space")

        LoggingService.shared.info("🗑 BulkImport database cleared and vacuumed", category: .database)
    }

    /// Check if a column exists in a table
    /// - Parameters:
    ///   - table: The table name to check
    ///   - column: The column name to look for
    /// - Returns: true if the column exists, false otherwise
    /// Internal access for DataSyncService extension
    func columnExists(table: String, column: String) -> Bool {
        guard let db = db else { return false }

        var stmt: OpaquePointer?
        let sql = "PRAGMA table_info(\(table));"

        guard sqlite3_prepare_v2(db, sql, -1, &stmt, nil) == SQLITE_OK else {
            return false
        }

        defer { sqlite3_finalize(stmt) }

        while sqlite3_step(stmt) == SQLITE_ROW {
            if let namePtr = sqlite3_column_text(stmt, 1) {
                let columnName = String(cString: namePtr)
                if columnName == column {
                    return true
                }
            }
        }

        return false
    }

    /// Add a column to a table if it doesn't exist (silent migration)
    /// - Parameters:
    ///   - table: The table name
    ///   - column: The column name
    ///   - type: The column type (e.g., "TEXT", "INTEGER", "REAL")
    ///   - defaultValue: Optional default value
    private func addColumnIfNotExists(table: String, column: String, type: String, defaultValue: String? = nil) {
        guard !columnExists(table: table, column: column) else { return }

        var sql = "ALTER TABLE \(table) ADD COLUMN \(column) \(type)"
        if let defaultValue = defaultValue {
            sql += " DEFAULT \(defaultValue)"
        }
        sql += ";"

        executeSQL(sql, operation: "Add column \(column) to \(table)", silent: true)
    }

    private func setupDatabase() {
        if sqlite3_open(dbPath, &db) == SQLITE_OK {
            // Emails table
            let createEmailsTable = """
                CREATE TABLE IF NOT EXISTS emails (
                    id INTEGER PRIMARY KEY AUTOINCREMENT,
                    email TEXT UNIQUE NOT NULL,
                    domain TEXT NOT NULL,
                    tld TEXT NOT NULL,
                    category TEXT DEFAULT 'other',
                    status TEXT DEFAULT 'pending',
                    imported_at REAL DEFAULT (julianday('now'))
                );
                CREATE INDEX IF NOT EXISTS idx_email ON emails(email);
                CREATE INDEX IF NOT EXISTS idx_domain ON emails(domain);
                CREATE INDEX IF NOT EXISTS idx_tld ON emails(tld);
                CREATE INDEX IF NOT EXISTS idx_category ON emails(category);
            """
            sqlite3_exec(db, createEmailsTable, nil, nil, nil)

            // Domains table
            let createDomainsTable = """
                CREATE TABLE IF NOT EXISTS domains (
                    id INTEGER PRIMARY KEY AUTOINCREMENT,
                    domain TEXT UNIQUE NOT NULL,
                    status TEXT DEFAULT 'pending',
                    dns_valid INTEGER,
                    ip_address TEXT,
                    http_status INTEGER,
                    response_time REAL,
                    checked_at REAL,
                    imported_at REAL DEFAULT (julianday('now'))
                );
                CREATE INDEX IF NOT EXISTS idx_dom ON domains(domain);
                CREATE INDEX IF NOT EXISTS idx_dom_status ON domains(status);
            """
            sqlite3_exec(db, createDomainsTable, nil, nil, nil)


Now I have a comprehensive understanding of the existing patterns. Let me now create a detailed implementation plan for the BRDataMiner app:


TodoWrite

Explore existing DomainChecker patterns and architecture

Exploring existing DomainChecker patterns

Design StreamingFileReader for 170GB file processing

Design data models (ExtractedData, ExtractionStats, etc.)

Design RegexExtractor service with Swift Regex patterns

Design CPF/CNPJ validation logic

Design CheckpointManager for pause/resume

Design UI views structure

Document complete implementation plan


Todos have been modified successfully. Ensure that you continue to use the todo list to track your progress. Please proceed with the current tasks if applicable

Explore existing DomainChecker patterns and architecture

Exploring existing DomainChecker patterns

Design StreamingFileReader for 170GB file processing

Design data models (ExtractedData, ExtractionStats, etc.)

Design RegexExtractor service with Swift Regex patterns

Design CPF/CNPJ validation logic

Design CheckpointManager for pause/resume

Design UI views structure

Document complete implementation plan

Design CPF/CNPJ validation logic

Design UI views structure

Document complete implementation plan


# BRDataMiner - Complete Implementation Plan

## Executive Summary

Based on my analysis of the existing DomainChecker project at `/Users/neog/Projects/tools/DomainChecker/` and other iOS apps in the workspace, I have designed a comprehensive implementation plan for BRDataMiner - a macOS SwiftUI app for mining Brazilian data from a 170GB text file.

---

## 1. Project Structure

```
/Users/neog/Apps/macOS/BRDataMiner/
├── BRDataMiner.xcodeproj/
├── BRDataMiner/
│   ├── BRDataMinerApp.swift              # @main entry point
│   ├── ContentView.swift                  # Main navigation
│   │
│   ├── Models/
│   │   ├── ExtractedData.swift           # Data types (Email, CPF, etc.)
│   │   ├── ExtractionSession.swift       # Session state
│   │   ├── ExtractionProgress.swift      # Progress tracking
│   │   ├── ExtractionConfig.swift        # User configuration
│   │   └── ExportFormat.swift            # JSON/CSV export options
│   │
│   ├── Services/
│   │   ├── StreamingFileReader.swift     # CRITICAL: Line-by-line file reading
│   │   ├── RegexExtractor.swift          # Pattern matching for all types
│   │   ├── DataValidator.swift           # CPF/CNPJ checksum validation
│   │   ├── CheckpointManager.swift       # Pause/resume state persistence
│   │   ├── ExtractionEngine.swift        # Main orchestrator (Actor)
│   │   ├── ExportService.swift           # JSON/CSV export
│   │   └── StorageManager.swift          # SQLite for extracted data
│   │
│   ├── Views/
│   │   ├── Setup/
│   │   │   ├── FileSelectionView.swift   # File picker for input
│   │   │   ├── OutputDirectoryView.swift # Output directory picker
│   │   │   └── ExtractionOptionsView.swift # Data type toggles
│   │   │
│   │   ├── Progress/
│   │   │   ├── ExtractionProgressView.swift # Main progress view
│   │   │   ├── ProgressRingView.swift    # Circular progress
│   │   │   ├── LiveStatsView.swift       # Real-time counters
│   │   │   └── ControlButtonsView.swift  # Pause/Resume/Cancel
│   │   │
│   │   ├── Results/
│   │   │   ├── ResultsDashboardView.swift # Statistics overview
│   │   │   ├── DataPreviewView.swift     # Preview extracted data
│   │   │   ├── ExportOptionsView.swift   # Export configuration
│   │   │   └── StatCardView.swift        # Reusable stat card
│   │   │
│   │   └── Components/
│   │       ├── MetricCard.swift          # Metric display component
│   │       ├── ProgressBar.swift         # Linear progress bar
│   │       └── StatusIndicator.swift     # Processing status
│   │
│   └── Resources/
│       ├── Assets.xcassets/
│       └── Info.plist
│
├── Package.swift                          # Swift Package (if using SPM)
└── CLAUDE.md                             # Project documentation
```

---

## 2. Data Models

### 2.1 ExtractedData.swift

```swift
import Foundation

/// Type of Brazilian data extracted
enum DataType: String, CaseIterable, Codable {
    case email = "email"
    case emailBR = "email_br"
    case domainBR = "domain_br"
    case cpf = "cpf"
    case cnpj = "cnpj"
    case phoneBR = "phone_br"
    case url = "url"

    var displayName: String {
        switch self {
        case .email: return "Email (International)"
        case .emailBR: return "Email (.br)"
        case .domainBR: return "Domain .br"
        case .cpf: return "CPF"
        case .cnpj: return "CNPJ"
        case .phoneBR: return "Phone BR"
        case .url: return "URL"
        }
    }

    var icon: String {
        switch self {
        case .email, .emailBR: return "envelope.fill"
        case .domainBR: return "globe.americas.fill"
        case .cpf: return "person.text.rectangle.fill"
        case .cnpj: return "building.2.fill"
        case .phoneBR: return "phone.fill"
        case .url: return "link"
        }
    }

    var color: Color {
        switch self {
        case .email: return .blue
        case .emailBR: return .green
        case .domainBR: return .teal
        case .cpf: return .orange
        case .cnpj: return .purple
        case .phoneBR: return .pink
        case .url: return .indigo
        }
    }
}

/// Single extracted item
struct ExtractedItem: Codable, Identifiable, Hashable {
    let id: UUID
    let value: String
    let type: DataType
    let lineNumber: Int64
    let extractedAt: Date
    let isValid: Bool  // For CPF/CNPJ validation

    init(value: String, type: DataType, lineNumber: Int64, isValid: Bool = true) {
        self.id = UUID()
        self.value = value
        self.type = type
        self.lineNumber = lineNumber
        self.extractedAt = Date()
        self.isValid = isValid
    }
}

/// Statistics for extraction session
struct ExtractionStatistics: Codable {
    var totalLines: Int64 = 0
    var processedLines: Int64 = 0
    var bytesProcessed: Int64 = 0
    var totalBytes: Int64 = 0

    var emailCount: Int = 0
    var emailBRCount: Int = 0
    var domainBRCount: Int = 0
    var cpfCount: Int = 0
    var cpfValidCount: Int = 0
    var cnpjCount: Int = 0
    var cnpjValidCount: Int = 0
    var phoneBRCount: Int = 0
    var urlCount: Int = 0

    var duplicatesSkipped: Int = 0
    var invalidSkipped: Int = 0

    var startTime: Date?
    var endTime: Date?

    var processingTimeSeconds: Double {
        guard let start = startTime else { return 0 }
        let end = endTime ?? Date()
        return end.timeIntervalSince(start)
    }

    var linesPerSecond: Double {
        guard processingTimeSeconds > 0 else { return 0 }
        return Double(processedLines) / processingTimeSeconds
    }

    var bytesPerSecond: Double {
        guard processingTimeSeconds > 0 else { return 0 }
        return Double(bytesProcessed) / processingTimeSeconds
    }

    var percentComplete: Double {
        guard totalBytes > 0 else { return 0 }
        return Double(bytesProcessed) / Double(totalBytes)
    }

    var estimatedSecondsRemaining: Int {
        guard bytesPerSecond > 0, bytesProcessed > 0 else { return 0 }
        let remainingBytes = totalBytes - bytesProcessed
        return Int(Double(remainingBytes) / bytesPerSecond)
    }

    var formattedETA: String {
        let seconds = estimatedSecondsRemaining
        if seconds <= 0 { return "--" }

        let hours = seconds / 3600
        let minutes = (seconds % 3600) / 60
        let secs = seconds % 60

        if hours > 0 {
            return String(format: "%dh %02dm", hours, minutes)
        } else if minutes > 0 {
            return String(format: "%dm %02ds", minutes, secs)
        } else {
            return String(format: "%ds", secs)
        }
    }
}
```

### 2.2 ExtractionSession.swift

```swift
import Foundation

/// State of extraction
enum ExtractionState: String, Codable {
    case idle
    case running
    case paused
    case completed
    case cancelled
    case error
}

/// Configuration for what to extract
struct ExtractionConfig: Codable {
    var extractEmails: Bool = true
    var extractEmailsBR: Bool = true
    var extractDomainsBR: Bool = true
    var extractCPF: Bool = true
    var validateCPF: Bool = true
    var extractCNPJ: Bool = true
    var validateCNPJ: Bool = true
    var extractPhonesBR: Bool = true
    var extractURLs: Bool = true
    var deduplicateResults: Bool = true
    var batchSize: Int = 10_000  // Lines per batch
}

/// Full session state
struct ExtractionSession: Codable, Identifiable {
    let id: UUID
    var sourceFilePath: String
    var sourceFileName: String
    var outputDirectoryPath: String
    var config: ExtractionConfig
    var state: ExtractionState
    var statistics: ExtractionStatistics
    var lastProcessedByteOffset: Int64
    var createdAt: Date
    var updatedAt: Date

    init(sourceFile: URL, outputDirectory: URL, config: ExtractionConfig = ExtractionConfig()) {
        self.id = UUID()
        self.sourceFilePath = sourceFile.path
        self.sourceFileName = sourceFile.lastPathComponent
        self.outputDirectoryPath = outputDirectory.path
        self.config = config
        self.state = .idle
        self.statistics = ExtractionStatistics()
        self.lastProcessedByteOffset = 0
        self.createdAt = Date()
        self.updatedAt = Date()
    }
}
```

---

## 3. Core Services Implementation

### 3.1 StreamingFileReader.swift (CRITICAL for 170GB)

```swift
import Foundation

/// Reads file line-by-line without loading into memory
/// Uses FileHandle for seeking and buffered reading
actor StreamingFileReader {
    private let fileURL: URL
    private var fileHandle: FileHandle?
    private var fileSize: Int64 = 0
    private var currentOffset: Int64 = 0
    private let bufferSize: Int = 64 * 1024  // 64KB buffer
    private var leftoverData: Data = Data()

    init(fileURL: URL) {
        self.fileURL = fileURL
    }

    /// Open file and get size
    func open() throws {
        // Request security-scoped access for external SSD
        guard fileURL.startAccessingSecurityScopedResource() else {
            throw StreamingError.accessDenied
        }

        fileHandle = try FileHandle(forReadingFrom: fileURL)

        // Get file size
        let attributes = try FileManager.default.attributesOfItem(atPath: fileURL.path)
        fileSize = attributes[.size] as? Int64 ?? 0
    }

    /// Close file handle
    func close() {
        try? fileHandle?.close()
        fileHandle = nil
        fileURL.stopAccessingSecurityScopedResource()
    }

    /// Seek to byte offset (for resume)
    func seek(to offset: Int64) throws {
        guard let handle = fileHandle else {
            throw StreamingError.fileNotOpen
        }
        try handle.seek(toOffset: UInt64(offset))
        currentOffset = offset
        leftoverData = Data()
    }

    /// Read next batch of lines
    /// Returns array of (line, byteOffset) tuples
    func readLines(count: Int) throws -> [(line: String, offset: Int64)] {
        guard let handle = fileHandle else {
            throw StreamingError.fileNotOpen
        }

        var lines: [(line: String, offset: Int64)] = []
        var lineStartOffset = currentOffset

        while lines.count < count {
            // Read buffer
            let readData: Data
            if leftoverData.isEmpty {
                guard let data = try handle.read(upToCount: bufferSize), !data.isEmpty else {
                    break  // EOF
                }
                readData = data
            } else {
                // Combine leftover with new data
                guard let newData = try handle.read(upToCount: bufferSize) else {
                    // Process remaining leftover
                    if let line = String(data: leftoverData, encoding: .utf8) {
                        lines.append((line, lineStartOffset))
                    }
                    leftoverData = Data()
                    break
                }
                readData = leftoverData + newData
                leftoverData = Data()
            }

            // Split by newlines
            var start = readData.startIndex
            for i in readData.indices {
                if readData[i] == UInt8(ascii: "\n") {
                    let lineData = readData[start..<i]
                    if let line = String(data: lineData, encoding: .utf8)?
                        .trimmingCharacters(in: .carriageReturn) {
                        lines.append((line, lineStartOffset))
                        lineStartOffset = currentOffset + Int64(i - readData.startIndex) + 1
                    }
                    start = readData.index(after: i)

                    if lines.count >= count {
                        // Save remaining as leftover
                        leftoverData = Data(readData[start...])
                        currentOffset += Int64(i - readData.startIndex) + 1
                        return lines
                    }
                }
            }

            // Save unprocessed data
            if start < readData.endIndex {
                leftoverData = Data(readData[start...])
            }
            currentOffset += Int64(readData.count)
        }

        return lines
    }

    /// Current byte position
    var position: Int64 { currentOffset }

    /// Total file size
    var totalSize: Int64 { fileSize }

    /// Errors
    enum StreamingError: Error, LocalizedError {
        case accessDenied
        case fileNotOpen
        case readError

        var errorDescription: String? {
            switch self {
            case .accessDenied: return "Cannot access file on external drive"
            case .fileNotOpen: return "File not opened"
            case .readError: return "Error reading file"
            }
        }
    }
}

extension CharacterSet {
    static let carriageReturn = CharacterSet(charactersIn: "\r")
}
```

### 3.2 RegexExtractor.swift

```swift
import Foundation

/// Extracts Brazilian data using optimized regex patterns
actor RegexExtractor {

    // MARK: - Compiled Regex Patterns (Swift 5.7+ Regex)

    // Email: standard pattern
    private let emailRegex = try! Regex(#"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}"#)

    // .br domain pattern
    private let brDomainRegex = try! Regex(#"[a-zA-Z0-9][a-zA-Z0-9.-]*\.(?:com|org|net|gov|edu|mil|jus|leg|mp|def|tur|inf|etc|nom|srv|wiki|blog|flog|vlog|art|pro|rec|ind|agr|adv|med|odo|psi|vet|cri)\.br"#)

    // CPF: XXX.XXX.XXX-XX or XXXXXXXXXXX (11 digits)
    private let cpfRegex = try! Regex(#"\b\d{3}\.?\d{3}\.?\d{3}[-.]?\d{2}\b"#)

    // CNPJ: XX.XXX.XXX/XXXX-XX or XXXXXXXXXXXXXX (14 digits)
    private let cnpjRegex = try! Regex(#"\b\d{2}\.?\d{3}\.?\d{3}/?0001[-.]?\d{2}\b"#)

    // Brazilian phone: (XX) XXXXX-XXXX, +55 XX XXXXX-XXXX, etc.
    private let phoneBRRegex = try! Regex(#"(?:\+55\s?)?(?:\(?\d{2}\)?\s?)(?:9?\d{4})[-.\s]?\d{4}"#)

    // URL pattern
    private let urlRegex = try! Regex(#"https?://[^\s<>\"'\])]+"#)

    // MARK: - Extraction Methods

    /// Extract all data types from a single line
    func extractAll(
        from line: String,
        lineNumber: Int64,
        config: ExtractionConfig
    ) -> [ExtractedItem] {
        var items: [ExtractedItem] = []

        if config.extractEmails || config.extractEmailsBR {
            items.append(contentsOf: extractEmails(from: line, lineNumber: lineNumber, config: config))
        }

        if config.extractDomainsBR {
            items.append(contentsOf: extractBRDomains(from: line, lineNumber: lineNumber))
        }

        if config.extractCPF {
            items.append(contentsOf: extractCPF(from: line, lineNumber: lineNumber, validate: config.validateCPF))
        }

        if config.extractCNPJ {
            items.append(contentsOf: extractCNPJ(from: line, lineNumber: lineNumber, validate: config.validateCNPJ))
        }

        if config.extractPhonesBR {
            items.append(contentsOf: extractPhonesBR(from: line, lineNumber: lineNumber))
        }

        if config.extractURLs {
            items.append(contentsOf: extractURLs(from: line, lineNumber: lineNumber))
        }

        return items
    }

    // MARK: - Individual Extractors

    private func extractEmails(from line: String, lineNumber: Int64, config: ExtractionConfig) -> [ExtractedItem] {
        var items: [ExtractedItem] = []

        for match in line.matches(of: emailRegex) {
            let email = String(match.output).lowercased()
            let isBR = email.hasSuffix(".br")

            if isBR && config.extractEmailsBR {
                items.append(ExtractedItem(value: email, type: .emailBR, lineNumber: lineNumber))
            } else if !isBR && config.extractEmails {
                items.append(ExtractedItem(value: email, type: .email, lineNumber: lineNumber))
            }
        }

        return items
    }

    private func extractBRDomains(from line: String, lineNumber: Int64) -> [ExtractedItem] {
        line.matches(of: brDomainRegex).map { match in
            ExtractedItem(value: String(match.output).lowercased(), type: .domainBR, lineNumber: lineNumber)
        }
    }

    private func extractCPF(from line: String, lineNumber: Int64, validate: Bool) -> [ExtractedItem] {
        line.matches(of: cpfRegex).compactMap { match in
            let raw = String(match.output)
            let normalized = normalizeNumeric(raw)

            guard normalized.count == 11 else { return nil }

            let isValid = validate ? DataValidator.validateCPF(normalized) : true
            return ExtractedItem(value: formatCPF(normalized), type: .cpf, lineNumber: lineNumber, isValid: isValid)
        }
    }

    private func extractCNPJ(from line: String, lineNumber: Int64, validate: Bool) -> [ExtractedItem] {
        line.matches(of: cnpjRegex).compactMap { match in
            let raw = String(match.output)
            let normalized = normalizeNumeric(raw)

            guard normalized.count == 14 else { return nil }

            let isValid = validate ? DataValidator.validateCNPJ(normalized) : true
            return ExtractedItem(value: formatCNPJ(normalized), type: .cnpj, lineNumber: lineNumber, isValid: isValid)
        }
    }

    private func extractPhonesBR(from line: String, lineNumber: Int64) -> [ExtractedItem] {
        line.matches(of: phoneBRRegex).map { match in
            let normalized = normalizePhone(String(match.output))
            ExtractedItem(value: normalized, type: .phoneBR, lineNumber: lineNumber)
        }
    }

    private func extractURLs(from line: String, lineNumber: Int64) -> [ExtractedItem] {
        line.matches(of: urlRegex).map { match in
            ExtractedItem(value: String(match.output), type: .url, lineNumber: lineNumber)
        }
    }

    // MARK: - Helpers

    private func normalizeNumeric(_ input: String) -> String {
        input.filter { $0.isNumber }
    }

    private func formatCPF(_ digits: String) -> String {
        guard digits.count == 11 else { return digits }
        let d = Array(digits)
        return "\(d[0...2].joined()).\(d[3...5].joined()).\(d[6...8].joined())-\(d[9...10].joined())"
    }

    private func formatCNPJ(_ digits: String) -> String {
        guard digits.count == 14 else { return digits }
        let d = Array(digits)
        return "\(d[0...1].joined()).\(d[2...4].joined()).\(d[5...7].joined())/\(d[8...11].joined())-\(d[12...13].joined())"
    }

    private func normalizePhone(_ input: String) -> String {
        let digits = input.filter { $0.isNumber }

        if digits.count == 11 {
            // (XX) XXXXX-XXXX
            let d = Array(digits)
            return "(\(d[0...1].joined())) \(d[2...6].joined())-\(d[7...10].joined())"
        } else if digits.count == 13 && digits.hasPrefix("55") {
            // +55 XX XXXXX-XXXX
            let d = Array(digits.dropFirst(2))
            return "+55 (\(d[0...1].joined())) \(d[2...6].joined())-\(d[7...10].joined())"
        }

        return input
    }
}

extension Array {
    func joined() -> String where Element == Character {
        String(self)
    }
}
```

### 3.3 DataValidator.swift

```swift
import Foundation

/// Validates CPF and CNPJ checksums
enum DataValidator {

    /// Validate CPF checksum (Brazilian personal ID)
    /// CPF has 2 check digits calculated using modulo 11
    static func validateCPF(_ cpf: String) -> Bool {
        let digits = cpf.filter { $0.isNumber }
        guard digits.count == 11 else { return false }

        // Reject known invalid patterns (all same digit)
        let allSame = Set(digits).count == 1
        if allSame { return false }

        let nums = digits.compactMap { Int(String($0)) }
        guard nums.count == 11 else { return false }

        // First check digit
        var sum = 0
        for i in 0..<9 {
            sum += nums[i] * (10 - i)
        }
        var remainder = sum % 11
        let digit1 = remainder < 2 ? 0 : 11 - remainder

        guard nums[9] == digit1 else { return false }

        // Second check digit
        sum = 0
        for i in 0..<10 {
            sum += nums[i] * (11 - i)
        }
        remainder = sum % 11
        let digit2 = remainder < 2 ? 0 : 11 - remainder

        return nums[10] == digit2
    }

    /// Validate CNPJ checksum (Brazilian company ID)
    /// CNPJ has 2 check digits calculated using modulo 11
    static func validateCNPJ(_ cnpj: String) -> Bool {
        let digits = cnpj.filter { $0.isNumber }
        guard digits.count == 14 else { return false }

        // Reject known invalid patterns (all same digit)
        let allSame = Set(digits).count == 1
        if allSame { return false }

        let nums = digits.compactMap { Int(String($0)) }
        guard nums.count == 14 else { return false }

        // First check digit
        let weights1 = [5, 4, 3, 2, 9, 8, 7, 6, 5, 4, 3, 2]
        var sum = 0
        for i in 0..<12 {
            sum += nums[i] * weights1[i]
        }
        var remainder = sum % 11
        let digit1 = remainder < 2 ? 0 : 11 - remainder

        guard nums[12] == digit1 else { return false }

        // Second check digit
        let weights2 = [6, 5, 4, 3, 2, 9, 8, 7, 6, 5, 4, 3, 2]
        sum = 0
        for i in 0..<13 {
            sum += nums[i] * weights2[i]
        }
        remainder = sum % 11
        let digit2 = remainder < 2 ? 0 : 11 - remainder

        return nums[13] == digit2
    }

    /// Basic email format validation
    static func validateEmail(_ email: String) -> Bool {
        let pattern = #"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$"#
        return email.range(of: pattern, options: .regularExpression) != nil
    }
}
```

### 3.4 CheckpointManager.swift

```swift
import Foundation

/// Manages checkpoint state for pause/resume functionality
/// Persists to disk for crash recovery
actor CheckpointManager {
    private let checkpointDirectory: URL
    private let encoder = JSONEncoder()
    private let decoder = JSONDecoder()

    init() {
        let appSupport = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask)[0]
        checkpointDirectory = appSupport.appendingPathComponent("BRDataMiner/Checkpoints")

        try? FileManager.default.createDirectory(at: checkpointDirectory, withIntermediateDirectories: true)
    }

    /// Checkpoint data structure
    struct Checkpoint: Codable {
        let sessionID: UUID
        let byteOffset: Int64
        let lineNumber: Int64
        let statistics: ExtractionStatistics
        let savedAt: Date

        // Deduplication sets (saved as arrays for Codable)
        var seenEmails: Set<String>
        var seenDomains: Set<String>
        var seenCPFs: Set<String>
        var seenCNPJs: Set<String>
        var seenPhones: Set<String>
        var seenURLs: Set<String>
    }

    /// Save checkpoint
    func saveCheckpoint(_ checkpoint: Checkpoint) throws {
        let url = checkpointDirectory.appendingPathComponent("\(checkpoint.sessionID.uuidString).checkpoint")
        let data = try encoder.encode(checkpoint)
        try data.write(to: url)
        print("Checkpoint saved: \(checkpoint.byteOffset) bytes, \(checkpoint.lineNumber) lines")
    }

    /// Load checkpoint for session
    func loadCheckpoint(sessionID: UUID) throws -> Checkpoint? {
        let url = checkpointDirectory.appendingPathComponent("\(sessionID.uuidString).checkpoint")
        guard FileManager.default.fileExists(atPath: url.path) else { return nil }

        let data = try Data(contentsOf: url)
        let checkpoint = try decoder.decode(Checkpoint.self, from: data)

        // Check if checkpoint is recent (within 7 days)
        let maxAge: TimeInterval = 7 * 24 * 60 * 60
        if Date().timeIntervalSince(checkpoint.savedAt) > maxAge {
            try deleteCheckpoint(sessionID: sessionID)
            return nil
        }

        return checkpoint
    }

    /// Delete checkpoint
    func deleteCheckpoint(sessionID: UUID) throws {
        let url = checkpointDirectory.appendingPathComponent("\(sessionID.uuidString).checkpoint")
        try? FileManager.default.removeItem(at: url)
    }

    /// Check if resumable checkpoint exists
    func hasResumableCheckpoint(sessionID: UUID) -> Bool {
        (try? loadCheckpoint(sessionID: sessionID)) != nil
    }
}
```

### 3.5 ExtractionEngine.swift (Main Orchestrator)

```swift
import Foundation

/// Main extraction engine using Actor pattern for thread safety
/// Orchestrates file reading, extraction, and storage
@MainActor
class ExtractionEngine: ObservableObject {
    // Published state
    @Published var session: ExtractionSession?
    @Published var state: ExtractionState = .idle
    @Published var statistics: ExtractionStatistics = ExtractionStatistics()
    @Published var currentLine: String = ""
    @Published var errorMessage: String?

    // Services
    private var fileReader: StreamingFileReader?
    private let extractor = RegexExtractor()
    private let checkpointManager = CheckpointManager()
    private var storageManager: StorageManager?

    // Control
    private var extractionTask: Task<Void, Never>?
    private var shouldPause = false
    private var shouldStop = false

    // Deduplication sets
    private var seenEmails = Set<String>()
    private var seenDomains = Set<String>()
    private var seenCPFs = Set<String>()
    private var seenCNPJs = Set<String>()
    private var seenPhones = Set<String>()
    private var seenURLs = Set<String>()

    // Configuration
    private let checkpointInterval = 100_000  // Save checkpoint every 100K lines

    // MARK: - Public Methods

    /// Start extraction from file
    func startExtraction(
        sourceFile: URL,
        outputDirectory: URL,
        config: ExtractionConfig
    ) async {
        // Create session
        var newSession = ExtractionSession(
            sourceFile: sourceFile,
            outputDirectory: outputDirectory,
            config: config
        )
        newSession.state = .running
        newSession.statistics.startTime = Date()

        self.session = newSession
        self.state = .running
        self.shouldPause = false
        self.shouldStop = false

        // Initialize services
        do {
            fileReader = StreamingFileReader(fileURL: sourceFile)
            try await fileReader?.open()
            statistics.totalBytes = await fileReader?.totalSize ?? 0

            storageManager = StorageManager(outputDirectory: outputDirectory, sessionID: newSession.id)
            try await storageManager?.initialize()

            // Check for resumable checkpoint
            if let checkpoint = try await checkpointManager.loadCheckpoint(sessionID: newSession.id) {
                await resumeFromCheckpoint(checkpoint)
            }

        } catch {
            self.errorMessage = error.localizedDescription
            self.state = .error
            return
        }

        // Start extraction task
        extractionTask = Task {
            await runExtractionLoop()
        }
    }

    /// Pause extraction
    func pause() {
        shouldPause = true
    }

    /// Resume extraction
    func resume() {
        shouldPause = false
        if state == .paused {
            state = .running
        }
    }

    /// Cancel extraction
    func cancel() {
        shouldStop = true
        extractionTask?.cancel()
    }

    // MARK: - Private Methods

    private func runExtractionLoop() async {
        guard let reader = fileReader, let config = session?.config else { return }

        var lineNumber: Int64 = statistics.processedLines
        var lastCheckpointLine: Int64 = lineNumber

        do {
            while !shouldStop {
                // Handle pause
                if shouldPause {
                    state = .paused
                    try await saveCheckpoint()
                    while shouldPause && !shouldStop {
                        try await Task.sleep(nanoseconds: 100_000_000)  // 100ms
                    }
                    if shouldStop { break }
                    state = .running
                }

                // Read batch of lines
                let batch = try await reader.readLines(count: config.batchSize)
                if batch.isEmpty { break }  // EOF

                // Process batch concurrently using TaskGroup
                await withTaskGroup(of: [ExtractedItem].self) { group in
                    // Split batch for concurrent processing (M4 has ~10 cores)
                    let chunkSize = max(1, batch.count / 8)

                    for chunk in stride(from: 0, to: batch.count, by: chunkSize) {
                        let end = min(chunk + chunkSize, batch.count)
                        let subBatch = Array(batch[chunk..<end])

                        group.addTask {
                            var items: [ExtractedItem] = []
                            for (line, _) in subBatch {
                                let extracted = await self.extractor.extractAll(
                                    from: line,
                                    lineNumber: lineNumber + Int64(chunk),
                                    config: config
                                )
                                items.append(contentsOf: extracted)
                            }
                            return items
                        }
                    }

                    // Collect results
                    for await items in group {
                        await processExtractedItems(items, config: config)
                    }
                }

                // Update statistics
                lineNumber += Int64(batch.count)
                statistics.processedLines = lineNumber
                statistics.bytesProcessed = await reader.position

                // Update UI (throttled)
                if lineNumber % 1000 == 0 {
                    await MainActor.run {
                        self.currentLine = batch.last?.line ?? ""
                    }
                }

                // Checkpoint periodically
                if lineNumber - lastCheckpointLine >= Int64(checkpointInterval) {
                    try await saveCheckpoint()
                    lastCheckpointLine = lineNumber
                }
            }

            // Finalize
            statistics.endTime = Date()
            state = shouldStop ? .cancelled : .completed

            // Final save
            try await storageManager?.flush()
            try await checkpointManager.deleteCheckpoint(sessionID: session!.id)

        } catch {
            errorMessage = error.localizedDescription
            state = .error
        }

        // Cleanup
        await fileReader?.close()
    }

    private func processExtractedItems(_ items: [ExtractedItem], config: ExtractionConfig) async {
        for item in items {
            // Deduplication
            if config.deduplicateResults {
                switch item.type {
                case .email, .emailBR:
                    if seenEmails.contains(item.value) {
                        statistics.duplicatesSkipped += 1
                        continue
                    }
                    seenEmails.insert(item.value)
                case .domainBR:
                    if seenDomains.contains(item.value) {
                        statistics.duplicatesSkipped += 1
                        continue
                    }
                    seenDomains.insert(item.value)
                case .cpf:
                    if seenCPFs.contains(item.value) {
                        statistics.duplicatesSkipped += 1
                        continue
                    }
                    seenCPFs.insert(item.value)
                case .cnpj:
                    if seenCNPJs.contains(item.value) {
                        statistics.duplicatesSkipped += 1
                        continue
                    }
                    seenCNPJs.insert(item.value)
                case .phoneBR:
                    if seenPhones.contains(item.value) {
                        statistics.duplicatesSkipped += 1
                        continue
                    }
                    seenPhones.insert(item.value)
                case .url:
                    if seenURLs.contains(item.value) {
                        statistics.duplicatesSkipped += 1
                        continue
                    }
                    seenURLs.insert(item.value)
                }
            }

            // Skip invalid CPF/CNPJ
            if !item.isValid && (item.type == .cpf || item.type == .cnpj) {
                statistics.invalidSkipped += 1
                continue
            }

            // Update statistics
            updateStatistics(for: item)

            // Store item
            try? await storageManager?.store(item)
        }
    }

    private func updateStatistics(for item: ExtractedItem) {
        switch item.type {
        case .email: statistics.emailCount += 1
        case .emailBR: statistics.emailBRCount += 1
        case .domainBR: statistics.domainBRCount += 1
        case .cpf:
            statistics.cpfCount += 1
            if item.isValid { statistics.cpfValidCount += 1 }
        case .cnpj:
            statistics.cnpjCount += 1
            if item.isValid { statistics.cnpjValidCount += 1 }
        case .phoneBR: statistics.phoneBRCount += 1
        case .url: statistics.urlCount += 1
        }
    }

    private func saveCheckpoint() async throws {
        guard let session = session else { return }

        let checkpoint = CheckpointManager.Checkpoint(
            sessionID: session.id,
            byteOffset: await fileReader?.position ?? 0,
            lineNumber: statistics.processedLines,
            statistics: statistics,
            savedAt: Date(),
            seenEmails: seenEmails,
            seenDomains: seenDomains,
            seenCPFs: seenCPFs,
            seenCNPJs: seenCNPJs,
            seenPhones: seenPhones,
            seenURLs: seenURLs
        )

        try await checkpointManager.saveCheckpoint(checkpoint)
    }

    private func resumeFromCheckpoint(_ checkpoint: CheckpointManager.Checkpoint) async {
        statistics = checkpoint.statistics
        seenEmails = checkpoint.seenEmails
        seenDomains = checkpoint.seenDomains
        seenCPFs = checkpoint.seenCPFs
        seenCNPJs = checkpoint.seenCNPJs
        seenPhones = checkpoint.seenPhones
        seenURLs = checkpoint.seenURLs

        try? await fileReader?.seek(to: checkpoint.byteOffset)
        print("Resumed from checkpoint: \(checkpoint.lineNumber) lines")
    }
}
```

### 3.6 StorageManager.swift

```swift
import Foundation
import SQLite3

/// Manages SQLite storage for extracted data
/// Uses batch inserts for performance
actor StorageManager {
    private var db: OpaquePointer?
    private let dbPath: String
    private let batchSize = 5000

    // Batch buffers
    private var emailBuffer: [ExtractedItem] = []
    private var domainBuffer: [ExtractedItem] = []
    private var cpfBuffer: [ExtractedItem] = []
    private var cnpjBuffer: [ExtractedItem] = []
    private var phoneBuffer: [ExtractedItem] = []
    private var urlBuffer: [ExtractedItem] = []

    init(outputDirectory: URL, sessionID: UUID) {
        dbPath = outputDirectory.appendingPathComponent("extracted_\(sessionID.uuidString).sqlite").path
    }

    func initialize() throws {
        guard sqlite3_open(dbPath, &db) == SQLITE_OK else {
            throw StorageError.cannotOpenDatabase
        }

        // Create tables
        let createTables = """
            CREATE TABLE IF NOT EXISTS emails (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                value TEXT UNIQUE,
                type TEXT,
                line_number INTEGER,
                extracted_at TEXT
            );
            CREATE TABLE IF NOT EXISTS domains (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                value TEXT UNIQUE,
                line_number INTEGER,
                extracted_at TEXT
            );
            CREATE TABLE IF NOT EXISTS cpf (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                value TEXT UNIQUE,
                is_valid INTEGER,
                line_number INTEGER,
                extracted_at TEXT
            );
            CREATE TABLE IF NOT EXISTS cnpj (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                value TEXT UNIQUE,
                is_valid INTEGER,
                line_number INTEGER,
                extracted_at TEXT
            );
            CREATE TABLE IF NOT EXISTS phones (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                value TEXT UNIQUE,
                line_number INTEGER,
                extracted_at TEXT
            );
            CREATE TABLE IF NOT EXISTS urls (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                value TEXT UNIQUE,
                line_number INTEGER,
                extracted_at TEXT
            );
            PRAGMA journal_mode=WAL;
            PRAGMA synchronous=NORMAL;
            PRAGMA cache_size=-64000;
        """

        var errMsg: UnsafeMutablePointer<CChar>?
        sqlite3_exec(db, createTables, nil, nil, &errMsg)
    }

    func store(_ item: ExtractedItem) throws {
        switch item.type {
        case .email, .emailBR:
            emailBuffer.append(item)
            if emailBuffer.count >= batchSize {
                try flushEmails()
            }
        case .domainBR:
            domainBuffer.append(item)
            if domainBuffer.count >= batchSize {
                try flushDomains()
            }
        case .cpf:
            cpfBuffer.append(item)
            if cpfBuffer.count >= batchSize {
                try flushCPFs()
            }
        case .cnpj:
            cnpjBuffer.append(item)
            if cnpjBuffer.count >= batchSize {
                try flushCNPJs()
            }
        case .phoneBR:
            phoneBuffer.append(item)
            if phoneBuffer.count >= batchSize {
                try flushPhones()
            }
        case .url:
            urlBuffer.append(item)
            if urlBuffer.count >= batchSize {
                try flushURLs()
            }
        }
    }

    func flush() throws {
        try flushEmails()
        try flushDomains()
        try flushCPFs()
        try flushCNPJs()
        try flushPhones()
        try flushURLs()
    }

    func close() {
        sqlite3_close(db)
    }

    // Batch insert methods...
    private func flushEmails() throws {
        guard !emailBuffer.isEmpty else { return }
        try batchInsert(
            items: emailBuffer,
            sql: "INSERT OR IGNORE INTO emails (value, type, line_number, extracted_at) VALUES (?, ?, ?, ?)"
        ) { stmt, item in
            sqlite3_bind_text(stmt, 1, item.value, -1, nil)
            sqlite3_bind_text(stmt, 2, item.type.rawValue, -1, nil)
            sqlite3_bind_int64(stmt, 3, item.lineNumber)
            sqlite3_bind_text(stmt, 4, ISO8601DateFormatter().string(from: item.extractedAt), -1, nil)
        }
        emailBuffer.removeAll(keepingCapacity: true)
    }

    // Similar methods for other types...

    private func batchInsert(
        items: [ExtractedItem],
        sql: String,
        bind: (OpaquePointer?, ExtractedItem) -> Void
    ) throws {
        sqlite3_exec(db, "BEGIN TRANSACTION", nil, nil, nil)

        var stmt: OpaquePointer?
        guard sqlite3_prepare_v2(db, sql, -1, &stmt, nil) == SQLITE_OK else {
            throw StorageError.prepareError
        }

        for item in items {
            bind(stmt, item)
            sqlite3_step(stmt)
            sqlite3_reset(stmt)
        }

        sqlite3_finalize(stmt)
        sqlite3_exec(db, "COMMIT", nil, nil, nil)
    }

    enum StorageError: Error {
        case cannotOpenDatabase
        case prepareError
    }
}
```

---

## 4. UI Views Structure

### 4.1 ContentView.swift (Main Navigation)

```swift
import SwiftUI

struct ContentView: View {
    @StateObject private var engine = ExtractionEngine()
    @State private var currentStep: SetupStep = .selectFile

    enum SetupStep {
        case selectFile
        case selectOutput
        case configure
        case processing
        case results
    }

    var body: some View {
        NavigationSplitView {
            // Sidebar with steps
            List(selection: $currentStep) {
                Label("Select File", systemImage: "doc.fill")
                    .tag(SetupStep.selectFile)
                Label("Output Folder", systemImage: "folder.fill")
                    .tag(SetupStep.selectOutput)
                Label("Configure", systemImage: "slider.horizontal.3")
                    .tag(SetupStep.configure)
                Label("Processing", systemImage: "gearshape.fill")
                    .tag(SetupStep.processing)
                Label("Results", systemImage: "chart.bar.fill")
                    .tag(SetupStep.results)
            }
            .listStyle(.sidebar)
        } detail: {
            switch currentStep {
            case .selectFile:
                FileSelectionView(engine: engine, nextStep: { currentStep = .selectOutput })
            case .selectOutput:
                OutputDirectoryView(engine: engine, nextStep: { currentStep = .configure })
            case .configure:
                ExtractionOptionsView(engine: engine, startExtraction: { currentStep = .processing })
            case .processing:
                ExtractionProgressView(engine: engine, onComplete: { currentStep = .results })
            case .results:
                ResultsDashboardView(engine: engine)
            }
        }
        .frame(minWidth: 900, minHeight: 600)
    }
}
```

### 4.2 ExtractionProgressView.swift

```swift
import SwiftUI

struct ExtractionProgressView: View {
    @ObservedObject var engine: ExtractionEngine
    var onComplete: () -> Void

    var body: some View {
        VStack(spacing: 24) {
            // Progress Ring
            ProgressRingView(progress: engine.statistics.percentComplete)
                .frame(width: 200, height: 200)

            // Stats Grid
            LazyVGrid(columns: [
                GridItem(.flexible()),
                GridItem(.flexible()),
                GridItem(.flexible()),
                GridItem(.flexible())
            ], spacing: 16) {
                StatCardView(title: "Emails BR", value: engine.statistics.emailBRCount, icon: "envelope.fill", color: .green)
                StatCardView(title: "Domains BR", value: engine.statistics.domainBRCount, icon: "globe.americas.fill", color: .teal)
                StatCardView(title: "CPF", value: engine.statistics.cpfValidCount, subtitle: "valid", icon: "person.text.rectangle.fill", color: .orange)
                StatCardView(title: "CNPJ", value: engine.statistics.cnpjValidCount, subtitle: "valid", icon: "building.2.fill", color: .purple)
                StatCardView(title: "Phones BR", value: engine.statistics.phoneBRCount, icon: "phone.fill", color: .pink)
                StatCardView(title: "URLs", value: engine.statistics.urlCount, icon: "link", color: .indigo)
                StatCardView(title: "Lines/sec", value: Int(engine.statistics.linesPerSecond), icon: "speedometer", color: .blue)
                StatCardView(title: "ETA", value: engine.statistics.formattedETA, icon: "clock", color: .gray)
            }

            // Current line preview
            Text(engine.currentLine)
                .font(.system(.caption, design: .monospaced))
                .lineLimit(1)
                .truncationMode(.middle)
                .foregroundStyle(.secondary)
                .frame(maxWidth: .infinity)
                .padding()
                .background(Color.gray.opacity(0.1))
                .clipShape(RoundedRectangle(cornerRadius: 8))

            // Control Buttons
            HStack(spacing: 16) {
                if engine.state == .running {
                    Button("Pause") {
                        engine.pause()
                    }
                    .buttonStyle(.bordered)
                } else if engine.state == .paused {
                    Button("Resume") {
                        engine.resume()
                    }
                    .buttonStyle(.borderedProminent)
                }

                Button("Cancel") {
                    engine.cancel()
                }
                .buttonStyle(.bordered)
                .foregroundStyle(.red)
            }

            // Progress text
            Text("\(formatBytes(engine.statistics.bytesProcessed)) / \(formatBytes(engine.statistics.totalBytes))")
                .font(.headline.monospacedDigit())
        }
        .padding(32)
        .onChange(of: engine.state) { oldState, newState in
            if newState == .completed {
                onComplete()
            }
        }
    }

    private func formatBytes(_ bytes: Int64) -> String {
        let formatter = ByteCountFormatter()
        formatter.countStyle = .file
        return formatter.string(fromByteCount: bytes)
    }
}
```

---

## 5. Performance Optimization Strategies

### 5.1 For 170GB File Processing

1. **Buffered Streaming** - Read file in 64KB chunks using FileHandle
2. **Line-by-Line Processing** - Never load full file into memory
3. **Concurrent Extraction** - Use TaskGroup with 8 parallel tasks (M4 optimization)
4. **Batch SQLite Inserts** - 5000 records per transaction
5. **WAL Mode** - SQLite Write-Ahead Logging for concurrent reads
6. **Memory-Efficient Sets** - Use Bloom filters for deduplication if sets grow too large

### 5.2 M4 Chip Optimization

```swift
// Use all available performance cores
let processingCores = ProcessInfo.processInfo.activeProcessorCount
let optimalConcurrency = min(processingCores, 8)

await withTaskGroup(of: [ExtractedItem].self) { group in
    for chunk in lines.chunked(into: lines.count / optimalConcurrency) {
        group.addTask(priority: .userInitiated) {
            // Process chunk
        }
    }
}
```

### 5.3 External SSD Considerations

1. **Security-Scoped Bookmarks** - Persist file access permissions
2. **Large Buffer Reads** - Minimize USB overhead with 64KB+ buffers
3. **Asynchronous I/O** - Don't block main thread on disk reads
4. **Progress Checkpoints** - Save every 100K lines for crash recovery

---

## 6. Step-by-Step Implementation Order

### Phase 1: Foundation (Day 1-2)
1. Create Xcode project for macOS SwiftUI
2. Implement data models (`ExtractedData.swift`, `ExtractionSession.swift`)
3. Implement `DataValidator.swift` (CPF/CNPJ validation)
4. Test validators with known valid/invalid numbers

### Phase 2: Core Services (Day 3-5)
5. Implement `StreamingFileReader.swift`
6. Implement `RegexExtractor.swift`
7. Implement `StorageManager.swift` (SQLite)
8. Implement `CheckpointManager.swift`
9. Test each service in isolation

### Phase 3: Orchestration (Day 6-7)
10. Implement `ExtractionEngine.swift`
11. Wire up all services
12. Add pause/resume functionality
13. Test with small files first

### Phase 4: UI (Day 8-10)
14. Create `ContentView.swift` navigation
15. Implement `FileSelectionView.swift`
16. Implement `ExtractionOptionsView.swift`
17. Implement `ExtractionProgressView.swift`
18. Implement `ResultsDashboardView.swift`
19. Add `ExportOptionsView.swift`

### Phase 5: Polish (Day 11-12)
20. Add export to JSON functionality
21. Add export to CSV functionality
22. Performance testing with large files
23. Memory profiling and optimization
24. Handle edge cases and errors

---

## 7. Output JSON Structure

```json
{
  "metadata": {
    "app_version": "1.0.0",
    "source_file": "/Volumes/ExternalSSD/data.txt",
    "source_file_size_bytes": 182536110080,
    "processed_at": "2025-12-28T15:30:00Z",
    "processing_time_seconds": 14400,
    "total_lines_processed": 500000000
  },
  "statistics": {
    "emails_international": 1500000,
    "emails_br": 250000,
    "domains_br": 125000,
    "cpf_total": 80000,
    "cpf_valid": 75000,
    "cnpj_total": 15000,
    "cnpj_valid": 14500,
    "phones_br": 45000,
    "urls": 200000,
    "duplicates_skipped": 150000,
    "invalid_skipped": 5500
  },
  "data": {
    "emails_br": ["contato@empresa.com.br", "..."],
    "emails_international": ["user@gmail.com", "..."],
    "domains_br": ["empresa.com.br", "..."],
    "cpf": ["123.456.789-09", "..."],
    "cnpj": ["12.345.678/0001-90", "..."],
    "phones_br": ["(11) 98765-4321", "..."],
    "urls": ["https://example.com", "..."]
  }
}
```

---

### Critical Files for Implementation

1. **/Users/neog/Projects/tools/DomainChecker/Services/ImportService.swift** - Reference for batch processing pattern with progress callbacks, deduplication, and Actor pattern

2. **/Users/neog/Projects/tools/DomainChecker/Services/DomainValidator.swift** - Reference for concurrent validation using TaskGroup with sliding window pattern, pause/resume state management

3. **/Users/neog/Projects/tools/DomainChecker/Services/SessionManager.swift** - Reference for checkpoint/resume functionality with UserDefaults persistence and 24-hour validity check

4. **/Users/neog/Projects/tools/DomainChecker/Views/Validation/ValidationProgressView.swift** - Reference for progress ring UI, live statistics display, and real-time counter animations

5. **/Users/neog/Projects/tools/DomainChecker/Services/ExportService.swift** - Reference for JSON/CSV export with options filtering, batch progress callbacks, and file sanitization