加载包含250万条记录的CSV文件

I have a large CSV file with every postcode in the UK, it comes to 2,558,797 records and I need to import it, manipulate the data by sorting it into a multi-dimensional array before saving the data in the multi-dimensional array in the database.

The problem is if I try and access the whole file I get an allowed memory exceeded exception. I can access about 128,000 records in any one go. Is there a way I can split the task down so that I can process the whole file? I've tried looking at fseek but that uses the number of bytes, not the number of rows, and I don't know how many bytes 128,000 rows is.

How can I process the entire file without hitting the memory limit? I've been trying to get this working for the last 6 hours and I've not had any joy.

This is my code so far:

    // This script takes a long time to run
ini_set('max_execution_time', 300);

// First we need to verify the files that have been uploaded.
$file = Validation::factory($_FILES);
$file->rule('import_update_file', 'Upload::not_empty');
$file->rule('import_update_file', 'Upload::valid');
$file->rule('import_update_file', 'Upload::size', array(':value', '8M'));
$file->rule('import_update_file', 'Upload::type', array(':value', array('zip')));
if (Request::current()->method() == Request::POST && $file->check())
{
    $file_name = date('Y-m-d-').'update.zip';
    $dir = Upload::save($file['import_update_file'], $file_name);
    if ($dir === false)
    {
        throw new Kohana_Exception('Unable to save uploaded file!', NULL, 1);
    }
    $zip = new ZipArchive;
    if ($zip->open($dir) !== TRUE)
    {
        throw new Kohana_Exception('Unable to open uploaded zip file! Error: '.$res, NULL, 1);
    }
    $zip->extractTo(realpath(Upload::$default_directory), array('localauthority.csv', 'postcode.csv'));
    $zip->close();

    if( ! file_exists(realpath(Upload::$default_directory).DIRECTORY_SEPARATOR.'localauthority.csv') OR 
        ! file_exists(realpath(Upload::$default_directory).DIRECTORY_SEPARATOR.'postcode.csv'))
    {
        throw new Kohana_Exception('Missing file from uploaded zip archive! Expected localauthority.csv and postcode.csv', NULL, 1);
    }
    $local_authorities = Request::factory('local_authority/read')->execute();

    // We start by combining the files, sorting the postcodes and local authority names under the local authority codes.
    $update = array();
    if (($fp = fopen(realpath(Upload::$default_directory).DIRECTORY_SEPARATOR.'localauthority.csv', 'r')) === FALSE)
    {
        throw new Kohana_Exception('Unable to open localauthority.csv file.', NULL, 1);
    }
    while (($line = fgetcsv($fp)) !== FALSE)
    {
        // Column 0 = Local Authority Code
        // Column 1 = Local Authority Name
        $update[$line[0]] = array(
            'name'      => $line[1],
            'postcodes' => array()
        );
    }
    fclose($fp);
    unlink(realpath(Upload::$default_directory).DIRECTORY_SEPARATOR.'localauthority.csv');

    if (($fp = fopen(realpath(Upload::$default_directory).DIRECTORY_SEPARATOR.'postcode.csv', 'r')) === FALSE)
    {
        throw new Kohana_Exception('Unable to open postcode.csv file.', NULL, 1);
    }
    $i = 1;
    while (($line = fgetcsv($fp)) !== FALSE && $i <= 128000)
    {
        $postcode = trim(substr($line[0], 0, 4));
        echo "Line ".sprintf("%03d", $i++) . ": Postcode: ".$line[0]."; Shortened Postcode: ".$postcode."; LAC: ".$line[1]."<br>";
        // Column 0 = Postcode
        // Column 1 = Local Authority Code
        if ( ! array_key_exists($line[1], $update))
        {
            echo $line[1]." not in array<br>";
            continue;
        }
        if ( ! in_array($postcode, $update[$line[1]]['postcodes']))
        {
            $update[$line[1]]['postcodes'][] = $postcode;
        }
    }
    fclose($fp);
    unlink(realpath(Upload::$default_directory).DIRECTORY_SEPARATOR.'postcode.csv');

    echo '<pre>'; var_dump($update); echo '</pre>';
}
else
{
    throw new Kohana_Exception('Invalid file uploaded!', NULL, 1);
}