Как сохранить данные 20k xml файлов в mysql в codeigniter в минимальное время? - PullRequest
0 голосов
/ 14 апреля 2020

Я хочу получить 20 тыс. xml файлов в мою базу данных за минимальное время, в настоящее время около 60-80 часов для обработки всех файлов. Есть ли способ сделать это за меньшее время?

public function all_report(){

$data = scandir(FCPATH . 'assets/xml');

$newdata = array_slice($data, 2);

if(!empty($newdata)){   
$limit;
$total_files = count($newdata);

if($total_files <= 600){
$limit = $total_files;  
}else{
$limit = 600;
}
$file = 1;
foreach(array_slice($newdata, 0, $limit) as $xmlfiles){

$path = base_url().'assets/xml/'.$xmlfiles;
$xml=simplexml_load_file($path);
if(!$xml){
echo "Not Found"; exit;
}

$survey_id = $xml->SurveyInfo->SurveyID;

$survey_data = array(
'survey_id'      =>$survey_id,
'section_id'     =>$xml->RoadSectionInfo->SectionID,
'file_name'      =>$xml->ProcessingInformation->RoadSectionFileName,
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end'   =>$xml->RoadSectionInfo->DistanceEnd_m
);

//$survey = $this->welcome_model->add_survey($survey_data);

if($xml->CrackInformation){

$crack_list = $xml->CrackInformation->CrackList->Crack;

$total_crack_area = 0;
$area_greater = 0;
$area_lessthan = 0;

foreach($crack_list as $crack){

$crack_data = array(
'survey_id'         =>$survey_id,
'section_id'        =>$xml->RoadSectionInfo->SectionID, 
'crack_id'          =>$crack->CrackID,
'length'            =>$crack->Length,
'weighted_depth'    =>$crack->WeightedDepth,
'weighted_width'    =>$crack->WeightedWidth
);

$this->welcome_model->add_crack($crack_data);

$total_node_width = 0;
$node_width = 0;

foreach($crack->Node as $node){

$node_width = $node->Width;

$node_data = array(
'survey_id' =>$survey_id,
'section_id'=>$xml->RoadSectionInfo->SectionID, 
'crack_id'  =>$crack->CrackID,
'x'         =>$node->X,
'y'         =>$node->Y,
'width'     =>$node_width,
'depth'     =>$node->Depth
);

$total_node_width +=  floatval($node->Width);
$this->welcome_model->add_node($node_data); 
}

//$node_to_meter = $total_node_width ;
$crack_area = floatval($crack->Length * 1000) * ($total_node_width/count($crack->Node));

if($crack_area < 100){
$area_lessthan++;   
}else{
$area_greater++;    
}   

$crack_area_data = array(
'survey_id'     =>$survey_id,
'section_id'    =>$xml->RoadSectionInfo->SectionID,
'crack_id'      =>$crack->CrackID,
'area'          =>$crack_area
);

$this->welcome_model->add_crack_area($crack_area_data); 
$total_crack_area += $crack_area;
}

$section_crack_area_calculation = array(
'survey_id'             =>$survey_id,
'section_id'            =>$xml->RoadSectionInfo->SectionID,
'section_crack_area'    =>$total_crack_area,
'section_area'          =>20,
'distance_begin'        =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end'          =>$xml->RoadSectionInfo->DistanceEnd_m
);
$this->welcome_model->section_crack_area_calculation($section_crack_area_calculation);  
}
else{

$crack_data = array(
'survey_id'         =>$survey_id,
'section_id'        =>$xml->RoadSectionInfo->SectionID, 
'crack_id'          =>0,
'length'            =>'0',
'weighted_depth'    =>'0',
'weighted_width'    =>'0'
);

$this->welcome_model->add_crack($crack_data);

$node_data = array(
'survey_id' =>$survey_id,
'section_id'=>$xml->RoadSectionInfo->SectionID, 
'crack_id'  =>0,
'x'         =>'0',
'y'         =>'0',
'width'     =>'0',
'depth'     =>'0'
);

$this->welcome_model->add_node($node_data); 

$crack_area_data = array(
'survey_id'     =>$survey_id,
'section_id'    =>$xml->RoadSectionInfo->SectionID,
'crack_id'      =>'0',
'area'          =>'0'
);

$this->welcome_model->add_crack_area($crack_area_data); 

$section_crack_area_calculation = array(
'survey_id'             =>$survey_id,
'section_id'            =>$xml->RoadSectionInfo->SectionID,
'section_crack_area'    =>'0',
'section_area'          =>20,
'distance_begin'        =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end'          =>$xml->RoadSectionInfo->DistanceEnd_m
);
$this->welcome_model->section_crack_area_calculation($section_crack_area_calculation);

}
/*--------------------------------- CRACK END------------------------------------*/

if($xml->RavelingInformation){

$ZoneReportList = $xml->RavelingInformation->ZoneReportList;
$zone_value = 0;

foreach($ZoneReportList->ZoneReport as $zonereport){
$zone_value += floatval($zonereport->RI_Area);
}

$raveling_data = array(
'survey_id'      =>$survey_id,
'section_id'     =>$xml->RoadSectionInfo->SectionID,
'ri_area'        =>$zone_value,
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end'   =>$xml->RoadSectionInfo->DistanceEnd_m 
);

$this->welcome_model->raveling($raveling_data); 
}else{

$raveling_data = array(
'survey_id'      =>$survey_id,
'section_id'     =>$xml->RoadSectionInfo->SectionID,
'ri_area'        =>'0',
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end'   =>$xml->RoadSectionInfo->DistanceEnd_m 
);

$this->welcome_model->raveling($raveling_data); 
}   
/*---------------------------------RAVELING END --------------------------------*/

$latitude = 0;
$longitude = 0;
$survey_date = 0;

if($xml->GPSInformation->GPSCoordinate){

$latitude = $xml->GPSInformation->GPSCoordinate->Longitude;
$longitude = $xml->GPSInformation->GPSCoordinate->Latitude; 
$survey_date = $xml->GPSInformation->GPSCoordinate->Date;   
}   

if($latitude == 0){
$result = $this->welcome_model->get_previous_info($xml->SurveyInfo->SurveyID);
$latitude = $result->latitude;  
}

if($longitude == 0){
$result = $this->welcome_model->get_previous_info($xml->SurveyInfo->SurveyID);
$longitude = $result->longitude;    
}   

if($survey_date == 0){
$result = $this->welcome_model->get_previous_info($xml->SurveyInfo->SurveyID);
$survey_date = $result->survey_date;    
}   

$survey_info = array(
'survey_id'     =>$xml->SurveyInfo->SurveyID,
'section_id'    =>$xml->RoadSectionInfo->SectionID,
'speed'         =>$xml->RoadSectionInfo->Speed_kmh,
'latitude'      =>$latitude,
'longitude'     =>$longitude,
'survey_date'   =>$survey_date,
);
$this->welcome_model->survey_info($survey_info);

if($xml->RutInformation){

$rut_data = $xml->RutInformation->RutMeasurement;
$total_left_depth = 0;
$total_right_depth = 0;
foreach($rut_data as $measurements){

$measurements_data = array(
'survey_id'      =>$survey_id,
'section_id'     =>$xml->RoadSectionInfo->SectionID,
'position'       =>$measurements->Position,
'lanside'        =>$measurements->LaneSide,
'depth'          =>$measurements->Depth,
'width'          =>$measurements->Width,    
'method'         =>$measurements->Method,
'type'           =>$measurements->Type,
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end'   =>$xml->RoadSectionInfo->DistanceEnd_m 
);

if($measurements->LaneSide == 'Left'){

$total_left_depth   += $measurements->Depth;    
}

if($measurements->LaneSide == 'Right'){
$total_right_depth  += $measurements->Depth;    
}

$this->welcome_model->rutting($measurements_data);      

}

    $rutting_data = array(
    'survey_id'         =>$xml->SurveyInfo->SurveyID,
    'section_id'        =>$xml->RoadSectionInfo->SectionID,
    'total_right_depth' =>$total_right_depth,
    'total_left_depth'  =>$total_left_depth,
    'average'           =>($total_right_depth + $total_left_depth) / 2,
    'distance_begin'    =>$xml->RoadSectionInfo->DistanceBegin_m,
    'distance_end'      =>$xml->RoadSectionInfo->DistanceEnd_m
    );
    $this->welcome_model->rutting_sum($rutting_data);   
}
else{

$measurements_data = array(
'survey_id'      =>$survey_id,
'section_id'     =>$xml->RoadSectionInfo->SectionID,
'position'       =>'0',
'lanside'        =>'',
'depth'          =>'0',
'width'          =>'0', 
'method'         =>'0',
'type'           =>'0',
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end'   =>$xml->RoadSectionInfo->DistanceEnd_m 
);
$this->welcome_model->rutting($measurements_data);

    $rutting_data = array(
    'survey_id'         =>$xml->SurveyInfo->SurveyID,
    'section_id'        =>$xml->RoadSectionInfo->SectionID,
    'total_right_depth' =>'0',
    'total_left_depth'  =>'0',
    'average'           =>'0',
    'distance_begin'    =>$xml->RoadSectionInfo->DistanceBegin_m,
    'distance_end'      =>$xml->RoadSectionInfo->DistanceEnd_m
    );
    $this->welcome_model->rutting_sum($rutting_data);   

}
/* ----------------------- RUTTING END --------------------------------------- */

if($xml->PotholesInformation){

$pothole_data = $xml->PotholesInformation->Pothole;
if($pothole_data){

$max_depth = 0;
$avg_depth = 0;
$area = 0;  
$total_potholes = 0;

foreach($pothole_data as $pothole){

$pothole_information = array(
'survey_id'      =>$survey_id,
'section_id'     =>$xml->RoadSectionInfo->SectionID,
'Pothole_id'     =>$pothole->PotholeID,
'maximum_depth'  =>$pothole->MaximumDepth,
'average_depth'  =>$pothole->AverageDepth,
'area'           =>$pothole->Area,  
'severity'       =>$pothole->Severity,
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end'   =>$xml->RoadSectionInfo->DistanceEnd_m 
);  
$this->welcome_model->pothole($pothole_information);

$max_depth += $pothole->MaximumDepth;
$avg_depth += $pothole->AverageDepth;
$area += $pothole->Area;

if($pothole->Area > 0.1){
$total_potholes++;  
}

}

$potholes_details = array(
'survey_id'     =>$survey_id,
'section_id'    =>$xml->RoadSectionInfo->SectionID,
'max_depth'     =>$max_depth,
'avg_depth'     =>$avg_depth,
'area'          =>$area,
'total_potholes'=>$total_potholes,
'distance_begin'=>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end'  =>$xml->RoadSectionInfo->DistanceEnd_m  
);  
$this->welcome_model->potholes_detail($potholes_details);
}
else{
$pothole_information = array(
'survey_id'      =>$survey_id,
'section_id'     =>$xml->RoadSectionInfo->SectionID,
'Pothole_id'     =>0,
'maximum_depth'  =>'0',
'average_depth'  =>'0',
'area'           =>'0', 
'severity'       =>'',
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end'   =>$xml->RoadSectionInfo->DistanceEnd_m 
);  
$this->welcome_model->pothole($pothole_information);

$potholes_details = array(
'survey_id'     =>$survey_id,
'section_id'    =>$xml->RoadSectionInfo->SectionID,
'max_depth'     =>'0',
'avg_depth'     =>'0',
'area'          =>'0',
'total_potholes'=>0,
'distance_begin'=>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end'  =>$xml->RoadSectionInfo->DistanceEnd_m  
);  
$this->welcome_model->potholes_detail($potholes_details);

}

}else{
$pothole_information = array(
'survey_id'      =>$survey_id,
'section_id'     =>$xml->RoadSectionInfo->SectionID,
'Pothole_id'     =>0,
'maximum_depth'  =>0,
'average_depth'  =>0,
'area'           =>0,   
'severity'       =>'',
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end'   =>$xml->RoadSectionInfo->DistanceEnd_m 
);  
$this->welcome_model->pothole($pothole_information);    

$potholes_details = array(
'survey_id'     =>$survey_id,
'section_id'    =>$xml->RoadSectionInfo->SectionID,
'max_depth'     =>'0',
'avg_depth'     =>'0',
'area'          =>'0',
'total_potholes'=>0,
'distance_begin'=>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end'  =>$xml->RoadSectionInfo->DistanceEnd_m  
);  
$this->welcome_model->potholes_detail($potholes_details);   

}   
/*--------------------------- POTHOLES END ------------------------------------*/

if($xml->MacroTextureInformation){
$band_report = $xml->MacroTextureInformation->MacroTextureMeasurement->BandReport;

$macro_info = array(
'survey_id' =>$xml->SurveyInfo->SurveyID,
'section_id'=>$xml->RoadSectionInfo->SectionID,
'position'  =>$xml->MacroTextureInformation->MacroTextureMeasurement->Position,
'length'    =>$xml->MacroTextureInformation->MacroTextureMeasurement->Length,
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end'   =>$xml->RoadSectionInfo->DistanceEnd_m 
);
$macro_id = $this->welcome_model->macro_info($macro_info);

if(!empty($band_report)){
$total_mtd = 0;
foreach($band_report as $band){

$macro_data = array(
'macro_id'       =>$macro_id,
'band_index'     =>$band->BandIndex,
'mtd'            =>$band->MTD
);
$total_mtd += $band->MTD;
$this->welcome_model->macro_band_report($macro_data);   

}

$macro_calculation_data = array(
'survey_id'      =>$xml->SurveyInfo->SurveyID,
'section_id'     =>$xml->RoadSectionInfo->SectionID,
'total_mtd'      =>$total_mtd,
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end'   =>$xml->RoadSectionInfo->DistanceEnd_m 
);
$this->welcome_model->macro_calculation($macro_calculation_data);   

}
}else{

$macro_info = array(
'survey_id' =>$xml->SurveyInfo->SurveyID,
'section_id'=>$xml->RoadSectionInfo->SectionID,
'position'  =>'0',
'length'    =>'0',
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end'   =>$xml->RoadSectionInfo->DistanceEnd_m 
);
$macro_id = $this->welcome_model->macro_info($macro_info);

$macro_data = array(
'macro_id'       =>$macro_id,
'band_index'     =>0,
'mtd'            =>'0'
);
$this->welcome_model->macro_band_report($macro_data);   

$macro_calculation_data = array(
'survey_id'      =>$xml->SurveyInfo->SurveyID,
'section_id'     =>$xml->RoadSectionInfo->SectionID,
'total_mtd'      =>'0',
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end'   =>$xml->RoadSectionInfo->DistanceEnd_m 
);
$this->welcome_model->macro_calculation($macro_calculation_data);   

}   
/*---------------------- MACRO TEXTURE END -------------------------------------*/

if($xml->PatchDetectionInformation){

$patch_report = $xml->PatchDetectionInformation->PatchData; 
    $total_patch_area = 0;
    foreach($patch_report as $patch){

    $patch_info = array(
    'survey_id'         =>$xml->SurveyInfo->SurveyID,
    'section_id'        =>$xml->RoadSectionInfo->SectionID,
    'patch_id'          =>$patch->PatchID,
    'area'              =>$patch->Area,
    'confident_score'   =>$patch->ConfidentScore,
    'severity_level'    =>$patch->SeverityLevel,        
    );
    $total_patch_area  += $patch->Area;
    $this->welcome_model->patch($patch_info);       
    }

    $patch_area_sum = array(
    'survey_id'         =>$xml->SurveyInfo->SurveyID,
    'section_id'        =>$xml->RoadSectionInfo->SectionID,
    'total_area'        =>$total_patch_area,
    'distance_begin'    =>$xml->RoadSectionInfo->DistanceBegin_m,
    'distance_end'      =>$xml->RoadSectionInfo->DistanceEnd_m
    );
    $this->welcome_model->patch_area_sum($patch_area_sum);
}else{

    $patch_info = array(
    'survey_id'         =>$xml->SurveyInfo->SurveyID,
    'section_id'        =>$xml->RoadSectionInfo->SectionID,
    'patch_id'          =>0,
    'area'              =>0,
    'confident_score'   =>0,
    'severity_level'    =>'',       
    );

    $this->welcome_model->patch($patch_info);

    $patch_area_sum = array(
    'survey_id'         =>$xml->SurveyInfo->SurveyID,
    'section_id'        =>$xml->RoadSectionInfo->SectionID,
    'total_area'        =>0,
    'distance_begin'    =>$xml->RoadSectionInfo->DistanceBegin_m,
    'distance_end'      =>$xml->RoadSectionInfo->DistanceEnd_m
    );
    $this->welcome_model->patch_area_sum($patch_area_sum);  
}   
/*------------------------------ PATCH END ----------------------------------- */

if($xml->BleedingInformation){

$bleeding_data = array(
'survey_id'         =>$xml->SurveyInfo->SurveyID,
'section_id'        =>$xml->RoadSectionInfo->SectionID,
'left'              =>$xml->BleedingInformation->BleedingPercent->Left,
'right'             =>$xml->BleedingInformation->BleedingPercent->Right,
'left_right_sum'    =>$xml->BleedingInformation->BleedingPercent->Left + $xml->BleedingInformation->BleedingPercent->Right,
'distance_begin'    =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end'      =>$xml->RoadSectionInfo->DistanceEnd_m  
);

$this->welcome_model->bleeding($bleeding_data); 
}
else{
$bleeding_data = array(
'survey_id'         =>$xml->SurveyInfo->SurveyID,
'section_id'        =>$xml->RoadSectionInfo->SectionID,
'left'              =>'0',
'right'             =>'0',
'left_right_sum'    =>'0',
'distance_begin'    =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end'      =>$xml->RoadSectionInfo->DistanceEnd_m  
);

$this->welcome_model->bleeding($bleeding_data);     
}   
/* ----------------------- BLEEDING END ---------------------------------------- */

if($xml->RoughnessInformation){ 
$roughness = $xml->RoughnessInformation->RoughnessMeasurements->Roughness;  
$i = 0;
foreach($roughness as $rough){  

if($latitude == 0){
$result = $this->welcome_model->get_previous_iri($xml->SurveyInfo->SurveyID);
$latitude = $result->latitude;  
}

if($longitude == 0){
$result = $this->welcome_model->get_previous_iri($xml->SurveyInfo->SurveyID);
$longitude = $result->longitude;    
}   

if($survey_date == 0){
$result = $this->welcome_model->get_previous_iri($xml->SurveyInfo->SurveyID);
$survey_date = $result->survey_date;    
}   

$roughness_data = array(
'survey_id'             =>$xml->SurveyInfo->SurveyID,
'section_id'            =>$xml->RoadSectionInfo->SectionID,
'iri'                   =>$rough->IRI,
'longitudinal_position' =>$rough->LongitudinalPositionY,
'speed'                 =>$xml->RoadSectionInfo->Speed_kmh,
'latitude'              =>$latitude,
'longitude'             =>$longitude,   
'distance_begin'        =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end'          =>$xml->RoadSectionInfo->DistanceEnd_m,
'survey_date'           =>$survey_date  
);  

if($i == 0){
$this->welcome_model->left_iri($roughness_data);
}else{
$this->welcome_model->right_iri($roughness_data);       
}

$i++;
}
}else{

$roughness_data = array(
'survey_id'             =>$xml->SurveyInfo->SurveyID,
'section_id'            =>$xml->RoadSectionInfo->SectionID,
'iri'                   =>'0',
'longitudinal_position' =>'0',
'speed'                 =>$xml->RoadSectionInfo->Speed_kmh,
'latitude'              =>$latitude,
'longitude'             =>$longitude,   
'distance_begin'        =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end'          =>$xml->RoadSectionInfo->DistanceEnd_m,
'survey_date'           =>$survey_date  
);  
$this->welcome_model->left_iri($roughness_data);
$this->welcome_model->right_iri($roughness_data);
}   
/*------------------------ ROUGHNESS END -------------------------------------- */

$joints_data = array();
if(!empty($xml->ConcreteJointInformation)){

$horizontal_joint = $xml->ConcreteJointInformation->JointList->Joint;
$vertical_joint  = $xml->ConcreteJointInformation->VerticalJointList->Joint;

$joints_data = array(
'survey_id'             =>$xml->SurveyInfo->SurveyID,
'section_id'            =>$xml->RoadSectionInfo->SectionID,
'distance_begin'        =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end'          =>$xml->RoadSectionInfo->DistanceEnd_m,
'horizontal_joints'     =>count($horizontal_joint),
'vertical_joints'       =>count($vertical_joint)
);
}
else{
$joints_data = array(
'survey_id'             =>$xml->SurveyInfo->SurveyID,
'section_id'            =>$xml->RoadSectionInfo->SectionID,
'distance_begin'        =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end'          =>$xml->RoadSectionInfo->DistanceEnd_m,
'horizontal_joints'     =>0,
'vertical_joints'       =>0
);      
}
$this->welcome_model->joints($joints_data);
/*---------------------------- JOINTS END ----------------------------------------- */  

$target_file = $_SERVER['DOCUMENT_ROOT'].'/road_inspection/assets/xml/'.$xmlfiles;
$target_folder = $_SERVER['DOCUMENT_ROOT'].'/road_inspection/assets/uploaded_xml/';

$ok =  rename($target_file, $target_folder.$xmlfiles);
if($ok){
$processed_data = array(
'survey_id'      =>$survey_id,
'section_id'     =>$xml->RoadSectionInfo->SectionID,
'file_name'      =>$xml->ProcessingInformation->RoadSectionFileName
);      
$this->welcome_model->processed_file($processed_data);
}
echo $file.'<br>';
$file++;
} 

}else{
    echo "No file found"; exit;
}       

}

в приведенном выше коде я извлекаю детали из файла xml по одному и отправляю детали в мою базу данных. Предложите мне лучший способ сохранить данные самым быстрым способом. Спасибо заранее

Ответы [ 2 ]

1 голос
/ 14 апреля 2020

Первым делом было бы уменьшить количество выполненных запросов. - Выполните 1 оператор выбора, чтобы получить несколько данных и сохранить их в переменной вместо выполнения 1000 маленьких запросов. Это увеличит потребляемую память, хотя. - Вставьте данные, используя пакеты. Вместо 1000 вставок выполните только одну. Найдите хорошее измерение для своей партии, выполнив несколько тестов. Проверьте, сколько нужно для простой вставки: вставка из 100 строк, 200 строк и т. Д. c.

0 голосов
/ 15 апреля 2020

20K файлов размером, скажем, 500Kb = 10Gb; инстинктивно трудно понять, почему это может занять не более часа.

К сожалению, я не знаю codeigniter и не могу следить за логикой c вашего кода, но если вы ' Вы готовы рассмотреть совершенно другой подход к загрузке базы данных SQL из XML, подумайте об использовании расширений SQL в процессоре Saxon XSLT: http://www.saxonica.com/documentation/index.html#! functions / sql

Сказав это, ваши результаты настолько не соответствуют ожиданиям, что почти наверняка есть способ повысить производительность вашей существующей технологии, я просто не могу сказать вам, что это такое.

Добро пожаловать на сайт PullRequest, где вы можете задавать вопросы и получать ответы от других членов сообщества.
...