Я хочу получить 20 тыс. xml файлов в мою базу данных за минимальное время, в настоящее время около 60-80 часов для обработки всех файлов. Есть ли способ сделать это за меньшее время?
public function all_report(){
$data = scandir(FCPATH . 'assets/xml');
$newdata = array_slice($data, 2);
if(!empty($newdata)){
$limit;
$total_files = count($newdata);
if($total_files <= 600){
$limit = $total_files;
}else{
$limit = 600;
}
$file = 1;
foreach(array_slice($newdata, 0, $limit) as $xmlfiles){
$path = base_url().'assets/xml/'.$xmlfiles;
$xml=simplexml_load_file($path);
if(!$xml){
echo "Not Found"; exit;
}
$survey_id = $xml->SurveyInfo->SurveyID;
$survey_data = array(
'survey_id' =>$survey_id,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'file_name' =>$xml->ProcessingInformation->RoadSectionFileName,
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end' =>$xml->RoadSectionInfo->DistanceEnd_m
);
//$survey = $this->welcome_model->add_survey($survey_data);
if($xml->CrackInformation){
$crack_list = $xml->CrackInformation->CrackList->Crack;
$total_crack_area = 0;
$area_greater = 0;
$area_lessthan = 0;
foreach($crack_list as $crack){
$crack_data = array(
'survey_id' =>$survey_id,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'crack_id' =>$crack->CrackID,
'length' =>$crack->Length,
'weighted_depth' =>$crack->WeightedDepth,
'weighted_width' =>$crack->WeightedWidth
);
$this->welcome_model->add_crack($crack_data);
$total_node_width = 0;
$node_width = 0;
foreach($crack->Node as $node){
$node_width = $node->Width;
$node_data = array(
'survey_id' =>$survey_id,
'section_id'=>$xml->RoadSectionInfo->SectionID,
'crack_id' =>$crack->CrackID,
'x' =>$node->X,
'y' =>$node->Y,
'width' =>$node_width,
'depth' =>$node->Depth
);
$total_node_width += floatval($node->Width);
$this->welcome_model->add_node($node_data);
}
//$node_to_meter = $total_node_width ;
$crack_area = floatval($crack->Length * 1000) * ($total_node_width/count($crack->Node));
if($crack_area < 100){
$area_lessthan++;
}else{
$area_greater++;
}
$crack_area_data = array(
'survey_id' =>$survey_id,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'crack_id' =>$crack->CrackID,
'area' =>$crack_area
);
$this->welcome_model->add_crack_area($crack_area_data);
$total_crack_area += $crack_area;
}
$section_crack_area_calculation = array(
'survey_id' =>$survey_id,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'section_crack_area' =>$total_crack_area,
'section_area' =>20,
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end' =>$xml->RoadSectionInfo->DistanceEnd_m
);
$this->welcome_model->section_crack_area_calculation($section_crack_area_calculation);
}
else{
$crack_data = array(
'survey_id' =>$survey_id,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'crack_id' =>0,
'length' =>'0',
'weighted_depth' =>'0',
'weighted_width' =>'0'
);
$this->welcome_model->add_crack($crack_data);
$node_data = array(
'survey_id' =>$survey_id,
'section_id'=>$xml->RoadSectionInfo->SectionID,
'crack_id' =>0,
'x' =>'0',
'y' =>'0',
'width' =>'0',
'depth' =>'0'
);
$this->welcome_model->add_node($node_data);
$crack_area_data = array(
'survey_id' =>$survey_id,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'crack_id' =>'0',
'area' =>'0'
);
$this->welcome_model->add_crack_area($crack_area_data);
$section_crack_area_calculation = array(
'survey_id' =>$survey_id,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'section_crack_area' =>'0',
'section_area' =>20,
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end' =>$xml->RoadSectionInfo->DistanceEnd_m
);
$this->welcome_model->section_crack_area_calculation($section_crack_area_calculation);
}
/*--------------------------------- CRACK END------------------------------------*/
if($xml->RavelingInformation){
$ZoneReportList = $xml->RavelingInformation->ZoneReportList;
$zone_value = 0;
foreach($ZoneReportList->ZoneReport as $zonereport){
$zone_value += floatval($zonereport->RI_Area);
}
$raveling_data = array(
'survey_id' =>$survey_id,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'ri_area' =>$zone_value,
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end' =>$xml->RoadSectionInfo->DistanceEnd_m
);
$this->welcome_model->raveling($raveling_data);
}else{
$raveling_data = array(
'survey_id' =>$survey_id,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'ri_area' =>'0',
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end' =>$xml->RoadSectionInfo->DistanceEnd_m
);
$this->welcome_model->raveling($raveling_data);
}
/*---------------------------------RAVELING END --------------------------------*/
$latitude = 0;
$longitude = 0;
$survey_date = 0;
if($xml->GPSInformation->GPSCoordinate){
$latitude = $xml->GPSInformation->GPSCoordinate->Longitude;
$longitude = $xml->GPSInformation->GPSCoordinate->Latitude;
$survey_date = $xml->GPSInformation->GPSCoordinate->Date;
}
if($latitude == 0){
$result = $this->welcome_model->get_previous_info($xml->SurveyInfo->SurveyID);
$latitude = $result->latitude;
}
if($longitude == 0){
$result = $this->welcome_model->get_previous_info($xml->SurveyInfo->SurveyID);
$longitude = $result->longitude;
}
if($survey_date == 0){
$result = $this->welcome_model->get_previous_info($xml->SurveyInfo->SurveyID);
$survey_date = $result->survey_date;
}
$survey_info = array(
'survey_id' =>$xml->SurveyInfo->SurveyID,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'speed' =>$xml->RoadSectionInfo->Speed_kmh,
'latitude' =>$latitude,
'longitude' =>$longitude,
'survey_date' =>$survey_date,
);
$this->welcome_model->survey_info($survey_info);
if($xml->RutInformation){
$rut_data = $xml->RutInformation->RutMeasurement;
$total_left_depth = 0;
$total_right_depth = 0;
foreach($rut_data as $measurements){
$measurements_data = array(
'survey_id' =>$survey_id,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'position' =>$measurements->Position,
'lanside' =>$measurements->LaneSide,
'depth' =>$measurements->Depth,
'width' =>$measurements->Width,
'method' =>$measurements->Method,
'type' =>$measurements->Type,
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end' =>$xml->RoadSectionInfo->DistanceEnd_m
);
if($measurements->LaneSide == 'Left'){
$total_left_depth += $measurements->Depth;
}
if($measurements->LaneSide == 'Right'){
$total_right_depth += $measurements->Depth;
}
$this->welcome_model->rutting($measurements_data);
}
$rutting_data = array(
'survey_id' =>$xml->SurveyInfo->SurveyID,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'total_right_depth' =>$total_right_depth,
'total_left_depth' =>$total_left_depth,
'average' =>($total_right_depth + $total_left_depth) / 2,
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end' =>$xml->RoadSectionInfo->DistanceEnd_m
);
$this->welcome_model->rutting_sum($rutting_data);
}
else{
$measurements_data = array(
'survey_id' =>$survey_id,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'position' =>'0',
'lanside' =>'',
'depth' =>'0',
'width' =>'0',
'method' =>'0',
'type' =>'0',
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end' =>$xml->RoadSectionInfo->DistanceEnd_m
);
$this->welcome_model->rutting($measurements_data);
$rutting_data = array(
'survey_id' =>$xml->SurveyInfo->SurveyID,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'total_right_depth' =>'0',
'total_left_depth' =>'0',
'average' =>'0',
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end' =>$xml->RoadSectionInfo->DistanceEnd_m
);
$this->welcome_model->rutting_sum($rutting_data);
}
/* ----------------------- RUTTING END --------------------------------------- */
if($xml->PotholesInformation){
$pothole_data = $xml->PotholesInformation->Pothole;
if($pothole_data){
$max_depth = 0;
$avg_depth = 0;
$area = 0;
$total_potholes = 0;
foreach($pothole_data as $pothole){
$pothole_information = array(
'survey_id' =>$survey_id,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'Pothole_id' =>$pothole->PotholeID,
'maximum_depth' =>$pothole->MaximumDepth,
'average_depth' =>$pothole->AverageDepth,
'area' =>$pothole->Area,
'severity' =>$pothole->Severity,
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end' =>$xml->RoadSectionInfo->DistanceEnd_m
);
$this->welcome_model->pothole($pothole_information);
$max_depth += $pothole->MaximumDepth;
$avg_depth += $pothole->AverageDepth;
$area += $pothole->Area;
if($pothole->Area > 0.1){
$total_potholes++;
}
}
$potholes_details = array(
'survey_id' =>$survey_id,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'max_depth' =>$max_depth,
'avg_depth' =>$avg_depth,
'area' =>$area,
'total_potholes'=>$total_potholes,
'distance_begin'=>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end' =>$xml->RoadSectionInfo->DistanceEnd_m
);
$this->welcome_model->potholes_detail($potholes_details);
}
else{
$pothole_information = array(
'survey_id' =>$survey_id,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'Pothole_id' =>0,
'maximum_depth' =>'0',
'average_depth' =>'0',
'area' =>'0',
'severity' =>'',
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end' =>$xml->RoadSectionInfo->DistanceEnd_m
);
$this->welcome_model->pothole($pothole_information);
$potholes_details = array(
'survey_id' =>$survey_id,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'max_depth' =>'0',
'avg_depth' =>'0',
'area' =>'0',
'total_potholes'=>0,
'distance_begin'=>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end' =>$xml->RoadSectionInfo->DistanceEnd_m
);
$this->welcome_model->potholes_detail($potholes_details);
}
}else{
$pothole_information = array(
'survey_id' =>$survey_id,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'Pothole_id' =>0,
'maximum_depth' =>0,
'average_depth' =>0,
'area' =>0,
'severity' =>'',
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end' =>$xml->RoadSectionInfo->DistanceEnd_m
);
$this->welcome_model->pothole($pothole_information);
$potholes_details = array(
'survey_id' =>$survey_id,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'max_depth' =>'0',
'avg_depth' =>'0',
'area' =>'0',
'total_potholes'=>0,
'distance_begin'=>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end' =>$xml->RoadSectionInfo->DistanceEnd_m
);
$this->welcome_model->potholes_detail($potholes_details);
}
/*--------------------------- POTHOLES END ------------------------------------*/
if($xml->MacroTextureInformation){
$band_report = $xml->MacroTextureInformation->MacroTextureMeasurement->BandReport;
$macro_info = array(
'survey_id' =>$xml->SurveyInfo->SurveyID,
'section_id'=>$xml->RoadSectionInfo->SectionID,
'position' =>$xml->MacroTextureInformation->MacroTextureMeasurement->Position,
'length' =>$xml->MacroTextureInformation->MacroTextureMeasurement->Length,
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end' =>$xml->RoadSectionInfo->DistanceEnd_m
);
$macro_id = $this->welcome_model->macro_info($macro_info);
if(!empty($band_report)){
$total_mtd = 0;
foreach($band_report as $band){
$macro_data = array(
'macro_id' =>$macro_id,
'band_index' =>$band->BandIndex,
'mtd' =>$band->MTD
);
$total_mtd += $band->MTD;
$this->welcome_model->macro_band_report($macro_data);
}
$macro_calculation_data = array(
'survey_id' =>$xml->SurveyInfo->SurveyID,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'total_mtd' =>$total_mtd,
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end' =>$xml->RoadSectionInfo->DistanceEnd_m
);
$this->welcome_model->macro_calculation($macro_calculation_data);
}
}else{
$macro_info = array(
'survey_id' =>$xml->SurveyInfo->SurveyID,
'section_id'=>$xml->RoadSectionInfo->SectionID,
'position' =>'0',
'length' =>'0',
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end' =>$xml->RoadSectionInfo->DistanceEnd_m
);
$macro_id = $this->welcome_model->macro_info($macro_info);
$macro_data = array(
'macro_id' =>$macro_id,
'band_index' =>0,
'mtd' =>'0'
);
$this->welcome_model->macro_band_report($macro_data);
$macro_calculation_data = array(
'survey_id' =>$xml->SurveyInfo->SurveyID,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'total_mtd' =>'0',
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end' =>$xml->RoadSectionInfo->DistanceEnd_m
);
$this->welcome_model->macro_calculation($macro_calculation_data);
}
/*---------------------- MACRO TEXTURE END -------------------------------------*/
if($xml->PatchDetectionInformation){
$patch_report = $xml->PatchDetectionInformation->PatchData;
$total_patch_area = 0;
foreach($patch_report as $patch){
$patch_info = array(
'survey_id' =>$xml->SurveyInfo->SurveyID,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'patch_id' =>$patch->PatchID,
'area' =>$patch->Area,
'confident_score' =>$patch->ConfidentScore,
'severity_level' =>$patch->SeverityLevel,
);
$total_patch_area += $patch->Area;
$this->welcome_model->patch($patch_info);
}
$patch_area_sum = array(
'survey_id' =>$xml->SurveyInfo->SurveyID,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'total_area' =>$total_patch_area,
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end' =>$xml->RoadSectionInfo->DistanceEnd_m
);
$this->welcome_model->patch_area_sum($patch_area_sum);
}else{
$patch_info = array(
'survey_id' =>$xml->SurveyInfo->SurveyID,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'patch_id' =>0,
'area' =>0,
'confident_score' =>0,
'severity_level' =>'',
);
$this->welcome_model->patch($patch_info);
$patch_area_sum = array(
'survey_id' =>$xml->SurveyInfo->SurveyID,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'total_area' =>0,
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end' =>$xml->RoadSectionInfo->DistanceEnd_m
);
$this->welcome_model->patch_area_sum($patch_area_sum);
}
/*------------------------------ PATCH END ----------------------------------- */
if($xml->BleedingInformation){
$bleeding_data = array(
'survey_id' =>$xml->SurveyInfo->SurveyID,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'left' =>$xml->BleedingInformation->BleedingPercent->Left,
'right' =>$xml->BleedingInformation->BleedingPercent->Right,
'left_right_sum' =>$xml->BleedingInformation->BleedingPercent->Left + $xml->BleedingInformation->BleedingPercent->Right,
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end' =>$xml->RoadSectionInfo->DistanceEnd_m
);
$this->welcome_model->bleeding($bleeding_data);
}
else{
$bleeding_data = array(
'survey_id' =>$xml->SurveyInfo->SurveyID,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'left' =>'0',
'right' =>'0',
'left_right_sum' =>'0',
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end' =>$xml->RoadSectionInfo->DistanceEnd_m
);
$this->welcome_model->bleeding($bleeding_data);
}
/* ----------------------- BLEEDING END ---------------------------------------- */
if($xml->RoughnessInformation){
$roughness = $xml->RoughnessInformation->RoughnessMeasurements->Roughness;
$i = 0;
foreach($roughness as $rough){
if($latitude == 0){
$result = $this->welcome_model->get_previous_iri($xml->SurveyInfo->SurveyID);
$latitude = $result->latitude;
}
if($longitude == 0){
$result = $this->welcome_model->get_previous_iri($xml->SurveyInfo->SurveyID);
$longitude = $result->longitude;
}
if($survey_date == 0){
$result = $this->welcome_model->get_previous_iri($xml->SurveyInfo->SurveyID);
$survey_date = $result->survey_date;
}
$roughness_data = array(
'survey_id' =>$xml->SurveyInfo->SurveyID,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'iri' =>$rough->IRI,
'longitudinal_position' =>$rough->LongitudinalPositionY,
'speed' =>$xml->RoadSectionInfo->Speed_kmh,
'latitude' =>$latitude,
'longitude' =>$longitude,
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end' =>$xml->RoadSectionInfo->DistanceEnd_m,
'survey_date' =>$survey_date
);
if($i == 0){
$this->welcome_model->left_iri($roughness_data);
}else{
$this->welcome_model->right_iri($roughness_data);
}
$i++;
}
}else{
$roughness_data = array(
'survey_id' =>$xml->SurveyInfo->SurveyID,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'iri' =>'0',
'longitudinal_position' =>'0',
'speed' =>$xml->RoadSectionInfo->Speed_kmh,
'latitude' =>$latitude,
'longitude' =>$longitude,
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end' =>$xml->RoadSectionInfo->DistanceEnd_m,
'survey_date' =>$survey_date
);
$this->welcome_model->left_iri($roughness_data);
$this->welcome_model->right_iri($roughness_data);
}
/*------------------------ ROUGHNESS END -------------------------------------- */
$joints_data = array();
if(!empty($xml->ConcreteJointInformation)){
$horizontal_joint = $xml->ConcreteJointInformation->JointList->Joint;
$vertical_joint = $xml->ConcreteJointInformation->VerticalJointList->Joint;
$joints_data = array(
'survey_id' =>$xml->SurveyInfo->SurveyID,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end' =>$xml->RoadSectionInfo->DistanceEnd_m,
'horizontal_joints' =>count($horizontal_joint),
'vertical_joints' =>count($vertical_joint)
);
}
else{
$joints_data = array(
'survey_id' =>$xml->SurveyInfo->SurveyID,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'distance_begin' =>$xml->RoadSectionInfo->DistanceBegin_m,
'distance_end' =>$xml->RoadSectionInfo->DistanceEnd_m,
'horizontal_joints' =>0,
'vertical_joints' =>0
);
}
$this->welcome_model->joints($joints_data);
/*---------------------------- JOINTS END ----------------------------------------- */
$target_file = $_SERVER['DOCUMENT_ROOT'].'/road_inspection/assets/xml/'.$xmlfiles;
$target_folder = $_SERVER['DOCUMENT_ROOT'].'/road_inspection/assets/uploaded_xml/';
$ok = rename($target_file, $target_folder.$xmlfiles);
if($ok){
$processed_data = array(
'survey_id' =>$survey_id,
'section_id' =>$xml->RoadSectionInfo->SectionID,
'file_name' =>$xml->ProcessingInformation->RoadSectionFileName
);
$this->welcome_model->processed_file($processed_data);
}
echo $file.'<br>';
$file++;
}
}else{
echo "No file found"; exit;
}
}
в приведенном выше коде я извлекаю детали из файла xml по одному и отправляю детали в мою базу данных. Предложите мне лучший способ сохранить данные самым быстрым способом. Спасибо заранее