Хранимая процедура выполняется очень медленно в SQL Server 2016 по сравнению с SQL Server 2019. Она выполняет некоторую общую проверку из временной промежуточной таблицы и вставляет данные в основные таблицы и таблицы сопоставления.
Таблицы сопоставления имеют кластерные индексы по первичным ключам {Table1Id, Table2Id, Table3Id}. Процедура должна обработать около 4 миллионов строк. Он завершает выполнение через 2 часа в SQL Server 2019, но занимает, вероятно, 6-8 часов в SQL Server 2016 с тем же файлом данных.
Описание сохраненного pro c -
Имеется таблица dbo.HouseholdTemp со всеми строками, которые нужно обработать, и ее выполнение
- Разрезайте каждую строку и вставьте одна часть в dbo.Locations и другая potion в таблице dbo.Individuals, если LocationGroupId или IndividualGroupId не существует или пуст.
- Сохраните сгенерированный идентификатор из этих двух таблиц и сопоставьте их в таблице dbo.Households {LocationId, IndividualId}.
- Выберите существующий MailingId из таблицы dbo.Mailings с помощью запроса JOIN.
- Вставьте еще одну строку в таблицу dbo.ListTypes, если комбинация типа и кода не существует, и получите идентификатор ListTypes.
- Вставьте строку в таблицу dbo.HouseholdMailings с MailingId, LocationId, IndividualId, ListTypeId.
План выполнения прилагается.
FYI, установка SQL выполняется на диске C, а файлы базы данных находятся на диске D. Пожалуйста, помогите!
План выполнения можно найти здесь: https://www.brentozar.com/pastetheplan/?id=BybjNm5a8
CREATE PROCEDURE [dbo].[usp_ProcessHouseholdDatav2]
AS
BEGIN TRY
-- Declare variables
DECLARE @DataIngestionLogId int = 0;
DECLARE @HouseholdCount bigint = 0;
DECLARE @n int = 0;
DECLARE @householdrecords int = 0;
DECLARE @householdrecordsexistscount int = 0;
DECLARE @householdmailingsrecords int = 0;
DECLARE @householdmailingsrecordsexistscount int = 0;
DECLARE @listtyperecords int = 0;
DECLARE @tempid int = 0;
DECLARE @locationid int = 0;
DECLARE @individualid int = 0;
DECLARE @mailingid int = 0;
DECLARE @jobnumber nvarchar(6) = N'';
DECLARE @mergedpanelcode nvarchar(10) = N'';
DECLARE @locationgroupid nvarchar(10) = N'';
DECLARE @individualgroupid nvarchar(10) = N'';
DECLARE @listtype varchar(3) = '';
DECLARE @listcode varchar(4) = '';
DECLARE @findernumber nvarchar(50) = N'';
DECLARE @modelrank nvarchar(2) = N'';
DECLARE @listtypeid int = 0;
DECLARE @starttime datetime = GETDATE();
INSERT INTO dbo.DataIngestionLogs (FileName,
Message,
CreatedDate)
VALUES ('usp_ProcessHouseholdData', 'PROCESS STARTED ' + CAST(@HouseholdCount AS nvarchar(10)) + ' records to be processed', GETDATE());
SET @DataIngestionLogId = SCOPE_IDENTITY();
WHILE EXISTS (SELECT TOP 1 1 FROM dbo.HouseholdTemp ht WHERE ht.ProcessedDate IS NULL /*AND @n < 1000000*/ )
BEGIN
-- Retrieve Id, JobNumber and MergedPanelCode from HouseholdTemp table
SELECT TOP 1
@tempid = ht.Id,
@jobnumber = ht.JobNumber,
@mergedpanelcode = ht.MergedPanelCode,
@locationgroupid = ht.LocationGroupID,
@individualgroupid = ht.IndividualGroupID,
@listtype = ht.ListType,
@listcode = ht.ListCode,
@findernumber = ht.FinderNumber,
@modelrank = ht.ModelRank
FROM dbo.HouseholdTemp ht
WHERE ht.ProcessedDate IS NULL;
IF (@locationgroupid IS NULL OR @locationgroupid = '')
BEGIN
INSERT INTO dbo.Locations (LocationGroupId,
Address1,
Address2,
City,
State,
Zip10,
ZipCode,
DeliveryPointCode,
FIPSCounty,
CountyName,
AddressType,
PrimaryNumber,
PreDirectional,
StreetName,
StreetSuffix,
PostDirectional,
UnitDesignator,
UnitNumber,
PMBNumber,
CarrierRoute,
PenetrationCode,
MetroName,
CreatedDate,
ModifiedDate)
SELECT TOP 1
ht.LocationGroupID AS LocationGroupID,
ht.Address1 AS Address1,
ht.Address2 AS Address2,
ht.City AS City,
ht.State AS State,
ht.Zip10 AS Zip10,
ht.ZipCode AS ZipCode,
ht.DeliveryPointCode AS DeliveryPointCode,
ht.FIPSCounty AS FIPSCounty,
ht.CountyName AS CountyName,
ht.AddressType AS AddressType,
ht.PrimaryNumber AS PrimaryNumber,
ht.PreDirectional AS PreDirectional,
ht.StreetName AS StreetName,
ht.StreetSuffix AS StreetSuffix,
ht.PostDirectional AS PostDirectional,
ht.UnitDesignator AS UnitDesignator,
ht.UnitNumber AS UnitNumber,
ht.PMBNumber AS PMBNumber,
ht.CarrierRoute AS CarrierRoute,
ht.PenetrationCode AS PenetrationCode,
ht.MetroName AS MetroName,
GETDATE(),
GETDATE()
FROM dbo.HouseholdTemp ht
WHERE ht.Id = @tempid;
SET @locationid = SCOPE_IDENTITY();
END;
ELSE
BEGIN
-- Check if record exists with LocationGroupId in Locations table
IF (NOT EXISTS (SELECT TOP 1
1
FROM dbo.Locations l
WHERE l.LocationGroupId = RTRIM(LTRIM(@locationgroupid))))
BEGIN
-- PRINT 'Location does not exists with LocationGroupId : ' + @locationgroupid + '--> Insert in Locations table....'
INSERT INTO dbo.Locations (LocationGroupId,
Address1,
Address2,
City,
State,
Zip10,
ZipCode,
DeliveryPointCode,
FIPSCounty,
CountyName,
AddressType,
PrimaryNumber,
PreDirectional,
StreetName,
StreetSuffix,
PostDirectional,
UnitDesignator,
UnitNumber,
PMBNumber,
CarrierRoute,
PenetrationCode,
MetroName,
CreatedDate,
ModifiedDate)
SELECT TOP 1
ht.LocationGroupID AS LocationGroupID,
ht.Address1 AS Address1,
ht.Address2 AS Address2,
ht.City AS City,
ht.State AS State,
ht.Zip10 AS Zip10,
ht.ZipCode AS ZipCode,
ht.DeliveryPointCode AS DeliveryPointCode,
ht.FIPSCounty AS FIPSCounty,
ht.CountyName AS CountyName,
ht.AddressType AS AddressType,
ht.PrimaryNumber AS PrimaryNumber,
ht.PreDirectional AS PreDirectional,
ht.StreetName AS StreetName,
ht.StreetSuffix AS StreetSuffix,
ht.PostDirectional AS PostDirectional,
ht.UnitDesignator AS UnitDesignator,
ht.UnitNumber AS UnitNumber,
ht.PMBNumber AS PMBNumber,
ht.CarrierRoute AS CarrierRoute,
ht.PenetrationCode AS PenetrationCode,
ht.MetroName AS MetroName,
GETDATE(),
GETDATE()
FROM dbo.HouseholdTemp ht
WHERE ht.Id = @tempid;
SET @locationid = SCOPE_IDENTITY();
END;
ELSE
BEGIN
SELECT TOP 1
@locationid = Id
FROM dbo.Locations l
WHERE l.LocationGroupId = RTRIM(LTRIM((@locationgroupid)));
END;
END;
IF (@individualgroupid IS NULL OR @individualgroupid = '')
BEGIN
INSERT INTO dbo.Individuals (IndividualGroupId,
FirstName,
LastName,
DateOfBirth,
EmailAddress,
Gender,
MaritalStatus,
CreatedDate,
ModifiedDate)
SELECT ht.IndividualGroupID AS IndividualGroupId,
ht.FirstName AS FirstName,
ht.LastName AS LastName,
CAST(ht.DateOfBirth AS date) AS DateOfBirth,
ht.EmailAddress AS EmailAddress,
ht.Gender AS Gender,
ht.MaritalStatus AS MaritalStatus,
GETDATE(),
GETDATE()
FROM dbo.HouseholdTemp ht
WHERE ht.Id = @tempid;
SET @individualid = SCOPE_IDENTITY();
END;
ELSE
BEGIN
-- Check if record exists with IndividualGroupId in Individuals table
IF (NOT EXISTS (SELECT TOP 1
1
FROM dbo.Individuals i
WHERE i.IndividualGroupId = RTRIM(LTRIM((@individualgroupid)))))
BEGIN
INSERT INTO dbo.Individuals (IndividualGroupId,
FirstName,
LastName,
DateOfBirth,
EmailAddress,
Gender,
MaritalStatus,
CreatedDate,
ModifiedDate)
SELECT ht.IndividualGroupID AS IndividualGroupId,
ht.FirstName AS FirstName,
ht.LastName AS LastName,
CAST(ht.DateOfBirth AS date) AS DateOfBirth,
ht.EmailAddress AS EmailAddress,
ht.Gender AS Gender,
ht.MaritalStatus AS MaritalStatus,
GETDATE(),
GETDATE()
FROM dbo.HouseholdTemp ht
WHERE ht.Id = @tempid;
SET @individualid = SCOPE_IDENTITY();
END;
ELSE
BEGIN
SELECT TOP 1
@individualid = Id
FROM dbo.Individuals i
WHERE i.IndividualGroupId = RTRIM(LTRIM(@individualgroupid));
END;
END;
IF (@locationid > 0 AND @individualid > 0)
BEGIN
BEGIN TRY
INSERT INTO dbo.Households (LocationId,
IndividualId,
CreatedDate)
VALUES (@locationid, @individualid, GETDATE());
SET @householdrecords = @householdrecords + 1;
END TRY
BEGIN CATCH
END CATCH;
-- Get Mailing Id from JobNumber and MergedPanelCode
SELECT TOP 1
@mailingid = m.Id
FROM dbo.Mailings m
INNER JOIN dbo.Jobs j ON j.Id = m.JobId
INNER JOIN dbo.MergedPanels mp ON mp.Id = m.MergedPanelId
WHERE j.Number = @jobnumber
AND mp.Code = @mergedpanelcode;
IF (@mailingid > 0)
BEGIN
IF ((@listtype IS NOT NULL
OR @listtype != '')
AND (@listcode IS NOT NULL
OR @listcode != ''))
BEGIN
IF NOT EXISTS (SELECT TOP 1
1
FROM dbo.ListTypes lt
WHERE lt.Type = @listtype
AND lt.Code = @listcode)
BEGIN
INSERT INTO dbo.ListTypes (Type,
Code,
CreatedDate,
ModifiedDate)
VALUES (@listtype, @listcode, GETDATE(), GETDATE());
SET @listtypeid = SCOPE_IDENTITY();
SET @listtyperecords = @listtyperecords + 1;
END;
ELSE
BEGIN
SELECT TOP 1
@listtypeid = Id
FROM dbo.ListTypes lt
WHERE lt.Type = @listtype
AND lt.Code = @listcode;
END;
END;
IF (@listtypeid > 0)
BEGIN TRY
INSERT INTO dbo.HouseholdMailings (MailingId,
LocationId,
IndividualId,
ListTypeId,
FinderNumber,
ModelRank,
CreatedDate)
VALUES (@mailingid, @locationid, @individualid, @listtypeid, @findernumber, @modelrank, GETDATE());
SET @listtypeid = 0;
SET @householdmailingsrecords = @householdmailingsrecords + 1;
END TRY
BEGIN CATCH
END CATCH;
END;
ELSE
BEGIN
SET @householdmailingsrecordsexistscount = @householdmailingsrecordsexistscount + 1;
END;
END;
-- Update the ProcessedDate in HouseholdTemp table
UPDATE dbo.HouseholdTemp
SET ProcessedDate = GETDATE()
WHERE Id = @tempid;
SET @n = @n + 1;
SET @locationid = 0;
SET @individualid = 0;
SET @mailingid = 0;
END;
INSERT INTO dbo.DataIngestionLogs (FileName,
Message,
ParentId,
CreatedDate)
VALUES ('usp_ProcessHouseholdData', 'PROCESS COMPLETED ' + CAST(@n AS nvarchar(10)) + ' records processed.', @DataIngestionLogId, GETDATE());
END TRY
BEGIN CATCH
PRINT 'Error insert...';
INSERT INTO dbo.DataIngestionErrors
VALUES (SUSER_SNAME(), ERROR_NUMBER(), ERROR_STATE(), ERROR_SEVERITY(), ERROR_LINE(), ERROR_PROCEDURE(), ERROR_MESSAGE(), GETDATE(), @DataIngestionLogId);
END CATCH;