Выполнить tNet действие в Azure Фабрика данных версии 1 (ADF1) не удается загрузить с atlas.microsoft.com - PullRequest
Новая
       65

Выполнить tNet действие в Azure Фабрика данных версии 1 (ADF1) не удается загрузить с atlas.microsoft.com

1 голос
/ 10 апреля 2020

У меня следующая проблема с использованием Azure Data Factory version 1 (ADF1). У меня есть DotNetActivity, которую я использую для загрузки информации с atlas.microsoft.com. Поскольку два дня go действие завершается неудачно, возвращается следующее сообщение: 

Error in Activity: Unknown error in module: System.Reflection.TargetInvocationException: Exception has been thrown by the target of an invocation. ---> System.Net.Http.HttpRequestException: An error occurred while sending the request. ---> System.Net.WebException: The underlying connection was closed: An unexpected error occurred on a send. ---> System.IO.IOException: Unable to read data from the transport connection: An existing connection was forcibly closed by the remote host. ---> System.Net.Sockets.SocketException: An existing connection was forcibly closed by the remote host
at System.Net.Sockets.NetworkStream.EndRead(IAsyncResult asyncResult)
--- End of inner exception stack trace ---
at System.Net.TlsStream.EndWrite(IAsyncResult asyncResult)
at System.Net.ConnectStream.WriteHeadersCallback(IAsyncResult ar)
--- End of inner exception stack trace ---
at System.Net.HttpWebRequest.EndGetResponse(IAsyncResult asyncResult)
at System.Net.Http.HttpClientHandler.GetResponseCallback(IAsyncResult ar)
--- End of inner exception stack trace ---
at System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess(Task task)
at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
at System.Runtime.CompilerServices.ConfiguredTaskAwaitable`1.ConfiguredTaskAwaiter.GetResult()
at Wen.Logging.Http.HttpClientExtensions.Send(HttpClient client, HttpMethod method, String url, HttpContent data, Dictionary`2 headers)
at Wen.Logging.Http.HttpClientExtensions.Get(HttpClient client, String url, Dictionary`2 headers)
at Delfos.CodeActivities.Activities.AzureMapClient.GetTimeZonesIds() in C:\projects\delfos\src\Delfos.CodeActivities\Activities\AzureMapClient.cs:line 52
at Delfos.CodeActivities.Activities.AzureMaps.LoadTimeZones.AdjustTimeZones(String connectionString) in C:\projects\delfos\src\Delfos.CodeActivities\Activities\AzureMaps\LoadTimeZones.cs:line 39
at Delfos.CodeActivities.Activities.AzureMaps.LoadTimeZones.Execute(IEnumerable`1 linkedServices, IEnumerable`1 datasets, Activity activity, IActivityLogger logger) in C:\projects\delfos\src\Delfos.CodeActivities\Activities\AzureMaps\LoadTimeZones.cs:line 32
at Microsoft.Azure.Management.DataFactories.Runtime.ActivityExecutor.Execute(Object job, String configuration, Action`1 logAction)
--- End of inner exception stack trace ---
at System.RuntimeMethodHandle.InvokeMethod(Object target, Object[] arguments, Signature sig, Boolean constructor)
at System.Reflection.RuntimeMethodInfo.UnsafeInvokeInternal(Object obj, Object[] parameters, Object[] arguments)
at System.Reflection.RuntimeMethodInfo.Invoke(Object obj, BindingFlags invokeAttr, Binder binder, Object[] parameters, CultureInfo culture)
at Microsoft.DataPipeline.Compute.HDInsightJobExecution.ReflectingActivityWrapper.Execute()
at Microsoft.DataPipeline.Compute.HDInsightJobExecution.JobWrapper.RunJob()
at Microsoft.DataPipeline.Compute.HDInsightJobExecution.Launcher.Main(String[] args)..

Это моя точка Net Активность: 

using System;
using System.Collections.Generic;
using Microsoft.Azure.Management.DataFactories.Models;
using Microsoft.Azure.Management.DataFactories.Runtime;
using System.Linq;
using System.Data.SqlClient;
using System.Data;

namespace Delfos.CodeActivities.Activities.AzureMaps
{
    public class LoadTimeZones : IDotNetActivity
    {
        private readonly DateTime startDate = new DateTime(2012, 1, 1);

        public IDictionary<string, string> Execute(IEnumerable<LinkedService> linkedServices,
            IEnumerable<Dataset> datasets,
            Activity activity,
            IActivityLogger logger)
        {
            var odsStagingSqlInputDataset = datasets
                                            .Where(dataset => dataset.Name == "dim-date-staging")
                                            .First();

            var odsStagingSqlInputLinkedService = linkedServices.Where(
                    linkedService =>
                    linkedService.Name ==
                    odsStagingSqlInputDataset.Properties.LinkedServiceName).First().Properties.TypeProperties
                    as AzureSqlDatabaseLinkedService;

            AdjustTimeZones(odsStagingSqlInputLinkedService.ConnectionString);

            return new Dictionary<string, string>();
        }

        public void AdjustTimeZones(string connectionString)
        {
            var transitionsYears = DateTime.Now.AddYears(1).Subtract(startDate).Days / 365;

            var timeZonesIds = AzureMapClient.GetTimeZonesIds();
            var usTimeZonesIds = timeZonesIds.Where(zone => zone.Id.StartsWith("US/") && zone.Id != "US/Arizona");
            var timeZoneTable = CreateTimeZoneDataTable();
            var marked = new Dictionary<string, bool>();

            foreach (var zoneId in usTimeZonesIds)
            {
                var timeZone = AzureMapClient.GetTimeZone(zoneId.Id, startDate, transitionsYears);

                if (marked.ContainsKey(timeZone.Names.Generic))
                    continue;

                marked.Add(timeZone.Names.Generic, true);

                timeZone.TimeTransitions.ForEach(t => timeZoneTable.Rows.Add(
                                timeZone.Names.Generic,
                                t.UtcStart,
                                t.UtcEnd,
                                t.StandardOffset.Add(t.DaylightSavings).Hours,
                                t.DaylightSavings.Hours > 0 ? true : false
                    ));
            }

            using (var connection = new SqlConnection(connectionString))
            {
                connection.Open();

                Util.ExecuteUpsertStoredProcedure("ods_ui.LoadTimeZones", "TimeZones", timeZoneTable, connection);
            }
        }

        private DataTable CreateTimeZoneDataTable()
        {
            var table = new DataTable();

            table.Columns.Add("TimeZone", typeof(string));
            table.Columns.Add("StartDate", typeof(DateTime));
            table.Columns.Add("EndDate", typeof(DateTime));
            table.Columns.Add("UTCdifferenceHours", typeof(int));
            table.Columns.Add("IsDaylightSavings", typeof(bool));

            return table;
        }
    }
}

, и этот класс я использую как помощник: 

using Delfos.CodeActivities.Models.AzureMaps;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Net;
using Wen.Logging.Http;

namespace Delfos.CodeActivities.Activities
{
    public static class AzureMapClient
    {
        private const string atlasUrl = "https://atlas.microsoft.com";
        private const string subscriptionKey = "<replaced_actual_value>";

        public static Models.AzureMaps.TimeZone GetTimeZone(string zone,
                                    DateTime transitionsFrom, 
                                    int transitionsYears = 1,
                                    string options = "all")
        {
            var baseUrl = new Uri(atlasUrl);
            var endpoint = $@"timezone/byId/json?subscription-key={subscriptionKey}&api-version=1.0&options={options}&query={zone}&transitionsFrom={transitionsFrom.ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'")}&transitionsYears={transitionsYears}";

            ServicePointManager.SecurityProtocol = SecurityProtocolType.Ssl3 | SecurityProtocolType.Tls12 | SecurityProtocolType.Tls11;

            using (var client = new HttpClient())
            {
                var response = client.Get(baseUrl.AbsoluteUri + endpoint);

                if (!response.IsSuccessStatusCode)
                    return new Models.AzureMaps.TimeZone();

                var respContent = response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
                var timeZonesContent = JObject.Parse(respContent).SelectToken("TimeZones").ToString();
                var timeZones = JsonConvert.DeserializeObject<List<Models.AzureMaps.TimeZone>>(timeZonesContent);

                return timeZones.FirstOrDefault();
            }
        }

        public static List<TimeZoneId> GetTimeZonesIds()
        {
            var baseUrl = new Uri(atlasUrl);
            var endpoint = $"timezone/enumIana/json?subscription-key={subscriptionKey}&api-version=1.0";

            ServicePointManager.SecurityProtocol = SecurityProtocolType.Ssl3 | SecurityProtocolType.Tls12 | SecurityProtocolType.Tls11;

            using (var client = new HttpClient())
            {
                var response = client.Get(baseUrl.AbsoluteUri + endpoint);

                if (!response.IsSuccessStatusCode)
                    return new List<TimeZoneId>();

                var respContent = response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
                var timeZonesIds = JsonConvert.DeserializeObject<List<TimeZoneId>>(respContent);

                return timeZonesIds;
                //return new List<TimeZoneId>();
            }
        }
    }
}

эта строка: 

 ServicePointManager.SecurityProtocol = SecurityProtocolType.Ssl3 | SecurityProtocolType.Tls12 | SecurityProtocolType.Tls11;

была моей последней попыткой, так как я обнаружил, что принудительное использование SecurityProtocol для TLS1.2 позволяет мне запускать весь код из моего локального проекта , но однажды в облаке сбои продолжались.

...