Почему WebRequest.AllowAutoRedirect не перенаправляет исходный URL-адрес на правильный URL-адрес загрузки?

Question

Я тестирую URL, который, как ожидается, будет автоматически перенаправлен на URL загрузки файла. Но он не будет перенаправлен на правильный URL-адрес https://s51.workupload.com/download/zMR4vIPW. Как я могу заставить это работать?

public static void Main()
{
    string url = "https://workupload.com/start/zMR4vIPW";
    HttpWebRequest req = WebRequest.Create(url) as HttpWebRequest;
    req.UserAgent = "Mozilla"; //Dummy useragent assigned
    HttpWebResponse response = req.GetResponse() as HttpWebResponse;

    //Expected url is https://s51.workupload.com/download/zMR4vIPW but it doesn't give that url
    Console.WriteLine("Redirected url: " + response.ResponseUri.OriginalString);

    foreach(var item in response.Headers.AllKeys)
        Console.WriteLine(response.Headers[item]);

}

Вывод:

Перенаправленный URL: https://workupload.com/file/zMR4vIPW

Accept-Encoding

3545 max-age = 0, без кеширования, обязательная повторная проверка text / html; charset = UTF-8

Ср, 19 февраля 2020 г. 19:30:54 GMT token = vdm35dfet0majftiosl236c1a7;

expires = Четверг, 20-2020-2020 01:30:54 GMT; Max-Age = 21600; путь = /;

домен = .workupload.com; HttpOnly Apache

Answer 1

Это решение должно работать для вашего случая:

using Newtonsoft.Json;
using System;
using System.IO;
using System.Net;
using System.Text;

namespace ConsoleApp1
{
    class Program
    {
        static void Main(string[] args)
        {
            string fileId = "zMR4vIPW";
            string userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.116 Safari/537.36";

            HttpWebRequest request;
            HttpWebResponse response;

            // 1: Get cookie from this request
            request = WebRequest.Create("https://workupload.com/file/" + fileId) as HttpWebRequest;
            request.UserAgent = userAgent;

            response = request.GetResponse() as HttpWebResponse;

            string setHeaderCookieValue = response.Headers["Set-Cookie"];
            string tokenCookieKeyValue = setHeaderCookieValue.Substring(0, setHeaderCookieValue.IndexOf(';'));

            response.Close();

            // 2: Trick the web server into thinking it's a real user request
            request = WebRequest.Create("https://workupload.com/start/" + fileId) as HttpWebRequest;
            request.UserAgent = userAgent;
            request.Headers["Cookie"] = tokenCookieKeyValue;

            response = request.GetResponse() as HttpWebResponse;
            response.Close();

            // 3: Call API to get file download URL
            request = WebRequest.Create("https://workupload.com/api/file/getDownloadServer/" + fileId) as HttpWebRequest;
            request.UserAgent = userAgent;
            request.Headers["Cookie"] = tokenCookieKeyValue;

            response = request.GetResponse() as HttpWebResponse;

            // 4: Get file download URL from JSON response
            using (Stream jsonStream = response.GetResponseStream())
            using (StreamReader jsonStreamReader = new StreamReader(jsonStream, Encoding.UTF8))
            {
                string responseString = jsonStreamReader.ReadToEnd();
                string downloadUrl = JsonConvert.DeserializeObject<dynamic>(responseString).data.url;

                response.Close();

                Console.WriteLine("Download URL: " + downloadUrl);

                // 5: Download the actual file
                request = WebRequest.Create(downloadUrl) as HttpWebRequest;
                request.UserAgent = userAgent;
                request.Headers["Cookie"] = tokenCookieKeyValue;

                response = request.GetResponse() as HttpWebResponse;

                using (Stream dataStream = response.GetResponseStream())
                using (MemoryStream memoryStream = new MemoryStream())
                {
                    dataStream.CopyTo(memoryStream);

                    byte[] fileContent = memoryStream.ToArray();

                    // For debugging purpose only, use FileStream for saving file to disk
                    File.WriteAllBytes("testing.rar", fileContent);
                }

                response.Close();
            }

            Console.WriteLine("Press any key to continue...");
            Console.ReadKey();
        }
    }
}