HTTP / 1.0 405 метод не разрешен и ложь с CURL

Question

Я пытаюсь получить данные с этого сайта: https://stubhub.com.

1- с file_get_contents:

$url= 'https://www.stubhub.com';
$html = file_get_contents($url);
echo $html;

Я получаю:

Warning: file_get_contents(https://stubhub.com): failed to open stream: HTTP request failed! HTTP/1.0 405 Method Not Allowed 

2- С CURL:

$url= 'https://www.stubhub.com';
$curl = curl_init();
curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, true);
curl_setopt($curl, CURLOPT_HEADER, true);
curl_setopt($curl, CURLOPT_AUTOREFERER, true);
curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($curl, CURLOPT_URL, $url);
curl_setopt($curl, CURLOPT_REFERER, $url);
curl_setopt($curl, CURLOPT_RETURNTRANSFER, TRUE);
$html = curl_exec($curl);
$response = curl_getinfo($curl, CURLINFO_HTTP_CODE);
curl_close($curl);
var_dump($html);
var_dump($response);

Но я получаю:

bool(false) int(0)

Я попытался добавить несколько заголовков, таких как User-Agent и proxy:

curl_setopt($curl, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows; U; Windows NT 6.1; rv:2.2) Gecko/20110201');
$proxy = '185.135.226.159:23500';
curl_setopt($curl, CURLOPT_PROXY, $proxy);

Но снова я получаю то же самое.

У меня есть allow_url_fopen=On, так что не так?

Answer 1

<code>function curl( $url=NULL, $options=NULL ){
    $cacert='c:/wwwroot/cacert.pem'; #   <----- download your own copy and configure this path
    $vbh = fopen('php://temp', 'w+');


    $res=array(
        'response'  =>  NULL,
        'info'      =>  array( 'http_code' => 100 ),
        'headers'   =>  NULL,
        'errors'    =>  NULL
    );
    if( is_null( $url ) ) return (object)$res;

    session_write_close();

    /* Initialise curl request object */
    $curl=curl_init();
    if( parse_url( $url,PHP_URL_SCHEME )=='https' ){
        curl_setopt( $curl, CURLOPT_SSL_VERIFYPEER, true );
        curl_setopt( $curl, CURLOPT_SSL_VERIFYHOST, 2 );
        curl_setopt( $curl, CURLOPT_CAINFO, $cacert );
    }

    /* Define standard options */
    curl_setopt( $curl, CURLOPT_URL,trim( $url ) );
    curl_setopt( $curl, CURLOPT_AUTOREFERER, true );
    curl_setopt( $curl, CURLOPT_FOLLOWLOCATION, true );
    curl_setopt( $curl, CURLOPT_FAILONERROR, true );
    curl_setopt( $curl, CURLOPT_HEADER, false );
    curl_setopt( $curl, CURLINFO_HEADER_OUT, false );
    curl_setopt( $curl, CURLOPT_RETURNTRANSFER, true );
    curl_setopt( $curl, CURLOPT_BINARYTRANSFER, true );
    curl_setopt( $curl, CURLOPT_CONNECTTIMEOUT, 20 );
    curl_setopt( $curl, CURLOPT_TIMEOUT, 60 );
    curl_setopt( $curl, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36' );
    curl_setopt( $curl, CURLOPT_MAXREDIRS, 10 );
    curl_setopt( $curl, CURLOPT_ENCODING, '' );

    curl_setopt( $curl, CURLOPT_VERBOSE, true );
    curl_setopt( $curl, CURLOPT_NOPROGRESS, true );
    curl_setopt( $curl, CURLOPT_STDERR, $vbh );



    /* Assign runtime parameters as options */
    if( isset( $options ) && is_array( $options ) ){
        foreach( $options as $param => $value ) curl_setopt( $curl, $param, $value );
    }

    /* Execute the request and store responses */
    $res=(object)array(
        'response'  =>  curl_exec( $curl ),
        'info'      =>  (object)curl_getinfo( $curl ),
        'errors'    =>  curl_error( $curl )
    );
    rewind( $vbh );
    $res->verbose=stream_get_contents( $vbh );
    fclose( $vbh );
    curl_close( $curl );
    return $res;
}


$url='https://www.stubhub.com/';
$res = curl( $url );
if( $res->info->http_code==200 ){
    printf('<pre>%s

', print_r ($ res-> info, true));printf ('

%s

', print_r ($ res-> verbose, true));}

Будет выведено:

stdClass Object
(
    [url] => https://www.stubhub.com/
    [content_type] => text/html
    [http_code] => 200
    [header_size] => 1304
    [request_size] => 214
    [filetime] => -1
    [ssl_verify_result] => 0
    [redirect_count] => 0
    [total_time] => 0.609
    [namelookup_time] => 0.25
    [connect_time] => 0.265
    [pretransfer_time] => 0.39
    [size_upload] => 0
    [size_download] => 1194
    [speed_download] => 1960
    [speed_upload] => 0
    [download_content_length] => 1194
    [upload_content_length] => -1
    [starttransfer_time] => 0.609
    [redirect_time] => 0
    [redirect_url] => 
    [primary_ip] => 23.43.75.46
    [certinfo] => Array
        (
        )

    [primary_port] => 443
    [local_ip] => 192.168.0.56
    [local_port] => 5042
)
*   Trying 23.43.75.46...
* TCP_NODELAY set
* Connected to www.stubhub.com (23.43.75.46) port 443 (#0)
* ALPN, offering http/1.1
* successfully set certificate verify locations:
  CAfile: c:/wwwroot/cacert.pem
  CApath: none
* SSL connection using TLSv1.2 / ECDHE-ECDSA-AES256-GCM-SHA384
* ALPN, server accepted to use http/1.1
* Server certificate:
*  subject: C=US; ST=California; L=San Francisco; O=Stubhub, Inc.; OU=Technology; CN=www.stubhub.com
*  start date: Jun 11 00:00:00 2018 GMT
*  expire date: Jan  9 12:00:00 2020 GMT
*  subjectAltName: host "www.stubhub.com" matched cert's "www.stubhub.com"
*  issuer: C=US; O=DigiCert Inc; CN=DigiCert ECC Secure Server CA
*  SSL certificate verify ok.
> GET / HTTP/1.1
Host: www.stubhub.com
User-Agent: Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36
Accept: */*
Accept-Encoding: deflate, gzip

< HTTP/1.1 200 OK
< Server: nginx
< Content-Type: text/html
< Expires: Thu, 01 Jan 1970 00:00:01 GMT
< Cache-Control: private, no-cache, no-store, must-revalidate
< Surrogate-Control: no-store, bypass-cache
< Content-Encoding: gzip
< X-EdgeConnect-MidMile-RTT: 163
< X-EdgeConnect-Origin-MEX-Latency: 24
< X-Akamai-Transformed: 9 624 0 pmb=mTOE,1mRUM,1
< Date: Sat, 20 Oct 2018 16:25:57 GMT
< Content-Length: 1194
< Connection: keep-alive
< Vary: Accept-Encoding
< Set-Cookie: DC=lvs31;Path=/;Domain=stubhub.com;Expires=Sat, 20-Oct-2018 16:55:56 GMT;Max-Age=1800
< Set-Cookie: akacd_PCF_Prod=1540053357~rv=98~id=53e183ee10a83152497c9102c8c7dee7; path=/; Expires=Sat, 20 Oct 2018 16:35:57 GMT
< Strict-Transport-Security: max-age=31536000; includeSubDomains
< Set-Cookie: _abck=10D08E1267D29C2EDBEA32445BD116805C7A3616AB3500001557CB5B9AD22713~-1~e+BGOJkoD/UwtPOWH75YXUSo6Kzyd7sF6nTkkw89JfE=~-1~-1; expires=Sun, 20 Oct 2019 16:25:57 GMT; max-age=31536000; path=/; domain=.stubhub.com
< Set-Cookie: bm_sz=7C06CFF7557E22DEC7855EC89DF628B0~QAAQFjZ6XGg5goBmAQAAIypMkhVJRZxwtVU8097T7Q8Z2TcGPZR0XRtAVFY3TBHGsR4EW51MqZlCAyk3cMPDJEmukVvLunM36/5Kn1gtoxarUtgkqBvlfudWZBJb2xc1rHdnMhdsAXoHWLaGt0NwROSXckDe48kkqu2Kw3suRgrWcqDlj7Y1akARK8OYnoa6; Domain=.stubhub.com; Path=/; Expires=Sat, 20 Oct 2018 20:25:56 GMT; Max-Age=14399; HttpOnly
< 
* Connection #0 to host www.stubhub.com left intact

Чтобы получить доступ к фактическому телу ответа, вы должны обработать $res->response - загрузить его в DOMDocument или что-то еще, что вы собираетесь сделать ..удачи