Тайм-аут операции по истечении 10000 миллисекунд с 11584 из 1526223 полученных байтов - PullRequest
Новая
       14

Тайм-аут операции по истечении 10000 миллисекунд с 11584 из 1526223 полученных байтов

0 голосов
/ 10 апреля 2019

Я пытаюсь получить от 1,4 до 2 МБ данных JSON с сервера, используя библиотеку curl в C.

Я увеличил размер буфера, время ожидания соединения 

curl_easy_setopt(conn->easy, CURLOPT_CONNECTTIMEOUT, 10);
curl_easy_setopt(conn->easy, CURLOPT_BUFFERSIZE, 524288);


#ifdef CURL_MAX_WRITE_SIZE
    #undef CURL_MAX_WRITE_SIZE
    #define CURL_MAX_WRITE_SIZE 524288
 #endif

Я сделал перехват на запущенном клиенте, и пакеты не были получены полностью.

Я также попробовал URL-адрес из браузера, и браузер может получать данные и отображать их.

Любая помощь по этому вопросу приветствуется.

Спасибо

Так что в основном мой код основан на примере https://curl.haxx.se/libcurl/c/crawler.html

Разница здесь в том, что timer_cb вызывается часто, а easy_handle освобождается, только если есть что почитать. 

static void timer_cb(int fd, short kind, void * userp)
{
    GlobalInfo * g = (GlobalInfo *)userp;
    CURLMcode rc;
    (void)fd;
    (void)kind;

    rc = curl_multi_socket_action(g->multi,
                                  CURL_SOCKET_TIMEOUT, 0, &g->still_running);
    mcode_or_die("timer_cb: curl_multi_socket_action", rc);
    check_multi_info(g);
}

Я прочитал ответ здесь в этом API, и new_conn_init создаст / malloc новый легкий дескриптор и вызовет curl_multi_add_handle. 

static void check_multi_info(GlobalInfo * g)
{
    char * eff_url;
    CURLMsg * msg;
    int msgs_left;
    ConnInfo * conn = NULL;
    CURL * easy;
.....
    while ((msg = curl_multi_info_read(g->multi, &msgs_left)))
    {
        if (msg->msg == CURLMSG_DONE)
        {
            easy = msg->easy_handle;
            res = msg->data.result;
            do
            {
                rc = ERROR;
                conn = NULL;
                contentType = NULL;
                iValidContent = 1;
    ..........

            }
            while (0);

            u8 ret = 0;

            curl_multi_remove_handle(g->multi, easy);
            curl_easy_cleanup(easy);

            if (conn)
               new_conn_init(&(conn->reqInfo), g, 1);

    }

    new_conn_init ()
    {
    conn->easy = curl_easy_init();
    curl_easy_setopt(conn->easy, CURLOPT_URL, conn->url);
    curl_easy_setopt(conn->easy, CURLOPT_BUFFERSIZE, 524288);
    curl_easy_setopt(conn->easy, CURLOPT_WRITEFUNCTION, write_cb);
    curl_easy_setopt(conn->easy, CURLOPT_WRITEDATA, &(conn->respData));
    //  curl_easy_setopt(conn->easy, CURLOPT_HEADERDATA, &(conn->headerData));
    //  curl_easy_setopt(conn->easy, CURLOPT_VERBOSE, 1L);
    curl_easy_setopt(conn->easy, CURLOPT_VERBOSE, (long)shmHttpClientConfig->debugLibCurl);
    curl_easy_setopt(conn->easy, CURLOPT_FORBID_REUSE, 1L);
    curl_easy_setopt(conn->easy, CURLOPT_ERRORBUFFER, conn->error);
    curl_easy_setopt(conn->easy, CURLOPT_PRIVATE, conn);
    curl_easy_setopt(conn->easy, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
    curl_easy_setopt(conn->easy, CURLOPT_DNS_CACHE_TIMEOUT, -1);
    curl_easy_setopt(conn->easy, CURLOPT_NOPROGRESS, 1);
    curl_easy_setopt(conn->easy, CURLOPT_CONNECTTIMEOUT, 10);
    rc = curl_multi_add_handle(g->multi, conn->easy);
}

Я провел некоторую отладку и похоже, что соединения закрываются довольно поздно, если вы видите опцию отладки curl. Я отправляю URL-запросы каждые 20 секунд. 

* Hostname 12.0.0.25 was found in DNS cache
*   Trying 12.0.0.25...
* Hostname 12.0.0.29 was found in DNS cache
* Name '12.0.0.29' family 2 resolved to '12.0.0.29' family 2
* Local port: 0
* Connected to 12.0.0.25 (12.0.0.25) port 1234 (#1)
> GET /xxxx HTTP/1.1
Host: 12.0.0.25:1234
Accept: */*

< HTTP/1.1 200 OK
< Connection: Keep-Alive
< Content-Length: 1526223
< Content-Type: application/json; charset=utf-8
< Date: Thu, 11 Apr 2019 11:37:53 GMT
< 
* Found bundle for host 12.0.0.25: 0x9f8d20
* Hostname 12.0.0.25 was found in DNS cache
*   Trying 12.0.0.25...
* Hostname 12.0.0.29 was found in DNS cache
* Name '12.0.0.29' family 2 resolved to '12.0.0.29' family 2
* Local port: 0
* Connected to 12.0.0.25 (12.0.0.25) port 1234 (#2)
> GET /xxxx HTTP/1.1
Host: 12.0.0.25:1234
Accept: */*

< HTTP/1.1 200 OK
< Connection: Keep-Alive
< Content-Length: 1526223
< Content-Type: application/json; charset=utf-8
< Date: Thu, 11 Apr 2019 11:38:13 GMT
< 
* Found bundle for host 12.0.0.25: 0x9f8d20
* Hostname 12.0.0.25 was found in DNS cache
*   Trying 12.0.0.25...
* Hostname 12.0.0.29 was found in DNS cache
* Name '12.0.0.29' family 2 resolved to '12.0.0.29' family 2
* Local port: 0
* Connected to 12.0.0.25 (12.0.0.25) port 1234 (#3)
> GET /xxxx HTTP/1.1
Host: 12.0.0.25:1234
Accept: */*

< HTTP/1.1 200 OK
< Connection: Keep-Alive
< Content-Length: 1526223
< Content-Type: application/json; charset=utf-8
< Date: Thu, 11 Apr 2019 11:38:33 GMT
< 
* Found bundle for host 12.0.0.25: 0x9f8d20
* Hostname 12.0.0.25 was found in DNS cache
*   Trying 12.0.0.25...
* Hostname 12.0.0.29 was found in DNS cache
* Name '12.0.0.29' family 2 resolved to '12.0.0.29' family 2
* Local port: 0
* Connected to 12.0.0.25 (12.0.0.25) port 1234 (#4)
> GET /xxxx HTTP/1.1
Host: 12.0.0.25:1234
Accept: */*

* Operation timed out after 10000 milliseconds with 160728 out of 1526223 bytes received
* Closing connection 1
< HTTP/1.1 200 OK
< Connection: Keep-Alive
< Content-Length: 1526223
< Content-Type: application/json; charset=utf-8
< Date: Thu, 11 Apr 2019 11:38:53 GMT
< 
* Closing connection 4
* Found bundle for host 12.0.0.25: 0x9f8d20
* Hostname 12.0.0.25 was found in DNS cache
*   Trying 12.0.0.25...
* Hostname 12.0.0.29 was found in DNS cache
* Name '12.0.0.29' family 2 resolved to '12.0.0.29' family 2
* Local port: 0
* Operation timed out after 10001 milliseconds with 11584 out of 1526223 bytes received
* Closing connection 2
* Connected to 12.0.0.25 (12.0.0.25) port 1234 (#5)
> GET /xxxx HTTP/1.1
Host: 12.0.0.25:1234
Accept: */*

< HTTP/1.1 200 OK
< Connection: Keep-Alive
< Content-Length: 1526223
< Content-Type: application/json; charset=utf-8
< Date: Thu, 11 Apr 2019 11:39:13 GMT
< 
* Closing connection 5
* Operation timed out after 10001 milliseconds with 845632 out of 1526223 bytes received
* Closing connection 3
* Hostname 12.0.0.25 was found in DNS cache
*   Trying 12.0.0.25...
* Hostname 12.0.0.29 was found in DNS cache
* Name '12.0.0.29' family 2 resolved to '12.0.0.29' family 2
* Local port: 0
* Connected to 12.0.0.25 (12.0.0.25) port 1234 (#6)
> GET /xxxx HTTP/1.1
Host: 12.0.0.25:1234
Accept: */*

< HTTP/1.1 200 OK
< Connection: Keep-Alive
< Content-Length: 1526223
< Content-Type: application/json; charset=utf-8
< Date: Thu, 11 Apr 2019 11:43:33 GMT
< 
* Found bundle for host 12.0.0.25: 0x9f31f0
* Hostname 12.0.0.25 was found in DNS cache
*   Trying 12.0.0.25...
* Hostname 12.0.0.29 was found in DNS cache
* Name '12.0.0.29' family 2 resolved to '12.0.0.29' family 2
* Local port: 0
* Connected to 12.0.0.25 (12.0.0.25) port 1234 (#7)
> GET /xxxx HTTP/1.1
Host: 12.0.0.25:1234
Accept: */*

< HTTP/1.1 200 OK
< Connection: Keep-Alive
< Content-Length: 1526223
< C
[root@BENU TEMP/MEG-1] ~# cat /opt/benu-data/admin/benu_libcurl_op.txt | more
* Hostname 12.0.0.25 was found in DNS cache
*   Trying 12.0.0.25...
* Hostname 12.0.0.29 was found in DNS cache
* Name '12.0.0.29' family 2 resolved to '12.0.0.29' family 2
* Local port: 0
* Connected to 12.0.0.25 (12.0.0.25) port 1234 (#1)
> GET /xxxx HTTP/1.1
Host: 12.0.0.25:1234
Accept: */*

< HTTP/1.1 200 OK
< Connection: Keep-Alive
< Content-Length: 1526223
< Content-Type: application/json; charset=utf-8
< Date: Thu, 11 Apr 2019 11:37:53 GMT
< 
* Found bundle for host 12.0.0.25: 0x9f8d20
* Hostname 12.0.0.25 was found in DNS cache
*   Trying 12.0.0.25...
* Hostname 12.0.0.29 was found in DNS cache
* Name '12.0.0.29' family 2 resolved to '12.0.0.29' family 2
* Local port: 0
* Connected to 12.0.0.25 (12.0.0.25) port 1234 (#2)
> GET /xxxx HTTP/1.1
Host: 12.0.0.25:1234
Accept: */*

< HTTP/1.1 200 OK
< Connection: Keep-Alive
< Content-Length: 1526223
< Content-Type: application/json; charset=utf-8
< Date: Thu, 11 Apr 2019 11:38:13 GMT
< 
* Found bundle for host 12.0.0.25: 0x9f8d20
* Hostname 12.0.0.25 was found in DNS cache
*   Trying 12.0.0.25...
* Hostname 12.0.0.29 was found in DNS cache
* Name '12.0.0.29' family 2 resolved to '12.0.0.29' family 2
* Local port: 0
* Connected to 12.0.0.25 (12.0.0.25) port 1234 (#3)
> GET /xxxx HTTP/1.1
Host: 12.0.0.25:1234
Accept: */*

< HTTP/1.1 200 OK
< Connection: Keep-Alive
< Content-Length: 1526223
< Content-Type: application/json; charset=utf-8
< Date: Thu, 11 Apr 2019 11:38:33 GMT
< 
* Found bundle for host 12.0.0.25: 0x9f8d20
* Hostname 12.0.0.25 was found in DNS cache
*   Trying 12.0.0.25...
* Hostname 12.0.0.29 was found in DNS cache
* Name '12.0.0.29' family 2 resolved to '12.0.0.29' family 2
* Local port: 0
* Connected to 12.0.0.25 (12.0.0.25) port 1234 (#4)
> GET /xxxx HTTP/1.1
Host: 12.0.0.25:1234
Accept: */*

* Operation timed out after 10000 milliseconds with 160728 out of 1526223 bytes received
* Closing connection 1
< HTTP/1.1 200 OK
< Connection: Keep-Alive
< Content-Length: 1526223
< Content-Type: application/json; charset=utf-8
< Date: Thu, 11 Apr 2019 11:38:53 GMT
< 
* Closing connection 4
* Found bundle for host 12.0.0.25: 0x9f8d20
* Hostname 12.0.0.25 was found in DNS cache
*   Trying 12.0.0.25...
* Hostname 12.0.0.29 was found in DNS cache
* Name '12.0.0.29' family 2 resolved to '12.0.0.29' family 2
* Local port: 0
* Operation timed out after 10001 milliseconds with 11584 out of 1526223 bytes received
* Closing connection 2
* Connected to 12.0.0.25 (12.0.0.25) port 1234 (#5)
> GET /xxxx HTTP/1.1
Host: 12.0.0.25:1234
Accept: */*
...