C Yahoo Quotes

/*
C Yahoo Quotes 0.1 alpha version

Req.: libmysql(MySQL), libcurl
Compilation: gcc `mysql_config --libs --cflags` `curl-config --libs --cflags`  cyahooquotes.c  -o cyahooquotes

Copyright (C) 2013 Dawid Mocek <dawid.mocek@gmail.com>

This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

*/

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <curl/curl.h>
#include <time.h>
#include <my_global.h>
#include <mysql.h>

#define YH_MAX_SYMBOLS      10000
#define YH_BASE_URL     "http://download.finance.yahoo.com/d/quotes.csv?"
#define YH_S_PARAM_GLUE     "+"
#define YH_S_PARAM_NAME     "&s="
#define YH_F_PARAM_NAME     "f="

#define MYSQL_DB        "dawid_market"
#define MYSQL_HOST      "localhost"
#define MYSQL_USER      "dawid"
#define MYSQL_PASS      ""
#define MYSQL_QUERY     "SELECT symbol_from_db(ticker, companyname) FROM market_data WHERE type != 'Test'"

int read_symbols_from_file(char *symbols_filename, char ***symbols_arr);
void perror_exit_failure(const char *mesg);
void sql_exit_failure(MYSQL *con);
void sql_free_res(MYSQL_RES *result);

typedef struct CURLResult
{
        char *memory;
        size_t size;
};


void sql_free_res(MYSQL_RES *result)
{
    mysql_free_result(result);
}

static int add_symbols_to_list_sql(MYSQL_RES *result, MYSQL *con, struct curl_slist **slist)
{
    if(result == NULL)
    {
    sql_exit_failure(con);
    }
    MYSQL_ROW row;
    int cnt = 0;
    int num = mysql_num_fields(result);
    while((row = mysql_fetch_row(result)))
    {
    int i;
    for(i = 0; i < num; i++)
    {
        //printf("%d, %s\n", cnt, row[i]);
        *slist = curl_slist_append(*slist, row[i]);
    }
    cnt++;
    }
    sql_free_res(result);
    return cnt;
}

void sql_con(MYSQL *con, const char *host, const char *db, const char *user, const char *pass)
{
    if(mysql_real_connect(con, host, user, pass, db, 0, NULL, 0) == NULL)
    {
    sql_exit_failure(con);
    }
}

MYSQL_RES *sql_query(MYSQL *con, const char *query)
{
    if(mysql_query(con, query))
    {
    sql_exit_failure(con);
    }
    MYSQL_RES *result;
    result = mysql_store_result(con);

    return result;
}

void sql_close(MYSQL *con)
{
    mysql_close(con);
}

void sql_exit_failure(MYSQL *con)
{
    fprintf(stderr, "%s\n", mysql_error(con));
    mysql_close(con);
    exit(EXIT_FAILURE);

}
/*
Function reads symbols from file to dynamic array
*/
static int add_symbols_to_list_file(const char *symbols_filename, struct curl_slist **slist)
{

    FILE *symbols_file = fopen(symbols_filename, "r");
    if(!symbols_file)
    {
    printf("Error occured while opening file: %s\n", symbols_filename);
    exit(EXIT_FAILURE);
    }

    char buffLine[BUFSIZ];
    int cnt = 0;
    /* Read line by line */
    while(fgets(buffLine, sizeof(buffLine), symbols_file))
    {
    if(cnt >= YH_MAX_SYMBOLS)
        break;

    char *symbol = strchr(buffLine,'\n');
    if(symbol) *symbol = 0;
    symbol = strdup(buffLine);

    // Add new symbol to list
    *slist = curl_slist_append(*slist, symbol);
    free(symbol);
    cnt++;
    }

    fclose(symbols_file);
    return cnt;
}

char *construct_s_param(struct curl_slist *symbols_list)
{
    char *s_param = (char*)malloc(sizeof(char));
    s_param[0]  = '\0';
    int len = 0;

    while(symbols_list != NULL)
    {
        len =  strlen(s_param) + strlen(symbols_list->data) + strlen((char*)YH_S_PARAM_GLUE) + 1;

        char *s_param_tmp = (char*)realloc(s_param,  len * sizeof(char));
        if(!s_param_tmp)
        {
        free(s_param_tmp);
        perror("Not enough memory [ Function: construct_s_param() Var: s_param_tmp ]\n");
        exit(EXIT_FAILURE);
        }
        s_param = s_param_tmp;
        strcat(s_param, symbols_list->data);
        strcat(s_param, (char*)YH_S_PARAM_GLUE);

        symbols_list = symbols_list->next;
    }
    s_param[len-2] = '\0';
    return s_param;
}

char *construct_url(const char *f_param, const char *s_param)
{

    char *full_url = (char*)malloc(((strlen((const char*)YH_BASE_URL) + strlen(s_param) + strlen((const char*)YH_F_PARAM_NAME) + strlen((const char*)YH_S_PARAM_NAME) + strlen(f_param))) * sizeof(char*));
    if(!full_url)
    {
        free(full_url);
        perror_exit_failure("Memory allocation failure. [ Function: construct_url() Var: full_url ]\n");
    }
    strcpy(full_url, (const char*)YH_BASE_URL);
    strcat(full_url, (const char*)YH_F_PARAM_NAME);
    strcat(full_url, f_param);
    strcat(full_url, (const char*)YH_S_PARAM_NAME);
    strcat(full_url, s_param);

    return full_url;
}

void  perror_exit_failure(const char *mesg)
{
    perror(mesg);
    exit(EXIT_FAILURE);
}

int construct_urls_list(struct curl_slist *symbols_list, struct curl_slist **urls_list, const char *f_param)
{
    int i = 1;
    int urls_list_size = 0;
    struct curl_slist *list_temp = NULL;

    while(symbols_list != NULL)
    {

    list_temp = curl_slist_append(list_temp, symbols_list->data);
    if(!list_temp)
        perror_exit_failure("Memory allocation failure. [ Function: construct_urls_list() Var: list_temp ]\n");

    if(i % 200 == 0)
    {
        char *s_param = construct_s_param(list_temp);
        char *url =  construct_url(f_param, s_param);

        *urls_list = curl_slist_append(*urls_list, url);

        if(!urls_list)
        perror_exit_failure("Memory allocation failure. [ Function: construct_urls_list() Var #1: *urls_list ]\n");

        curl_slist_free_all(list_temp);
        list_temp = NULL;
        free(url);
        free(s_param);
        urls_list_size++;
        i = 0;
    }
    i++;
    symbols_list = symbols_list->next;

    }

    if(i > 0 && list_temp != NULL)
    {
        char *s_param = construct_s_param(list_temp);
        char *url =  construct_url(f_param, s_param);

        *urls_list = curl_slist_append(*urls_list, url);

        if(!urls_list)
        perror_exit_failure("Memory allocation failure. [ Function: construct_urls_list() Var #2: *urls_list ]\n");

        curl_slist_free_all(list_temp);
        list_temp = NULL;
        free(url);
        free(s_param);
        urls_list_size++;

    }
    return urls_list_size;
}

void construct_curl_results_arr(struct CURLResult ***curl_results, int urls_list_size)
{
        int i = 0, cnt = 0;
        for(i = 0; i < urls_list_size; i++)
        {
            //Zwiekszam dynamiczna tablice curl_results
            struct  CURLResult **curl_results_tmp = (struct CURLResult**)realloc(*curl_results, ++cnt * sizeof(struct CURLResult*));
            *curl_results = curl_results_tmp;

           // Tworze nowa strukture mem
            struct CURLResult *mem = (struct CURLResult*)malloc(sizeof(struct CURLResult));

           // Przypisuje jej domyslne wartosci
           mem->memory = (char*)malloc(1);
           mem->size = 0;

           (*curl_results)[i] = mem;
        }
}
void free_curl_results_arr(struct CURLResult ***curl_results, int urls_list_size)
{
    int i = 0;
    for(i = 0; i < urls_list_size; i++)
    {
    free((*curl_results)[i]->memory);
    (*curl_results)[i]->memory = NULL;
    free((*curl_results)[i]);
    }
    free(*curl_results);
    *curl_results = NULL;
}

static size_t write_to_memory(void *contents, size_t size, size_t nmemb, void *userp)
{
    size_t realsize = 0;
    realsize = size * nmemb;

    struct CURLResult *mem = (struct CURLResult *)userp;

    mem->memory = (char*)realloc(mem->memory, mem->size + realsize + 1);

    memcpy(&(mem->memory[mem->size]), contents, realsize);
    mem->size += realsize;
    mem->memory[mem->size] = 0;

    return realsize;
}

void prepare_curl_handles(CURL ***curl_handlers, struct CURLResult ***curl_results, struct curl_slist *urls_list)
{
    int i = 0;
        while(urls_list != NULL)
        {
            CURL *http_handle;
            http_handle = curl_easy_init();


           curl_easy_setopt(http_handle, CURLOPT_URL,  urls_list->data);


           curl_easy_setopt(http_handle, CURLOPT_WRITEDATA, (void*)(*curl_results)[i]);

           curl_easy_setopt(http_handle, CURLOPT_WRITEFUNCTION, write_to_memory);

           CURL **curl_handlers_tmp = (CURL**)realloc(*curl_handlers, ++i * sizeof(CURL*));
           *curl_handlers = curl_handlers_tmp;

           (*curl_handlers)[i-1] = http_handle;
           urls_list = urls_list->next;


        }

}

void cleanup_curl_handles(CURL ***curl_handles, int urls_list_size)
{
    int i = 0;
    for(i = 0; i < urls_list_size; i++)
    {
        curl_easy_cleanup((*curl_handles)[i]);
    }
    free(*curl_handles);
    *curl_handles = NULL;
}

void print_curl_slist(struct curl_slist *slist)
{
    while(slist != NULL)
    {
    printf("%s\n\n", slist->data);
    slist = slist->next;
    }
}
void print_curl_results(struct CURLResult **curl_results, int urls_list_size)
{
    int i;
    for(i = 0; i < urls_list_size; i++)
    {
    printf("%s", curl_results[i]->memory);
    }
}

void curl_finish(CURLM *multi_handle, CURL ***curl_handles, int urls_list_size)
{
    int i;

    curl_multi_cleanup(multi_handle);
    multi_handle = NULL;

    for(i= 0; i < urls_list_size; i++)
    {
        curl_easy_cleanup((*curl_handles)[i]);
        (*curl_handles)[i] = NULL;
    }

    free((*curl_handles));
    curl_handles = NULL;

}

int main(void)
{
    MYSQL *con = NULL;
    MYSQL_RES *result = NULL;

    struct curl_slist *symbols_list = NULL;
    struct curl_slist *urls_list = NULL;
    struct CURLResult **curl_results = NULL;
    CURLM *multi_handle;
    int still_running = 0;
    CURL **curl_handles = NULL;

    char f_param[] = "svp2ghjkpol1na2";

    con = mysql_init(NULL);
    sql_con(con, (const char*)MYSQL_HOST, (const char*)MYSQL_DB, (const char*)MYSQL_USER, (const char*)MYSQL_PASS);
    result = sql_query(con, (const char*)MYSQL_QUERY);
    add_symbols_to_list_sql(result, con, &symbols_list);
    sql_close(con);


    int urls_list_size = construct_urls_list(symbols_list, &urls_list, f_param);
    construct_curl_results_arr(&curl_results, urls_list_size);

    prepare_curl_handles(&curl_handles, &curl_results, urls_list);

    curl_global_init(CURL_GLOBAL_ALL);
    multi_handle = curl_multi_init();
    int i;
    for(i = 0; i < urls_list_size; i++)
    {
    curl_multi_add_handle(multi_handle, curl_handles[i]);
    }


    curl_multi_perform(multi_handle, &still_running);

    do {
    struct timeval timeout;
    int rc; // select() return code

    fd_set fdread;
    fd_set fdwrite;
    fd_set fdexcep;
    int maxfd = -1;

    long curl_timeo = -1;

    FD_ZERO(&fdread);
    FD_ZERO(&fdwrite);
    FD_ZERO(&fdexcep);

    //set a suitable timeout to play around with
    timeout.tv_sec = 1;
    timeout.tv_usec = 0;

    curl_multi_timeout(multi_handle, &curl_timeo);
    if(curl_timeo >= 0)
    {
        timeout.tv_sec = curl_timeo / 1000;
        if(timeout.tv_sec > 1)
        timeout.tv_sec = 1;
        else
        timeout.tv_usec = (curl_timeo % 1000) * 1000;
    }

    // get file descriptors from the transfers
    curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);

    // In a real-world program you OF COURSE check the return code of the
    //  function calls.  On success, the value of maxfd is guaranteed to be
    //  greater or equal than -1.  We call select(maxfd + 1, ...), specially in
    //  case of (maxfd == -1), we call select(0, ...), which is basically equal
    //  to sleep.
    if((maxfd != -1 ))
    {
        rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout);
    }
    else
    {
#ifdef _WIN32
            Sleep(timeout.tv_sec * 1000);
#else
            select(0 , NULL, NULL, NULL, &timeout);
#endif
            rc =1;
    }
    switch(rc)
    {
        case -1:
        printf("%d\n", rc);
        break;
        case 0:
        default:
        // timeout or readable/writable sockets
        curl_multi_perform(multi_handle, &still_running);
        break;
    }
    } while(still_running);

    curl_multi_cleanup(multi_handle);
    cleanup_curl_handles(&curl_handles, urls_list_size);
    curl_global_cleanup();
    print_curl_results(curl_results, urls_list_size);
    free_curl_results_arr(&curl_results, urls_list_size);
    curl_slist_free_all(symbols_list);
    curl_slist_free_all(urls_list);

return 0;
}