#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <pthread.h>
#include <curl/curl.h>
#include <iostream>



#define CONNECT_TIMEOUT 30
#define TOTAL_TIMEOUT 90


using namespace std;


FILE * input;

pthread_mutex_t locker = PTHREAD_MUTEX_INITIALIZER;

struct FetchBuffer{

  char *buffer;
  size_t size;
};


size_t  writeToBuffer(void *ptr, size_t size, size_t nmemb, void *data){

  size_t realsize = size * nmemb;
  struct FetchBuffer *mem = (struct FetchBuffer*) data;

  mem->buffer = (char*)realloc(mem->buffer,mem->size+realsize+1);
  
  if (mem->buffer) {
    memcpy(mem->buffer+mem->size, ptr, realsize);
    mem->size += realsize;
    mem->buffer[mem->size] = '\0';
  }else{
    cerr<<"can not realloc memory in writeToBuffer"<<endl;
    exit(1);
  }
 
  return realsize;
}


CURL* initialFetchAgent(struct FetchBuffer* content,curl_slist* headers, char* errorMessage){

  CURL* singleAgent = curl_easy_init();
  assert(singleAgent != NULL);
  curl_easy_setopt(singleAgent,CURLOPT_FOLLOWLOCATION,0);
  curl_easy_setopt(singleAgent,CURLOPT_FAILONERROR,1); 
  curl_easy_setopt(singleAgent,CURLOPT_CONNECTTIMEOUT,CONNECT_TIMEOUT);
  curl_easy_setopt(singleAgent,CURLOPT_TIMEOUT,TOTAL_TIMEOUT);
  curl_easy_setopt(singleAgent,CURLOPT_FILETIME, 1);
  curl_easy_setopt(singleAgent,CURLOPT_WRITEFUNCTION, writeToBuffer);
  curl_easy_setopt(singleAgent,CURLOPT_HTTPHEADER, headers);
  curl_easy_setopt(singleAgent,CURLOPT_UNRESTRICTED_AUTH,1);
  curl_easy_setopt(singleAgent,CURLOPT_ERRORBUFFER, errorMessage);
  curl_easy_setopt(singleAgent,CURLOPT_WRITEDATA, content);
  //curl_easy_setopt(singleAgent,CURLOPT_MAXREDIRS,4);
  curl_easy_setopt(singleAgent,CURLOPT_NOSIGNAL,1);

  return singleAgent;

}


 
 void *pull_one_url(void *)
 {
   
   CURL *curl;
   char line[4096];
   char* url;


   curl_slist * headers = NULL;
   headers = curl_slist_append(headers, "Accept: text/html"); 
   char* errorMessage = (char*) malloc(CURL_ERROR_SIZE);
   bzero(errorMessage,CURL_ERROR_SIZE);

   FetchBuffer * content = (struct FetchBuffer *)malloc(sizeof(struct FetchBuffer));
   content->buffer = NULL;
   content->size = 0;

   while(true){
     
     //get url
     pthread_mutex_lock(&locker);
     if(!fgets(line,4096,input)){
       curl_easy_cleanup(curl);
       pthread_mutex_unlock(&locker);
       break;
     }

     url = new char[strlen(line)+1];
     strcpy(url,line);

     pthread_mutex_unlock(&locker);

     //get page
     curl = initialFetchAgent(content,headers,errorMessage);
     curl_easy_setopt(curl,CURLOPT_URL,url);
     curl_easy_perform(curl); // ignores error 
     curl_easy_cleanup(curl);


     //clean up
     delete [] url;

     bzero(errorMessage,strlen(errorMessage));
     if(content->buffer != NULL){
       free(content->buffer);
       content->buffer = NULL;
       content->size = 0;
     }
   }   
   return NULL;
 }

int main(int argc, char **argv){

  
  int i;
  int error;
  char line[4096];
  input = fopen(argv[1],"r");
  int numThread = atoi(argv[2]);
  pthread_t* tid = new pthread_t[numThread];

  for(i=0; i< numThread; i++) {
    error = pthread_create(&tid[i],
			   NULL, /* default attributes please */
			   pull_one_url,NULL);
    if(0 != error)
      fprintf(stderr, "Couldn't run thread number %d, errno %d\n", i, error);
    else
      ;
  }
  
  /* now wait for all threads to terminate */
  for(i=0; i< numThread; i++) {
    error = pthread_join(tid[i], NULL);
    fprintf(stderr, "Thread %d terminated\n", i);
  }

  fclose(input);
  return 0;
}