#include <iostream>
#include <pthread.h>
#include <netdb.h>
#include <netinet/tcp.h>

#include <curl/curl.h>

#include <openssl/lhash.h>
#include <openssl/crypto.h>
#include <openssl/buffer.h>
#include <openssl/x509.h>
#include <openssl/ssl.h>
#include <openssl/err.h>
#include <openssl/rand.h>

using namespace std;

//locks
pthread_mutex_t urlLock = PTHREAD_MUTEX_INITIALIZER;

static pthread_mutex_t *lock_cs;

FILE* input;
int myCount, threadNumber;

unsigned long pthreads_thread_id(void){
  
  unsigned long ret;
  
  ret=(unsigned long)pthread_self();
  return(ret);
}



void pthreads_locking_callback(int mode, int type, char *file,
			       int line){
  if (mode & CRYPTO_LOCK){
    pthread_mutex_lock(&(lock_cs[type]));
  }else{
    pthread_mutex_unlock(&(lock_cs[type]));
  }
}

struct FetchBuffer{

  char *buffer;
  size_t size;
};


size_t  writeToBuffer(void *ptr, size_t size, size_t nmemb, void *data){

  size_t realsize = size * nmemb;
  struct FetchBuffer *mem = (struct FetchBuffer*) data;

  mem->buffer = (char*)realloc(mem->buffer,mem->size+realsize+1);
  
  if (mem->buffer) {
    memcpy(mem->buffer+mem->size, ptr, realsize);
    mem->size += realsize;
    mem->buffer[mem->size] = '\0';
  }
 
  return realsize;
}



CURL* initialFetchAgent(struct FetchBuffer* content,curl_slist* headers, char* errorMessage){

  CURL* singleAgent = curl_easy_init();
  assert(singleAgent != NULL);
  curl_easy_setopt(singleAgent,CURLOPT_FOLLOWLOCATION,1);
  curl_easy_setopt(singleAgent,CURLOPT_FAILONERROR,1); 
  curl_easy_setopt(singleAgent,CURLOPT_CONNECTTIMEOUT,30);
  curl_easy_setopt(singleAgent,CURLOPT_TIMEOUT,90);
  curl_easy_setopt(singleAgent,CURLOPT_FILETIME, 1);
  curl_easy_setopt(singleAgent,CURLOPT_WRITEFUNCTION, writeToBuffer);
  curl_easy_setopt(singleAgent,CURLOPT_HTTPHEADER, headers);
  curl_easy_setopt(singleAgent,CURLOPT_UNRESTRICTED_AUTH,1);
  curl_easy_setopt(singleAgent,CURLOPT_ERRORBUFFER, errorMessage);
  curl_easy_setopt(singleAgent,CURLOPT_WRITEDATA, content);
  curl_easy_setopt(singleAgent,CURLOPT_MAXREDIRS,10);
  curl_easy_setopt(singleAgent,CURLOPT_DNS_CACHE_TIMEOUT,600);
  //must set no signal for multi-threads
  curl_easy_setopt(singleAgent,CURLOPT_NOSIGNAL,1);

  return singleAgent;
}


void* singleFetch(void *){

  curl_slist * headers = NULL;
  headers = curl_slist_append(headers, "Accept: text/html"); 
  char* errorMessage = (char*) malloc(CURL_ERROR_SIZE);
  bzero(errorMessage,CURL_ERROR_SIZE);

  //web page will store in the struct first.
  FetchBuffer * content = (struct FetchBuffer *)malloc(sizeof(struct FetchBuffer));
  content->buffer = NULL;
  content->size = 0;
  CURL* singleAgent;
  
  //url and its'id will store here
  char* url = NULL;
  CURLcode result;
 
  char line[1024];
 
  //initial a single agent. it will perform the actual fetch.
  singleAgent = initialFetchAgent(content,headers,errorMessage);


  while(true){

    
    //get a url.
    pthread_mutex_lock(&urlLock);
    if(fgets(line,1024,input)){
      int urlLen = strlen(line);

      if(1 == urlLen){
	pthread_mutex_unlock(&urlLock);
	break;
      }
      line[urlLen-1] = '\0';
      url = (char*)malloc(urlLen);
      strcpy(url,line);
      cerr<<"downloading "<<++myCount<<" file\r"<<flush;
    }else{
      pthread_mutex_unlock(&urlLock);
      break;
    }
    pthread_mutex_unlock(&urlLock);

    cout<<url<<endl;
    //give the url name to the single agent
    curl_easy_setopt(singleAgent,CURLOPT_URL, url);
    //go fetch
    result = curl_easy_perform(singleAgent);

   
    //clean up
    bzero(errorMessage,strlen(errorMessage));
    free(content->buffer);
    content->buffer = NULL;
    content->size = 0;
    
    free(url);
    url = NULL;
  }
  
  curl_easy_cleanup(singleAgent);

 
  return NULL;
}


void run(){

  pthread_t agents[threadNumber];

  //set up lock function for ssl. required by openssl with curl
  lock_cs=(pthread_mutex_t*)OPENSSL_malloc(CRYPTO_num_locks() * sizeof(pthread_mutex_t));
  for (int i=0; i<CRYPTO_num_locks(); i++){
    pthread_mutex_init(&(lock_cs[i]),NULL);
  }
  
  CRYPTO_set_id_callback((unsigned long (*)())pthreads_thread_id);
  CRYPTO_set_locking_callback((void (*)(int,int,const char*,int))pthreads_locking_callback);


  curl_global_init(CURL_GLOBAL_ALL);

  //create threads(singel agent) to download pages
  for(int i = 0; i < threadNumber; ++i){
    if(pthread_create(&agents[i],NULL,singleFetch,NULL)<0){
      cerr<<"fetch agent creation failed"<<endl;
      exit(1);
    }
  }

  for(int i = 0; i < threadNumber; ++i){
    pthread_join(agents[i],NULL);
  }

  //clean up ssl locks
  CRYPTO_set_locking_callback(NULL);
  for (int i=0; i<CRYPTO_num_locks(); i++){
    pthread_mutex_destroy(&(lock_cs[i]));
  }
  OPENSSL_free(lock_cs);

}

int main(int argc, char* argv[]){

  if(argc != 3){
    cerr<<"Usage: download fileName threadNumber"<<endl;
    exit(1);
  }

  input = fopen(argv[1],"r");
 
  myCount = 0;
  threadNumber = atoi(argv[2]);

  run();

  fclose(input);

}