From 6fc007925bf65b54db717541b3466c5b697147e2 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Tue, 30 Mar 2010 00:40:22 +0200 Subject: [PATCH] a new option WILDCARDMATCH (FTP only for now) --- include/curl/curl.h | 42 +++ lib/Makefile.inc | 8 +- lib/fileinfo.c | 49 ++++ lib/fileinfo.h | 32 +++ lib/ftp.c | 687 ++++++++++++++++++++++++++++++++++++++++++++++++++- lib/ftp.h | 4 + lib/strerror.c | 3 + lib/transfer.c | 36 +++- lib/url.c | 22 ++ lib/urldata.h | 5 + lib/wildcard.c | 58 +++++ lib/wildcard.h | 54 ++++ 12 files changed, 985 insertions(+), 15 deletions(-) create mode 100644 lib/fileinfo.c create mode 100644 lib/fileinfo.h create mode 100644 lib/wildcard.c create mode 100644 lib/wildcard.h diff --git a/include/curl/curl.h b/include/curl/curl.h index e635968..146c26f 100644 --- a/include/curl/curl.h +++ b/include/curl/curl.h @@ -198,6 +198,35 @@ typedef size_t (*curl_write_callback)(char *buffer, size_t nitems, void *outstream); +/* callback type for wildcard downloading pattern matching - if the string + matches the pattern, return zero; otherwise return non-zero value */ +typedef int (*curl_fnmatch_callback)(const char *pattern, const char *string); + +/* enumeration of file types */ +typedef enum { + CURLFILETYPE_FILE, + CURLFILETYPE_DIRECTORY, + CURLFILETYPE_SIMLINK, + CURLFILETYPE_DEVICE, + CURLFILETYPE_UNKNOWNTYPE +} Curl_file_type; + +/* File info passed to curl_startunit_callback. Based on the info an individual + file can be rejected. */ +typedef struct { + char *filename; + Curl_file_type filetype; + curl_off_t filesize; +} Curl_fileinfo; + +/* wildcard callback - called before download of an individual file, + returned non-zero value causes the file to be skipped over */ +typedef long (*curl_startunit_callback)(const Curl_fileinfo *, void *ptr, + int remains); +/* wildcard callback - called as soon as download of an individual file + finishes */ +typedef long (*curl_endunit_callback)(void *ptr); + /* These are the return codes for the seek callbacks */ #define CURL_SEEKFUNC_OK 0 #define CURL_SEEKFUNC_FAIL 1 /* fail the entire transfer */ @@ -409,6 +438,7 @@ typedef enum { CURLE_FTP_PRET_FAILED, /* 84 - a PRET command failed */ CURLE_RTSP_CSEQ_ERROR, /* 85 - mismatch of RTSP CSeq numbers */ CURLE_RTSP_SESSION_ERROR, /* 86 - mismatch of RTSP Session Identifiers */ + CURLE_FTP_BAD_FILE_LIST, /* 87 - unable to parse FTP file list */ CURL_LAST /* never use! */ } CURLcode; @@ -1316,6 +1346,18 @@ typedef enum { /* Let the application define a custom write method for RTP data */ CINIT(INTERLEAVEFUNCTION, FUNCTIONPOINT, 196), + /* Turn on wildcard pattern matching */ + CINIT(WILDCARDMATCH, LONG, 197), + + /* Wildcard, callback called before downloading of an individual file started */ + CINIT(STARTUNIT, FUNCTIONPOINT, 198), + + /* Wildcard - callback called after the file was downloaded, or skipped */ + CINIT(ENDUNIT, FUNCTIONPOINT, 199), + + /* Change fnmatch callback for wildcard downloading */ + CINIT(FNMATCH, FUNCTIONPOINT, 200), + CURLOPT_LASTENTRY /* the last unused */ } CURLoption; diff --git a/lib/Makefile.inc b/lib/Makefile.inc index 9803c8c..d15b2aa 100644 --- a/lib/Makefile.inc +++ b/lib/Makefile.inc @@ -11,8 +11,8 @@ CSOURCES = file.c timeval.c base64.c hostip.c progress.c formdata.c \ inet_ntop.c parsedate.c select.c gtls.c sslgen.c tftp.c splay.c \ strdup.c socks.c ssh.c nss.c qssl.c rawstr.c curl_addrinfo.c \ socks_gssapi.c socks_sspi.c curl_sspi.c slist.c nonblock.c \ - curl_memrchr.c imap.c pop3.c smtp.c pingpong.c rtsp.c curl_threads.c \ - warnless.c + curl_memrchr.c imap.c pop3.c smtp.c pingpong.c rtsp.c curl_threads.c \ + wildcard.c fileinfo.c warnless.c HHEADERS = arpa_telnet.h netrc.h file.h timeval.h qssl.h hostip.h \ progress.h formdata.h cookie.h http.h sendf.h ftp.h url.h dict.h \ @@ -24,5 +24,5 @@ HHEADERS = arpa_telnet.h netrc.h file.h timeval.h qssl.h hostip.h \ transfer.h select.h easyif.h multiif.h parsedate.h sslgen.h gtls.h \ tftp.h sockaddr.h splay.h strdup.h setup_once.h socks.h ssh.h nssg.h \ curl_base64.h rawstr.h curl_addrinfo.h curl_sspi.h slist.h nonblock.h \ - curl_memrchr.h imap.h pop3.h smtp.h pingpong.h rtsp.h curl_threads.h \ - warnless.h + curl_memrchr.h imap.h pop3.h smtp.h pingpong.h rtsp.h curl_threads.h \ + wildcard.h fileinfo.h warnless.h diff --git a/lib/fileinfo.c b/lib/fileinfo.c new file mode 100644 index 0000000..b4ea50e --- /dev/null +++ b/lib/fileinfo.c @@ -0,0 +1,49 @@ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 2010, Daniel Stenberg, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +#include +#include "strdup.h" +#include "fileinfo.h" + +void Curl_fileinfo_dtor(void *user, void *element) +{ + Curl_fileinfo *finfo = element; + (void) user; + if (!finfo) + return; + + free(finfo->filename); + free(finfo); +} + +Curl_fileinfo *Curl_fileinfo_copy(const Curl_fileinfo *src) +{ + Curl_fileinfo *ptr = malloc(sizeof(Curl_fileinfo)); + if(!ptr) + return NULL; + *ptr = *src; + ptr->filename = strdup(src->filename); + if (!ptr->filename) + return NULL; + else + return ptr; +} diff --git a/lib/fileinfo.h b/lib/fileinfo.h new file mode 100644 index 0000000..a4e30e1 --- /dev/null +++ b/lib/fileinfo.h @@ -0,0 +1,32 @@ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 2010, Daniel Stenberg, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +#ifndef __FILEINFO_H +#define __FILEINFO_H + +#include + +void Curl_fileinfo_dtor(void *, void *); + +Curl_fileinfo *Curl_fileinfo_copy(const Curl_fileinfo *src); + +#endif /* __FILEINFO_H */ diff --git a/lib/ftp.c b/lib/ftp.c index 0ec5243..f03b898 100644 --- a/lib/ftp.c +++ b/lib/ftp.c @@ -71,6 +71,7 @@ #include "http.h" /* for HTTP proxy tunnel stuff */ #include "socks.h" #include "ftp.h" +#include "fileinfo.h" #if defined(HAVE_KRB4) || defined(HAVE_GSSAPI) #include "krb4.h" @@ -110,6 +111,77 @@ #define ftp_pasv_verbose(a,b,c,d) do { } while(0) #endif +typedef enum { + PL_FILETYPE = 0, + PL_PERMISSION, + PL_NUM_OF_HLINKS, + PL_USER, + PL_GROUP, + PL_SIZE, + PL_TIME, + PL_FILENAME, + PL_ERROR +} parselist_states; + +typedef enum { + PL_NUM_OF_HLINKS_INIT = 0, + PL_NUM_OF_HLINKS_COUNT +} parselist_substate_num_of_hardlinks; + +typedef enum { + PL_USER_INIT = 0, + PL_USER_NAME +} parselist_substate_user; + +typedef enum { + PL_GROUP_INIT = 0, + PL_GROUP_NAME +} parselist_substate_group; + +typedef enum { + PL_SIZE_INIT = 0, + PL_SIZE_SIZE, + PL_SIZE_DEVICESTEP, + PL_SIZE_DEVICE +} parselist_substate_size; + +typedef enum { + PL_TIME_INIT = 0, + PL_TIME_DAY, + PL_TIME_PREMONTH, + PL_TIME_MONTH, + PL_TIME_PREYEAR_OR_TIME, + PL_TIME_YEAR_OR_TIME +} parselist_substate_time; + +typedef union { + parselist_substate_num_of_hardlinks num_of_hardlinks; + parselist_substate_user user; + parselist_substate_group group; + parselist_substate_size size; + parselist_substate_time time; + +} parselist_substates; + +#define FTP_BUFFER_ALLOCSIZE 80 /* has to be longer then username can be */ + +/* This struct is used in wildcard downloading - for parsing LIST response + */ +struct ftp_parselist_data { + parselist_states state; + parselist_substates substate; + char *buffer; + size_t bufferlength; + size_t bufferin; + curl_write_callback old_fwritefunc; + FILE *old_file_descriptor; + CURLcode error; + /* Curl_filelist *filelist; */ + struct curl_llist *filelist; + Curl_fileinfo file_data; + char c; +}; + /* Local API functions */ static CURLcode ftp_sendquote(struct connectdata *conn, struct curl_slist *quote); @@ -143,6 +215,15 @@ static int ftp_getsock(struct connectdata *conn, static CURLcode ftp_doing(struct connectdata *conn, bool *dophase_done); static CURLcode ftp_setup_connection(struct connectdata * conn); +static size_t ftp_get_filelist(char *buffer, size_t size, + size_t nmemb, void *userp); + +static CURLcode free_wc_data(struct connectdata *conn); +static CURLcode init_wc_data(struct connectdata *conn); +static CURLcode wc_statemach(struct connectdata *conn); + +static CURLcode ftp_state_post_retr_size(struct connectdata *conn, + curl_off_t filesize); /* easy-to-use macro: */ #define FTPSENDF(x,y,z) if((result = Curl_ftpsendf(x,y,z)) != CURLE_OK) \ @@ -1455,8 +1536,14 @@ static CURLcode ftp_state_quote(struct connectdata *conn, if(ftp->transfer != FTPTRANSFER_BODY) state(conn, FTP_STOP); else { - PPSENDF(&ftpc->pp, "SIZE %s", ftpc->file); - state(conn, FTP_RETR_SIZE); + if(ftpc->known_filesize != -1) { + Curl_pgrsSetDownloadSize(data, ftpc->known_filesize); + result = ftp_state_post_retr_size(conn, ftpc->known_filesize); + } + else { + PPSENDF(&ftpc->pp, "SIZE %s", ftpc->file); + state(conn, FTP_RETR_SIZE); + } } break; case FTP_STOR_PREQUOTE: @@ -2842,6 +2929,8 @@ static CURLcode ftp_init(struct connectdata *conn) if(TRUE == isBadFtpString(ftp->passwd)) return CURLE_URL_MALFORMAT; + conn->proto.ftpc.known_filesize = -1; /* unknown size for now */ + return CURLE_OK; } @@ -3005,6 +3094,12 @@ static CURLcode ftp_done(struct connectdata *conn, CURLcode status, if(ftpc->prevpath) free(ftpc->prevpath); + if(data->set.wildcardmatch) { + if(data->set.endunit && ftpc->file) { + data->set.endunit(data->set.out); + } + } + /* get the "raw" path */ path = curl_easy_unescape(data, path_to_use, 0, NULL); if(!path) { @@ -3387,6 +3482,357 @@ static CURLcode ftp_nextconnect(struct connectdata *conn) } +static void ftp_parsinglist_time(struct ftp_parselist_data* parser) +{ + char c = parser->c; + switch (parser->substate.time) { + case PL_TIME_INIT: + if(isspace(c)) { /* skip free space */ } + else { + parser->buffer[0] = c; + parser->bufferin = 1; + parser->substate.time = PL_TIME_DAY; + } + break; + + case PL_TIME_DAY: + if(!isspace(c)) { + parser->buffer[parser->bufferin++] = c; + if(parser->bufferin > FTP_BUFFER_ALLOCSIZE - 1) { + parser->error = CURLE_FTP_BAD_FILE_LIST; + } + } + else { + parser->buffer[parser->bufferin++] = ' '; + parser->substate.time = PL_TIME_PREMONTH; + if(parser->bufferin > FTP_BUFFER_ALLOCSIZE - 1) + parser->error = CURLE_FTP_BAD_FILE_LIST; + } + break; + + case PL_TIME_PREMONTH: /* month */ + if(isspace(c)) { /* skip free space */ } + else { + parser->buffer[parser->bufferin++] = c; + if(parser->bufferin > FTP_BUFFER_ALLOCSIZE - 1) + parser->error = CURLE_FTP_BAD_FILE_LIST; + parser->substate.time = PL_TIME_MONTH; + } + break; + + case PL_TIME_MONTH: + if(!isspace(c)) { + parser->buffer[parser->bufferin++] = c; + if(parser->bufferin > FTP_BUFFER_ALLOCSIZE - 1) + parser->error = CURLE_FTP_BAD_FILE_LIST; + } + else { + parser->buffer[parser->bufferin++] = ' '; + parser->substate.time = PL_TIME_PREYEAR_OR_TIME; + if(parser->bufferin > FTP_BUFFER_ALLOCSIZE - 1) + parser->error = CURLE_FTP_BAD_FILE_LIST; + } + break; + + case PL_TIME_PREYEAR_OR_TIME: + if(isspace(c)) {} + else { + parser->buffer[parser->bufferin++] = c; + parser->substate.time = PL_TIME_YEAR_OR_TIME; + if(parser->bufferin > FTP_BUFFER_ALLOCSIZE - 1) + parser->error = CURLE_FTP_BAD_FILE_LIST; + } + break; + + case PL_TIME_YEAR_OR_TIME: + if(!isspace(c)) { + parser->buffer[parser->bufferin++] = c; + if(parser->bufferin > FTP_BUFFER_ALLOCSIZE - 1) + parser->error = CURLE_FTP_BAD_FILE_LIST; + } + else { + /* date is readed */ + parser->buffer[parser->bufferin++] = 0; + if(parser->bufferin > FTP_BUFFER_ALLOCSIZE - 1) + parser->error = CURLE_FTP_BAD_FILE_LIST; + + /* now we can deal with date */ + + parser->state = PL_FILENAME; + parser->buffer[0] = 0; + parser->bufferin = 0; + } + break; + } +} + +static size_t ftp_get_filelist(char *buffer, size_t size, + size_t nmemb, void *connptr) +{ + size_t bufflen = size*nmemb; + struct connectdata *conn = (struct connectdata *)connptr; + struct ftp_conn *ftpc = &conn->proto.ftpc; + struct ftp_parselist_data *parser = ftpc->parselist; + unsigned int i; + + if(parser->error) { /* error in previous call */ + return bufflen; + } + + for (i = 0; i < bufflen; i++) { + char c = buffer[i]; + parser->c = c; + switch (parser->state) { + case PL_FILETYPE: + switch (c) { + case '-': parser->file_data.filetype = CURLFILETYPE_FILE; break; + case 'd': parser->file_data.filetype = CURLFILETYPE_DIRECTORY; break; + case 's': parser->file_data.filetype = CURLFILETYPE_SIMLINK; break; + case 'c': + case 'b': parser->file_data.filetype = CURLFILETYPE_DEVICE; break; + default: parser->file_data.filetype = CURLFILETYPE_UNKNOWNTYPE; break; + } + parser->state = PL_PERMISSION; + parser->bufferin = 0; + parser->buffer[0] = 0; + break; + + case PL_PERMISSION: + if(parser->bufferin < 8) { + parser->buffer[parser->bufferin++] = c; + } + else if(parser->bufferin == 8) { + parser->buffer[parser->bufferin] = c; + parser->buffer[parser->bufferin+1] = 0; + + /* now could be possible to parse permissions .. */ + + parser->state = PL_NUM_OF_HLINKS; + parser->bufferin = 0; + parser->substate.num_of_hardlinks = PL_NUM_OF_HLINKS_INIT; + parser->buffer[0] = 0; + } + break; + + case PL_NUM_OF_HLINKS: + switch (parser->substate.num_of_hardlinks) { + case PL_NUM_OF_HLINKS_INIT: + if(!isspace(c)) { + parser->buffer[0] = c; + parser->bufferin = 1; + parser->substate.num_of_hardlinks = PL_NUM_OF_HLINKS_COUNT; + } /* else skip free space */ + break; + case PL_NUM_OF_HLINKS_COUNT: + if(!isspace(c)) { + if(isdigit(c)) { + parser->buffer[parser->bufferin++] = c; + } + else { + parser->error = CURLE_FTP_BAD_FILE_LIST; + return bufflen; + } + + if(parser->bufferin > FTP_BUFFER_ALLOCSIZE - 1) { + parser->error = CURLE_FTP_BAD_FILE_LIST; + return bufflen; + } + } + else { /* end of filesize */ + parser->buffer[parser->bufferin] = 0; + + /* now could be possible to work with number of hardlinks .. */ + + parser->buffer[0] = 0; + parser->substate.user = PL_USER_INIT; + parser->bufferin = 0; + parser->state = PL_USER; + } + break; + } + break; + + case PL_USER: + switch (parser->substate.user) { + case PL_USER_INIT: + if(isspace(c)) { /* skip spaces */ } + else { + parser->buffer[0] = c; + parser->bufferin = 1; + parser->substate.user = PL_USER_NAME; + } + break; + case PL_USER_NAME: + if(!isspace(c)) { + parser->buffer[parser->bufferin++] = c; + if(parser->bufferin > FTP_BUFFER_ALLOCSIZE -1) { + parser->error = CURLE_FTP_BAD_FILE_LIST; + return bufflen; + } + } + else { /* end of username */ + parser->buffer[parser->bufferin] = 0; + + /* now we can work with username */ + + parser->buffer[parser->bufferin] = 0; + parser->bufferin = 0; + parser->substate.group = PL_GROUP_INIT; + parser->state = PL_GROUP; + } + break; + } + break; + + case PL_GROUP: + switch (parser->substate.group) { + case PL_GROUP_INIT: + if(isspace(c)) { /* skip spaces */ } + else { + parser->buffer[0] = c; + parser->bufferin = 1; + parser->substate.group = PL_GROUP_NAME; + } + break; + case PL_GROUP_NAME: + if(!isspace(c)) { + parser->buffer[parser->bufferin++] = c; + if(parser->bufferin > FTP_BUFFER_ALLOCSIZE -1) { + parser->error = CURLE_FTP_BAD_FILE_LIST; + return bufflen; + } + } + else { /* end of username */ + parser->buffer[parser->bufferin] = 0; + + /* now we can work with groupname */ + + parser->buffer[parser->bufferin] = 0; + parser->bufferin = 0; + parser->substate.size = PL_SIZE_INIT; + parser->state = PL_SIZE; + } + break; + } + break; + + case PL_SIZE: + switch (parser->substate.size) { + case PL_SIZE_INIT: + if(isspace(c)) { /* skip space */ } + else if(isdigit(c)) { + parser->buffer[0] = c; + parser->bufferin = 1; + parser->substate.size = PL_SIZE_SIZE; + } + else { + parser->error = CURLE_FTP_BAD_FILE_LIST; + return bufflen; + } + break; + case PL_SIZE_SIZE: + if(c == ',') { + parser->buffer[parser->bufferin++] = c; + parser->substate.size = PL_SIZE_DEVICESTEP; + } + else if(isspace(c)) { + parser->buffer[parser->bufferin] = 0; + /* now we get filesize */ + parser->file_data.filesize = + curlx_strtoofft(parser->buffer, NULL, 0); + parser->buffer[0] = 0; + parser->substate.time = PL_TIME_INIT; + parser->bufferin = 0; + parser->state = PL_TIME; + } + else { + parser->buffer[parser->bufferin++] = c; + } + break; + + case PL_SIZE_DEVICESTEP: /* devices */ + if(isspace(c)) { /* skip free space */ } + else if(isdigit(c)) { + parser->buffer[parser->bufferin++] = c; + if(parser->bufferin > FTP_BUFFER_ALLOCSIZE - 1) { + parser->error = CURLE_FTP_BAD_FILE_LIST; + return bufflen; + } + parser->substate.size = PL_SIZE_DEVICE; + } + break; + case PL_SIZE_DEVICE: + if(isdigit(c)) { + parser->buffer[parser->bufferin++] = c; + if(parser->bufferin > FTP_BUFFER_ALLOCSIZE - 1) { + parser->error = CURLE_FTP_BAD_FILE_LIST; + return bufflen; + } + } + else if(isspace(c)) { + parser->buffer[parser->bufferin] = 0; + /* now we can deal with size information about device */ + parser->buffer[0] = 0; + parser->substate.time = PL_TIME_INIT; + parser->bufferin = 0; + parser->state = PL_TIME; + } + break; + } + break; + + case PL_TIME: + ftp_parsinglist_time(parser); + if(parser->error) + return bufflen; + break; + + case PL_FILENAME: + if(c != '\n') { + parser->buffer[parser->bufferin++] = c; + if(parser->bufferin > parser->bufferlength - 1) { + parser->buffer = realloc(parser->buffer, + parser->bufferlength + FTP_BUFFER_ALLOCSIZE); + if(!parser->buffer) { + parser->error = CURLE_OUT_OF_MEMORY; + return bufflen; + } + parser->bufferlength += FTP_BUFFER_ALLOCSIZE; + } + } + else { + Curl_fileinfo *finfo; + parser->buffer[parser->bufferin] = 0; + parser->file_data.filename = parser->buffer; + + if(strcmp(parser->buffer, ".") != 0 && + strcmp(parser->buffer, "..") != 0) { + finfo = Curl_fileinfo_copy(&(parser->file_data)); + parser->error = Curl_llist_insert_next(parser->filelist, + parser->filelist->tail, finfo); + if(parser->error) + parser->error = CURLE_OK; + else + parser->error = CURLE_OUT_OF_MEMORY; + + if(parser->error) + return bufflen; + } + + parser->state = PL_FILETYPE; + parser->buffer[0] = 0; + parser->bufferin = 0; + parser->file_data.filename = NULL; + } + break; + + case PL_ERROR: + break; + } + } + + return bufflen; +} /*********************************************************************** * @@ -3435,6 +3881,225 @@ CURLcode ftp_perform(struct connectdata *conn, return result; } +static CURLcode init_wc_data(struct connectdata *conn) +{ + struct ftp_conn *ftpc = &conn->proto.ftpc; + char *last_slash = strrchr(conn->data->state.path, '/'); + char *path = conn->data->state.path; + struct WildcardData *wildcard = &(conn->data->wildcard); + CURLcode ret = CURLE_OK; + wildcard->current_file_size = -1; + + if(last_slash) { + last_slash++; + if(last_slash[0] == '\0') { + /* only list => disable wildcardmatch */ + conn->data->set.wildcardmatch = 0L; + /* free_wc_data(conn); */ + ret = ftp_parse_url_path(conn); + return ret; + } + else { + wildcard->pattern = strdup(last_slash); + last_slash[0] = '\0'; /* cut file from path */ + } + } + else { + /* there is only 'wildcard pattern' or nothing */ + if(path[0]) { + wildcard->pattern = strdup(path); + path[0] = '\0'; + } + else { /* only list */ + conn->data->set.wildcardmatch = 0L; + /* free_wc_data(conn); */ + ret = ftp_parse_url_path(conn); + return ret; + } + } + + /* INITIALIZE parselist structure! */ + ftpc->parselist = malloc(sizeof(struct ftp_parselist_data)); + if(!ftpc->parselist) + return CURLE_OUT_OF_MEMORY; + bzero(ftpc->parselist, sizeof(struct ftp_parselist_data)); + ftpc->parselist->state = PL_FILETYPE; + + /* ftpc->parselist->filelist = Curl_filelist_new(); */ + ftpc->parselist->filelist = + Curl_llist_alloc((curl_llist_dtor)Curl_fileinfo_dtor); + if(!ftpc->parselist->filelist) + return CURLE_OUT_OF_MEMORY; + + if(!ftpc->parselist->filelist) + return CURLE_OUT_OF_MEMORY; + ftpc->parselist->buffer = malloc(FTP_BUFFER_ALLOCSIZE); + if(!ftpc->parselist->buffer) + return CURLE_OUT_OF_MEMORY; + ftpc->parselist->bufferlength = FTP_BUFFER_ALLOCSIZE; + memset(ftpc->parselist->buffer, 0, FTP_BUFFER_ALLOCSIZE); + + /* wildcard does not support NOCWD option (assert it?) */ + if(conn->data->set.ftp_filemethod == FTPFILE_NOCWD) + conn->data->set.ftp_filemethod = FTPFILE_MULTICWD; + + ret = ftp_parse_url_path(conn); + if(ret) return ret; + + /* backup old write_function */ + ftpc->parselist->old_fwritefunc = conn->data->set.fwrite_func; + /* parsing write function */ + conn->data->set.fwrite_func = ftp_get_filelist; + /* backup old file descriptor */ + ftpc->parselist->old_file_descriptor = conn->data->set.out; + /* let the writefunc callback know what curl pointer is working with */ + conn->data->set.out = (FILE *)conn; + + wildcard->path = strdup(conn->data->state.path); + wildcard->filelist = ftpc->parselist->filelist; + + infof(conn->data, "Wildcard - Parsing started\n"); + + return CURLE_OK; +} + +static CURLcode free_wc_data(struct connectdata *conn) +{ + struct WildcardData *wildcard = &(conn->data->wildcard); + struct ftp_conn *ftpc = &conn->proto.ftpc; + + /* Curl_filelist_free(wildcard->filelist); */ + Curl_llist_destroy(wildcard->filelist, NULL); + free(wildcard->path); + free(wildcard->pattern); + + free(ftpc->parselist->buffer); + free(ftpc->parselist); + + wildcard->state = CURLWC_INIT; + + return CURLE_OK; +} + + +static CURLcode wc_statemach(struct connectdata *conn) +{ + struct WildcardData *wildcard = &(conn->data->wildcard); + struct ftp_conn *ftpc = &conn->proto.ftpc; + char *tmp_path; + CURLcode ret = CURLE_OK; + long userresponse = 0; + + switch (wildcard->state) { + case CURLWC_INIT: + ret = init_wc_data(conn); + if(ret) return ret; + wildcard->state = CURLWC_MATCHING; + break; + + case CURLWC_MATCHING: + /* restore write func */ + conn->data->set.fwrite_func = ftpc->parselist->old_fwritefunc; + conn->data->set.out = ftpc->parselist->old_file_descriptor; + wildcard->state = CURLWC_DOWNLOADING; + + if(ftpc->parselist->error) { /* error found in LIST parsing */ + wildcard->state = CURLWC_CLEAN; + return wc_statemach(conn); + } + + /* keep only pattern-corresponding filenames */ + ret = curl_wildcard_filter(wildcard); + if(ret) { + infof(conn->data, "Wildcard - no file matching pattern \"%s\"\n", + wildcard->pattern); + free_wc_data(conn); + return ret; + } + + ret = wc_statemach(conn); + break; + + case CURLWC_DOWNLOADING: { + Curl_fileinfo *finfo = wildcard->filelist->head->ptr; + + tmp_path = malloc(strlen(conn->data->state.path) + + strlen(finfo->filename) + 1); + if(!tmp_path) return CURLE_OUT_OF_MEMORY; + tmp_path[0] = 0; + + strcat(tmp_path, wildcard->path); + strcat(tmp_path, finfo->filename); + + if(conn->data->state.pathbuffer) + free(conn->data->state.pathbuffer); + conn->data->state.pathbuffer = tmp_path; + conn->data->state.path = tmp_path; + + infof(conn->data, "Wildcard - START of \"%s\"\n", finfo->filename); + if(conn->data->set.startunit) { + userresponse = conn->data->set.startunit( + finfo, conn->data->set.out, wildcard->filelist->size); + if(userresponse) { + infof(conn->data, "Wildcard - \"%s\" skipped by user\n", + finfo->filename); + wildcard->state = CURLWC_SKIP; + return wc_statemach(conn); + } + } + + if(finfo->filetype != CURLFILETYPE_FILE) { + wildcard->state = CURLWC_SKIP; + return wc_statemach(conn); + } + + ret = ftp_parse_url_path(conn); + + ftpc->known_filesize = finfo->filesize; + + Curl_llist_remove(wildcard->filelist, wildcard->filelist->head, NULL); + + if(wildcard->filelist->size == 0) { /* remains only one file to down. */ + wildcard->state = CURLWC_CLEAN; + /* after that will be ftp_do called once again and no transfer + will be done because of CURLWC_CLEAN state */ + return CURLE_OK; + } + } break; + + case CURLWC_SKIP: { + Curl_fileinfo *f = wildcard->filelist->head->ptr; + if(conn->data->set.endunit) { + conn->data->set.endunit(conn->data->set.out); + } + Curl_llist_remove(wildcard->filelist, wildcard->filelist->head, NULL); + + if(wildcard->filelist->size == 0) { + wildcard->state = CURLWC_CLEAN; + ret = wc_statemach(conn); + } + else { + wildcard->state = CURLWC_DOWNLOADING; + ret = wc_statemach(conn); + } + } break; + + case CURLWC_CLEAN: + if(ftpc->parselist->error) { + ret = ftpc->parselist->error; + } + free_wc_data(conn); + wildcard->state = CURLWC_DONE; + break; + + case CURLWC_DONE: + break; + } + + return ret; +} + + /*********************************************************************** * * ftp_do() @@ -3461,9 +4126,21 @@ static CURLcode ftp_do(struct connectdata *conn, bool *done) if(retcode) return retcode; - retcode = ftp_parse_url_path(conn); - if(retcode) - return retcode; + if(conn->data->set.wildcardmatch) { + retcode = wc_statemach(conn); + if(conn->data->wildcard.state == CURLWC_SKIP || + conn->data->wildcard.state == CURLWC_DONE) { + /* do not call ftp_regular_transfer */ + return CURLE_OK; + } + if(retcode) /* error, loop or skipping the file */ + return retcode; + } + else { /* no wildcard FSM needed */ + retcode = ftp_parse_url_path(conn); + if(retcode) + return retcode; + } retcode = ftp_regular_transfer(conn, done); diff --git a/lib/ftp.h b/lib/ftp.h index 7a4f89e..e949fe7 100644 --- a/lib/ftp.h +++ b/lib/ftp.h @@ -107,6 +107,7 @@ struct FTP { curl_off_t downloadsize; }; +struct ftp_parselist_data; /* ftp_conn is used for struct connection-oriented data in the connectdata struct */ @@ -135,6 +136,9 @@ struct ftp_conn { int count3; /* general purpose counter for the state machine */ ftpstate state; /* always use ftp.c:state() to change state! */ char * server_os; /* The target server operating system. */ + struct ftp_parselist_data *parselist; /* help pointer needed to parse LIST */ + curl_off_t known_filesize; /* file size is different from -1, if wildcard + LIST parsing was done and wc_statemach set it */ }; #endif /* HEADER_CURL_FTP_H */ diff --git a/lib/strerror.c b/lib/strerror.c index 673e89c..8480138 100644 --- a/lib/strerror.c +++ b/lib/strerror.c @@ -275,6 +275,9 @@ curl_easy_strerror(CURLcode error) case CURLE_RTSP_SESSION_ERROR: return "RTSP session error"; + case CURLE_FTP_BAD_FILE_LIST: + return "unable to parse FTP file list"; + /* error codes not used by current libcurl */ case CURLE_OBSOLETE4: case CURLE_OBSOLETE10: diff --git a/lib/transfer.c b/lib/transfer.c index 2d50405..8b975b9 100644 --- a/lib/transfer.c +++ b/lib/transfer.c @@ -2009,12 +2009,7 @@ CURLcode Curl_retry_request(struct connectdata *conn, return CURLE_OK; } -/* - * Curl_perform() is the internal high-level function that gets called by the - * external curl_easy_perform() function. It inits, performs and cleans up a - * single file transfer. - */ -CURLcode Curl_perform(struct SessionHandle *data) +static CURLcode Curl_do_perform(struct SessionHandle *data) { CURLcode res; CURLcode res2; @@ -2049,6 +2044,15 @@ CURLcode Curl_perform(struct SessionHandle *data) res = Curl_do(&conn, &do_done); if(res == CURLE_OK) { + if(conn->data->set.wildcardmatch) { + if(conn->data->wildcard.state == CURLWC_DONE || + conn->data->wildcard.state == CURLWC_SKIP) { + /* keep connection open for application to use the socket */ + conn->bits.close = FALSE; + res = Curl_done(&conn, CURLE_OK, FALSE); + break; + } + } res = Transfer(conn); /* now fetch that URL please */ if((res == CURLE_OK) || (res == CURLE_RECV_ERROR)) { bool retry = FALSE; @@ -2166,6 +2170,26 @@ CURLcode Curl_perform(struct SessionHandle *data) } /* + * Curl_perform() is the internal high-level function that gets called by the + * external curl_easy_perform() function. It inits, performs and cleans up a + * single file transfer. + */ +CURLcode Curl_perform(struct SessionHandle *data) +{ + CURLcode res = Curl_do_perform(data); + if(!data->set.wildcardmatch) + return res; + + /* wildcard loop */ + while(!res && data->wildcard.state != CURLWC_DONE) + res = Curl_do_perform(data); + + /* wildcard download finished or failed */ + data->wildcard.state = CURLWC_INIT; + return res; +} + +/* * Curl_setup_transfer() is called to setup some basic properties for the * upcoming transfer. */ diff --git a/lib/url.c b/lib/url.c index 357f213..6a91aff 100644 --- a/lib/url.c +++ b/lib/url.c @@ -763,6 +763,10 @@ CURLcode Curl_init_userdefined(struct UserDefined *set) res = setstropt(&set->str[STRING_SSL_CAPATH], (char *) CURL_CA_PATH); #endif + set->wildcardmatch = 0L; + set->startunit = NULL; + set->endunit = NULL; + return res; } @@ -831,6 +835,9 @@ CURLcode Curl_open(struct SessionHandle **curl) data->progress.flags |= PGRS_HIDE; data->state.current_speed = -1; /* init to negative == impossible */ + data->wildcard.state = CURLWC_INIT; + data->wildcard.filelist = NULL; + data->wildcard.fnmatch = curl_fnmatch; /* This no longer creates a connection cache here. It is instead made on the first call to curl_easy_perform() or when the handle is added to a multi stack. */ @@ -2440,6 +2447,21 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option, /* Set the user defined RTP write function */ data->set.fwrite_rtp = va_arg(param, curl_write_callback); break; + + case CURLOPT_WILDCARDMATCH: + data->set.wildcardmatch = va_arg(param, long); + break; + case CURLOPT_STARTUNIT: + data->set.startunit = va_arg(param, curl_startunit_callback); + break; + case CURLOPT_ENDUNIT: + data->set.endunit = va_arg(param, curl_endunit_callback); + break; + + case CURLOPT_FNMATCH: + data->wildcard.fnmatch = va_arg(param, curl_fnmatch_callback); + break; + default: /* unknown tag and its companion, just ignore: */ result = CURLE_FAILED_INIT; /* correct this */ diff --git a/lib/urldata.h b/lib/urldata.h index ad172d6..99bb8bc 100644 --- a/lib/urldata.h +++ b/lib/urldata.h @@ -148,6 +148,7 @@ #include "ssh.h" #include "http.h" #include "rtsp.h" +#include "wildcard.h" #ifdef HAVE_GSSAPI # ifdef HAVE_GSSGNU @@ -1372,6 +1373,9 @@ struct UserDefined { /* Common RTSP header options */ Curl_RtspReq rtspreq; /* RTSP request type */ long rtspversion; /* like httpversion, for RTSP */ + bool wildcardmatch; /* enable wildcard matching */ + curl_startunit_callback startunit; /* called before transfer starts */ + curl_endunit_callback endunit; /* called after transfer stopped */ }; struct Names { @@ -1413,6 +1417,7 @@ struct SessionHandle { struct Progress progress; /* for all the progress meter data */ struct UrlState state; /* struct for fields used for state info and other dynamic purposes */ + struct WildcardData wildcard; /* wildcard download state info */ struct PureInfo info; /* stats, reports and info data */ #if defined(CURL_DOES_CONVERSIONS) && defined(HAVE_ICONV) iconv_t outbound_cd; /* for translating to the network encoding */ diff --git a/lib/wildcard.c b/lib/wildcard.c new file mode 100644 index 0000000..30847d0 --- /dev/null +++ b/lib/wildcard.c @@ -0,0 +1,58 @@ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 2010, Daniel Stenberg, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +#include +#include "wildcard.h" +#include "fileinfo.h" +#include "llist.h" + +int curl_fnmatch(const char *pattern, const char *string) +{ + return fnmatch(pattern, string, 0); +} + +CURLcode curl_wildcard_filter(struct WildcardData *wc) +{ + struct curl_llist *tmp_filelist = + Curl_llist_alloc((curl_llist_dtor)Curl_fileinfo_dtor); + if (!tmp_filelist) return CURLE_OUT_OF_MEMORY; + + while (wc->filelist->size > 0) { + Curl_fileinfo *data = Curl_fileinfo_copy(wc->filelist->head->ptr); + if (wc->fnmatch(wc->pattern, data->filename) == 0) { + Curl_llist_insert_next(tmp_filelist, tmp_filelist->tail, data); + } + else { + Curl_fileinfo_dtor(NULL, data); + } + Curl_llist_remove(wc->filelist, wc->filelist->head, NULL); + } + + Curl_llist_destroy(wc->filelist, NULL); + wc->filelist = tmp_filelist; + + if (wc->filelist->size == 0) { + return CURLE_REMOTE_FILE_NOT_FOUND; + } + else + return CURLE_OK; +} diff --git a/lib/wildcard.h b/lib/wildcard.h new file mode 100644 index 0000000..74e0adc --- /dev/null +++ b/lib/wildcard.h @@ -0,0 +1,54 @@ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 2010, Daniel Stenberg, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +#ifndef __WILDCARD_H +#define __WILDCARD_H + +#include + +/* list of wildcard process states */ +typedef enum { + CURLWC_INIT = 0, + CURLWC_MATCHING, + CURLWC_DOWNLOADING, + CURLWC_CLEAN, + CURLWC_SKIP, + CURLWC_DONE +} curl_wildcard_states; + +/* struct keeping information about wildcard download process */ +struct WildcardData { + curl_wildcard_states state; + char *path; + char *pattern; + struct curl_llist *filelist; + curl_off_t current_file_size; + curl_fnmatch_callback fnmatch; +}; + +/* default pattern matching function */ +int curl_fnmatch(const char *pattern, const char *string); + +/* on the basis of the wcd->pattern filters this function wcd->filelist */ +CURLcode curl_wildcard_filter(struct WildcardData *wcd); + +#endif /* __WILDCARD_H */ -- 1.6.6.1